Tweepy: Scraping Twitter After the API Price Hike

import requests
from bs4 import BeautifulSoup

def scrape_with_nitter(username):
    """
    Use public Nitter instances to avoid API entirely
    Note: Instances go down frequently, need fallback list
    """
    instances = [
        "nitter.net",
        "nitter.poast.org",
        "nitter.privacydev.net"
    ]

    for instance in instances:
        try:
            url = f"https://{instance}/{username}"
            headers = {'User-Agent': 'Mozilla/5.0'}
            response = requests.get(url, headers=headers, timeout=10)

            if response.status_code == 200:
                soup = BeautifulSoup(response.text, 'html.parser')
                # Parse tweets from HTML
                tweets = soup.find_all('div', class_='timeline-item')
                return [extract_tweet_data(t) for t in tweets]

        except Exception as e:
            continue

    return None

import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
import time

def scrape_with_selenium(url):
    """
    Only use for small-scale scraping
    Twitter detects automation quickly
    """
    driver = uc.Chrome()
    driver.get(url)

    # Wait for manual login if needed
    time.sleep(10)

    # Scroll to load more tweets
    for _ in range(5):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)

    # Extract data
    tweets = driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
    # Parse and return...

    driver.quit()

# twitter_scraper.py - Production configuration

import tweepy
import time
from typing import List, Generator
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class TwitterScraper:
    def __init__(self, bearer_token: str, consumer_key: str = None,
                 consumer_secret: str = None, access_token: str = None,
                 access_token_secret: str = None):
        """
        Initialize with OAuth 1.0a user context for better rate limits
        Falls back to bearer-only if user context not provided
        """
        self.client = tweepy.Client(
            bearer_token=bearer_token,
            consumer_key=consumer_key,
            consumer_secret=consumer_secret,
            access_token=access_token,
            access_token_secret=access_token_secret,
            wait_on_rate_limit=True
        )

    def fetch_tweets(self, query: str, max_tweets: int = 1000) -> Generator:
        """
        Fetch tweets with built-in rate limit protection

        Args:
            query: Search query
            max_tweets: Maximum tweets to fetch

        Yields:
            Tweet objects
        """
        collected = 0
        next_token = None

        while collected < max_tweets:
            try:
                response = self.client.search_recent_tweets(
                    query=query,
                    max_results=min(100, max_tweets - collected),
                    next_token=next_token,
                    tweet_fields=['created_at', 'public_metrics', 'author_id']
                )

                if not response.data:
                    logger.info("No more tweets available")
                    break

                for tweet in response.data:
                    yield tweet
                    collected += 1

                next_token = response.meta.get('next_token')
                if not next_token:
                    break

                # Conservative delay between requests
                time.sleep(3)

            except tweepy.errors.TooManyRequests:
                logger.warning("Rate limit hit, waiting 15 min")
                time.sleep(900)

            except Exception as e:
                logger.error(f"Error fetching tweets: {e}")
                time.sleep(60)

# Usage
if __name__ == "__main__":
    scraper = TwitterScraper(
        bearer_token="YOUR_TOKEN",
        consumer_key="YOUR_KEY",
        consumer_secret="YOUR_SECRET"
    )

    for tweet in scraper.fetch_tweets("python programming", max_tweets=500):
        print(f"@{tweet.author_id}: {tweet.text[:100]}...")

# Check your current rate limit status
curl -X GET "https://api.twitter.com/2/users/me?user.fields=public_metrics" \
  -H "Authorization: Bearer $BEARER_TOKEN"

# Test authentication
curl -X GET "https://api.twitter.com/2/tweets/search/recent?query=test" \
  -H "Authorization: Bearer $BEARER_TOKEN"

# Monitor headers for rate limit info
# Look for: x-rate-limit-remaining, x-rate-limit-reset