fix: YouTube timeout bump to 90s + Reddit 429 fail-fast
YouTube: Add youtube_future timeout key (60/90/120s for quick/default/deep) separate from the shared future timeout. YouTube needs more time because it does search + parallel transcript fetching. Previously, 20 videos + 5 transcripts exceeded the 60s budget and all results were discarded. Reddit 429: Propagate rate-limit errors instead of swallowing them. Enrichment now uses 10s timeout / 1 retry (was 30s / 3 retries). On first 429, cancel remaining enrichment and skip Phase 2 Reddit. Total time wasted on 429 drops from ~75s to ~12s. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -25,15 +25,30 @@ def extract_reddit_path(url: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def fetch_thread_data(url: str, mock_data: Optional[Dict] = None) -> Optional[Dict[str, Any]]:
|
||||
class RedditRateLimitError(Exception):
|
||||
"""Raised when Reddit returns HTTP 429 (rate limited)."""
|
||||
pass
|
||||
|
||||
|
||||
def fetch_thread_data(
|
||||
url: str,
|
||||
mock_data: Optional[Dict] = None,
|
||||
timeout: int = 30,
|
||||
retries: int = 3,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch Reddit thread JSON data.
|
||||
|
||||
Args:
|
||||
url: Reddit thread URL
|
||||
mock_data: Mock data for testing
|
||||
timeout: HTTP timeout per attempt in seconds
|
||||
retries: Number of retries on failure
|
||||
|
||||
Returns:
|
||||
Thread data dict or None on failure
|
||||
|
||||
Raises:
|
||||
RedditRateLimitError: When Reddit returns 429 (caller should bail)
|
||||
"""
|
||||
if mock_data is not None:
|
||||
return mock_data
|
||||
@@ -43,9 +58,11 @@ def fetch_thread_data(url: str, mock_data: Optional[Dict] = None) -> Optional[Di
|
||||
return None
|
||||
|
||||
try:
|
||||
data = http.get_reddit_json(path)
|
||||
data = http.get_reddit_json(path, timeout=timeout, retries=retries)
|
||||
return data
|
||||
except http.HTTPError:
|
||||
except http.HTTPError as e:
|
||||
if e.status_code == 429:
|
||||
raise RedditRateLimitError(f"Reddit rate limited (429) fetching {url}") from e
|
||||
return None
|
||||
|
||||
|
||||
@@ -178,20 +195,27 @@ def extract_comment_insights(comments: List[Dict], limit: int = 7) -> List[str]:
|
||||
def enrich_reddit_item(
|
||||
item: Dict[str, Any],
|
||||
mock_thread_data: Optional[Dict] = None,
|
||||
timeout: int = 10,
|
||||
retries: int = 1,
|
||||
) -> Dict[str, Any]:
|
||||
"""Enrich a Reddit item with real engagement data.
|
||||
|
||||
Args:
|
||||
item: Reddit item dict
|
||||
mock_thread_data: Mock data for testing
|
||||
timeout: HTTP timeout per attempt (default 10s for enrichment)
|
||||
retries: Number of retries (default 1 — fail fast for enrichment)
|
||||
|
||||
Returns:
|
||||
Enriched item dict
|
||||
|
||||
Raises:
|
||||
RedditRateLimitError: Propagated so caller can bail on remaining items
|
||||
"""
|
||||
url = item.get("url", "")
|
||||
|
||||
# Fetch thread data
|
||||
thread_data = fetch_thread_data(url, mock_thread_data)
|
||||
# Fetch thread data (RedditRateLimitError propagates to caller)
|
||||
thread_data = fetch_thread_data(url, mock_thread_data, timeout=timeout, retries=retries)
|
||||
if not thread_data:
|
||||
return item
|
||||
|
||||
|
||||
Reference in New Issue
Block a user