fix: YouTube timeout bump to 90s + Reddit 429 fail-fast

YouTube: Add youtube_future timeout key (60/90/120s for quick/default/deep)
separate from the shared future timeout. YouTube needs more time because
it does search + parallel transcript fetching. Previously, 20 videos +
5 transcripts exceeded the 60s budget and all results were discarded.

Reddit 429: Propagate rate-limit errors instead of swallowing them.
Enrichment now uses 10s timeout / 1 retry (was 30s / 3 retries).
On first 429, cancel remaining enrichment and skip Phase 2 Reddit.
Total time wasted on 429 drops from ~75s to ~12s.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-02-15 00:48:26 -08:00
parent c3640931ed
commit 20a859ecec
4 changed files with 61 additions and 15 deletions
+29 -5
View File
@@ -25,15 +25,30 @@ def extract_reddit_path(url: str) -> Optional[str]:
return None
def fetch_thread_data(url: str, mock_data: Optional[Dict] = None) -> Optional[Dict[str, Any]]:
class RedditRateLimitError(Exception):
"""Raised when Reddit returns HTTP 429 (rate limited)."""
pass
def fetch_thread_data(
url: str,
mock_data: Optional[Dict] = None,
timeout: int = 30,
retries: int = 3,
) -> Optional[Dict[str, Any]]:
"""Fetch Reddit thread JSON data.
Args:
url: Reddit thread URL
mock_data: Mock data for testing
timeout: HTTP timeout per attempt in seconds
retries: Number of retries on failure
Returns:
Thread data dict or None on failure
Raises:
RedditRateLimitError: When Reddit returns 429 (caller should bail)
"""
if mock_data is not None:
return mock_data
@@ -43,9 +58,11 @@ def fetch_thread_data(url: str, mock_data: Optional[Dict] = None) -> Optional[Di
return None
try:
data = http.get_reddit_json(path)
data = http.get_reddit_json(path, timeout=timeout, retries=retries)
return data
except http.HTTPError:
except http.HTTPError as e:
if e.status_code == 429:
raise RedditRateLimitError(f"Reddit rate limited (429) fetching {url}") from e
return None
@@ -178,20 +195,27 @@ def extract_comment_insights(comments: List[Dict], limit: int = 7) -> List[str]:
def enrich_reddit_item(
item: Dict[str, Any],
mock_thread_data: Optional[Dict] = None,
timeout: int = 10,
retries: int = 1,
) -> Dict[str, Any]:
"""Enrich a Reddit item with real engagement data.
Args:
item: Reddit item dict
mock_thread_data: Mock data for testing
timeout: HTTP timeout per attempt (default 10s for enrichment)
retries: Number of retries (default 1 — fail fast for enrichment)
Returns:
Enriched item dict
Raises:
RedditRateLimitError: Propagated so caller can bail on remaining items
"""
url = item.get("url", "")
# Fetch thread data
thread_data = fetch_thread_data(url, mock_thread_data)
# Fetch thread data (RedditRateLimitError propagates to caller)
thread_data = fetch_thread_data(url, mock_thread_data, timeout=timeout, retries=retries)
if not thread_data:
return item