sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: YouTube timeout bump to 90s + Reddit 429 fail-fast

Browse Source 
YouTube: Add youtube_future timeout key (60/90/120s for quick/default/deep)
separate from the shared future timeout. YouTube needs more time because
it does search + parallel transcript fetching. Previously, 20 videos +
5 transcripts exceeded the 60s budget and all results were discarded.

Reddit 429: Propagate rate-limit errors instead of swallowing them.
Enrichment now uses 10s timeout / 1 retry (was 30s / 3 retries).
On first 429, cancel remaining enrichment and skip Phase 2 Reddit.
Total time wasted on 429 drops from ~75s to ~12s.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-02-15 00:48:26 -08:00
parent c3640931ed
commit 20a859ecec
4 changed files with 61 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/last30days.py
+24 -7
View File
@@ -38,9 +38,9 @@ _child_pids: set = set()
_child_pids_lock = threading.Lock()
TIMEOUT_PROFILES = {
"quick": {"global": 90, "future": 30, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
"quick": {"global": 90, "future": 30, "youtube_future": 60, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "youtube_future": 90, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "youtube_future": 120, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
}
@@ -364,6 +364,7 @@ def _run_supplemental(
depth: str,
x_source: str,
progress: ui.ProgressDisplay = None,
skip_reddit: bool = False,
) -> tuple:
"""Run Phase 2 supplemental searches based on entities from Phase 1.
@@ -379,6 +380,7 @@ def _run_supplemental(
depth: Research depth
x_source: 'bird' or 'xai'
progress: Optional progress display
skip_reddit: If True, skip Reddit supplemental (e.g. rate-limited)
Returns:
Tuple of (supplemental_reddit, supplemental_x)
@@ -401,7 +403,7 @@ def _run_supplemental(
)
has_handles = entities["x_handles"] and x_source == "bird"
has_subs = entities["reddit_subreddits"]
has_subs = entities["reddit_subreddits"] and not skip_reddit
if not has_handles and not has_subs:
return [], []
@@ -642,12 +644,13 @@ def run_research(
progress.end_x(len(x_items))
if youtube_future:
yt_timeout = timeouts.get("youtube_future", future_timeout)
try:
youtube_items, youtube_error = youtube_future.result(timeout=future_timeout)
youtube_items, youtube_error = youtube_future.result(timeout=yt_timeout)
if youtube_error and progress:
progress.show_error(f"YouTube error: {youtube_error}")
except TimeoutError:
youtube_error = f"YouTube search timed out after {future_timeout}s"
youtube_error = f"YouTube search timed out after {yt_timeout}s"
if progress:
progress.show_error(youtube_error)
except Exception as e:
@@ -677,6 +680,7 @@ def run_research(
enrich_max = timeouts["enrich_max_items"]
enrich_total_timeout = timeouts["enrich_total"]
items_to_enrich = reddit_items[:enrich_max]
rate_limited = False # Set True if Reddit returns 429 during enrichment
if items_to_enrich:
if progress:
@@ -696,7 +700,9 @@ def run_research(
raw_reddit_enriched.append(reddit_items[i])
else:
# Parallel enrichment with bounded concurrency and total timeout
# Uses short HTTP timeout (10s) and 1 retry to fail fast on 429
completed_count = 0
rate_limited = False
with ThreadPoolExecutor(max_workers=5) as enrich_pool:
futures = {
enrich_pool.submit(reddit_enrich.enrich_reddit_item, item): i
@@ -710,6 +716,16 @@ def run_research(
progress.update_reddit_enrich(completed_count, len(items_to_enrich))
try:
reddit_items[idx] = future.result(timeout=timeouts["enrich_per"])
except reddit_enrich.RedditRateLimitError:
rate_limited = True
if progress:
progress.show_error(
"Reddit rate-limited (429) — skipping remaining enrichment"
)
# Cancel remaining futures and bail
for f in futures:
f.cancel()
break
except Exception as e:
if progress:
progress.show_error(
@@ -731,11 +747,12 @@ def run_research(
progress.end_reddit_enrich()
# Phase 2: Supplemental search based on entities from Phase 1
# Skip on --quick (speed matters) and mock mode
# Skip on --quick (speed matters), mock mode, or if Reddit is rate-limiting
if depth != "quick" and not mock and (reddit_items or x_items):
sup_reddit, sup_x = _run_supplemental(
topic, reddit_items, x_items,
from_date, to_date, depth, x_source, progress,
skip_reddit=rate_limited,
)
if sup_reddit:
reddit_items.extend(sup_reddit)
scripts/lib/http.py
+4 -2
View File
@@ -124,11 +124,13 @@ def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]]
return request("POST", url, headers=headers, json_data=json_data, **kwargs)
def get_reddit_json(path: str) -> Dict[str, Any]:
def get_reddit_json(path: str, timeout: int = DEFAULT_TIMEOUT, retries: int = MAX_RETRIES) -> Dict[str, Any]:
"""Fetch Reddit thread JSON.
Args:
path: Reddit path (e.g., /r/subreddit/comments/id/title)
timeout: HTTP timeout per attempt in seconds
retries: Number of retries on failure
Returns:
Parsed JSON response
@@ -149,4 +151,4 @@ def get_reddit_json(path: str) -> Dict[str, Any]:
"Accept": "application/json",
}
return get(url, headers=headers)
return get(url, headers=headers, timeout=timeout, retries=retries)
scripts/lib/openai_reddit.py
+4 -1
View File
@@ -235,7 +235,7 @@ def search_subreddits(
"Accept": "application/json",
}
data = http.get(full_url, headers=headers, timeout=15)
data = http.get(full_url, headers=headers, timeout=15, retries=1)
# Reddit search returns {"data": {"children": [...]}}
children = data.get("data", {}).get("children", [])
@@ -267,6 +267,9 @@ def search_subreddits(
except http.HTTPError as e:
_log_info(f"Subreddit search failed for r/{sub}: {e}")
if e.status_code == 429:
_log_info("Reddit rate-limited (429) — skipping remaining subreddits")
break
except Exception as e:
_log_info(f"Subreddit search error for r/{sub}: {e}")
scripts/lib/reddit_enrich.py
+29 -5
View File
@@ -25,15 +25,30 @@ def extract_reddit_path(url: str) -> Optional[str]:
return None
def fetch_thread_data(url: str, mock_data: Optional[Dict] = None) -> Optional[Dict[str, Any]]:
class RedditRateLimitError(Exception):
"""Raised when Reddit returns HTTP 429 (rate limited)."""
pass
def fetch_thread_data(
url: str,
mock_data: Optional[Dict] = None,
timeout: int = 30,
retries: int = 3,
) -> Optional[Dict[str, Any]]:
"""Fetch Reddit thread JSON data.
Args:
url: Reddit thread URL
mock_data: Mock data for testing
timeout: HTTP timeout per attempt in seconds
retries: Number of retries on failure
Returns:
Thread data dict or None on failure
Raises:
RedditRateLimitError: When Reddit returns 429 (caller should bail)
"""
if mock_data is not None:
return mock_data
@@ -43,9 +58,11 @@ def fetch_thread_data(url: str, mock_data: Optional[Dict] = None) -> Optional[Di
return None
try:
data = http.get_reddit_json(path)
data = http.get_reddit_json(path, timeout=timeout, retries=retries)
return data
except http.HTTPError:
except http.HTTPError as e:
if e.status_code == 429:
raise RedditRateLimitError(f"Reddit rate limited (429) fetching {url}") from e
return None
@@ -178,20 +195,27 @@ def extract_comment_insights(comments: List[Dict], limit: int = 7) -> List[str]:
def enrich_reddit_item(
item: Dict[str, Any],
mock_thread_data: Optional[Dict] = None,
timeout: int = 10,
retries: int = 1,
) -> Dict[str, Any]:
"""Enrich a Reddit item with real engagement data.
Args:
item: Reddit item dict
mock_thread_data: Mock data for testing
timeout: HTTP timeout per attempt (default 10s for enrichment)
retries: Number of retries (default 1 — fail fast for enrichment)
Returns:
Enriched item dict
Raises:
RedditRateLimitError: Propagated so caller can bail on remaining items
"""
url = item.get("url", "")
# Fetch thread data
thread_data = fetch_thread_data(url, mock_thread_data)
# Fetch thread data (RedditRateLimitError propagates to caller)
thread_data = fetch_thread_data(url, mock_thread_data, timeout=timeout, retries=retries)
if not thread_data:
return item