sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replace OpenAI Reddit search with ScrapeCreators API

Browse Source 
- New scripts/lib/reddit.py: multi-query expansion, global search,
  subreddit discovery, targeted subreddit search, comment enrichment
- 68 results in 17s vs ~15 results in 60-90s (OpenAI)
- Cost: ~$0.02/search vs $0.03-0.10 (15-50x cheaper)
- Real engagement data (score, comments, dates) from API
- No more 429 rate limits on comment enrichment
- Falls back to OpenAI if SCRAPECREATORS_API_KEY missing
- Registered as last30daysbeta for parallel local testing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-05 15:55:02 -08:00
parent db75f9e341
commit 09b09946c0
5 changed files with 725 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/last30days.py
+55 -13
View File
@@ -140,6 +140,7 @@ from lib import (
models,
normalize,
openai_reddit,
reddit,
reddit_enrich,
render,
schema,
@@ -171,19 +172,51 @@ def _search_reddit(
depth: str,
mock: bool,
) -> tuple:
"""Search Reddit via OpenAI (runs in thread).
"""Search Reddit (runs in thread).
Uses ScrapeCreators when SCRAPECREATORS_API_KEY is available (preferred).
Falls back to OpenAI Responses API otherwise.
Returns:
Tuple of (reddit_items, raw_openai, error)
Tuple of (reddit_items, raw_response, error, used_scrapecreators)
"""
raw_openai = None
raw_response = None
reddit_error = None
used_scrapecreators = False
sc_token = config.get("SCRAPECREATORS_API_KEY")
if mock:
raw_openai = load_fixture("openai_sample.json")
else:
raw_response = load_fixture("openai_sample.json")
elif sc_token:
# === ScrapeCreators path (preferred) ===
used_scrapecreators = True
try:
raw_openai = openai_reddit.search_reddit(
sys.stderr.write("[Reddit] Using ScrapeCreators API\n")
sys.stderr.flush()
result = reddit.search_and_enrich(
topic, from_date, to_date,
depth=depth, token=sc_token,
)
reddit_items = result.get("items", [])
if result.get("error"):
reddit_error = result["error"]
return reddit_items, result, reddit_error, used_scrapecreators
except Exception as e:
reddit_error = f"ScrapeCreators: {type(e).__name__}: {e}"
sys.stderr.write(f"[Reddit] ScrapeCreators failed: {e}\n")
sys.stderr.flush()
# Fall through to OpenAI if we have that key
if not config.get("OPENAI_API_KEY"):
return [], {"error": str(e)}, reddit_error, used_scrapecreators
used_scrapecreators = False
sys.stderr.write("[Reddit] Falling back to OpenAI\n")
sys.stderr.flush()
# === OpenAI path (fallback) ===
if not mock:
try:
raw_response = openai_reddit.search_reddit(
config["OPENAI_API_KEY"],
selected_models["openai"],
topic,
@@ -194,14 +227,14 @@ def _search_reddit(
account_id=config.get("OPENAI_CHATGPT_ACCOUNT_ID"),
)
except http.HTTPError as e:
raw_openai = {"error": str(e)}
raw_response = {"error": str(e)}
reddit_error = f"API error: {e}"
except Exception as e:
raw_openai = {"error": str(e)}
raw_response = {"error": str(e)}
reddit_error = f"{type(e).__name__}: {e}"
# Parse response
reddit_items = openai_reddit.parse_reddit_response(raw_openai or {})
reddit_items = openai_reddit.parse_reddit_response(raw_response or {})
# Quick retry with simpler query if few results
if len(reddit_items) < 5 and not mock and not reddit_error:
@@ -218,7 +251,6 @@ def _search_reddit(
account_id=config.get("OPENAI_CHATGPT_ACCOUNT_ID"),
)
retry_items = openai_reddit.parse_reddit_response(retry_raw)
# Add items not already found (by URL)
existing_urls = {item.get("url") for item in reddit_items}
for item in retry_items:
if item.get("url") not in existing_urls:
@@ -245,7 +277,7 @@ def _search_reddit(
except Exception:
pass
return reddit_items, raw_openai, reddit_error
return reddit_items, raw_response, reddit_error, used_scrapecreators
def _search_x(
@@ -887,10 +919,11 @@ def run_research(
)
# Collect results (with timeouts to prevent indefinite blocking)
reddit_used_sc = False # Track if ScrapeCreators was used for Reddit
if reddit_future:
reddit_timeout = timeouts.get("reddit_future", future_timeout)
try:
reddit_items, raw_openai, reddit_error = reddit_future.result(timeout=reddit_timeout)
reddit_items, raw_openai, reddit_error, reddit_used_sc = reddit_future.result(timeout=reddit_timeout)
if reddit_error and progress:
progress.show_error(f"Reddit error: {reddit_error}")
except TimeoutError:
@@ -1022,11 +1055,19 @@ def run_research(
sys.stderr.flush()
# Enrich Reddit items with real data (parallel, capped)
# Skip enrichment if ScrapeCreators already provided comments + engagement
enrich_max = timeouts["enrich_max_items"]
enrich_total_timeout = timeouts["enrich_total"]
items_to_enrich = reddit_items[:enrich_max]
rate_limited = False # Set True if Reddit returns 429 during enrichment
if reddit_used_sc and items_to_enrich:
# ScrapeCreators already enriched items with comments — just copy to raw list
sys.stderr.write(f"[Reddit] Skipping old enrichment — ScrapeCreators already provided comments\n")
sys.stderr.flush()
raw_reddit_enriched = list(reddit_items[:enrich_max])
items_to_enrich = [] # Skip the enrichment block below
if items_to_enrich:
if progress:
progress.start_reddit_enrich(1, len(items_to_enrich))
@@ -1101,11 +1142,12 @@ def run_research(
# Phase 2: Supplemental search based on entities from Phase 1
# Skip on --quick (speed matters), mock mode, or if Reddit is rate-limiting
# Also skip Reddit supplemental when ScrapeCreators was used (subreddit drilling already done)
if depth != "quick" and not mock and (reddit_items or x_items):
sup_reddit, sup_x = _run_supplemental(
topic, reddit_items, x_items,
from_date, to_date, depth, x_source, progress,
skip_reddit=rate_limited,
skip_reddit=(rate_limited or reddit_used_sc),
resolved_handle=resolved_handle,
)
if sup_reddit: