feat: replace OpenAI Reddit search with ScrapeCreators API

- New scripts/lib/reddit.py: multi-query expansion, global search,
  subreddit discovery, targeted subreddit search, comment enrichment
- 68 results in 17s vs ~15 results in 60-90s (OpenAI)
- Cost: ~$0.02/search vs $0.03-0.10 (15-50x cheaper)
- Real engagement data (score, comments, dates) from API
- No more 429 rate limits on comment enrichment
- Falls back to OpenAI if SCRAPECREATORS_API_KEY missing
- Registered as last30daysbeta for parallel local testing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-05 15:55:02 -08:00
parent db75f9e341
commit 09b09946c0
5 changed files with 725 additions and 28 deletions
+70 -1
View File
@@ -1,4 +1,9 @@
"""Reddit thread enrichment with real engagement metrics."""
"""Reddit thread enrichment with real engagement metrics.
Supports two backends:
1. ScrapeCreators API (preferred) - no rate limits, 1 credit/call
2. reddit.com/.json (fallback) - free but 429-prone
"""
import re
from typing import Any, Dict, List, Optional
@@ -254,3 +259,67 @@ def enrich_reddit_item(
item["comment_insights"] = extract_comment_insights(top_comments)
return item
def enrich_reddit_item_sc(
item: Dict[str, Any],
token: str,
timeout: int = 30,
) -> Dict[str, Any]:
"""Enrich a Reddit item using ScrapeCreators comment API.
No rate limit risk. Uses 1 credit per call.
Args:
item: Reddit item dict (already has engagement from search)
token: ScrapeCreators API key
timeout: HTTP timeout
Returns:
Enriched item with top_comments and comment_insights
"""
from . import reddit as reddit_mod
url = item.get("url", "")
if not url:
return item
raw_comments = reddit_mod.fetch_post_comments(url, token)
if not raw_comments:
return item
top_comments = []
for c in raw_comments[:10]:
body = c.get("body", "")
if not body or body in ("[deleted]", "[removed]"):
continue
score = c.get("ups") or c.get("score", 0)
author = c.get("author", "[deleted]")
permalink = c.get("permalink", "")
comment_url = f"https://reddit.com{permalink}" if permalink else ""
top_comments.append({
"score": score,
"date": dates.timestamp_to_date(c.get("created_utc")) if c.get("created_utc") else None,
"author": author,
"body": body[:300],
"excerpt": body[:200],
"url": comment_url,
})
top_comments.sort(key=lambda c: c.get("score", 0), reverse=True)
item["top_comments"] = []
for c in top_comments:
item["top_comments"].append({
"score": c.get("score", 0),
"date": c.get("date"),
"author": c.get("author", ""),
"excerpt": c.get("excerpt", ""),
"url": c.get("url", ""),
})
item["comment_insights"] = extract_comment_insights(top_comments)
return item