feat: replace OpenAI Reddit search with ScrapeCreators API

- New scripts/lib/reddit.py: multi-query expansion, global search,
  subreddit discovery, targeted subreddit search, comment enrichment
- 68 results in 17s vs ~15 results in 60-90s (OpenAI)
- Cost: ~$0.02/search vs $0.03-0.10 (15-50x cheaper)
- Real engagement data (score, comments, dates) from API
- No more 429 rate limits on comment enrichment
- Falls back to OpenAI if SCRAPECREATORS_API_KEY missing
- Registered as last30daysbeta for parallel local testing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-05 15:55:02 -08:00
parent db75f9e341
commit 09b09946c0
5 changed files with 725 additions and 28 deletions
+33 -9
View File
@@ -220,18 +220,42 @@ def config_exists() -> bool:
return CONFIG_FILE.exists()
def is_reddit_available(config: Dict[str, Any]) -> bool:
"""Check if Reddit search is available.
Reddit can use either ScrapeCreators (preferred) or OpenAI.
"""
has_sc = bool(config.get('SCRAPECREATORS_API_KEY'))
has_openai = bool(config.get('OPENAI_API_KEY')) and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK
return has_sc or has_openai
def get_reddit_source(config: Dict[str, Any]) -> Optional[str]:
"""Determine which Reddit backend to use.
Priority: ScrapeCreators (cheaper, faster) > OpenAI (legacy)
Returns: 'scrapecreators', 'openai', or None
"""
if config.get('SCRAPECREATORS_API_KEY'):
return 'scrapecreators'
if config.get('OPENAI_API_KEY') and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK:
return 'openai'
return None
def get_available_sources(config: Dict[str, Any]) -> str:
"""Determine which sources are available based on API keys.
Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
"""
has_openai = bool(config.get('OPENAI_API_KEY')) and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK
has_reddit = is_reddit_available(config)
has_xai = bool(config.get('XAI_API_KEY'))
has_web = has_web_search_keys(config)
if has_openai and has_xai:
if has_reddit and has_xai:
return 'all' if has_web else 'both'
elif has_openai:
elif has_reddit:
return 'reddit-web' if has_web else 'reddit'
elif has_xai:
return 'x-web' if has_web else 'x'
@@ -263,11 +287,11 @@ def get_web_search_source(config: Dict[str, Any]) -> Optional[str]:
def get_missing_keys(config: Dict[str, Any]) -> str:
"""Determine which sources are missing (accounting for Bird).
"""Determine which sources are missing (accounting for Bird and ScrapeCreators).
Returns: 'all', 'both', 'reddit', 'x', 'web', or 'none'
"""
has_openai = bool(config.get('OPENAI_API_KEY')) and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK
has_reddit = is_reddit_available(config)
has_xai = bool(config.get('XAI_API_KEY'))
has_web = has_web_search_keys(config)
@@ -277,14 +301,14 @@ def get_missing_keys(config: Dict[str, Any]) -> str:
has_x = has_xai or has_bird
if has_openai and has_x and has_web:
if has_reddit and has_x and has_web:
return 'none'
elif has_openai and has_x:
elif has_reddit and has_x:
return 'web' # Missing web search keys
elif has_openai:
elif has_reddit:
return 'x' # Missing X source (and possibly web)
elif has_x:
return 'reddit' # Missing OpenAI key (and possibly web)
return 'reddit' # Missing Reddit source (and possibly web)
else:
return 'all' # Missing everything