feat(x): add ScrapeCreators as X/Twitter search backend
One SCRAPECREATORS_API_KEY now covers Reddit, TikTok, Instagram, AND X. Priority: Bird (free) > xAI API > ScrapeCreators (shared key). New module scrapecreators_x.py follows the same pattern as tiktok.py. Updated env.py source routing and last30days.py orchestrator dispatch. Includes 20 unit tests. Fixes #55. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+26
-6
@@ -151,6 +151,7 @@ from lib import (
|
||||
render,
|
||||
schema,
|
||||
score,
|
||||
scrapecreators_x,
|
||||
ui,
|
||||
tiktok,
|
||||
instagram,
|
||||
@@ -357,6 +358,25 @@ def _search_x(
|
||||
|
||||
return x_items, raw_response, x_error
|
||||
|
||||
# Use ScrapeCreators if specified
|
||||
if x_source == "scrapecreators":
|
||||
try:
|
||||
raw_response = scrapecreators_x.search_x(
|
||||
topic, from_date, to_date,
|
||||
depth=depth,
|
||||
token=config.get("SCRAPECREATORS_API_KEY"),
|
||||
)
|
||||
except Exception as e:
|
||||
raw_response = {"error": str(e)}
|
||||
x_error = f"{type(e).__name__}: {e}"
|
||||
|
||||
x_items = scrapecreators_x.parse_x_response(raw_response or {})
|
||||
|
||||
if raw_response and isinstance(raw_response, dict) and raw_response.get("error") and not x_error:
|
||||
x_error = raw_response["error"]
|
||||
|
||||
return x_items, raw_response, x_error
|
||||
|
||||
# Use xAI (original behavior)
|
||||
try:
|
||||
raw_response = xai_x.search_x(
|
||||
@@ -1481,14 +1501,14 @@ def main():
|
||||
# Check available sources (accounting for Bird auto-detection)
|
||||
available = env.get_available_sources(config)
|
||||
|
||||
# Override available if Bird is ready
|
||||
if x_source == 'bird':
|
||||
# Override available if Bird or ScrapeCreators provides X
|
||||
if x_source in ('bird', 'scrapecreators'):
|
||||
if available == 'reddit':
|
||||
available = 'both' # Now have both Reddit + X (via Bird)
|
||||
available = 'both' # Now have both Reddit + X
|
||||
elif available == 'reddit-web':
|
||||
available = 'all' # Reddit + X (via Bird) + Web
|
||||
available = 'all' # Reddit + X + Web
|
||||
elif available == 'web':
|
||||
available = 'x-web' # X via Bird + Web
|
||||
available = 'x-web' # X + Web
|
||||
|
||||
# Mock mode can work without keys
|
||||
if args.mock:
|
||||
@@ -1724,7 +1744,7 @@ def main():
|
||||
if x_source_status["bird_installed"]:
|
||||
source_info["x_skip_reason"] = "Bird installed but not authenticated — log into x.com in browser"
|
||||
else:
|
||||
source_info["x_skip_reason"] = "No Bird CLI or XAI_API_KEY (Node.js 22+ needed for Bird)"
|
||||
source_info["x_skip_reason"] = "No Bird CLI, XAI_API_KEY, or SCRAPECREATORS_API_KEY"
|
||||
if not has_ytdlp:
|
||||
source_info["youtube_skip_reason"] = "yt-dlp not installed — fix: brew install yt-dlp"
|
||||
elif has_ytdlp and not report.youtube:
|
||||
|
||||
Reference in New Issue
Block a user