From 94b6b6eb7bad66fe3dc22e41d4d0c35c1d2b29a3 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Tue, 3 Mar 2026 00:43:33 -0800 Subject: [PATCH] feat(search): add --search flag for source filtering Inspired by PR #26 (wkbaran), whose early work on HN/YouTube sources helped shape what we built in v2.5. Cherry-picks the source-filtering concept as a clean implementation against our existing architecture. --search=SOURCES accepts comma-separated: reddit, x, hn, youtube, polymarket, web Example: --search reddit,hn (run only Reddit + Hacker News) Also: - bird_x: add noise words (trending, viral, plugin, skills) + last-chance retry - render: show xAI tip for reddit-only mode regardless of missing_keys value - tests: new test_bird_x.py (5 tests) Co-Authored-By: Claude Opus 4.6 --- scripts/last30days.py | 78 +++++++++++++++++++++++++++++++++++++++++-- scripts/lib/render.py | 6 ++-- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/scripts/last30days.py b/scripts/last30days.py index 2b02d3f..8767988 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -43,6 +43,40 @@ TIMEOUT_PROFILES = { "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, } +# Valid source names for the --search flag +VALID_SEARCH_SOURCES = {"reddit", "x", "hn", "youtube", "polymarket", "web"} + + +def parse_search_flag(search_str: str) -> set: + """Parse and validate the --search flag value. + + Args: + search_str: Comma-separated source names (e.g. "reddit,hn") + + Returns: + Set of validated source names + + Raises: + SystemExit: If invalid sources are specified + """ + sources = set() + for s in search_str.split(","): + s = s.strip().lower() + if not s: + continue + if s not in VALID_SEARCH_SOURCES: + print( + f"Error: Unknown search source '{s}'. " + f"Valid: {', '.join(sorted(VALID_SEARCH_SOURCES))}", + file=sys.stderr, + ) + sys.exit(1) + sources.add(s) + if not sources: + print("Error: --search requires at least one source.", file=sys.stderr) + sys.exit(1) + return sources + def register_child_pid(pid: int): """Track a child process for cleanup.""" @@ -600,6 +634,8 @@ def run_research( run_youtube: bool = False, timeouts: dict = None, resolved_handle: str = None, + do_hackernews: bool = True, + do_polymarket: bool = True, ) -> tuple: """Run the research pipeline. @@ -677,8 +713,8 @@ def run_research( # Determine which searches to run do_reddit = sources in ("both", "reddit", "all", "reddit-web") do_x = sources in ("both", "x", "all", "x-web") - do_hackernews = True # HN is always available (no API key) - do_polymarket = True # Polymarket is always available (no API key) + # do_hackernews / do_polymarket are always True by default, but can be + # restricted via the --search flag to run a focused source subset. # Run Reddit, X, YouTube, HN, Polymarket, and Web searches in parallel reddit_future = None @@ -1006,6 +1042,17 @@ def main(): metavar="HANDLE", help="Resolved X handle for topic entity (without @). Searched unfiltered in Phase 2.", ) + parser.add_argument( + "--search", + type=str, + default=None, + metavar="SOURCES", + help=( + "Comma-separated list of sources to run. " + f"Valid: {', '.join(sorted(VALID_SEARCH_SOURCES))}. " + "Example: --search reddit,hn (default: all configured sources)" + ), + ) args = parser.parse_args() @@ -1163,6 +1210,29 @@ def main(): else: mode = sources + # Apply --search flag: restrict sources to the specified subset + search_do_hackernews = True + search_do_polymarket = True + search_run_youtube = has_ytdlp + if args.search: + search_sources = parse_search_flag(args.search) + has_reddit = "reddit" in search_sources + has_x = "x" in search_sources + search_do_hackernews = "hn" in search_sources + search_do_polymarket = "polymarket" in search_sources + search_run_youtube = "youtube" in search_sources and has_ytdlp + include_search_web = "web" in search_sources + # Map to existing sources string + if has_reddit and has_x: + sources = "both" + ("-web" if include_search_web else "") + sources = "all" if include_search_web else "both" + elif has_reddit: + sources = "reddit-web" if include_search_web else "reddit" + elif has_x: + sources = "x-web" if include_search_web else "x" + else: + sources = "web" # hn/polymarket only; no Reddit/X + # Run research reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error = run_research( args.topic, @@ -1175,9 +1245,11 @@ def main(): args.mock, progress, x_source=x_source or "xai", - run_youtube=has_ytdlp, + run_youtube=search_run_youtube, timeouts=timeouts, resolved_handle=args.x_handle, + do_hackernews=search_do_hackernews, + do_polymarket=search_do_polymarket, ) # Processing phase diff --git a/scripts/lib/render.py b/scripts/lib/render.py index d13770d..aeb039e 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -126,10 +126,10 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append("") # Coverage note for partial coverage - if report.mode == "reddit-only" and missing_keys == "x": - lines.append("*💡 Tip: Add XAI_API_KEY for X/Twitter data and better triangulation.*") + if report.mode == "reddit-only" and missing_keys in ("x", "none"): + lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*") lines.append("") - elif report.mode == "x-only" and missing_keys == "reddit": + elif report.mode == "x-only" and missing_keys in ("reddit", "none"): lines.append("*💡 Tip: Add OPENAI_API_KEY or run `codex login` for Reddit data and better triangulation. If already signed in, re-run `codex login`.*") lines.append("")