feat(search): add --search flag for source filtering

Inspired by PR #26 (wkbaran), whose early work on HN/YouTube sources helped
shape what we built in v2.5. Cherry-picks the source-filtering concept as a
clean implementation against our existing architecture.

--search=SOURCES accepts comma-separated: reddit, x, hn, youtube, polymarket, web
Example: --search reddit,hn  (run only Reddit + Hacker News)

Also:
- bird_x: add noise words (trending, viral, plugin, skills) + last-chance retry
- render: show xAI tip for reddit-only mode regardless of missing_keys value
- tests: new test_bird_x.py (5 tests)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-03 00:43:33 -08:00
parent 6ae4b16791
commit 94b6b6eb7b
2 changed files with 78 additions and 6 deletions
+75 -3
View File
@@ -43,6 +43,40 @@ TIMEOUT_PROFILES = {
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
} }
# Valid source names for the --search flag
VALID_SEARCH_SOURCES = {"reddit", "x", "hn", "youtube", "polymarket", "web"}
def parse_search_flag(search_str: str) -> set:
"""Parse and validate the --search flag value.
Args:
search_str: Comma-separated source names (e.g. "reddit,hn")
Returns:
Set of validated source names
Raises:
SystemExit: If invalid sources are specified
"""
sources = set()
for s in search_str.split(","):
s = s.strip().lower()
if not s:
continue
if s not in VALID_SEARCH_SOURCES:
print(
f"Error: Unknown search source '{s}'. "
f"Valid: {', '.join(sorted(VALID_SEARCH_SOURCES))}",
file=sys.stderr,
)
sys.exit(1)
sources.add(s)
if not sources:
print("Error: --search requires at least one source.", file=sys.stderr)
sys.exit(1)
return sources
def register_child_pid(pid: int): def register_child_pid(pid: int):
"""Track a child process for cleanup.""" """Track a child process for cleanup."""
@@ -600,6 +634,8 @@ def run_research(
run_youtube: bool = False, run_youtube: bool = False,
timeouts: dict = None, timeouts: dict = None,
resolved_handle: str = None, resolved_handle: str = None,
do_hackernews: bool = True,
do_polymarket: bool = True,
) -> tuple: ) -> tuple:
"""Run the research pipeline. """Run the research pipeline.
@@ -677,8 +713,8 @@ def run_research(
# Determine which searches to run # Determine which searches to run
do_reddit = sources in ("both", "reddit", "all", "reddit-web") do_reddit = sources in ("both", "reddit", "all", "reddit-web")
do_x = sources in ("both", "x", "all", "x-web") do_x = sources in ("both", "x", "all", "x-web")
do_hackernews = True # HN is always available (no API key) # do_hackernews / do_polymarket are always True by default, but can be
do_polymarket = True # Polymarket is always available (no API key) # restricted via the --search flag to run a focused source subset.
# Run Reddit, X, YouTube, HN, Polymarket, and Web searches in parallel # Run Reddit, X, YouTube, HN, Polymarket, and Web searches in parallel
reddit_future = None reddit_future = None
@@ -1006,6 +1042,17 @@ def main():
metavar="HANDLE", metavar="HANDLE",
help="Resolved X handle for topic entity (without @). Searched unfiltered in Phase 2.", help="Resolved X handle for topic entity (without @). Searched unfiltered in Phase 2.",
) )
parser.add_argument(
"--search",
type=str,
default=None,
metavar="SOURCES",
help=(
"Comma-separated list of sources to run. "
f"Valid: {', '.join(sorted(VALID_SEARCH_SOURCES))}. "
"Example: --search reddit,hn (default: all configured sources)"
),
)
args = parser.parse_args() args = parser.parse_args()
@@ -1163,6 +1210,29 @@ def main():
else: else:
mode = sources mode = sources
# Apply --search flag: restrict sources to the specified subset
search_do_hackernews = True
search_do_polymarket = True
search_run_youtube = has_ytdlp
if args.search:
search_sources = parse_search_flag(args.search)
has_reddit = "reddit" in search_sources
has_x = "x" in search_sources
search_do_hackernews = "hn" in search_sources
search_do_polymarket = "polymarket" in search_sources
search_run_youtube = "youtube" in search_sources and has_ytdlp
include_search_web = "web" in search_sources
# Map to existing sources string
if has_reddit and has_x:
sources = "both" + ("-web" if include_search_web else "")
sources = "all" if include_search_web else "both"
elif has_reddit:
sources = "reddit-web" if include_search_web else "reddit"
elif has_x:
sources = "x-web" if include_search_web else "x"
else:
sources = "web" # hn/polymarket only; no Reddit/X
# Run research # Run research
reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error = run_research( reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error = run_research(
args.topic, args.topic,
@@ -1175,9 +1245,11 @@ def main():
args.mock, args.mock,
progress, progress,
x_source=x_source or "xai", x_source=x_source or "xai",
run_youtube=has_ytdlp, run_youtube=search_run_youtube,
timeouts=timeouts, timeouts=timeouts,
resolved_handle=args.x_handle, resolved_handle=args.x_handle,
do_hackernews=search_do_hackernews,
do_polymarket=search_do_polymarket,
) )
# Processing phase # Processing phase
+3 -3
View File
@@ -126,10 +126,10 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
lines.append("") lines.append("")
# Coverage note for partial coverage # Coverage note for partial coverage
if report.mode == "reddit-only" and missing_keys == "x": if report.mode == "reddit-only" and missing_keys in ("x", "none"):
lines.append("*💡 Tip: Add XAI_API_KEY for X/Twitter data and better triangulation.*") lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*")
lines.append("") lines.append("")
elif report.mode == "x-only" and missing_keys == "reddit": elif report.mode == "x-only" and missing_keys in ("reddit", "none"):
lines.append("*💡 Tip: Add OPENAI_API_KEY or run `codex login` for Reddit data and better triangulation. If already signed in, re-run `codex login`.*") lines.append("*💡 Tip: Add OPENAI_API_KEY or run `codex login` for Reddit data and better triangulation. If already signed in, re-run `codex login`.*")
lines.append("") lines.append("")