feat(search): add --search flag for source filtering
Inspired by PR #26 (wkbaran), whose early work on HN/YouTube sources helped shape what we built in v2.5. Cherry-picks the source-filtering concept as a clean implementation against our existing architecture. --search=SOURCES accepts comma-separated: reddit, x, hn, youtube, polymarket, web Example: --search reddit,hn (run only Reddit + Hacker News) Also: - bird_x: add noise words (trending, viral, plugin, skills) + last-chance retry - render: show xAI tip for reddit-only mode regardless of missing_keys value - tests: new test_bird_x.py (5 tests) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+75
-3
@@ -43,6 +43,40 @@ TIMEOUT_PROFILES = {
|
|||||||
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Valid source names for the --search flag
|
||||||
|
VALID_SEARCH_SOURCES = {"reddit", "x", "hn", "youtube", "polymarket", "web"}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_search_flag(search_str: str) -> set:
|
||||||
|
"""Parse and validate the --search flag value.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
search_str: Comma-separated source names (e.g. "reddit,hn")
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Set of validated source names
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
SystemExit: If invalid sources are specified
|
||||||
|
"""
|
||||||
|
sources = set()
|
||||||
|
for s in search_str.split(","):
|
||||||
|
s = s.strip().lower()
|
||||||
|
if not s:
|
||||||
|
continue
|
||||||
|
if s not in VALID_SEARCH_SOURCES:
|
||||||
|
print(
|
||||||
|
f"Error: Unknown search source '{s}'. "
|
||||||
|
f"Valid: {', '.join(sorted(VALID_SEARCH_SOURCES))}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
sources.add(s)
|
||||||
|
if not sources:
|
||||||
|
print("Error: --search requires at least one source.", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return sources
|
||||||
|
|
||||||
|
|
||||||
def register_child_pid(pid: int):
|
def register_child_pid(pid: int):
|
||||||
"""Track a child process for cleanup."""
|
"""Track a child process for cleanup."""
|
||||||
@@ -600,6 +634,8 @@ def run_research(
|
|||||||
run_youtube: bool = False,
|
run_youtube: bool = False,
|
||||||
timeouts: dict = None,
|
timeouts: dict = None,
|
||||||
resolved_handle: str = None,
|
resolved_handle: str = None,
|
||||||
|
do_hackernews: bool = True,
|
||||||
|
do_polymarket: bool = True,
|
||||||
) -> tuple:
|
) -> tuple:
|
||||||
"""Run the research pipeline.
|
"""Run the research pipeline.
|
||||||
|
|
||||||
@@ -677,8 +713,8 @@ def run_research(
|
|||||||
# Determine which searches to run
|
# Determine which searches to run
|
||||||
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
|
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
|
||||||
do_x = sources in ("both", "x", "all", "x-web")
|
do_x = sources in ("both", "x", "all", "x-web")
|
||||||
do_hackernews = True # HN is always available (no API key)
|
# do_hackernews / do_polymarket are always True by default, but can be
|
||||||
do_polymarket = True # Polymarket is always available (no API key)
|
# restricted via the --search flag to run a focused source subset.
|
||||||
|
|
||||||
# Run Reddit, X, YouTube, HN, Polymarket, and Web searches in parallel
|
# Run Reddit, X, YouTube, HN, Polymarket, and Web searches in parallel
|
||||||
reddit_future = None
|
reddit_future = None
|
||||||
@@ -1006,6 +1042,17 @@ def main():
|
|||||||
metavar="HANDLE",
|
metavar="HANDLE",
|
||||||
help="Resolved X handle for topic entity (without @). Searched unfiltered in Phase 2.",
|
help="Resolved X handle for topic entity (without @). Searched unfiltered in Phase 2.",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--search",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
metavar="SOURCES",
|
||||||
|
help=(
|
||||||
|
"Comma-separated list of sources to run. "
|
||||||
|
f"Valid: {', '.join(sorted(VALID_SEARCH_SOURCES))}. "
|
||||||
|
"Example: --search reddit,hn (default: all configured sources)"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
@@ -1163,6 +1210,29 @@ def main():
|
|||||||
else:
|
else:
|
||||||
mode = sources
|
mode = sources
|
||||||
|
|
||||||
|
# Apply --search flag: restrict sources to the specified subset
|
||||||
|
search_do_hackernews = True
|
||||||
|
search_do_polymarket = True
|
||||||
|
search_run_youtube = has_ytdlp
|
||||||
|
if args.search:
|
||||||
|
search_sources = parse_search_flag(args.search)
|
||||||
|
has_reddit = "reddit" in search_sources
|
||||||
|
has_x = "x" in search_sources
|
||||||
|
search_do_hackernews = "hn" in search_sources
|
||||||
|
search_do_polymarket = "polymarket" in search_sources
|
||||||
|
search_run_youtube = "youtube" in search_sources and has_ytdlp
|
||||||
|
include_search_web = "web" in search_sources
|
||||||
|
# Map to existing sources string
|
||||||
|
if has_reddit and has_x:
|
||||||
|
sources = "both" + ("-web" if include_search_web else "")
|
||||||
|
sources = "all" if include_search_web else "both"
|
||||||
|
elif has_reddit:
|
||||||
|
sources = "reddit-web" if include_search_web else "reddit"
|
||||||
|
elif has_x:
|
||||||
|
sources = "x-web" if include_search_web else "x"
|
||||||
|
else:
|
||||||
|
sources = "web" # hn/polymarket only; no Reddit/X
|
||||||
|
|
||||||
# Run research
|
# Run research
|
||||||
reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error = run_research(
|
reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error = run_research(
|
||||||
args.topic,
|
args.topic,
|
||||||
@@ -1175,9 +1245,11 @@ def main():
|
|||||||
args.mock,
|
args.mock,
|
||||||
progress,
|
progress,
|
||||||
x_source=x_source or "xai",
|
x_source=x_source or "xai",
|
||||||
run_youtube=has_ytdlp,
|
run_youtube=search_run_youtube,
|
||||||
timeouts=timeouts,
|
timeouts=timeouts,
|
||||||
resolved_handle=args.x_handle,
|
resolved_handle=args.x_handle,
|
||||||
|
do_hackernews=search_do_hackernews,
|
||||||
|
do_polymarket=search_do_polymarket,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Processing phase
|
# Processing phase
|
||||||
|
|||||||
@@ -126,10 +126,10 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
|||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# Coverage note for partial coverage
|
# Coverage note for partial coverage
|
||||||
if report.mode == "reddit-only" and missing_keys == "x":
|
if report.mode == "reddit-only" and missing_keys in ("x", "none"):
|
||||||
lines.append("*💡 Tip: Add XAI_API_KEY for X/Twitter data and better triangulation.*")
|
lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
elif report.mode == "x-only" and missing_keys == "reddit":
|
elif report.mode == "x-only" and missing_keys in ("reddit", "none"):
|
||||||
lines.append("*💡 Tip: Add OPENAI_API_KEY or run `codex login` for Reddit data and better triangulation. If already signed in, re-run `codex login`.*")
|
lines.append("*💡 Tip: Add OPENAI_API_KEY or run `codex login` for Reddit data and better triangulation. If already signed in, re-run `codex login`.*")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user