feat(engine): add native web search, --store, and --diagnose to main engine

- _search_web() dispatches to Parallel AI / Brave / OpenRouter based on config
- Web results flow through full pipeline: normalize → score → dedupe
- --diagnose shows all source availability (API keys, Bird, YouTube, web backends)
- --store persists findings to SQLite via store.py for watchlist/briefing system
- run_research() now returns web_items alongside reddit/x/youtube
- web_needed flag only set when no native web backend is available

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

scripts/last30days.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-last30days - Research a topic from the last 30 days on Reddit + X.
+last30days - Research a topic from the last 30 days on Reddit + X + YouTube + Web.
 
 Usage:
     python3 last30days.py <topic> [options]
@@ -12,6 +12,8 @@ Options:
     --quick             Faster research with fewer sources (8-12 each)
     --deep              Comprehensive research with more sources (50-70 Reddit, 40-60 X)
     --debug             Enable verbose debug logging
+    --store             Persist findings to SQLite database
+    --diagnose          Show source availability diagnostics and exit
 """
 
 import argparse
@@ -235,6 +237,58 @@ def _search_youtube(
     return youtube_items, youtube_error
 
 
+def _search_web(
+    topic: str,
+    config: dict,
+    from_date: str,
+    to_date: str,
+    depth: str,
+) -> tuple:
+    """Search the web via native API backend (runs in thread).
+
+    Uses the best available backend: Parallel AI > Brave > OpenRouter.
+
+    Returns:
+        Tuple of (web_items, web_error)
+        web_items are raw dicts ready for websearch.normalize_websearch_items()
+    """
+    from lib import brave_search, parallel_search, openrouter_search
+
+    backend = env.get_web_search_source(config)
+    if not backend:
+        return [], "No web search API keys configured"
+
+    web_error = None
+    raw_results = []
+
+    try:
+        if backend == "parallel":
+            raw_results = parallel_search.search_web(
+                topic, from_date, to_date, config["PARALLEL_API_KEY"], depth=depth,
+            )
+        elif backend == "brave":
+            raw_results = brave_search.search_web(
+                topic, from_date, to_date, config["BRAVE_API_KEY"], depth=depth,
+            )
+        elif backend == "openrouter":
+            raw_results = openrouter_search.search_web(
+                topic, from_date, to_date, config["OPENROUTER_API_KEY"], depth=depth,
+            )
+    except Exception as e:
+        return [], f"{type(e).__name__}: {e}"
+
+    # Add IDs and date_confidence for websearch.normalize_websearch_items()
+    for i, item in enumerate(raw_results):
+        item.setdefault("id", f"W{i+1}")
+        if item.get("date") and not item.get("date_confidence"):
+            item["date_confidence"] = "med"
+        elif not item.get("date"):
+            item["date_confidence"] = "low"
+        item.setdefault("why_relevant", "")
+
+    return raw_results, web_error
+
+
 def _run_supplemental(
     topic: str,
     reddit_items: list,
@@ -374,26 +428,49 @@ def run_research(
     """Run the research pipeline.
 
     Returns:
-        Tuple of (reddit_items, x_items, youtube_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error)
+        Tuple of (reddit_items, x_items, youtube_items, web_items, web_needed,
+                  raw_openai, raw_xai, raw_reddit_enriched,
+                  reddit_error, x_error, youtube_error, web_error)
 
-    Note: web_needed is True when web search should be performed by the assistant.
+    Note: web_needed is True when web search should be performed by the assistant
-    The script outputs a marker and the assistant handles web search in its session.
+    (i.e., no native web search API keys are configured). When native web search
+    runs, web_items will be populated and web_needed will be False.
     """
     reddit_items = []
     x_items = []
     youtube_items = []
+    web_items = []
     raw_openai = None
     raw_xai = None
     raw_reddit_enriched = []
     reddit_error = None
     x_error = None
     youtube_error = None
+    web_error = None
 
-    # Check if WebSearch is needed (always needed in web-only mode)
+    # Determine web search mode
-    web_needed = sources in ("all", "web", "reddit-web", "x-web")
+    do_web = sources in ("all", "web", "reddit-web", "x-web")
+    web_backend = env.get_web_search_source(config) if do_web else None
+    web_needed = do_web and not web_backend
 
-    # Web-only mode: no API calls needed, assistant handles everything
+    # Web-only mode
     if sources == "web":
+        if web_backend:
+            # Native web search available — run it
+            sys.stderr.write(f"[web] Searching via {web_backend}\n")
+            sys.stderr.flush()
+            try:
+                web_items, web_error = _search_web(topic, config, from_date, to_date, depth)
+                if web_error and progress:
+                    progress.show_error(f"Web error: {web_error}")
+            except Exception as e:
+                web_error = f"{type(e).__name__}: {e}"
+                if progress:
+                    progress.show_error(f"Web error: {e}")
+            sys.stderr.write(f"[web] {len(web_items)} results\n")
+            sys.stderr.flush()
+        else:
+            # No native backend — assistant handles WebSearch
             if progress:
                 progress.start_web_only()
                 progress.end_web_only()
@@ -411,17 +488,18 @@ def run_research(
                     progress.show_error(f"YouTube error: {e}")
             if progress:
                 progress.end_youtube(len(youtube_items))
-        return reddit_items, x_items, youtube_items, True, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error
+        return reddit_items, x_items, youtube_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, web_error
 
     # Determine which searches to run
     do_reddit = sources in ("both", "reddit", "all", "reddit-web")
     do_x = sources in ("both", "x", "all", "x-web")
 
-    # Run Reddit, X, and YouTube searches in parallel
+    # Run Reddit, X, YouTube, and Web searches in parallel
     reddit_future = None
     x_future = None
     youtube_future = None
-    max_workers = 2 + (1 if run_youtube else 0)
+    web_future = None
+    max_workers = 2 + (1 if run_youtube else 0) + (1 if web_backend else 0)
 
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
         # Submit searches
@@ -448,6 +526,13 @@ def run_research(
                 _search_youtube, topic, from_date, to_date, depth
             )
 
+        if web_backend:
+            sys.stderr.write(f"[web] Searching via {web_backend}\n")
+            sys.stderr.flush()
+            web_future = executor.submit(
+                _search_web, topic, config, from_date, to_date, depth
+            )
+
         # Collect results
         if reddit_future:
             try:
@@ -485,6 +570,18 @@ def run_research(
             if progress:
                 progress.end_youtube(len(youtube_items))
 
+        if web_future:
+            try:
+                web_items, web_error = web_future.result()
+                if web_error and progress:
+                    progress.show_error(f"Web error: {web_error}")
+            except Exception as e:
+                web_error = f"{type(e).__name__}: {e}"
+                if progress:
+                    progress.show_error(f"Web error: {e}")
+            sys.stderr.write(f"[web] {len(web_items)} results\n")
+            sys.stderr.flush()
+
     # Enrich Reddit items with real data (sequential, but with error handling per-item)
     if reddit_items:
         if progress:
@@ -522,7 +619,7 @@ def run_research(
         if sup_x:
             x_items.extend(sup_x)
 
-    return reddit_items, x_items, youtube_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error
+    return reddit_items, x_items, youtube_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, web_error
 
 
 def main():
@@ -576,6 +673,16 @@ def main():
         metavar="N",
         help="Number of days to look back (1-30, default: 30)",
     )
+    parser.add_argument(
+        "--store",
+        action="store_true",
+        help="Persist findings to SQLite database (~/.local/share/last30days/research.db)",
+    )
+    parser.add_argument(
+        "--diagnose",
+        action="store_true",
+        help="Show source availability diagnostics and exit",
+    )
 
     args = parser.parse_args()
 
@@ -597,12 +704,6 @@ def main():
     else:
         depth = "default"
 
-    # Validate topic first (matches original NUX)
-    if not args.topic:
-        print("Error: Please provide a topic to research.", file=sys.stderr)
-        print("Usage: python3 last30days.py <topic> [options]", file=sys.stderr)
-        sys.exit(1)
-
     # Load config
     config = env.get_config()
 
@@ -613,6 +714,31 @@ def main():
     # Auto-detect yt-dlp for YouTube search
     has_ytdlp = env.is_ytdlp_available()
 
+    # --diagnose: show source availability and exit
+    if args.diagnose:
+        web_source = env.get_web_search_source(config)
+        diag = {
+            "openai": bool(config.get("OPENAI_API_KEY")),
+            "xai": bool(config.get("XAI_API_KEY")),
+            "x_source": x_source_status["source"],
+            "bird_installed": x_source_status["bird_installed"],
+            "bird_authenticated": x_source_status["bird_authenticated"],
+            "bird_username": x_source_status.get("bird_username"),
+            "youtube": has_ytdlp,
+            "web_search_backend": web_source,
+            "parallel_ai": bool(config.get("PARALLEL_API_KEY")),
+            "brave": bool(config.get("BRAVE_API_KEY")),
+            "openrouter": bool(config.get("OPENROUTER_API_KEY")),
+        }
+        print(json.dumps(diag, indent=2))
+        sys.exit(0)
+
+    # Validate topic (--diagnose doesn't need one)
+    if not args.topic:
+        print("Error: Please provide a topic to research.", file=sys.stderr)
+        print("Usage: python3 last30days.py <topic> [options]", file=sys.stderr)
+        sys.exit(1)
+
     # Initialize progress display with topic
     progress = ui.ProgressDisplay(args.topic, show_banner=True)
 
@@ -689,7 +815,7 @@ def main():
         mode = sources
 
     # Run research
-    reddit_items, x_items, youtube_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error = run_research(
+    reddit_items, x_items, youtube_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, web_error = run_research(
         args.topic,
         sources,
         config,
@@ -710,27 +836,32 @@ def main():
     normalized_reddit = normalize.normalize_reddit_items(reddit_items, from_date, to_date)
     normalized_x = normalize.normalize_x_items(x_items, from_date, to_date)
     normalized_youtube = normalize.normalize_youtube_items(youtube_items, from_date, to_date) if youtube_items else []
+    normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
 
     # Hard date filter: exclude items with verified dates outside the range
     # This is the safety net - even if prompts let old content through, this filters it
     filtered_reddit = normalize.filter_by_date_range(normalized_reddit, from_date, to_date)
     filtered_x = normalize.filter_by_date_range(normalized_x, from_date, to_date)
     filtered_youtube = normalize.filter_by_date_range(normalized_youtube, from_date, to_date) if normalized_youtube else []
+    filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else []
 
     # Score items
     scored_reddit = score.score_reddit_items(filtered_reddit)
     scored_x = score.score_x_items(filtered_x)
     scored_youtube = score.score_youtube_items(filtered_youtube) if filtered_youtube else []
+    scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
 
     # Sort items
     sorted_reddit = score.sort_items(scored_reddit)
     sorted_x = score.sort_items(scored_x)
     sorted_youtube = score.sort_items(scored_youtube) if scored_youtube else []
+    sorted_web = score.sort_items(scored_web) if scored_web else []
 
     # Dedupe items
     deduped_reddit = dedupe.dedupe_reddit(sorted_reddit)
     deduped_x = dedupe.dedupe_x(sorted_x)
     deduped_youtube = dedupe.dedupe_youtube(sorted_youtube) if sorted_youtube else []
+    deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
 
     # Minimum result guarantee: if all Reddit results were filtered out but
     # we had raw results, keep top 3 by relevance regardless of score
@@ -753,9 +884,11 @@ def main():
     report.reddit = deduped_reddit
     report.x = deduped_x
     report.youtube = deduped_youtube
+    report.web = deduped_web
     report.reddit_error = reddit_error
     report.x_error = x_error
     report.youtube_error = youtube_error
+    report.web_error = web_error
 
     # Generate context snippet
     report.context_snippet_md = render.render_context_snippet(report)
@@ -772,6 +905,68 @@ def main():
     # Output result
     output_result(report, args.emit, web_needed, args.topic, from_date, to_date, missing_keys, args.days)
 
+    # Persist findings to SQLite if requested
+    if args.store:
+        import store as store_mod
+        store_mod.init_db()
+        topic_row = store_mod.add_topic(args.topic)
+        topic_id = topic_row["id"]
+        run_id = store_mod.record_run(topic_id, source_mode=mode, status="completed")
+
+        findings = []
+        for item in deduped_reddit:
+            findings.append({
+                "source": "reddit",
+                "url": item.url,
+                "title": item.title,
+                "author": item.subreddit,
+                "content": item.title,
+                "engagement_score": item.engagement.score if item.engagement else 0,
+                "relevance_score": item.relevance,
+            })
+        for item in deduped_x:
+            findings.append({
+                "source": "x",
+                "url": item.url,
+                "title": item.text[:100],
+                "author": item.author_handle,
+                "content": item.text,
+                "engagement_score": item.engagement.likes if item.engagement else 0,
+                "relevance_score": item.relevance,
+            })
+        for item in deduped_youtube:
+            findings.append({
+                "source": "youtube",
+                "url": item.url,
+                "title": item.title,
+                "author": item.channel_name,
+                "content": item.transcript_snippet[:500] if item.transcript_snippet else item.title,
+                "engagement_score": item.engagement.views if item.engagement and item.engagement.views else 0,
+                "relevance_score": item.relevance,
+            })
+        for item in deduped_web:
+            findings.append({
+                "source": "web",
+                "url": item.url,
+                "title": item.title,
+                "author": item.source_domain,
+                "content": item.snippet,
+                "engagement_score": 0,
+                "relevance_score": item.relevance,
+            })
+
+        counts = store_mod.store_findings(run_id, topic_id, findings)
+        store_mod.update_run(
+            run_id,
+            status="completed",
+            findings_new=counts["new"],
+            findings_updated=counts["updated"],
+        )
+        sys.stderr.write(
+            f"[store] Saved {counts['new']} new, {counts['updated']} updated findings\n"
+        )
+        sys.stderr.flush()
+
 
 def output_result(
     report: schema.Report,