feat: Smart supplemental search — Phase 2 entity-aware drill-down

After the initial broad search (Phase 1), extract key entities from results
and run targeted secondary searches to surface content the broad pass missed:

- New entity_extract.py: parses @handles, #hashtags, subreddits from results
- bird_x.py: search_handles() does targeted from:handle searches via Bird CLI
- openai_reddit.py: search_subreddits() uses Reddit's free .json search endpoint
- last30days.py: Phase 2 orchestration runs after enrichment, merges + dedupes

Tested with "kanye west" (+9 Reddit, +1 X) and "claude code skills" (+6 Reddit, +1 X).
Phase 2 is skipped on --quick mode. Default caps at 3 handles/subs, deep at 5.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

scripts/last30days.py

@@ -30,6 +30,7 @@ from lib import (
     bird_x,
     dates,
     dedupe,
+    entity_extract,
     env,
     http,
     models,
@@ -205,6 +206,129 @@ def _search_x(
     return x_items, raw_response, x_error
 
 
+def _run_supplemental(
+    topic: str,
+    reddit_items: list,
+    x_items: list,
+    from_date: str,
+    to_date: str,
+    depth: str,
+    x_source: str,
+    progress: ui.ProgressDisplay = None,
+) -> tuple:
+    """Run Phase 2 supplemental searches based on entities from Phase 1.
+
+    Extracts handles/subreddits from initial results, then runs targeted
+    searches to find additional content the broad search missed.
+
+    Args:
+        topic: Original search topic
+        reddit_items: Phase 1 Reddit items (raw dicts)
+        x_items: Phase 1 X items (raw dicts)
+        from_date: Start date
+        to_date: End date
+        depth: Research depth
+        x_source: 'bird' or 'xai'
+        progress: Optional progress display
+
+    Returns:
+        Tuple of (supplemental_reddit, supplemental_x)
+    """
+    # Depth-dependent caps
+    if depth == "default":
+        max_handles = 3
+        max_subs = 3
+        count_per = 3
+    else:  # deep
+        max_handles = 5
+        max_subs = 5
+        count_per = 5
+
+    # Extract entities from Phase 1 results
+    entities = entity_extract.extract_entities(
+        reddit_items, x_items,
+        max_handles=max_handles,
+        max_subreddits=max_subs,
+    )
+
+    has_handles = entities["x_handles"] and x_source == "bird"
+    has_subs = entities["reddit_subreddits"]
+
+    if not has_handles and not has_subs:
+        return [], []
+
+    parts = []
+    if has_handles:
+        parts.append(f"@{', @'.join(entities['x_handles'][:3])}")
+    if has_subs:
+        parts.append(f"r/{', r/'.join(entities['reddit_subreddits'][:3])}")
+    sys.stderr.write(f"[Phase 2] Drilling into {' + '.join(parts)}\n")
+    sys.stderr.flush()
+
+    supplemental_reddit = []
+    supplemental_x = []
+
+    # Collect existing URLs to avoid adding duplicates before dedupe
+    existing_urls = set()
+    for item in reddit_items:
+        existing_urls.add(item.get("url", ""))
+    for item in x_items:
+        existing_urls.add(item.get("url", ""))
+
+    # Run supplemental searches in parallel
+    reddit_future = None
+    x_future = None
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        if has_subs:
+            reddit_future = executor.submit(
+                openai_reddit.search_subreddits,
+                entities["reddit_subreddits"],
+                topic,
+                from_date,
+                to_date,
+                count_per,
+            )
+
+        if has_handles:
+            x_future = executor.submit(
+                bird_x.search_handles,
+                entities["x_handles"],
+                topic,
+                from_date,
+                count_per,
+            )
+
+        if reddit_future:
+            try:
+                raw_reddit = reddit_future.result()
+                # Filter out URLs already found in Phase 1
+                supplemental_reddit = [
+                    item for item in raw_reddit
+                    if item.get("url", "") not in existing_urls
+                ]
+            except Exception as e:
+                sys.stderr.write(f"[Phase 2] Supplemental Reddit error: {e}\n")
+
+        if x_future:
+            try:
+                raw_x = x_future.result()
+                supplemental_x = [
+                    item for item in raw_x
+                    if item.get("url", "") not in existing_urls
+                ]
+            except Exception as e:
+                sys.stderr.write(f"[Phase 2] Supplemental X error: {e}\n")
+
+    if supplemental_reddit or supplemental_x:
+        sys.stderr.write(
+            f"[Phase 2] +{len(supplemental_reddit)} Reddit, +{len(supplemental_x)} X\n"
+        )
+        sys.stderr.flush()
+
+    return supplemental_reddit, supplemental_x
+
+
 def run_research(
     topic: str,
     sources: str,
@@ -319,6 +443,18 @@ def run_research(
         if progress:
             progress.end_reddit_enrich()
 
+    # Phase 2: Supplemental search based on entities from Phase 1
+    # Skip on --quick (speed matters) and mock mode
+    if depth != "quick" and not mock and (reddit_items or x_items):
+        sup_reddit, sup_x = _run_supplemental(
+            topic, reddit_items, x_items,
+            from_date, to_date, depth, x_source, progress,
+        )
+        if sup_reddit:
+            reddit_items.extend(sup_reddit)
+        if sup_x:
+            x_items.extend(sup_x)
+
     return reddit_items, x_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error
 
 

scripts/lib/bird_x.py

@@ -190,6 +190,70 @@ def search_x(
         return {"error": str(e), "items": []}
 
 
+def search_handles(
+    handles: List[str],
+    topic: str,
+    from_date: str,
+    count_per: int = 5,
+) -> List[Dict[str, Any]]:
+    """Search specific X handles for topic-related content.
+
+    Runs targeted Bird searches using `from:handle topic` syntax.
+    Used in Phase 2 supplemental search after entity extraction.
+
+    Args:
+        handles: List of X handles to search (without @)
+        topic: Search topic (core subject, not full verbose query)
+        from_date: Start date (YYYY-MM-DD)
+        count_per: Results to request per handle
+
+    Returns:
+        List of raw item dicts (same format as parse_bird_response output).
+    """
+    all_items = []
+    core_topic = _extract_core_subject(topic)
+
+    for handle in handles:
+        handle = handle.lstrip("@")
+        query = f"from:{handle} {core_topic} since:{from_date}"
+
+        cmd = [
+            "bird", "search",
+            query,
+            "-n", str(count_per),
+            "--json",
+        ]
+
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=15,  # Short timeout per handle
+            )
+
+            if result.returncode != 0:
+                _log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
+                continue
+
+            output = result.stdout.strip()
+            if not output:
+                continue
+
+            response = json.loads(output)
+            items = parse_bird_response(response)
+            all_items.extend(items)
+
+        except subprocess.TimeoutExpired:
+            _log(f"Handle search timed out for @{handle}")
+        except json.JSONDecodeError:
+            _log(f"Invalid JSON from handle search for @{handle}")
+        except Exception as e:
+            _log(f"Handle search error for @{handle}: {e}")
+
+    return all_items
+
+
 def parse_bird_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
     """Parse Bird response to match xai_x output format.
 

scripts/lib/entity_extract.py

@@ -0,0 +1,127 @@
+"""Entity extraction from Phase 1 search results for supplemental searches."""
+
+import re
+from collections import Counter
+from typing import Any, Dict, List
+
+# Handles that appear too frequently to be useful for targeted search.
+# These are generic/platform accounts, not topic-specific voices.
+GENERIC_HANDLES = {
+    "elonmusk", "openai", "google", "microsoft", "apple", "meta",
+    "github", "youtube", "x", "twitter", "reddit", "wikipedia",
+    "nytimes", "washingtonpost", "cnn", "bbc", "reuters",
+    "verified", "jack", "sundarpichai",
+}
+
+
+def extract_entities(
+    reddit_items: List[Dict[str, Any]],
+    x_items: List[Dict[str, Any]],
+    max_handles: int = 5,
+    max_hashtags: int = 3,
+    max_subreddits: int = 5,
+) -> Dict[str, List[str]]:
+    """Extract key entities from Phase 1 results for supplemental searches.
+
+    Parses X results for @handles and #hashtags, Reddit results for subreddit
+    names and cross-referenced communities.
+
+    Args:
+        reddit_items: Raw Reddit item dicts from Phase 1
+        x_items: Raw X item dicts from Phase 1
+        max_handles: Maximum handles to return
+        max_hashtags: Maximum hashtags to return
+        max_subreddits: Maximum subreddits to return
+
+    Returns:
+        Dict with keys: x_handles, x_hashtags, reddit_subreddits
+    """
+    handles = _extract_x_handles(x_items)
+    hashtags = _extract_x_hashtags(x_items)
+    subreddits = _extract_subreddits(reddit_items)
+
+    return {
+        "x_handles": handles[:max_handles],
+        "x_hashtags": hashtags[:max_hashtags],
+        "reddit_subreddits": subreddits[:max_subreddits],
+    }
+
+
+def _extract_x_handles(x_items: List[Dict[str, Any]]) -> List[str]:
+    """Extract and rank @handles from X results.
+
+    Sources handles from:
+    1. author_handle field (who posted)
+    2. @mentions in post text (who they're talking about/to)
+
+    Returns handles ranked by frequency, filtered for generic accounts.
+    """
+    handle_counts = Counter()
+
+    for item in x_items:
+        # Author handle
+        author = item.get("author_handle", "").strip().lstrip("@").lower()
+        if author and author not in GENERIC_HANDLES:
+            handle_counts[author] += 1
+
+        # @mentions in text
+        text = item.get("text", "")
+        mentions = re.findall(r'@(\w{1,15})', text)
+        for mention in mentions:
+            mention_lower = mention.lower()
+            if mention_lower not in GENERIC_HANDLES:
+                handle_counts[mention_lower] += 1
+
+    # Return all handles ranked by frequency
+    return [h for h, _ in handle_counts.most_common()]
+
+
+def _extract_x_hashtags(x_items: List[Dict[str, Any]]) -> List[str]:
+    """Extract and rank #hashtags from X results.
+
+    Returns hashtags ranked by frequency.
+    """
+    hashtag_counts = Counter()
+
+    for item in x_items:
+        text = item.get("text", "")
+        tags = re.findall(r'#(\w{2,30})', text)
+        for tag in tags:
+            hashtag_counts[tag.lower()] += 1
+
+    # Return all hashtags ranked by frequency
+    return [f"#{t}" for t, _ in hashtag_counts.most_common()]
+
+
+def _extract_subreddits(reddit_items: List[Dict[str, Any]]) -> List[str]:
+    """Extract and rank subreddits from Reddit results.
+
+    Sources from:
+    1. subreddit field on each result
+    2. Cross-references in comment text (e.g., "check out r/localLLaMA")
+
+    Returns subreddits ranked by frequency.
+    """
+    sub_counts = Counter()
+
+    for item in reddit_items:
+        # Primary subreddit
+        sub = item.get("subreddit", "").strip().lstrip("r/")
+        if sub:
+            sub_counts[sub] += 1
+
+        # Cross-references in comment insights
+        for insight in item.get("comment_insights", []):
+            cross_refs = re.findall(r'r/(\w{2,30})', insight)
+            for ref in cross_refs:
+                sub_counts[ref] += 1
+
+        # Cross-references in top comments
+        for comment in item.get("top_comments", []):
+            excerpt = comment.get("excerpt", "")
+            cross_refs = re.findall(r'r/(\w{2,30})', excerpt)
+            for ref in cross_refs:
+                sub_counts[ref] += 1
+
+    # Return subreddits ranked by frequency
+    return [sub for sub, _ in sub_counts.most_common()]

scripts/lib/openai_reddit.py

@@ -198,6 +198,87 @@ def search_reddit(
     raise http.HTTPError("No models available")
 
 
+def search_subreddits(
+    subreddits: List[str],
+    topic: str,
+    from_date: str,
+    to_date: str,
+    count_per: int = 5,
+) -> List[Dict[str, Any]]:
+    """Search specific subreddits via Reddit's free JSON endpoint.
+
+    No API key needed. Uses reddit.com/r/{sub}/search/.json endpoint.
+    Used in Phase 2 supplemental search after entity extraction.
+
+    Args:
+        subreddits: List of subreddit names (without r/)
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        count_per: Results to request per subreddit
+
+    Returns:
+        List of raw item dicts (same format as parse_reddit_response output).
+    """
+    all_items = []
+    core = _extract_core_subject(topic)
+
+    for sub in subreddits:
+        sub = sub.lstrip("r/")
+        try:
+            url = f"https://www.reddit.com/r/{sub}/search/.json"
+            params = f"q={_url_encode(core)}&restrict_sr=on&sort=new&limit={count_per}&raw_json=1"
+            full_url = f"{url}?{params}"
+
+            headers = {
+                "User-Agent": http.USER_AGENT,
+                "Accept": "application/json",
+            }
+
+            data = http.get(full_url, headers=headers, timeout=15)
+
+            # Reddit search returns {"data": {"children": [...]}}
+            children = data.get("data", {}).get("children", [])
+            for i, child in enumerate(children):
+                if child.get("kind") != "t3":  # t3 = link/submission
+                    continue
+                post = child.get("data", {})
+                permalink = post.get("permalink", "")
+                if not permalink:
+                    continue
+
+                item = {
+                    "id": f"RS{len(all_items)+1}",
+                    "title": str(post.get("title", "")).strip(),
+                    "url": f"https://www.reddit.com{permalink}",
+                    "subreddit": str(post.get("subreddit", sub)).strip(),
+                    "date": None,
+                    "why_relevant": f"Found in r/{sub} supplemental search",
+                    "relevance": 0.65,  # Slightly lower default for supplemental
+                }
+
+                # Parse date from created_utc
+                created_utc = post.get("created_utc")
+                if created_utc:
+                    from . import dates as dates_mod
+                    item["date"] = dates_mod.timestamp_to_date(created_utc)
+
+                all_items.append(item)
+
+        except http.HTTPError as e:
+            _log_info(f"Subreddit search failed for r/{sub}: {e}")
+        except Exception as e:
+            _log_info(f"Subreddit search error for r/{sub}: {e}")
+
+    return all_items
+
+
+def _url_encode(text: str) -> str:
+    """Simple URL encoding for query parameters."""
+    import urllib.parse
+    return urllib.parse.quote_plus(text)
+
+
 def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
     """Parse OpenAI response to extract Reddit items.