From d66758659703ffd65a659f67271ce2d2e7f12c5a Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:07:50 -0700
Subject: [PATCH 01/18] Add shared query.py and relevance.py modules

Consolidate duplicated _extract_core_subject() (7 copies across bird_x,
reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into
query.extract_core_subject() with parameterized noise set, max_words,
and suffix stripping.

Consolidate duplicated _tokenize/_compute_relevance/STOPWORDS/SYNONYMS
(4 copies across youtube_yt, tiktok, instagram, scrapecreators_x) into
relevance.token_overlap_relevance() with hashtag-aware matching.

Integration into per-module imports follows in next commits.
---
 scripts/lib/query.py     |  98 +++++++++++++++++++++++++++++
 scripts/lib/relevance.py |  95 ++++++++++++++++++++++++++++
 tests/test_query.py      | 130 +++++++++++++++++++++++++++++++++++++++
 tests/test_relevance.py  | 122 ++++++++++++++++++++++++++++++++++++
 4 files changed, 445 insertions(+)
 create mode 100644 scripts/lib/query.py
 create mode 100644 scripts/lib/relevance.py
 create mode 100644 tests/test_query.py
 create mode 100644 tests/test_relevance.py

diff --git a/scripts/lib/query.py b/scripts/lib/query.py
new file mode 100644
index 0000000..1d48875
--- /dev/null
+++ b/scripts/lib/query.py
@@ -0,0 +1,98 @@
+"""Shared query utilities for /last30days search modules.
+
+Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
+youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
+parameterized function. Each platform calls with its own overrides.
+"""
+
+from typing import FrozenSet, List, Optional, Set
+
+# Common multi-word prefixes stripped from all queries (identical across modules)
+PREFIXES = [
+    'what are the best', 'what is the best', 'what are the latest',
+    'what are people saying about', 'what do people think about',
+    'how do i use', 'how to use', 'how to',
+    'what are', 'what is', 'tips for', 'best practices for',
+]
+
+# Multi-word suffixes (used by bird_x)
+SUFFIXES = [
+    'best practices', 'use cases', 'prompt techniques',
+    'prompting techniques', 'prompting tips',
+]
+
+# Base noise words shared across most modules
+NOISE_WORDS = frozenset({
+    # Articles/prepositions/conjunctions
+    'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
+    'of', 'in', 'on', 'for', 'with', 'about', 'to',
+    # Question words
+    'how', 'what', 'which', 'who', 'why', 'when', 'where',
+    'does', 'should', 'could', 'would',
+    # Research/meta descriptors
+    'best', 'top', 'good', 'great', 'awesome', 'killer',
+    'latest', 'new', 'news', 'update', 'updates',
+    'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
+    'practices', 'features', 'guide', 'tutorial',
+    'recommendations', 'advice', 'review', 'reviews',
+    'usecases', 'examples', 'comparison', 'versus', 'vs',
+    'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
+    # Prompting meta words
+    'prompt', 'prompts', 'prompting', 'techniques', 'tips',
+    'tricks', 'methods', 'strategies', 'approaches',
+    # Action words
+    'using', 'uses', 'use',
+    # Misc filler
+    'people', 'saying', 'think', 'said', 'lately',
+})
+
+
+def extract_core_subject(
+    topic: str,
+    *,
+    noise: Optional[FrozenSet[str]] = None,
+    max_words: Optional[int] = None,
+    strip_suffixes: bool = False,
+) -> str:
+    """Extract core subject from a verbose search query.
+
+    Strips common question/meta prefixes and noise words to produce a
+    compact search-friendly query. Platforms customize via parameters.
+
+    Args:
+        topic: Raw user query
+        noise: Override noise word set (default: NOISE_WORDS)
+        max_words: Cap result to N words (default: no cap)
+        strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this)
+
+    Returns:
+        Cleaned query string
+    """
+    text = topic.lower().strip()
+    if not text:
+        return text
+
+    # Phase 1: Strip multi-word prefixes (longest first, stop after first match)
+    for p in PREFIXES:
+        if text.startswith(p + ' '):
+            text = text[len(p):].strip()
+            break
+
+    # Phase 2: Strip multi-word suffixes (opt-in)
+    if strip_suffixes:
+        for s in SUFFIXES:
+            if text.endswith(' ' + s):
+                text = text[:-len(s)].strip()
+                break
+
+    # Phase 3: Filter individual noise words
+    noise_set = noise if noise is not None else NOISE_WORDS
+    words = text.split()
+    filtered = [w for w in words if w not in noise_set]
+
+    # Apply word cap if requested
+    if max_words is not None and filtered:
+        filtered = filtered[:max_words]
+
+    result = ' '.join(filtered) if filtered else text
+    return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
diff --git a/scripts/lib/relevance.py b/scripts/lib/relevance.py
new file mode 100644
index 0000000..f936b70
--- /dev/null
+++ b/scripts/lib/relevance.py
@@ -0,0 +1,95 @@
+"""Shared relevance scoring for /last30days search modules.
+
+Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS
+from youtube_yt, tiktok, instagram, and scrapecreators_x into one module.
+"""
+
+import re
+from typing import List, Optional, Set
+
+# Stopwords for relevance computation (common English words that dilute token overlap)
+STOPWORDS = frozenset({
+    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
+    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
+    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
+    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
+    'all', 'just', 'get', 'has', 'have', 'was', 'will',
+})
+
+# Synonym groups for relevance scoring (bidirectional expansion)
+# Superset of all platform-specific synonym dicts
+SYNONYMS = {
+    'hip': {'rap', 'hiphop'},
+    'hop': {'rap', 'hiphop'},
+    'rap': {'hip', 'hop', 'hiphop'},
+    'hiphop': {'rap', 'hip', 'hop'},
+    'js': {'javascript'},
+    'javascript': {'js'},
+    'ts': {'typescript'},
+    'typescript': {'ts'},
+    'ai': {'artificial', 'intelligence'},
+    'ml': {'machine', 'learning'},
+    'react': {'reactjs'},
+    'reactjs': {'react'},
+    'svelte': {'sveltejs'},
+    'sveltejs': {'svelte'},
+    'vue': {'vuejs'},
+    'vuejs': {'vue'},
+}
+
+
+def tokenize(text: str) -> Set[str]:
+    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.
+
+    Expands tokens with synonyms for better cross-domain matching.
+    """
+    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
+    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
+    expanded = set(tokens)
+    for t in tokens:
+        if t in SYNONYMS:
+            expanded.update(SYNONYMS[t])
+    return expanded
+
+
+def token_overlap_relevance(
+    query: str,
+    text: str,
+    hashtags: Optional[List[str]] = None,
+) -> float:
+    """Compute relevance as ratio of query tokens found in text.
+
+    Uses ratio overlap (intersection / query_length) so short queries
+    score higher when fully represented in the text. Floors at 0.1.
+
+    Args:
+        query: Search query
+        text: Content text to match against
+        hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated
+            hashtags are split to match query tokens (e.g. "claudecode" matches "claude").
+
+    Returns:
+        Float between 0.1 and 1.0 (0.5 for empty queries)
+    """
+    q_tokens = tokenize(query)
+
+    # Combine text and hashtags for matching
+    combined = text
+    if hashtags:
+        combined = f"{text} {' '.join(hashtags)}"
+    t_tokens = tokenize(combined)
+
+    # Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code")
+    if hashtags:
+        for tag in hashtags:
+            tag_lower = tag.lower()
+            for qt in q_tokens:
+                if qt in tag_lower and qt != tag_lower:
+                    t_tokens.add(qt)
+
+    if not q_tokens:
+        return 0.5  # Neutral fallback for empty/stopword-only queries
+
+    overlap = len(q_tokens & t_tokens)
+    ratio = overlap / len(q_tokens)
+    return max(0.1, min(1.0, ratio))
diff --git a/tests/test_query.py b/tests/test_query.py
new file mode 100644
index 0000000..a081f4f
--- /dev/null
+++ b/tests/test_query.py
@@ -0,0 +1,130 @@
+"""Tests for query.py — shared query utilities."""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.query import NOISE_WORDS, extract_core_subject
+
+
+class TestExtractCoreSubject(unittest.TestCase):
+    """Tests for extract_core_subject() with default noise set."""
+
+    def test_strips_what_are_prefix(self):
+        self.assertEqual(extract_core_subject("what are the best AI tools"), "ai")
+
+    def test_strips_how_to_prefix(self):
+        self.assertEqual(extract_core_subject("how to use cursor IDE"), "cursor ide")
+
+    def test_strips_what_do_people_think(self):
+        result = extract_core_subject("what do people think about React Server Components")
+        self.assertEqual(result, "react server components")
+
+    def test_preserves_product_name(self):
+        self.assertEqual(extract_core_subject("cursor IDE"), "cursor ide")
+
+    def test_strips_trailing_punctuation(self):
+        result = extract_core_subject("what is Claude?")
+        self.assertFalse(result.endswith("?"))
+
+    def test_empty_string(self):
+        self.assertEqual(extract_core_subject(""), "")
+
+    def test_all_noise_returns_original(self):
+        # When all words are noise, fall back to original text
+        result = extract_core_subject("best latest new")
+        self.assertTrue(len(result) > 0)
+
+    def test_only_first_prefix_stripped(self):
+        # "how to" should match, stripping once, not recursively
+        result = extract_core_subject("how to use how to debug")
+        self.assertIn("debug", result)
+
+
+class TestMaxWords(unittest.TestCase):
+    """Tests for max_words parameter."""
+
+    def test_max_words_caps_output(self):
+        result = extract_core_subject(
+            "multi agent reinforcement learning framework",
+            max_words=5,
+        )
+        self.assertLessEqual(len(result.split()), 5)
+
+    def test_max_words_none_no_cap(self):
+        result = extract_core_subject("cursor IDE react native components")
+        # Without max_words, no cap applied
+        self.assertGreaterEqual(len(result.split()), 3)
+
+    def test_max_words_fallback_on_empty(self):
+        # All words filtered + max_words should fall back to original
+        result = extract_core_subject("best top latest", max_words=3)
+        self.assertTrue(len(result) > 0)
+
+
+class TestStripSuffixes(unittest.TestCase):
+    """Tests for strip_suffixes parameter."""
+
+    def test_strips_best_practices(self):
+        result = extract_core_subject(
+            "claude code best practices",
+            strip_suffixes=True,
+        )
+        self.assertNotIn("practices", result)
+
+    def test_strips_use_cases(self):
+        result = extract_core_subject(
+            "react hooks use cases",
+            strip_suffixes=True,
+        )
+        self.assertNotIn("cases", result)
+
+    def test_no_strip_without_flag(self):
+        result = extract_core_subject("claude code best practices")
+        # "best" and "practices" are noise words so they get filtered anyway
+        # but the suffix phase doesn't run
+        self.assertIn("claude", result)
+
+
+class TestCustomNoise(unittest.TestCase):
+    """Tests for noise override parameter."""
+
+    def test_custom_noise_keeps_tips(self):
+        # YouTube keeps tips/tricks/tutorial — pass a noise set without them
+        youtube_noise = frozenset({
+            'best', 'top', 'good', 'great', 'awesome', 'killer',
+            'latest', 'new', 'news', 'update', 'updates',
+            'trending', 'hottest', 'popular', 'viral',
+            'practices', 'features',
+            'recommendations', 'advice',
+            'prompt', 'prompts', 'prompting',
+            'methods', 'strategies', 'approaches',
+        })
+        result = extract_core_subject("best react tips", noise=youtube_noise)
+        self.assertIn("tips", result)
+
+    def test_default_noise_removes_tips(self):
+        result = extract_core_subject("best react tips")
+        self.assertNotIn("tips", result)
+
+
+class TestNoiseWordsCompleteness(unittest.TestCase):
+    """Verify NOISE_WORDS superset covers all platform sets."""
+
+    def test_question_words_present(self):
+        for w in ('who', 'why', 'when', 'where', 'does', 'should', 'could', 'would'):
+            self.assertIn(w, NOISE_WORDS, f"Missing question word: {w}")
+
+    def test_core_filler_present(self):
+        for w in ('the', 'a', 'an', 'is', 'are', 'for', 'with', 'about'):
+            self.assertIn(w, NOISE_WORDS)
+
+    def test_research_meta_present(self):
+        for w in ('best', 'top', 'latest', 'trending', 'popular'):
+            self.assertIn(w, NOISE_WORDS)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_relevance.py b/tests/test_relevance.py
new file mode 100644
index 0000000..c641e76
--- /dev/null
+++ b/tests/test_relevance.py
@@ -0,0 +1,122 @@
+"""Tests for relevance.py — shared relevance scoring.
+
+Migrated from test_youtube_relevance.py + new hashtag/synonym tests.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.relevance import STOPWORDS, SYNONYMS, token_overlap_relevance, tokenize
+
+
+class TestTokenize(unittest.TestCase):
+    """Tests for tokenize()."""
+
+    def test_removes_stopwords(self):
+        tokens = tokenize("how to use the AI tools")
+        self.assertNotIn("how", tokens)
+        self.assertNotIn("to", tokens)
+        self.assertNotIn("the", tokens)
+        self.assertIn("ai", tokens)
+        self.assertIn("tools", tokens)
+
+    def test_lowercases(self):
+        tokens = tokenize("Python REACT")
+        self.assertIn("python", tokens)
+        self.assertIn("react", tokens)
+
+    def test_strips_punctuation(self):
+        tokens = tokenize("hello, world!")
+        self.assertIn("hello", tokens)
+        self.assertIn("world", tokens)
+
+    def test_drops_single_char(self):
+        tokens = tokenize("a b c python")
+        self.assertNotIn("a", tokens)
+        self.assertNotIn("b", tokens)
+        self.assertNotIn("c", tokens)
+        self.assertIn("python", tokens)
+
+    def test_expands_synonyms(self):
+        tokens = tokenize("ai tools")
+        self.assertIn("artificial", tokens)
+        self.assertIn("intelligence", tokens)
+
+    def test_expands_js_synonym(self):
+        tokens = tokenize("js framework")
+        self.assertIn("javascript", tokens)
+
+    def test_expands_svelte(self):
+        tokens = tokenize("svelte app")
+        self.assertIn("sveltejs", tokens)
+
+    def test_expands_vue(self):
+        tokens = tokenize("vue components")
+        self.assertIn("vuejs", tokens)
+
+
+class TestTokenOverlapRelevance(unittest.TestCase):
+    """Tests for token_overlap_relevance()."""
+
+    def test_high_relevance_exact_match(self):
+        rel = token_overlap_relevance("claude code", "Claude Code tricks and tips")
+        self.assertGreater(rel, 0.7)
+
+    def test_low_relevance_no_match(self):
+        rel = token_overlap_relevance("claude code tips", "Best AI tools for coding")
+        self.assertLess(rel, 0.5)
+
+    def test_empty_query_returns_neutral(self):
+        rel = token_overlap_relevance("", "Some video title")
+        self.assertEqual(rel, 0.5)
+
+    def test_floor_at_0_1(self):
+        rel = token_overlap_relevance("quantum physics", "cat dancing video")
+        self.assertGreaterEqual(rel, 0.1)
+
+    def test_full_match_returns_1(self):
+        rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners")
+        self.assertEqual(rel, 1.0)
+
+    def test_partial_match(self):
+        rel = token_overlap_relevance("react native tutorial", "React Native Guide")
+        self.assertGreater(rel, 0.3)
+        self.assertLess(rel, 1.0)
+
+    def test_synonym_boosts_relevance(self):
+        # "js" should match "javascript" via synonym expansion
+        rel_with_syn = token_overlap_relevance("js framework", "javascript framework comparison")
+        rel_without = token_overlap_relevance("python framework", "javascript framework comparison")
+        self.assertGreater(rel_with_syn, rel_without)
+
+    def test_stopword_only_query(self):
+        rel = token_overlap_relevance("the a is", "some content here")
+        self.assertEqual(rel, 0.5)
+
+
+class TestHashtagRelevance(unittest.TestCase):
+    """Tests for hashtag-aware relevance (TikTok/Instagram pattern)."""
+
+    def test_hashtag_boost(self):
+        rel_no_hash = token_overlap_relevance("claude code", "random video about stuff")
+        rel_with_hash = token_overlap_relevance(
+            "claude code", "random video about stuff", ["claudecode", "ai"]
+        )
+        self.assertGreater(rel_with_hash, rel_no_hash)
+
+    def test_concatenated_hashtag_splitting(self):
+        # "claudecode" should match "claude" from query via substring check
+        rel = token_overlap_relevance("claude", "video", ["claudecode"])
+        self.assertGreater(rel, 0.5)
+
+    def test_none_hashtags_same_as_no_hashtags(self):
+        rel1 = token_overlap_relevance("test query", "test content", None)
+        rel2 = token_overlap_relevance("test query", "test content")
+        self.assertEqual(rel1, rel2)
+
+
+if __name__ == "__main__":
+    unittest.main()

From fa42a5d03156726c8ded0b00af08b930057dd398 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:08:29 -0700
Subject: [PATCH 02/18] Add urllib fallback for TikTok/Instagram when requests
 unavailable

Previously tiktok.py and instagram.py returned an error when the
requests library was not installed. Reddit already had an http.get()
fallback using stdlib urllib. Apply the same pattern so all three
ScrapeCreators modules work without requests installed.
---
 scripts/lib/instagram.py | 42 ++++++++++++++++++++++++++--------------
 scripts/lib/tiktok.py    | 42 ++++++++++++++++++++++++++--------------
 2 files changed, 54 insertions(+), 30 deletions(-)

diff --git a/scripts/lib/instagram.py b/scripts/lib/instagram.py
index 3861a93..bbf7e39 100644
--- a/scripts/lib/instagram.py
+++ b/scripts/lib/instagram.py
@@ -17,6 +17,8 @@ try:
 except ImportError:
     _requests = None
 
+from . import http
+
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
 
 # Depth configurations: how many results to fetch / captions to extract
@@ -207,26 +209,36 @@ def search_instagram(
     if not token:
         return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
 
-    if not _requests:
-        return {"items": [], "error": "requests library not installed"}
-
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     core_topic = _extract_core_subject(topic)
 
     _log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
-    try:
-        resp = _requests.get(
-            f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
-            params={"query": core_topic},
-            headers=_sc_headers(token),
-            timeout=30,
-        )
-        resp.raise_for_status()
-        data = resp.json()
-    except Exception as e:
-        _log(f"ScrapeCreators error: {e}")
-        return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    if not _requests:
+        _log("requests library not installed, falling back to urllib")
+        try:
+            from urllib.parse import urlencode
+            params = urlencode({"query": core_topic})
+            url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
+            headers = _sc_headers(token)
+            headers["User-Agent"] = http.USER_AGENT
+            data = http.get(url, headers=headers, timeout=30, retries=2)
+        except Exception as e:
+            _log(f"ScrapeCreators error (urllib): {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    else:
+        try:
+            resp = _requests.get(
+                f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
+                params={"query": core_topic},
+                headers=_sc_headers(token),
+                timeout=30,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception as e:
+            _log(f"ScrapeCreators error: {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Items are in the 'reels' array (ScrapeCreators v2 response)
     raw_items = data.get("reels") or data.get("items") or data.get("data") or []
diff --git a/scripts/lib/tiktok.py b/scripts/lib/tiktok.py
index 3f5f704..2459ba2 100644
--- a/scripts/lib/tiktok.py
+++ b/scripts/lib/tiktok.py
@@ -17,6 +17,8 @@ try:
 except ImportError:
     _requests = None
 
+from . import http
+
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
 
 # Depth configurations: how many results to fetch / captions to extract
@@ -204,26 +206,36 @@ def search_tiktok(
     if not token:
         return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
 
-    if not _requests:
-        return {"items": [], "error": "requests library not installed"}
-
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     core_topic = _extract_core_subject(topic)
 
     _log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
-    try:
-        resp = _requests.get(
-            f"{SCRAPECREATORS_BASE}/search/keyword",
-            params={"query": core_topic, "sort_by": "relevance"},
-            headers=_sc_headers(token),
-            timeout=30,
-        )
-        resp.raise_for_status()
-        data = resp.json()
-    except Exception as e:
-        _log(f"ScrapeCreators error: {e}")
-        return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    if not _requests:
+        _log("requests library not installed, falling back to urllib")
+        try:
+            from urllib.parse import urlencode
+            params = urlencode({"query": core_topic, "sort_by": "relevance"})
+            url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
+            headers = _sc_headers(token)
+            headers["User-Agent"] = http.USER_AGENT
+            data = http.get(url, headers=headers, timeout=30, retries=2)
+        except Exception as e:
+            _log(f"ScrapeCreators error (urllib): {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    else:
+        try:
+            resp = _requests.get(
+                f"{SCRAPECREATORS_BASE}/search/keyword",
+                params={"query": core_topic, "sort_by": "relevance"},
+                headers=_sc_headers(token),
+                timeout=30,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception as e:
+            _log(f"ScrapeCreators error: {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Items are nested under aweme_info
     raw_entries = data.get("search_item_list") or data.get("data") or []

From dc88c215be5b79547192989b540f41faa78da48d Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:12:46 -0700
Subject: [PATCH 03/18] Integrate shared query.py into per-source modules

Replace duplicated _extract_core_subject() in bird_x, reddit, youtube_yt,
tiktok, instagram, bluesky, and scrapecreators_x with thin wrappers that
delegate to query.extract_core_subject() with platform-specific noise sets.

Each module preserves its current behavior exactly:
- bird_x: max_words=5, strip_suffixes=True, full noise set
- youtube_yt: keeps tips/tricks/tutorial/guide/review (content types)
- reddit: preserves original smaller noise set
- tiktok/instagram: same small noise set
- bluesky/scrapecreators_x: minimal noise set

Existing tests pass without modification since _extract_core_subject()
still exists as a callable on each module.
---
 scripts/lib/bird_x.py           | 52 ++-------------------------------
 scripts/lib/bluesky.py          | 20 +++----------
 scripts/lib/instagram.py        | 30 ++++---------------
 scripts/lib/reddit.py           | 23 +++------------
 scripts/lib/scrapecreators_x.py | 20 +++----------
 scripts/lib/tiktok.py           | 30 ++++---------------
 scripts/lib/youtube_yt.py       | 32 +++++---------------
 7 files changed, 32 insertions(+), 175 deletions(-)

diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py
index 66dc13c..7d541fc 100644
--- a/scripts/lib/bird_x.py
+++ b/scripts/lib/bird_x.py
@@ -54,56 +54,10 @@ def _extract_core_subject(topic: str) -> str:
 
     X search is literal keyword AND matching — all words must appear.
     Aggressively strip question/meta/research words to keep only the
-    core product/concept name (2-3 words max).
+    core product/concept name (max 5 words).
     """
-    text = topic.lower().strip()
-
-    # Phase 1: Strip multi-word prefixes (longest first)
-    prefixes = [
-        'what are the best', 'what is the best', 'what are the latest',
-        'what are people saying about', 'what do people think about',
-        'how do i use', 'how to use', 'how to',
-        'what are', 'what is', 'tips for', 'best practices for',
-    ]
-    for p in prefixes:
-        if text.startswith(p + ' '):
-            text = text[len(p):].strip()
-            break
-
-    # Phase 2: Strip multi-word suffixes
-    suffixes = [
-        'best practices', 'use cases', 'prompt techniques',
-        'prompting techniques', 'prompting tips',
-    ]
-    for s in suffixes:
-        if text.endswith(' ' + s):
-            text = text[:-len(s)].strip()
-            break
-
-    # Phase 3: Filter individual noise words
-    _noise = {
-        # Question/filler words
-        'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
-        'of', 'in', 'on', 'for', 'with', 'about', 'to',
-        'people', 'saying', 'think', 'said', 'lately',
-        # Research/meta descriptors
-        'best', 'top', 'good', 'great', 'awesome', 'killer',
-        'latest', 'new', 'news', 'update', 'updates',
-        'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
-        'practices', 'features', 'guide', 'tutorial',
-        'recommendations', 'advice', 'review', 'reviews',
-        'usecases', 'examples', 'comparison', 'versus', 'vs',
-        'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
-        # Prompting meta words
-        'prompt', 'prompts', 'prompting', 'techniques', 'tips',
-        'tricks', 'methods', 'strategies', 'approaches',
-        # Action words
-        'using', 'uses', 'use',
-    }
-    words = text.split()
-    result = [w for w in words if w not in _noise]
-
-    return ' '.join(result[:3]) or topic.lower().strip()  # Max 3 words
+    from .query import NOISE_WORDS, extract_core_subject
+    return extract_core_subject(topic, max_words=5, strip_suffixes=True)
 
 
 def is_bird_installed() -> bool:
diff --git a/scripts/lib/bluesky.py b/scripts/lib/bluesky.py
index 8a098a7..9bfcd43 100644
--- a/scripts/lib/bluesky.py
+++ b/scripts/lib/bluesky.py
@@ -67,26 +67,14 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
 
 def _extract_core_subject(topic: str) -> str:
     """Extract core subject from verbose query for Bluesky search."""
-    text = topic.lower().strip()
-    prefixes = [
-        'what are the best', 'what is the best', 'what are the latest',
-        'what are people saying about', 'what do people think about',
-        'how do i use', 'how to use', 'how to',
-        'what are', 'what is', 'tips for', 'best practices for',
-    ]
-    for p in prefixes:
-        if text.startswith(p + ' '):
-            text = text[len(p):].strip()
-    noise = {
+    from .query import extract_core_subject
+    _BSKY_NOISE = frozenset({
         'best', 'top', 'good', 'great', 'awesome',
         'latest', 'new', 'news', 'update', 'updates',
         'trending', 'hottest', 'popular', 'viral',
         'practices', 'features', 'recommendations', 'advice',
-    }
-    words = text.split()
-    filtered = [w for w in words if w not in noise]
-    result = ' '.join(filtered) if filtered else text
-    return result.rstrip('?!.')
+    })
+    return extract_core_subject(topic, noise=_BSKY_NOISE)
 
 
 def _parse_date(item: Dict[str, Any]) -> Optional[str]:
diff --git a/scripts/lib/instagram.py b/scripts/lib/instagram.py
index bbf7e39..0f39fa1 100644
--- a/scripts/lib/instagram.py
+++ b/scripts/lib/instagram.py
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
 
 
 def _extract_core_subject(topic: str) -> str:
-    """Extract core subject from verbose query for Instagram search.
-
-    Strips meta/research words to keep only the core product/concept name.
-    """
-    text = topic.lower().strip()
-
-    # Strip multi-word prefixes
-    prefixes = [
-        'what are the best', 'what is the best', 'what are the latest',
-        'what are people saying about', 'what do people think about',
-        'how do i use', 'how to use', 'how to',
-        'what are', 'what is', 'tips for', 'best practices for',
-    ]
-    for p in prefixes:
-        if text.startswith(p + ' '):
-            text = text[len(p):].strip()
-
-    # Strip individual noise words
-    noise = {
+    """Extract core subject from verbose query for Instagram search."""
+    from .query import extract_core_subject
+    _INSTAGRAM_NOISE = frozenset({
         'best', 'top', 'good', 'great', 'awesome', 'killer',
         'latest', 'new', 'news', 'update', 'updates',
         'trending', 'hottest', 'popular', 'viral',
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
         'recommendations', 'advice',
         'prompt', 'prompts', 'prompting',
         'methods', 'strategies', 'approaches',
-    }
-    words = text.split()
-    filtered = [w for w in words if w not in noise]
-
-    result = ' '.join(filtered) if filtered else text
-    return result.rstrip('?!.')
+    })
+    return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
 
 
 def _log(msg: str):
diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py
index b88a3d5..f9e56df 100644
--- a/scripts/lib/reddit.py
+++ b/scripts/lib/reddit.py
@@ -48,7 +48,9 @@ DEPTH_CONFIG = {
     },
 }
 
-# Stopwords for query extraction
+from .query import extract_core_subject as _query_extract
+
+# Reddit-specific noise words (preserves original smaller set)
 NOISE_WORDS = frozenset({
     'best', 'top', 'good', 'great', 'awesome', 'killer',
     'latest', 'new', 'news', 'update', 'updates',
@@ -82,24 +84,7 @@ def _extract_core_subject(topic: str) -> str:
 
     Strips meta/research words to keep only the core product/concept name.
     """
-    text = topic.lower().strip()
-
-    # Strip multi-word prefixes
-    prefixes = [
-        'what are the best', 'what is the best', 'what are the latest',
-        'what are people saying about', 'what do people think about',
-        'how do i use', 'how to use', 'how to',
-        'what are', 'what is', 'tips for', 'best practices for',
-    ]
-    for p in prefixes:
-        if text.startswith(p + ' '):
-            text = text[len(p):].strip()
-
-    words = text.split()
-    filtered = [w for w in words if w not in NOISE_WORDS]
-
-    result = ' '.join(filtered) if filtered else text
-    return result.rstrip('?!.')
+    return _query_extract(topic, noise=NOISE_WORDS)
 
 
 def expand_reddit_queries(topic: str, depth: str) -> List[str]:
diff --git a/scripts/lib/scrapecreators_x.py b/scripts/lib/scrapecreators_x.py
index 3abd7ef..727a6db 100644
--- a/scripts/lib/scrapecreators_x.py
+++ b/scripts/lib/scrapecreators_x.py
@@ -66,26 +66,14 @@ def _compute_relevance(query: str, text: str) -> float:
 
 def _extract_core_subject(topic: str) -> str:
     """Extract core subject from verbose query for Twitter search."""
-    text = topic.lower().strip()
-    prefixes = [
-        'what are the best', 'what is the best', 'what are the latest',
-        'what are people saying about', 'what do people think about',
-        'how do i use', 'how to use', 'how to',
-        'what are', 'what is', 'tips for', 'best practices for',
-    ]
-    for p in prefixes:
-        if text.startswith(p + ' '):
-            text = text[len(p):].strip()
-    noise = {
+    from .query import extract_core_subject
+    _SC_X_NOISE = frozenset({
         'best', 'top', 'good', 'great', 'awesome',
         'latest', 'new', 'news', 'update', 'updates',
         'trending', 'hottest', 'popular', 'viral',
         'practices', 'features', 'recommendations', 'advice',
-    }
-    words = text.split()
-    filtered = [w for w in words if w not in noise]
-    result = ' '.join(filtered) if filtered else text
-    return result.rstrip('?!.')
+    })
+    return extract_core_subject(topic, noise=_SC_X_NOISE)
 
 
 def _log(msg: str):
diff --git a/scripts/lib/tiktok.py b/scripts/lib/tiktok.py
index 2459ba2..27c4ec3 100644
--- a/scripts/lib/tiktok.py
+++ b/scripts/lib/tiktok.py
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
 
 
 def _extract_core_subject(topic: str) -> str:
-    """Extract core subject from verbose query for TikTok search.
-
-    Strips meta/research words to keep only the core product/concept name.
-    """
-    text = topic.lower().strip()
-
-    # Strip multi-word prefixes
-    prefixes = [
-        'what are the best', 'what is the best', 'what are the latest',
-        'what are people saying about', 'what do people think about',
-        'how do i use', 'how to use', 'how to',
-        'what are', 'what is', 'tips for', 'best practices for',
-    ]
-    for p in prefixes:
-        if text.startswith(p + ' '):
-            text = text[len(p):].strip()
-
-    # Strip individual noise words
-    noise = {
+    """Extract core subject from verbose query for TikTok search."""
+    from .query import extract_core_subject
+    _TIKTOK_NOISE = frozenset({
         'best', 'top', 'good', 'great', 'awesome', 'killer',
         'latest', 'new', 'news', 'update', 'updates',
         'trending', 'hottest', 'popular', 'viral',
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
         'recommendations', 'advice',
         'prompt', 'prompts', 'prompting',
         'methods', 'strategies', 'approaches',
-    }
-    words = text.split()
-    filtered = [w for w in words if w not in noise]
-
-    result = ' '.join(filtered) if filtered else text
-    return result.rstrip('?!.')
+    })
+    return extract_core_subject(topic, noise=_TIKTOK_NOISE)
 
 
 def _log(msg: str):
diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py
index c560d49..b171d41 100644
--- a/scripts/lib/youtube_yt.py
+++ b/scripts/lib/youtube_yt.py
@@ -110,26 +110,12 @@ def is_ytdlp_installed() -> bool:
 def _extract_core_subject(topic: str) -> str:
     """Extract core subject from verbose query for YouTube search.
 
-    Strips meta/research words to keep only the core product/concept name,
-    similar to bird_x.py's approach.
+    NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
+    are intentionally KEPT — they're YouTube content types that improve search.
     """
-    text = topic.lower().strip()
-
-    # Strip multi-word prefixes
-    prefixes = [
-        'what are the best', 'what is the best', 'what are the latest',
-        'what are people saying about', 'what do people think about',
-        'how do i use', 'how to use', 'how to',
-        'what are', 'what is', 'tips for', 'best practices for',
-    ]
-    for p in prefixes:
-        if text.startswith(p + ' '):
-            text = text[len(p):].strip()
-
-    # Strip individual noise words
-    # NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
-    # are intentionally KEPT — they're YouTube content types that improve search
-    noise = {
+    from .query import extract_core_subject
+    # YouTube-specific noise set: smaller than default, keeps content-type words
+    _YT_NOISE = frozenset({
         'best', 'top', 'good', 'great', 'awesome', 'killer',
         'latest', 'new', 'news', 'update', 'updates',
         'trending', 'hottest', 'popular', 'viral',
@@ -137,12 +123,8 @@ def _extract_core_subject(topic: str) -> str:
         'recommendations', 'advice',
         'prompt', 'prompts', 'prompting',
         'methods', 'strategies', 'approaches',
-    }
-    words = text.split()
-    filtered = [w for w in words if w not in noise]
-
-    result = ' '.join(filtered) if filtered else text
-    return result.rstrip('?!.')
+    })
+    return extract_core_subject(topic, noise=_YT_NOISE)
 
 
 def search_youtube(

From 96948cc7c0480dfcea48ae5a2be6b479cb84034c Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:14:39 -0700
Subject: [PATCH 04/18] Deduplicate relevance code across
 youtube/tiktok/instagram/scrapecreators_x

Replace duplicated STOPWORDS, SYNONYMS, _tokenize, and _compute_relevance
in four modules with imports from the shared relevance.py module.

Existing tests pass unchanged since modules re-export the functions
under the same names via import aliases.
---
 scripts/lib/instagram.py        | 71 +++------------------------------
 scripts/lib/scrapecreators_x.py | 46 ++++-----------------
 scripts/lib/tiktok.py           | 71 +++------------------------------
 scripts/lib/youtube_yt.py       | 65 +++---------------------------
 4 files changed, 25 insertions(+), 228 deletions(-)

diff --git a/scripts/lib/instagram.py b/scripts/lib/instagram.py
index 0f39fa1..cb166fd 100644
--- a/scripts/lib/instagram.py
+++ b/scripts/lib/instagram.py
@@ -31,71 +31,12 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-# Stopwords for relevance computation (shared with tiktok.py pattern)
-STOPWORDS = frozenset({
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
-})
-
-# Synonym groups for relevance scoring
-SYNONYMS = {
-    'hip': {'rap', 'hiphop'},
-    'hop': {'rap', 'hiphop'},
-    'rap': {'hip', 'hop', 'hiphop'},
-    'hiphop': {'rap', 'hip', 'hop'},
-    'js': {'javascript'},
-    'javascript': {'js'},
-    'ts': {'typescript'},
-    'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'},
-    'reactjs': {'react'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> float:
-    """Compute relevance as ratio of query tokens found in text + hashtags.
-
-    Uses ratio overlap (intersection / query_length). Hashtags provide
-    an Instagram-specific relevance boost. Floors at 0.1.
-    """
-    q_tokens = _tokenize(query)
-
-    # Combine text and hashtags for matching
-    combined = text
-    if hashtags:
-        combined = f"{text} {' '.join(hashtags)}"
-    t_tokens = _tokenize(combined)
-
-    # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code")
-    if hashtags:
-        for tag in hashtags:
-            tag_lower = tag.lower()
-            for qt in q_tokens:
-                if qt in tag_lower and qt != tag_lower:
-                    t_tokens.add(qt)
-
-    if not q_tokens:
-        return 0.5  # Neutral fallback
-
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
+from .relevance import (
+    STOPWORDS,
+    SYNONYMS,
+    token_overlap_relevance as _compute_relevance,
+    tokenize as _tokenize,
+)
 
 
 def _extract_core_subject(topic: str) -> str:
diff --git a/scripts/lib/scrapecreators_x.py b/scripts/lib/scrapecreators_x.py
index 727a6db..27bbb89 100644
--- a/scripts/lib/scrapecreators_x.py
+++ b/scripts/lib/scrapecreators_x.py
@@ -7,10 +7,9 @@ Requires SCRAPECREATORS_API_KEY in config.
 API docs: https://scrapecreators.com/docs
 """
 
-import re
 import sys
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, List, Optional
 
 try:
     import requests as _requests
@@ -25,43 +24,12 @@ DEPTH_CONFIG = {
     "deep":    {"results_per_page": 40},
 }
 
-STOPWORDS = frozenset({
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
-})
-
-SYNONYMS = {
-    'js': {'javascript'}, 'javascript': {'js'},
-    'ts': {'typescript'}, 'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'}, 'reactjs': {'react'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, text: str) -> float:
-    """Compute relevance as ratio of query tokens found in text. Floors at 0.1."""
-    q_tokens = _tokenize(query)
-    t_tokens = _tokenize(text)
-    if not q_tokens:
-        return 0.5
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
+from .relevance import (
+    STOPWORDS,
+    SYNONYMS,
+    token_overlap_relevance as _compute_relevance,
+    tokenize as _tokenize,
+)
 
 
 def _extract_core_subject(topic: str) -> str:
diff --git a/scripts/lib/tiktok.py b/scripts/lib/tiktok.py
index 27c4ec3..56473f6 100644
--- a/scripts/lib/tiktok.py
+++ b/scripts/lib/tiktok.py
@@ -31,71 +31,12 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-# Stopwords for relevance computation (shared with youtube_yt.py pattern)
-STOPWORDS = frozenset({
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
-})
-
-# Synonym groups for relevance scoring
-SYNONYMS = {
-    'hip': {'rap', 'hiphop'},
-    'hop': {'rap', 'hiphop'},
-    'rap': {'hip', 'hop', 'hiphop'},
-    'hiphop': {'rap', 'hip', 'hop'},
-    'js': {'javascript'},
-    'javascript': {'js'},
-    'ts': {'typescript'},
-    'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'},
-    'reactjs': {'react'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> float:
-    """Compute relevance as ratio of query tokens found in text + hashtags.
-
-    Uses ratio overlap (intersection / query_length). Hashtags provide
-    a TikTok-specific relevance boost. Floors at 0.1.
-    """
-    q_tokens = _tokenize(query)
-
-    # Combine text and hashtags for matching
-    combined = text
-    if hashtags:
-        combined = f"{text} {' '.join(hashtags)}"
-    t_tokens = _tokenize(combined)
-
-    # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code")
-    if hashtags:
-        for tag in hashtags:
-            tag_lower = tag.lower()
-            for qt in q_tokens:
-                if qt in tag_lower and qt != tag_lower:
-                    t_tokens.add(qt)
-
-    if not q_tokens:
-        return 0.5  # Neutral fallback
-
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
+from .relevance import (
+    STOPWORDS,
+    SYNONYMS,
+    token_overlap_relevance as _compute_relevance,
+    tokenize as _tokenize,
+)
 
 
 def _extract_core_subject(topic: str) -> str:
diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py
index b171d41..680de91 100644
--- a/scripts/lib/youtube_yt.py
+++ b/scripts/lib/youtube_yt.py
@@ -35,65 +35,12 @@ TRANSCRIPT_LIMITS = {
 # Max words to keep from each transcript
 TRANSCRIPT_MAX_WORDS = 500
 
-# Stopwords for relevance computation (common English words that dilute token overlap)
-STOPWORDS = frozenset({
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
-})
-
-
-# Synonym groups for relevance scoring (bidirectional expansion)
-SYNONYMS = {
-    'hip': {'rap', 'hiphop'},
-    'hop': {'rap', 'hiphop'},
-    'rap': {'hip', 'hop', 'hiphop'},
-    'hiphop': {'rap', 'hip', 'hop'},
-    'js': {'javascript'},
-    'javascript': {'js'},
-    'ts': {'typescript'},
-    'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'},
-    'reactjs': {'react'},
-    'svelte': {'sveltejs'},
-    'sveltejs': {'svelte'},
-    'vue': {'vuejs'},
-    'vuejs': {'vue'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.
-    Expands tokens with synonyms for better cross-domain matching."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    # Expand synonyms
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, title: str) -> float:
-    """Compute relevance as ratio of query tokens found in title.
-
-    Uses ratio overlap (intersection / query_length) so short queries
-    score higher when fully represented in the title. Floors at 0.1.
-    """
-    q_tokens = _tokenize(query)
-    t_tokens = _tokenize(title)
-
-    if not q_tokens:
-        return 0.5  # Neutral fallback for empty/stopword-only queries
-
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
+from .relevance import (
+    STOPWORDS,
+    SYNONYMS,
+    token_overlap_relevance as _compute_relevance,
+    tokenize as _tokenize,
+)
 
 
 def _log(msg: str):

From 38caae328804d0ec6465bf4812c1be57cbd0da9a Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:14:45 -0700
Subject: [PATCH 05/18] Add mise.toml for Python version pinning and
 implementation plan

Pin Python 3.12 via mise for consistent local development.
Add plan document for the query/relevance consolidation work.
---
 ...ctor-query-relevance-consolidation-plan.md | 80 +++++++++++++++++++
 mise.toml                                     |  2 +
 2 files changed, 82 insertions(+)
 create mode 100644 docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
 create mode 100644 mise.toml

diff --git a/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md b/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
new file mode 100644
index 0000000..e82d684
--- /dev/null
+++ b/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
@@ -0,0 +1,80 @@
+# Search Pipeline: Query & Relevance Consolidation
+
+## Strategy: Two-phase delivery
+
+**Phase 1 — Upstream PR** (`refactor/query-relevance-consolidation` -> `mvanhorn/last30days-skill:main`)
+Pure refactors and bug fixes anyone would want. No behavior changes.
+
+**Phase 2 — Fork-only** (`feat/search-quality` on `j-sperling/last30days-skill`)
+Opinionated behavior changes: computed relevance scores, platform-specific query optimization, post-retrieval filtering.
+
+---
+
+## Phase 1: Upstream PR (refactors + bug fixes)
+
+### Step 1: New `query.py` — shared query utilities
+- Consolidate 7 duplicated `_extract_core_subject()` (bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into one parameterized function
+- `extract_core_subject(topic, noise=None, max_words=None, strip_suffixes=False)` — platform modules pass their own noise set and options to preserve current behavior
+- Shared `PREFIXES` list (identical across all 7), shared `NOISE_WORDS` base set
+- Each platform imports `extract_core_subject` and calls with its own overrides (e.g. bird_x passes `max_words=5, strip_suffixes=True`; youtube keeps tips/tricks/tutorial in its noise exclusion)
+- Fix reddit.py prefix-loop missing `break` (apply all matching prefixes vs only first)
+- Skip polymarket.py (too different — handles "last N days", preserves title case)
+- Tests: `tests/test_query.py`
+
+### Step 2: New `relevance.py` — shared relevance scoring
+- Consolidate `_tokenize`, `_compute_relevance`, `STOPWORDS`, `SYNONYMS` from youtube_yt/tiktok/instagram
+- `token_overlap_relevance(query, text, hashtags=None) -> float` — zero-dep, superset of all three implementations (hashtag substring matching from tiktok/instagram, synonym expansion from youtube)
+- Unified `SYNONYMS` dict (youtube superset: includes svelte/vue entries missing from tiktok/instagram)
+- Tests: `tests/test_relevance.py` (migrate from `test_youtube_relevance.py` + new hashtag tests)
+
+### Step 6: urllib fallback for TikTok/Instagram (independent bug fix)
+- `tiktok.py`: add `http.get()`/`http.post()` fallback when `_requests is None`
+- `instagram.py`: same pattern
+- Copies pattern from reddit.py's existing fallback
+
+### Step 3: Integrate `query.py` into per-source modules (pure refactor)
+- `bird_x.py`: replace lines 52-106 with import, call `extract_core_subject(topic, max_words=5, strip_suffixes=True, noise=BIRD_NOISE)`
+- `reddit.py`: replace `NOISE_WORDS` + `_extract_core_subject` with query import; `expand_reddit_queries` imports from query.py too
+- `youtube_yt.py`: replace `_extract_core_subject` with import, pass youtube-specific noise set (keeps tips/tricks/tutorial/guide/review)
+- `tiktok.py`, `instagram.py`, `bluesky.py`: same replacement with their noise sets
+- Update tests: 12+ test methods across 6 test files reference `module._extract_core_subject()` — either re-export from original modules or update test imports
+
+### Step 8: Deduplicate relevance code in youtube/tiktok/instagram (pure refactor)
+- `youtube_yt.py`: remove `STOPWORDS`, `SYNONYMS`, `_tokenize`, `_compute_relevance`; import from `relevance.py`
+- `tiktok.py`: same
+- `instagram.py`: same
+- Update tests: `test_youtube_relevance.py`, `test_tiktok.py`, `test_instagram_sc.py`, `test_scrapecreators_x.py` reference `module._tokenize`/`module._compute_relevance` — re-export or update imports
+
+### Commit order: 1 → 2 → 6 → 3 → 8
+
+---
+
+## Phase 2: Fork-only (behavior changes)
+
+### Step 4: Replace hardcoded relevance with computed scores
+- `bird_x.py:471` — `"relevance": 0.7` → `token_overlap_relevance(core_topic, text)`
+- `reddit.py:223` — `"relevance": 0.7` → `token_overlap_relevance(core, title + " " + selftext)`
+- `hackernews.py:139-141` — blend: `0.6 * rank_score + 0.4 * token_overlap`
+
+### Step 5: Platform-specific query optimization
+- `detect_query_type(topic)` — heuristic classifier (product/concept/opinion/how_to/comparison), added here not Phase 1
+- `extract_compound_terms(topic)` — detect hyphenated/title-case terms, return quoted
+- `bird_x.py`: OR-group construction for multi-concept queries, OR-based retry before word-dropping fallback
+- `reddit.py`: conditional opinion/review suffix only for product/opinion queries (uses `detect_query_type`)
+- `hackernews.py`: add `numericFilters: points>5`, `restrictSearchableAttributes=title`, use `extract_core_subject()` instead of raw topic
+- `youtube_yt.py`: add `--dateafter YYYYMMDD` (from_date already in signature)
+
+### Step 7: Post-retrieval relevance filtering in orchestrator
+- `last30days.py` (after dedup): filter items with `relevance < 0.3` per source (only when list has >3 items)
+- Extend fallback guarantee to all sources: keep top 3 by relevance if all filtered
+- `rerank_with_embeddings()` — optional, env-var gated (`OPENAI_API_KEY` or `GOOGLE_API_KEY`), uses existing `http.py`, graceful fallback to token overlap
+
+### Commit order: 4 → 5 → 7
+
+---
+
+## Verify
+
+```bash
+cd ~/projects/last30days-skill && python3 -m unittest discover -s tests -v
+```
diff --git a/mise.toml b/mise.toml
new file mode 100644
index 0000000..a190abb
--- /dev/null
+++ b/mise.toml
@@ -0,0 +1,2 @@
+[tools]
+python = "3.12"

From c5be117701cc7cdf12c4ec4b3f81392b966ed0fb Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:21:52 -0700
Subject: [PATCH 06/18] Replace hardcoded 0.7 relevance with computed
 token-overlap scores

- bird_x: parse_bird_response now accepts query param and computes
  token_overlap_relevance against tweet text
- reddit: _normalize_post computes relevance from query vs title+selftext
- hackernews: blends 60% Algolia rank + 40% token overlap + engagement

This makes the 45%-weight relevance factor in score.py actually
differentiate results instead of being a constant.
---
 scripts/last30days.py     |  4 ++--
 scripts/lib/bird_x.py     | 13 ++++++++-----
 scripts/lib/hackernews.py | 16 ++++++++++++----
 scripts/lib/reddit.py     | 21 ++++++++++++++-------
 4 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/scripts/last30days.py b/scripts/last30days.py
index 70f6550..ea3f753 100644
--- a/scripts/last30days.py
+++ b/scripts/last30days.py
@@ -353,7 +353,7 @@ def _search_x(
             raw_response = {"error": str(e)}
             x_error = f"{type(e).__name__}: {e}"
 
-        x_items = bird_x.parse_bird_response(raw_response or {})
+        x_items = bird_x.parse_bird_response(raw_response or {}, query=topic)
 
         # Check for error in response (Bird returns list on success, dict on error)
         if raw_response and isinstance(raw_response, dict) and raw_response.get("error") and not x_error:
@@ -508,7 +508,7 @@ def _search_hackernews(
     except Exception as e:
         return [], f"{type(e).__name__}: {e}"
 
-    hn_items = hackernews.parse_hackernews_response(response)
+    hn_items = hackernews.parse_hackernews_response(response, query=topic)
 
     if response.get("error"):
         hn_error = response["error"]
diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py
index 7d541fc..0270a8c 100644
--- a/scripts/lib/bird_x.py
+++ b/scripts/lib/bird_x.py
@@ -14,6 +14,8 @@ from pathlib import Path
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple
 
+from .relevance import token_overlap_relevance as _compute_relevance
+
 # Path to the vendored bird-search wrapper
 _BIRD_SEARCH_MJS = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
 
@@ -233,7 +235,7 @@ def search_x(
     response = _run_bird_search(query, count, timeout)
 
     # Check if we got results
-    items = parse_bird_response(response)
+    items = parse_bird_response(response, query=core_topic)
 
     # Retry with fewer keywords if 0 results and query has 3+ words
     core_words = core_topic.split()
@@ -242,7 +244,7 @@ def search_x(
         _log(f"0 results for '{core_topic}', retrying with '{shorter}'")
         query = f"{shorter} since:{from_date}"
         response = _run_bird_search(query, count, timeout)
-        items = parse_bird_response(response)
+        items = parse_bird_response(response, query=core_topic)
 
     # Last-chance retry: use strongest remaining token (often the product name)
     if not items and core_words:
@@ -329,7 +331,7 @@ def search_handles(
                 continue
 
             response = json.loads(output)
-            items = parse_bird_response(response)
+            items = parse_bird_response(response, query=core_topic)
             all_items.extend(items)
 
         except json.JSONDecodeError:
@@ -340,11 +342,12 @@ def search_handles(
     return all_items
 
 
-def parse_bird_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
     """Parse Bird response to match xai_x output format.
 
     Args:
         response: Raw Bird JSON response
+        query: Original search query for relevance scoring
 
     Returns:
         List of normalized item dicts matching xai_x.parse_x_response() format.
@@ -422,7 +425,7 @@ def parse_bird_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
             "date": date,
             "engagement": engagement if any(v is not None for v in engagement.values()) else None,
             "why_relevant": "",  # Bird doesn't provide relevance explanations
-            "relevance": 0.7,  # Default relevance, let score.py re-rank
+            "relevance": _compute_relevance(query, str(tweet.get("text", ""))) if query else 0.7,
         }
 
         items.append(item)
diff --git a/scripts/lib/hackernews.py b/scripts/lib/hackernews.py
index 1ce3792..01f544a 100644
--- a/scripts/lib/hackernews.py
+++ b/scripts/lib/hackernews.py
@@ -12,6 +12,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, Dict, List, Optional
 
 from . import http
+from .relevance import token_overlap_relevance
 
 ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
 ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date"
@@ -111,9 +112,13 @@ def search_hackernews(
     return response
 
 
-def parse_hackernews_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
     """Parse Algolia response into normalized item dicts.
 
+    Args:
+        response: Algolia search response
+        query: Original search query for token-overlap relevance scoring
+
     Returns:
         List of item dicts ready for normalization.
     """
@@ -134,11 +139,14 @@ def parse_hackernews_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
         article_url = hit.get("url") or ""
         hn_url = f"https://news.ycombinator.com/item?id={object_id}"
 
-        # Relevance: Algolia rank position gives a base, engagement boosts it
-        # Position 0 = most relevant from Algolia
+        # Relevance: blend Algolia rank with token-overlap content matching
         rank_score = max(0.3, 1.0 - (i * 0.02))  # 1.0 -> 0.3 over 35 items
         engagement_boost = min(0.2, math.log1p(points) / 40)
-        relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
+        if query:
+            content_score = token_overlap_relevance(query, hit.get("title", ""))
+            relevance = min(1.0, 0.6 * rank_score + 0.4 * content_score + engagement_boost)
+        else:
+            relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
 
         items.append({
             "object_id": object_id,
diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py
index f9e56df..8957464 100644
--- a/scripts/lib/reddit.py
+++ b/scripts/lib/reddit.py
@@ -49,6 +49,7 @@ DEPTH_CONFIG = {
 }
 
 from .query import extract_core_subject as _query_extract
+from .relevance import token_overlap_relevance
 
 # Reddit-specific noise words (preserves original smaller set)
 NOISE_WORDS = frozenset({
@@ -184,7 +185,7 @@ def _parse_date(created_utc) -> Optional[str]:
         return None
 
 
-def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global") -> Dict[str, Any]:
+def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global", query: str = "") -> Dict[str, Any]:
     """Normalize a ScrapeCreators Reddit post to our internal format."""
     permalink = post.get("permalink", "")
     url = f"https://www.reddit.com{permalink}" if permalink else post.get("url", "")
@@ -193,10 +194,16 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global"
     if url and "reddit.com" not in url:
         url = ""
 
+    title = str(post.get("title", "")).strip()
+    selftext = str(post.get("selftext", ""))
+
+    # Compute relevance from query-to-content overlap (or default 0.7)
+    relevance = token_overlap_relevance(query, title + " " + selftext) if query else 0.7
+
     return {
         "id": f"R{idx}",
         "reddit_id": post.get("id", ""),
-        "title": str(post.get("title", "")).strip(),
+        "title": title,
         "url": url,
         "subreddit": str(post.get("subreddit", "")).strip(),
         "date": _parse_date(post.get("created_utc")),
@@ -205,7 +212,7 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global"
             "num_comments": post.get("num_comments", 0),
             "upvote_ratio": post.get("upvote_ratio"),
         },
-        "relevance": 0.7,
+        "relevance": relevance,
         "why_relevant": f"Reddit {source_label} search",
         "selftext": str(post.get("selftext", ""))[:500],
     }
@@ -416,23 +423,23 @@ def search_reddit(
         _log(f"  -> {len(posts)} results")
         all_raw_posts.extend(posts)
 
-    # Normalize all posts
+    # Normalize all posts (with query for relevance scoring)
+    core = _extract_core_subject(topic)
     all_items = []
     for i, post in enumerate(all_raw_posts):
-        item = _normalize_post(post, i + 1, "global")
+        item = _normalize_post(post, i + 1, "global", query=core)
         all_items.append(item)
 
     # === Phase 3: Subreddit Discovery + Targeted Search ===
     discovered_subs = discover_subreddits(all_raw_posts, topic=topic, max_subs=config["subreddit_searches"])
     _log(f"Discovered subreddits: {discovered_subs}")
 
-    core = _extract_core_subject(topic)
     for sub in discovered_subs[:config["subreddit_searches"]]:
         _log(f"Subreddit search: r/{sub} for '{core}'")
         sub_posts = _subreddit_search(sub, core, token, sort="relevance", timeframe=timeframe)
         _log(f"  -> {len(sub_posts)} results from r/{sub}")
         for j, post in enumerate(sub_posts):
-            item = _normalize_post(post, len(all_items) + j + 1, f"r/{sub}")
+            item = _normalize_post(post, len(all_items) + j + 1, f"r/{sub}", query=core)
             all_items.append(item)
 
     # === Phase 4: Deduplicate ===

From 1002f1f0205e1d94f0fa03cf9ffc5ea1d9cb9199 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:24:44 -0700
Subject: [PATCH 07/18] Add platform-specific query optimizations

- hackernews: use extract_core_subject instead of raw topic, add
  points>5 filter and restrictSearchableAttributes=title to reduce
  noise from URL-match and low-signal posts
- youtube: add --dateafter parameter to yt-dlp for server-side date
  filtering (Python soft filter still handles fallback)
- reddit: skip opinion/review query variant for how_to/comparison
  queries where it adds noise
- bird_x: add OR-group retry with compound terms before falling back
  to word-dropping (uses X OR operator for multi-concept queries)
- query.py: add detect_query_type() and extract_compound_terms()
---
 scripts/lib/bird_x.py     | 14 ++++++++-
 scripts/lib/hackernews.py | 12 ++++---
 scripts/lib/query.py      | 66 +++++++++++++++++++++++++++++++++++++++
 scripts/lib/reddit.py     |  6 ++--
 scripts/lib/youtube_yt.py |  7 +++--
 tests/test_query.py       | 46 ++++++++++++++++++++++++++-
 6 files changed, 140 insertions(+), 11 deletions(-)

diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py
index 0270a8c..dd9fa9f 100644
--- a/scripts/lib/bird_x.py
+++ b/scripts/lib/bird_x.py
@@ -237,8 +237,20 @@ def search_x(
     # Check if we got results
     items = parse_bird_response(response, query=core_topic)
 
-    # Retry with fewer keywords if 0 results and query has 3+ words
+    # Retry with OR groups for multi-word queries (X supports OR operator)
     core_words = core_topic.split()
+    if not items and len(core_words) >= 2:
+        from .query import extract_compound_terms
+        compounds = extract_compound_terms(topic)
+        if compounds:
+            # Build OR-group query: ("multi-agent" OR "agent simulation") since:DATE
+            or_parts = ' OR '.join(f'"{t}"' for t in compounds[:3])
+            _log(f"0 results for '{core_topic}', retrying with OR groups: {or_parts}")
+            query = f"({or_parts}) since:{from_date}"
+            response = _run_bird_search(query, count, timeout)
+            items = parse_bird_response(response, query=core_topic)
+
+    # Retry with fewer keywords if still 0 results and query has 3+ words
     if not items and len(core_words) > 2:
         shorter = ' '.join(core_words[:2])
         _log(f"0 results for '{core_topic}', retrying with '{shorter}'")
diff --git a/scripts/lib/hackernews.py b/scripts/lib/hackernews.py
index 01f544a..5b6fab1 100644
--- a/scripts/lib/hackernews.py
+++ b/scripts/lib/hackernews.py
@@ -12,6 +12,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, Dict, List, Optional
 
 from . import http
+from .query import extract_core_subject
 from .relevance import token_overlap_relevance
 
 ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
@@ -85,14 +86,17 @@ def search_hackernews(
     from_ts = _date_to_unix(from_date)
     to_ts = _date_to_unix(to_date) + 86400  # Include the end date
 
-    _log(f"Searching for '{topic}' (since {from_date}, count={count})")
+    # Use extracted core subject instead of raw topic for cleaner Algolia matching
+    core = extract_core_subject(topic)
+    _log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
 
-    # Use relevance-sorted search (better for topic matching)
+    # Use relevance-sorted search with minimum engagement filter
     params = {
-        "query": topic,
+        "query": core,
         "tags": "story",
-        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts}",
+        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>5",
         "hitsPerPage": str(count),
+        "restrictSearchableAttributes": "title",
     }
 
     from urllib.parse import urlencode
diff --git a/scripts/lib/query.py b/scripts/lib/query.py
index 1d48875..e7bda9a 100644
--- a/scripts/lib/query.py
+++ b/scripts/lib/query.py
@@ -5,6 +5,7 @@ youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
 parameterized function. Each platform calls with its own overrides.
 """
 
+import re
 from typing import FrozenSet, List, Optional, Set
 
 # Common multi-word prefixes stripped from all queries (identical across modules)
@@ -96,3 +97,68 @@ def extract_core_subject(
 
     result = ' '.join(filtered) if filtered else text
     return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
+
+
+# ---- Query type detection (heuristic, no LLM) ----
+
+_OPINION_SIGNALS = frozenset({
+    'worth', 'thoughts', 'opinion', 'opinions', 'review', 'reviews',
+    'recommend', 'recommendation', 'recommendations', 'should',
+    'anyone', 'anybody', 'experience', 'experiences',
+})
+
+_HOW_TO_SIGNALS = frozenset({
+    'how', 'setup', 'configure', 'install', 'tutorial', 'guide',
+    'step', 'steps', 'instructions',
+})
+
+_COMPARISON_SIGNALS = frozenset({
+    'vs', 'versus', 'compared', 'comparison', 'better', 'alternative',
+    'alternatives', 'difference', 'differences',
+})
+
+_PRODUCT_SIGNALS = frozenset({
+    'pricing', 'price', 'cost', 'plan', 'plans', 'tier', 'tiers',
+    'buy', 'purchase', 'subscription', 'trial', 'free',
+})
+
+
+def detect_query_type(topic: str) -> str:
+    """Classify query intent without an LLM.
+
+    Returns one of: "product", "concept", "opinion", "how_to", "comparison".
+    Used to adapt per-platform query construction.
+    """
+    words = set(topic.lower().split())
+
+    if words & _COMPARISON_SIGNALS:
+        return "comparison"
+    if words & _HOW_TO_SIGNALS or topic.lower().startswith("how "):
+        return "how_to"
+    if words & _OPINION_SIGNALS:
+        return "opinion"
+    if words & _PRODUCT_SIGNALS:
+        return "product"
+    return "concept"
+
+
+def extract_compound_terms(topic: str) -> List[str]:
+    """Detect multi-word terms that should be quoted in search queries.
+
+    Identifies:
+    - Hyphenated terms: "multi-agent", "vc-backed"
+    - Title-cased multi-word names: "Claude Code", "React Native"
+
+    Returns list of terms suitable for quoting (e.g., '"multi-agent"').
+    """
+    terms: List[str] = []
+
+    # Hyphenated terms
+    for match in re.finditer(r'\b\w+-\w+(?:-\w+)*\b', topic):
+        terms.append(match.group())
+
+    # Title-cased sequences (2+ capitalized words in a row)
+    for match in re.finditer(r'(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+', topic):
+        terms.append(match.group())
+
+    return terms
diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py
index 8957464..bf0695d 100644
--- a/scripts/lib/reddit.py
+++ b/scripts/lib/reddit.py
@@ -48,7 +48,7 @@ DEPTH_CONFIG = {
     },
 }
 
-from .query import extract_core_subject as _query_extract
+from .query import detect_query_type, extract_core_subject as _query_extract
 from .relevance import token_overlap_relevance
 
 # Reddit-specific noise words (preserves original smaller set)
@@ -107,7 +107,9 @@ def expand_reddit_queries(topic: str, depth: str) -> List[str]:
     if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
         queries.append(original_clean)
 
-    if depth in ("default", "deep"):
+    # Add opinion/review variant except for how_to/comparison queries
+    qtype = detect_query_type(topic)
+    if depth in ("default", "deep") and qtype not in ("how_to", "comparison"):
         queries.append(f"{core} worth it OR thoughts OR review")
 
     if depth == "deep":
diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py
index 680de91..e6b23ba 100644
--- a/scripts/lib/youtube_yt.py
+++ b/scripts/lib/youtube_yt.py
@@ -100,15 +100,16 @@ def search_youtube(
     _log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})")
 
     # yt-dlp search with full metadata (no --flat-playlist so dates are real).
-    # No --dateafter — we filter by date in Python with a soft fallback,
-    # because YouTube search returns relevance-sorted results and strict date
-    # filtering returns 0 for evergreen topics like "thumbnail tips".
+    # --dateafter helps yt-dlp filter server-side, but Python soft filter
+    # (below) handles the fallback for evergreen topics with 0 recent results.
+    dateafter = from_date.replace("-", "")  # YYYYMMDD format for yt-dlp
     cmd = [
         "yt-dlp",
         f"ytsearch{count}:{core_topic}",
         "--dump-json",
         "--no-warnings",
         "--no-download",
+        "--dateafter", dateafter,
     ]
 
     preexec = os.setsid if hasattr(os, 'setsid') else None
diff --git a/tests/test_query.py b/tests/test_query.py
index a081f4f..1cf8b49 100644
--- a/tests/test_query.py
+++ b/tests/test_query.py
@@ -6,7 +6,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
 
-from lib.query import NOISE_WORDS, extract_core_subject
+from lib.query import NOISE_WORDS, detect_query_type, extract_compound_terms, extract_core_subject
 
 
 class TestExtractCoreSubject(unittest.TestCase):
@@ -126,5 +126,49 @@ class TestNoiseWordsCompleteness(unittest.TestCase):
             self.assertIn(w, NOISE_WORDS)
 
 
+class TestDetectQueryType(unittest.TestCase):
+    """Tests for detect_query_type()."""
+
+    def test_comparison(self):
+        self.assertEqual(detect_query_type("React vs Vue"), "comparison")
+
+    def test_how_to(self):
+        self.assertEqual(detect_query_type("how to deploy on Vercel"), "how_to")
+
+    def test_opinion(self):
+        self.assertEqual(detect_query_type("cursor IDE worth it"), "opinion")
+
+    def test_product(self):
+        self.assertEqual(detect_query_type("cursor IDE pricing"), "product")
+
+    def test_concept_default(self):
+        self.assertEqual(detect_query_type("multi-agent reinforcement learning"), "concept")
+
+    def test_how_prefix(self):
+        self.assertEqual(detect_query_type("how does Claude work"), "how_to")
+
+
+class TestExtractCompoundTerms(unittest.TestCase):
+    """Tests for extract_compound_terms()."""
+
+    def test_hyphenated(self):
+        terms = extract_compound_terms("multi-agent reinforcement learning")
+        self.assertIn("multi-agent", terms)
+
+    def test_title_case(self):
+        terms = extract_compound_terms("Claude Code and React Native")
+        self.assertTrue(any("Claude Code" in t for t in terms))
+        self.assertTrue(any("React Native" in t for t in terms))
+
+    def test_no_compounds(self):
+        terms = extract_compound_terms("python tutorial")
+        self.assertEqual(len(terms), 0)
+
+    def test_multiple_hyphens(self):
+        terms = extract_compound_terms("vc-backed start-up")
+        self.assertIn("vc-backed", terms)
+        self.assertIn("start-up", terms)
+
+
 if __name__ == "__main__":
     unittest.main()

From 046795c4ae0cb694d87948f3be6acfb7def66086 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:26:45 -0700
Subject: [PATCH 08/18] Add post-retrieval relevance filtering across all
 sources

Filter items with relevance < 0.3 per source after dedup, but only
when list has >3 items. Extends the Reddit-only minimum-result
guarantee to all sources: keeps top 3 by relevance if all filtered.

This works with the computed relevance scores from the previous commit
to actually remove off-topic results from the final report.
---
 scripts/last30days.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/scripts/last30days.py b/scripts/last30days.py
index ea3f753..60c6608 100644
--- a/scripts/last30days.py
+++ b/scripts/last30days.py
@@ -1829,12 +1829,28 @@ def main():
     deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
     deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
 
-    # Minimum result guarantee: if all Reddit results were filtered out but
-    # we had raw results, keep top 3 by relevance regardless of score
-    if not deduped_reddit and normalized_reddit:
-        print("[REDDIT WARNING] All results scored below threshold, keeping top 3 by relevance", file=sys.stderr)
-        by_relevance = sorted(normalized_reddit, key=lambda item: item.relevance, reverse=True)
-        deduped_reddit = by_relevance[:3]
+    # Post-retrieval relevance filter: drop low-relevance items per source
+    # Only filter when there are enough items (>3) to avoid empty results
+    def _relevance_filter(items, source_name, threshold=0.3):
+        """Filter items below relevance threshold with minimum-result guarantee."""
+        if len(items) <= 3:
+            return items
+        passed = [i for i in items if getattr(i, 'relevance', 0.7) >= threshold]
+        if not passed:
+            # Keep top 3 by relevance if all filtered
+            print(f"[{source_name} WARNING] All results below relevance {threshold}, keeping top 3", file=sys.stderr)
+            by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.7), reverse=True)
+            return by_rel[:3]
+        return passed
+
+    deduped_reddit = _relevance_filter(deduped_reddit, "REDDIT")
+    deduped_x = _relevance_filter(deduped_x, "X")
+    deduped_youtube = _relevance_filter(deduped_youtube, "YOUTUBE")
+    deduped_tiktok = _relevance_filter(deduped_tiktok, "TIKTOK")
+    deduped_ig = _relevance_filter(deduped_ig, "INSTAGRAM")
+    deduped_hn = _relevance_filter(deduped_hn, "HN")
+    deduped_bsky = _relevance_filter(deduped_bsky, "BLUESKY")
+    deduped_ts = _relevance_filter(deduped_ts, "TRUTHSOCIAL")
 
     # Cross-source linking: annotate items that discuss the same story
     dedupe.cross_source_link(

From 6c402f66b7e8347f06705bee9d96edb76acb3ba0 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:45:42 -0700
Subject: [PATCH 09/18] Update plan to reflect single upstream PR strategy

---
 ...ctor-query-relevance-consolidation-plan.md | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md b/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
index e82d684..31f814c 100644
--- a/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
+++ b/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
@@ -1,16 +1,17 @@
 # Search Pipeline: Query & Relevance Consolidation
 
-## Strategy: Two-phase delivery
+## Strategy: Single upstream PR
 
-**Phase 1 — Upstream PR** (`refactor/query-relevance-consolidation` -> `mvanhorn/last30days-skill:main`)
-Pure refactors and bug fixes anyone would want. No behavior changes.
+**Branch**: `refactor/query-relevance-consolidation` -> `mvanhorn/last30days-skill:main`
+**PR**: https://github.com/mvanhorn/last30days-skill/pull/65
 
-**Phase 2 — Fork-only** (`feat/search-quality` on `j-sperling/last30days-skill`)
-Opinionated behavior changes: computed relevance scores, platform-specific query optimization, post-retrieval filtering.
+All changes (refactors + behavior improvements) combined into one upstream PR.
+Originally planned as two phases, but the search quality improvements are
+broadly useful, not opinionated — merged into a single contribution.
 
 ---
 
-## Phase 1: Upstream PR (refactors + bug fixes)
+## Refactors (commits 1-5)
 
 ### Step 1: New `query.py` — shared query utilities
 - Consolidate 7 duplicated `_extract_core_subject()` (bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into one parameterized function
@@ -49,7 +50,7 @@ Opinionated behavior changes: computed relevance scores, platform-specific query
 
 ---
 
-## Phase 2: Fork-only (behavior changes)
+## Search quality improvements (commits 6-8)
 
 ### Step 4: Replace hardcoded relevance with computed scores
 - `bird_x.py:471` — `"relevance": 0.7` → `token_overlap_relevance(core_topic, text)`
@@ -73,6 +74,10 @@ Opinionated behavior changes: computed relevance scores, platform-specific query
 
 ---
 
+## Status: COMPLETE
+
+All 8 commits pushed to `refactor/query-relevance-consolidation`. PR #65 updated.
+
 ## Verify
 
 ```bash

From 036bcd2ae335a528f0e8adb0ad1573d424fadec8 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 18:40:07 -0700
Subject: [PATCH 10/18] Address review feedback: deduplicate query_type, clean
 unused imports, fix defaults

- Remove duplicate detect_query_type from query.py (divergent 5-type version);
  canonical 7-type version lives in query_type.py
- Fix reddit.py import to use query_type.detect_query_type
- Clean unused STOPWORDS/SYNONYMS/tokenize imports from youtube_yt, instagram,
  tiktok, scrapecreators_x, bird_x after relevance consolidation
- Fix _relevance_filter default from 0.7 to 0.0 (items without relevance
  should not silently pass the filter)
- Remove --dateafter from yt-dlp (returns 0 results for evergreen topics)
- Remove restrictSearchableAttributes from HN search (misses Ask/Show HN)
- Lower HN points filter from >5 to >2 (avoids filtering niche posts)
- Add error logging to select_openai_model HTTP failures
- Remove mise.toml and internal planning doc from repo
- Update module docstrings to describe current purpose, not migration history
- Update tests to import from canonical relevance module
---
 ...ctor-query-relevance-consolidation-plan.md | 85 -------------------
 mise.toml                                     |  2 -
 scripts/last30days.py                         |  4 +-
 scripts/lib/bird_x.py                         |  2 +-
 scripts/lib/hackernews.py                     |  7 +-
 scripts/lib/instagram.py                      |  7 +-
 scripts/lib/models.py                         |  4 +
 scripts/lib/query.py                          | 51 +----------
 scripts/lib/reddit.py                         |  3 +-
 scripts/lib/relevance.py                      |  5 +-
 scripts/lib/scrapecreators_x.py               |  7 +-
 scripts/lib/tiktok.py                         |  7 +-
 scripts/lib/youtube_yt.py                     | 14 +--
 tests/test_instagram_sc.py                    | 13 +--
 tests/test_models.py                          |  6 ++
 tests/test_query.py                           | 23 +----
 tests/test_scrapecreators_x.py                |  9 +-
 tests/test_youtube_relevance.py               |  2 +-
 18 files changed, 44 insertions(+), 207 deletions(-)
 delete mode 100644 docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
 delete mode 100644 mise.toml

diff --git a/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md b/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
deleted file mode 100644
index 31f814c..0000000
--- a/docs/plans/2026-03-11-refactor-query-relevance-consolidation-plan.md
+++ /dev/null
@@ -1,85 +0,0 @@
-# Search Pipeline: Query & Relevance Consolidation
-
-## Strategy: Single upstream PR
-
-**Branch**: `refactor/query-relevance-consolidation` -> `mvanhorn/last30days-skill:main`
-**PR**: https://github.com/mvanhorn/last30days-skill/pull/65
-
-All changes (refactors + behavior improvements) combined into one upstream PR.
-Originally planned as two phases, but the search quality improvements are
-broadly useful, not opinionated — merged into a single contribution.
-
----
-
-## Refactors (commits 1-5)
-
-### Step 1: New `query.py` — shared query utilities
-- Consolidate 7 duplicated `_extract_core_subject()` (bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into one parameterized function
-- `extract_core_subject(topic, noise=None, max_words=None, strip_suffixes=False)` — platform modules pass their own noise set and options to preserve current behavior
-- Shared `PREFIXES` list (identical across all 7), shared `NOISE_WORDS` base set
-- Each platform imports `extract_core_subject` and calls with its own overrides (e.g. bird_x passes `max_words=5, strip_suffixes=True`; youtube keeps tips/tricks/tutorial in its noise exclusion)
-- Fix reddit.py prefix-loop missing `break` (apply all matching prefixes vs only first)
-- Skip polymarket.py (too different — handles "last N days", preserves title case)
-- Tests: `tests/test_query.py`
-
-### Step 2: New `relevance.py` — shared relevance scoring
-- Consolidate `_tokenize`, `_compute_relevance`, `STOPWORDS`, `SYNONYMS` from youtube_yt/tiktok/instagram
-- `token_overlap_relevance(query, text, hashtags=None) -> float` — zero-dep, superset of all three implementations (hashtag substring matching from tiktok/instagram, synonym expansion from youtube)
-- Unified `SYNONYMS` dict (youtube superset: includes svelte/vue entries missing from tiktok/instagram)
-- Tests: `tests/test_relevance.py` (migrate from `test_youtube_relevance.py` + new hashtag tests)
-
-### Step 6: urllib fallback for TikTok/Instagram (independent bug fix)
-- `tiktok.py`: add `http.get()`/`http.post()` fallback when `_requests is None`
-- `instagram.py`: same pattern
-- Copies pattern from reddit.py's existing fallback
-
-### Step 3: Integrate `query.py` into per-source modules (pure refactor)
-- `bird_x.py`: replace lines 52-106 with import, call `extract_core_subject(topic, max_words=5, strip_suffixes=True, noise=BIRD_NOISE)`
-- `reddit.py`: replace `NOISE_WORDS` + `_extract_core_subject` with query import; `expand_reddit_queries` imports from query.py too
-- `youtube_yt.py`: replace `_extract_core_subject` with import, pass youtube-specific noise set (keeps tips/tricks/tutorial/guide/review)
-- `tiktok.py`, `instagram.py`, `bluesky.py`: same replacement with their noise sets
-- Update tests: 12+ test methods across 6 test files reference `module._extract_core_subject()` — either re-export from original modules or update test imports
-
-### Step 8: Deduplicate relevance code in youtube/tiktok/instagram (pure refactor)
-- `youtube_yt.py`: remove `STOPWORDS`, `SYNONYMS`, `_tokenize`, `_compute_relevance`; import from `relevance.py`
-- `tiktok.py`: same
-- `instagram.py`: same
-- Update tests: `test_youtube_relevance.py`, `test_tiktok.py`, `test_instagram_sc.py`, `test_scrapecreators_x.py` reference `module._tokenize`/`module._compute_relevance` — re-export or update imports
-
-### Commit order: 1 → 2 → 6 → 3 → 8
-
----
-
-## Search quality improvements (commits 6-8)
-
-### Step 4: Replace hardcoded relevance with computed scores
-- `bird_x.py:471` — `"relevance": 0.7` → `token_overlap_relevance(core_topic, text)`
-- `reddit.py:223` — `"relevance": 0.7` → `token_overlap_relevance(core, title + " " + selftext)`
-- `hackernews.py:139-141` — blend: `0.6 * rank_score + 0.4 * token_overlap`
-
-### Step 5: Platform-specific query optimization
-- `detect_query_type(topic)` — heuristic classifier (product/concept/opinion/how_to/comparison), added here not Phase 1
-- `extract_compound_terms(topic)` — detect hyphenated/title-case terms, return quoted
-- `bird_x.py`: OR-group construction for multi-concept queries, OR-based retry before word-dropping fallback
-- `reddit.py`: conditional opinion/review suffix only for product/opinion queries (uses `detect_query_type`)
-- `hackernews.py`: add `numericFilters: points>5`, `restrictSearchableAttributes=title`, use `extract_core_subject()` instead of raw topic
-- `youtube_yt.py`: add `--dateafter YYYYMMDD` (from_date already in signature)
-
-### Step 7: Post-retrieval relevance filtering in orchestrator
-- `last30days.py` (after dedup): filter items with `relevance < 0.3` per source (only when list has >3 items)
-- Extend fallback guarantee to all sources: keep top 3 by relevance if all filtered
-- `rerank_with_embeddings()` — optional, env-var gated (`OPENAI_API_KEY` or `GOOGLE_API_KEY`), uses existing `http.py`, graceful fallback to token overlap
-
-### Commit order: 4 → 5 → 7
-
----
-
-## Status: COMPLETE
-
-All 8 commits pushed to `refactor/query-relevance-consolidation`. PR #65 updated.
-
-## Verify
-
-```bash
-cd ~/projects/last30days-skill && python3 -m unittest discover -s tests -v
-```
diff --git a/mise.toml b/mise.toml
deleted file mode 100644
index a190abb..0000000
--- a/mise.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-[tools]
-python = "3.12"
diff --git a/scripts/last30days.py b/scripts/last30days.py
index 60c6608..db99ccc 100644
--- a/scripts/last30days.py
+++ b/scripts/last30days.py
@@ -1835,11 +1835,11 @@ def main():
         """Filter items below relevance threshold with minimum-result guarantee."""
         if len(items) <= 3:
             return items
-        passed = [i for i in items if getattr(i, 'relevance', 0.7) >= threshold]
+        passed = [i for i in items if getattr(i, 'relevance', 0.0) >= threshold]
         if not passed:
             # Keep top 3 by relevance if all filtered
             print(f"[{source_name} WARNING] All results below relevance {threshold}, keeping top 3", file=sys.stderr)
-            by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.7), reverse=True)
+            by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.0), reverse=True)
             return by_rel[:3]
         return passed
 
diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py
index dd9fa9f..5146f37 100644
--- a/scripts/lib/bird_x.py
+++ b/scripts/lib/bird_x.py
@@ -58,7 +58,7 @@ def _extract_core_subject(topic: str) -> str:
     Aggressively strip question/meta/research words to keep only the
     core product/concept name (max 5 words).
     """
-    from .query import NOISE_WORDS, extract_core_subject
+    from .query import extract_core_subject
     return extract_core_subject(topic, max_words=5, strip_suffixes=True)
 
 
diff --git a/scripts/lib/hackernews.py b/scripts/lib/hackernews.py
index 5b6fab1..cda271b 100644
--- a/scripts/lib/hackernews.py
+++ b/scripts/lib/hackernews.py
@@ -90,13 +90,14 @@ def search_hackernews(
     core = extract_core_subject(topic)
     _log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
 
-    # Use relevance-sorted search with minimum engagement filter
+    # Use relevance-sorted search with minimum engagement filter.
+    # NOTE: restrictSearchableAttributes=title omitted intentionally — it would
+    # miss Ask HN/Show HN threads where the topic appears in the body.
     params = {
         "query": core,
         "tags": "story",
-        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>5",
+        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
         "hitsPerPage": str(count),
-        "restrictSearchableAttributes": "title",
     }
 
     from urllib.parse import urlencode
diff --git a/scripts/lib/instagram.py b/scripts/lib/instagram.py
index cb166fd..450b9e4 100644
--- a/scripts/lib/instagram.py
+++ b/scripts/lib/instagram.py
@@ -31,12 +31,7 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _extract_core_subject(topic: str) -> str:
diff --git a/scripts/lib/models.py b/scripts/lib/models.py
index dd25ad0..4f51a11 100644
--- a/scripts/lib/models.py
+++ b/scripts/lib/models.py
@@ -53,6 +53,10 @@ def is_search_capable_model(model_id: str) -> bool:
     Includes mini variants (same structured extraction quality, lower cost).
     Excludes: nano (no web_search), gpt-4o-mini (no domain filtering),
     chat/codex/pro/preview/turbo/search (specialized variants).
+
+    Note: gpt-5 with reasoning effort="minimal" does NOT support web_search
+    (per OpenAI docs). We never set reasoning params — our usage is pure
+    tool invocation + JSON extraction — so gpt-5 is safe to include here.
     """
     model_lower = model_id.lower()
 
diff --git a/scripts/lib/query.py b/scripts/lib/query.py
index e7bda9a..77f35f8 100644
--- a/scripts/lib/query.py
+++ b/scripts/lib/query.py
@@ -1,9 +1,5 @@
-"""Shared query utilities for /last30days search modules.
-
-Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
-youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
-parameterized function. Each platform calls with its own overrides.
-"""
+"""Shared query preprocessing utilities: noise-word stripping, core subject
+extraction, and compound term detection. Used by all search modules."""
 
 import re
 from typing import FrozenSet, List, Optional, Set
@@ -99,49 +95,6 @@ def extract_core_subject(
     return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
 
 
-# ---- Query type detection (heuristic, no LLM) ----
-
-_OPINION_SIGNALS = frozenset({
-    'worth', 'thoughts', 'opinion', 'opinions', 'review', 'reviews',
-    'recommend', 'recommendation', 'recommendations', 'should',
-    'anyone', 'anybody', 'experience', 'experiences',
-})
-
-_HOW_TO_SIGNALS = frozenset({
-    'how', 'setup', 'configure', 'install', 'tutorial', 'guide',
-    'step', 'steps', 'instructions',
-})
-
-_COMPARISON_SIGNALS = frozenset({
-    'vs', 'versus', 'compared', 'comparison', 'better', 'alternative',
-    'alternatives', 'difference', 'differences',
-})
-
-_PRODUCT_SIGNALS = frozenset({
-    'pricing', 'price', 'cost', 'plan', 'plans', 'tier', 'tiers',
-    'buy', 'purchase', 'subscription', 'trial', 'free',
-})
-
-
-def detect_query_type(topic: str) -> str:
-    """Classify query intent without an LLM.
-
-    Returns one of: "product", "concept", "opinion", "how_to", "comparison".
-    Used to adapt per-platform query construction.
-    """
-    words = set(topic.lower().split())
-
-    if words & _COMPARISON_SIGNALS:
-        return "comparison"
-    if words & _HOW_TO_SIGNALS or topic.lower().startswith("how "):
-        return "how_to"
-    if words & _OPINION_SIGNALS:
-        return "opinion"
-    if words & _PRODUCT_SIGNALS:
-        return "product"
-    return "concept"
-
-
 def extract_compound_terms(topic: str) -> List[str]:
     """Detect multi-word terms that should be quoted in search queries.
 
diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py
index bf0695d..8714ed1 100644
--- a/scripts/lib/reddit.py
+++ b/scripts/lib/reddit.py
@@ -48,7 +48,8 @@ DEPTH_CONFIG = {
     },
 }
 
-from .query import detect_query_type, extract_core_subject as _query_extract
+from .query import extract_core_subject as _query_extract
+from .query_type import detect_query_type
 from .relevance import token_overlap_relevance
 
 # Reddit-specific noise words (preserves original smaller set)
diff --git a/scripts/lib/relevance.py b/scripts/lib/relevance.py
index f936b70..eb57b61 100644
--- a/scripts/lib/relevance.py
+++ b/scripts/lib/relevance.py
@@ -1,7 +1,6 @@
-"""Shared relevance scoring for /last30days search modules.
+"""Shared token-overlap relevance scoring for search result ranking.
 
-Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS
-from youtube_yt, tiktok, instagram, and scrapecreators_x into one module.
+Tokenizes text, expands synonyms, and computes query-to-content overlap ratios.
 """
 
 import re
diff --git a/scripts/lib/scrapecreators_x.py b/scripts/lib/scrapecreators_x.py
index 27bbb89..6266019 100644
--- a/scripts/lib/scrapecreators_x.py
+++ b/scripts/lib/scrapecreators_x.py
@@ -24,12 +24,7 @@ DEPTH_CONFIG = {
     "deep":    {"results_per_page": 40},
 }
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _extract_core_subject(topic: str) -> str:
diff --git a/scripts/lib/tiktok.py b/scripts/lib/tiktok.py
index 56473f6..25fb270 100644
--- a/scripts/lib/tiktok.py
+++ b/scripts/lib/tiktok.py
@@ -31,12 +31,7 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _extract_core_subject(topic: str) -> str:
diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py
index e6b23ba..5ac8137 100644
--- a/scripts/lib/youtube_yt.py
+++ b/scripts/lib/youtube_yt.py
@@ -35,12 +35,7 @@ TRANSCRIPT_LIMITS = {
 # Max words to keep from each transcript
 TRANSCRIPT_MAX_WORDS = 500
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _log(msg: str):
@@ -100,16 +95,15 @@ def search_youtube(
     _log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})")
 
     # yt-dlp search with full metadata (no --flat-playlist so dates are real).
-    # --dateafter helps yt-dlp filter server-side, but Python soft filter
-    # (below) handles the fallback for evergreen topics with 0 recent results.
-    dateafter = from_date.replace("-", "")  # YYYYMMDD format for yt-dlp
+    # NOTE: --dateafter intentionally omitted — YouTube search returns
+    # relevance-sorted results and strict date filtering returns 0 for
+    # evergreen topics. Python soft filter (below) handles date filtering.
     cmd = [
         "yt-dlp",
         f"ytsearch{count}:{core_topic}",
         "--dump-json",
         "--no-warnings",
         "--no-download",
-        "--dateafter", dateafter,
     ]
 
     preexec = os.setsid if hasattr(os, 'setsid') else None
diff --git a/tests/test_instagram_sc.py b/tests/test_instagram_sc.py
index 0076c80..59fa3a8 100644
--- a/tests/test_instagram_sc.py
+++ b/tests/test_instagram_sc.py
@@ -8,34 +8,35 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
 
 from lib import instagram
+from lib.relevance import tokenize as _tokenize
 
 
 class TestTokenize(unittest.TestCase):
-    """Tests for _tokenize()."""
+    """Tests for tokenize() from relevance module."""
 
     def test_strips_stopwords(self):
-        tokens = instagram._tokenize("how to use the AI tools")
+        tokens = _tokenize("how to use the AI tools")
         self.assertNotIn("how", tokens)
         self.assertNotIn("the", tokens)
         self.assertNotIn("to", tokens)
 
     def test_expands_synonyms(self):
-        tokens = instagram._tokenize("ai tools")
+        tokens = _tokenize("ai tools")
         self.assertTrue("artificial" in tokens or "intelligence" in tokens)
 
     def test_removes_single_char(self):
-        tokens = instagram._tokenize("a b c python")
+        tokens = _tokenize("a b c python")
         self.assertNotIn("a", tokens)
         self.assertNotIn("b", tokens)
         self.assertIn("python", tokens)
 
     def test_lowercases(self):
-        tokens = instagram._tokenize("Python REACT")
+        tokens = _tokenize("Python REACT")
         self.assertIn("python", tokens)
         self.assertIn("react", tokens)
 
     def test_strips_punctuation(self):
-        tokens = instagram._tokenize("hello, world!")
+        tokens = _tokenize("hello, world!")
         self.assertIn("hello", tokens)
         self.assertIn("world", tokens)
 
diff --git a/tests/test_models.py b/tests/test_models.py
index 36db570..e68a08a 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -30,6 +30,12 @@ class TestParseVersion(unittest.TestCase):
 
 class TestIsSearchCapableModel(unittest.TestCase):
     def test_gpt5_is_capable(self):
+        """gpt-5 supports web_search when reasoning is not set to 'minimal'.
+
+        Per OpenAI docs, gpt-5 with reasoning effort="minimal" does NOT
+        support web_search. We never set reasoning params (our usage is
+        tool invocation + JSON extraction only), so gpt-5 is safe here.
+        """
         self.assertTrue(models.is_search_capable_model("gpt-5"))
 
     def test_gpt52_is_capable(self):
diff --git a/tests/test_query.py b/tests/test_query.py
index 1cf8b49..55c759c 100644
--- a/tests/test_query.py
+++ b/tests/test_query.py
@@ -6,7 +6,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
 
-from lib.query import NOISE_WORDS, detect_query_type, extract_compound_terms, extract_core_subject
+from lib.query import NOISE_WORDS, extract_compound_terms, extract_core_subject
 
 
 class TestExtractCoreSubject(unittest.TestCase):
@@ -126,27 +126,6 @@ class TestNoiseWordsCompleteness(unittest.TestCase):
             self.assertIn(w, NOISE_WORDS)
 
 
-class TestDetectQueryType(unittest.TestCase):
-    """Tests for detect_query_type()."""
-
-    def test_comparison(self):
-        self.assertEqual(detect_query_type("React vs Vue"), "comparison")
-
-    def test_how_to(self):
-        self.assertEqual(detect_query_type("how to deploy on Vercel"), "how_to")
-
-    def test_opinion(self):
-        self.assertEqual(detect_query_type("cursor IDE worth it"), "opinion")
-
-    def test_product(self):
-        self.assertEqual(detect_query_type("cursor IDE pricing"), "product")
-
-    def test_concept_default(self):
-        self.assertEqual(detect_query_type("multi-agent reinforcement learning"), "concept")
-
-    def test_how_prefix(self):
-        self.assertEqual(detect_query_type("how does Claude work"), "how_to")
-
 
 class TestExtractCompoundTerms(unittest.TestCase):
     """Tests for extract_compound_terms()."""
diff --git a/tests/test_scrapecreators_x.py b/tests/test_scrapecreators_x.py
index e899555..0ebb429 100644
--- a/tests/test_scrapecreators_x.py
+++ b/tests/test_scrapecreators_x.py
@@ -6,26 +6,27 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
 from lib import scrapecreators_x
+from lib.relevance import tokenize as _tokenize
 
 
 class TestTokenize(unittest.TestCase):
     def test_lowercases(self):
-        tokens = scrapecreators_x._tokenize("Claude AI")
+        tokens = _tokenize("Claude AI")
         self.assertIn("claude", tokens)
 
     def test_strips_stopwords(self):
-        tokens = scrapecreators_x._tokenize("the best AI tool")
+        tokens = _tokenize("the best AI tool")
         self.assertNotIn("the", tokens)
         self.assertIn("best", tokens)  # 'best' is not a stopword in tokenizer
 
     def test_removes_single_char(self):
-        tokens = scrapecreators_x._tokenize("a b cd ef")
+        tokens = _tokenize("a b cd ef")
         self.assertNotIn("a", tokens)
         self.assertNotIn("b", tokens)
         self.assertIn("cd", tokens)
 
     def test_expands_synonyms(self):
-        tokens = scrapecreators_x._tokenize("ai research")
+        tokens = _tokenize("ai research")
         self.assertIn("artificial", tokens)
         self.assertIn("intelligence", tokens)
 
diff --git a/tests/test_youtube_relevance.py b/tests/test_youtube_relevance.py
index cc55d28..432cb69 100644
--- a/tests/test_youtube_relevance.py
+++ b/tests/test_youtube_relevance.py
@@ -7,7 +7,7 @@ from pathlib import Path
 # Add lib to path
 sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
 
-from lib.youtube_yt import _compute_relevance, _tokenize
+from lib.relevance import token_overlap_relevance as _compute_relevance, tokenize as _tokenize
 
 
 class TestTokenize(unittest.TestCase):

From d8d2b97716506970ee2ce1d3b5e1629c6785d5e3 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 19:01:33 -0700
Subject: [PATCH 11/18] Gitignore mise.toml instead of removing it

Dev environment tool config is useful locally but shouldn't be tracked.
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 68bb607..7833a3b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ variants/open/references/research.md
 .entire/
 __pycache__/
 *.pyc
+mise.toml

From cbee987f65174eecd6d6ada0725d0e4825db3ccc Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 19:09:06 -0700
Subject: [PATCH 12/18] Extract relevance_filter, add Bluesky/TruthSocial type
 hint + test coverage

- Extract _relevance_filter from last30days.py closure to score.relevance_filter()
  for testability
- Add BlueskyItem/TruthSocialItem to sort_items() type hint (was missing despite
  being in _ITEM_SOURCE_MAP)
- Add tests: Bluesky/TruthSocial engagement scoring, sort_items mixed sources,
  relevance_filter behavior (threshold, minimum-result guarantee, missing attr),
  select_openai_model HTTP 401/403 error paths
---
 scripts/last30days.py |  29 +++-----
 scripts/lib/score.py  |  20 +++++-
 tests/test_models.py  |  21 ++++++
 tests/test_score.py   | 155 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 203 insertions(+), 22 deletions(-)

diff --git a/scripts/last30days.py b/scripts/last30days.py
index db99ccc..29365ad 100644
--- a/scripts/last30days.py
+++ b/scripts/last30days.py
@@ -1830,27 +1830,14 @@ def main():
     deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
 
     # Post-retrieval relevance filter: drop low-relevance items per source
-    # Only filter when there are enough items (>3) to avoid empty results
-    def _relevance_filter(items, source_name, threshold=0.3):
-        """Filter items below relevance threshold with minimum-result guarantee."""
-        if len(items) <= 3:
-            return items
-        passed = [i for i in items if getattr(i, 'relevance', 0.0) >= threshold]
-        if not passed:
-            # Keep top 3 by relevance if all filtered
-            print(f"[{source_name} WARNING] All results below relevance {threshold}, keeping top 3", file=sys.stderr)
-            by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.0), reverse=True)
-            return by_rel[:3]
-        return passed
-
-    deduped_reddit = _relevance_filter(deduped_reddit, "REDDIT")
-    deduped_x = _relevance_filter(deduped_x, "X")
-    deduped_youtube = _relevance_filter(deduped_youtube, "YOUTUBE")
-    deduped_tiktok = _relevance_filter(deduped_tiktok, "TIKTOK")
-    deduped_ig = _relevance_filter(deduped_ig, "INSTAGRAM")
-    deduped_hn = _relevance_filter(deduped_hn, "HN")
-    deduped_bsky = _relevance_filter(deduped_bsky, "BLUESKY")
-    deduped_ts = _relevance_filter(deduped_ts, "TRUTHSOCIAL")
+    deduped_reddit = score.relevance_filter(deduped_reddit, "REDDIT")
+    deduped_x = score.relevance_filter(deduped_x, "X")
+    deduped_youtube = score.relevance_filter(deduped_youtube, "YOUTUBE")
+    deduped_tiktok = score.relevance_filter(deduped_tiktok, "TIKTOK")
+    deduped_ig = score.relevance_filter(deduped_ig, "INSTAGRAM")
+    deduped_hn = score.relevance_filter(deduped_hn, "HN")
+    deduped_bsky = score.relevance_filter(deduped_bsky, "BLUESKY")
+    deduped_ts = score.relevance_filter(deduped_ts, "TRUTHSOCIAL")
 
     # Cross-source linking: annotate items that discuss the same story
     dedupe.cross_source_link(
diff --git a/scripts/lib/score.py b/scripts/lib/score.py
index ffd108d..9ac8502 100644
--- a/scripts/lib/score.py
+++ b/scripts/lib/score.py
@@ -715,7 +715,7 @@ _ITEM_SOURCE_MAP = {
 _DEFAULT_TIEBREAKER = {"reddit": 0, "x": 1, "youtube": 2, "tiktok": 3, "instagram": 4, "hn": 5, "bluesky": 6, "truthsocial": 7, "polymarket": 8, "web": 9}
 
 
-def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem]], query_type: QueryType = None) -> List:
+def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.BlueskyItem, schema.TruthSocialItem, schema.PolymarketItem]], query_type: QueryType = None) -> List:
     """Sort items by score (descending), then date, then source tiebreaker.
 
     Tiebreaker (tertiary sort key, after score and date): source priority
@@ -749,3 +749,21 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
         return (score, date_key, source_priority, text)
 
     return sorted(items, key=sort_key)
+
+
+def relevance_filter(items, source_name: str, threshold: float = 0.3):
+    """Filter items below relevance threshold with minimum-result guarantee.
+
+    Items with no relevance attribute are treated as 0.0 (fail the filter).
+    If all items are below threshold, keeps the top 3 by relevance.
+    Lists with 3 or fewer items are returned unchanged.
+    """
+    import sys
+    if len(items) <= 3:
+        return items
+    passed = [i for i in items if getattr(i, 'relevance', 0.0) >= threshold]
+    if not passed:
+        print(f"[{source_name} WARNING] All results below relevance {threshold}, keeping top 3", file=sys.stderr)
+        by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.0), reverse=True)
+        return by_rel[:3]
+    return passed
diff --git a/tests/test_models.py b/tests/test_models.py
index e68a08a..f093f27 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -140,6 +140,27 @@ class TestSelectOpenAIModel(unittest.TestCase):
         self.assertEqual(result, "gpt-4.1-mini")
 
 
+class TestSelectOpenAIModelErrorPaths(unittest.TestCase):
+    def setUp(self):
+        from lib import cache
+        cache.MODEL_CACHE_FILE.unlink(missing_ok=True)
+
+    def test_http_error_returns_fallback(self):
+        """HTTPError during model fetch should return fallback, not crash."""
+        from unittest.mock import patch
+        from lib import http
+        with patch('lib.http.get', side_effect=http.HTTPError("Unauthorized", status_code=401)):
+            result = models.select_openai_model("bad-key", policy="auto")
+        self.assertEqual(result, models.OPENAI_FALLBACK_MODELS[0])
+
+    def test_http_403_returns_fallback(self):
+        from unittest.mock import patch
+        from lib import http
+        with patch('lib.http.get', side_effect=http.HTTPError("Forbidden", status_code=403)):
+            result = models.select_openai_model("bad-key", policy="auto")
+        self.assertEqual(result, models.OPENAI_FALLBACK_MODELS[0])
+
+
 class TestSelectXAIModel(unittest.TestCase):
     def test_latest_policy(self):
         result = models.select_xai_model(
diff --git a/tests/test_score.py b/tests/test_score.py
index c5d010e..436258e 100644
--- a/tests/test_score.py
+++ b/tests/test_score.py
@@ -192,5 +192,160 @@ class TestInstagramEngagement(unittest.TestCase):
         )
 
 
+class TestBlueskyEngagement(unittest.TestCase):
+    """Tests for compute_bluesky_engagement_raw()."""
+
+    def test_basic(self):
+        eng = schema.Engagement(likes=100, reposts=25, replies=15, quotes=5)
+        raw = score.compute_bluesky_engagement_raw(eng)
+        self.assertIsNotNone(raw)
+        self.assertGreater(raw, 0)
+
+    def test_likes_dominate(self):
+        likes_heavy = schema.Engagement(likes=1000, reposts=0, replies=0, quotes=0)
+        reposts_heavy = schema.Engagement(likes=0, reposts=1000, replies=0, quotes=0)
+        self.assertGreater(
+            score.compute_bluesky_engagement_raw(likes_heavy),
+            score.compute_bluesky_engagement_raw(reposts_heavy),
+        )
+
+    def test_none_engagement(self):
+        self.assertIsNone(score.compute_bluesky_engagement_raw(None))
+
+    def test_no_likes_no_reposts(self):
+        eng = schema.Engagement(replies=10)
+        self.assertIsNone(score.compute_bluesky_engagement_raw(eng))
+
+
+class TestTruthSocialEngagement(unittest.TestCase):
+    """Tests for compute_truthsocial_engagement_raw()."""
+
+    def test_basic(self):
+        eng = schema.Engagement(likes=100, reposts=25, replies=15)
+        raw = score.compute_truthsocial_engagement_raw(eng)
+        self.assertIsNotNone(raw)
+        self.assertGreater(raw, 0)
+
+    def test_likes_dominate(self):
+        likes_heavy = schema.Engagement(likes=1000, reposts=0, replies=0)
+        reposts_heavy = schema.Engagement(likes=0, reposts=1000, replies=0)
+        self.assertGreater(
+            score.compute_truthsocial_engagement_raw(likes_heavy),
+            score.compute_truthsocial_engagement_raw(reposts_heavy),
+        )
+
+    def test_none_engagement(self):
+        self.assertIsNone(score.compute_truthsocial_engagement_raw(None))
+
+
+class TestScoreBlueskyItems(unittest.TestCase):
+    """Tests for score_bluesky_items()."""
+
+    def test_scores_items(self):
+        items = [
+            schema.BlueskyItem(
+                id="bsky1", text="Test", url="https://bsky.app/1",
+                author_handle="user.bsky.social", display_name="User",
+                engagement=schema.Engagement(likes=50, reposts=10, replies=5, quotes=2),
+                relevance=0.8,
+            ),
+        ]
+        result = score.score_bluesky_items(items)
+        self.assertEqual(len(result), 1)
+        self.assertGreater(result[0].score, 0)
+
+    def test_empty_list(self):
+        self.assertEqual(score.score_bluesky_items([]), [])
+
+
+class TestScoreTruthSocialItems(unittest.TestCase):
+    """Tests for score_truthsocial_items()."""
+
+    def test_scores_items(self):
+        items = [
+            schema.TruthSocialItem(
+                id="ts1", text="Test", url="https://truthsocial.com/1",
+                author_handle="@user", display_name="User",
+                engagement=schema.Engagement(likes=50, reposts=10, replies=5),
+                relevance=0.8,
+            ),
+        ]
+        result = score.score_truthsocial_items(items)
+        self.assertEqual(len(result), 1)
+        self.assertGreater(result[0].score, 0)
+
+    def test_empty_list(self):
+        self.assertEqual(score.score_truthsocial_items([]), [])
+
+
+class TestSortItemsMixedSources(unittest.TestCase):
+    """Test sort_items with Bluesky and TruthSocial items."""
+
+    def test_bluesky_item_sorts(self):
+        items = [
+            schema.RedditItem(id="R1", title="Reddit", url="", subreddit="", score=30),
+            schema.BlueskyItem(id="B1", text="Bluesky", url="", author_handle="u.bsky.social", display_name="U", score=90),
+        ]
+        result = score.sort_items(items)
+        self.assertEqual(result[0].id, "B1")
+
+    def test_truthsocial_item_sorts(self):
+        items = [
+            schema.RedditItem(id="R1", title="Reddit", url="", subreddit="", score=30),
+            schema.TruthSocialItem(id="T1", text="TS", url="", author_handle="@u", display_name="U", score=90),
+        ]
+        result = score.sort_items(items)
+        self.assertEqual(result[0].id, "T1")
+
+
+class TestRelevanceFilter(unittest.TestCase):
+    """Tests for relevance_filter()."""
+
+    def _make_items(self, relevances):
+        """Helper: create RedditItems with given relevance values."""
+        return [
+            schema.RedditItem(id=f"R{i}", title=f"Item {i}", url="", subreddit="", relevance=r)
+            for i, r in enumerate(relevances)
+        ]
+
+    def test_filters_below_threshold(self):
+        items = self._make_items([0.8, 0.1, 0.5, 0.2])
+        result = score.relevance_filter(items, "TEST", threshold=0.3)
+        self.assertEqual(len(result), 2)
+        self.assertTrue(all(i.relevance >= 0.3 for i in result))
+
+    def test_small_list_unchanged(self):
+        items = self._make_items([0.1, 0.05, 0.02])
+        result = score.relevance_filter(items, "TEST")
+        self.assertEqual(len(result), 3)
+
+    def test_all_below_threshold_keeps_top_3(self):
+        items = self._make_items([0.1, 0.25, 0.05, 0.2, 0.15])
+        result = score.relevance_filter(items, "TEST", threshold=0.3)
+        self.assertEqual(len(result), 3)
+        # Should be sorted by relevance: 0.25, 0.2, 0.15
+        self.assertEqual(result[0].relevance, 0.25)
+        self.assertEqual(result[1].relevance, 0.2)
+
+    def test_empty_list(self):
+        result = score.relevance_filter([], "TEST")
+        self.assertEqual(result, [])
+
+    def test_items_without_relevance_attr_treated_as_zero(self):
+        """Objects lacking a relevance attribute get 0.0, failing the filter."""
+        class BareItem:
+            def __init__(self, id):
+                self.id = id
+        items = [
+            schema.RedditItem(id="R0", title="Has relevance", url="", subreddit="", relevance=0.8),
+            BareItem("B1"),
+            BareItem("B2"),
+            BareItem("B3"),
+        ]
+        result = score.relevance_filter(items, "TEST", threshold=0.3)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0].id, "R0")
+
+
 if __name__ == "__main__":
     unittest.main()

From 946af84f9a4381f4e204acdf6dab579ae212abd1 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Fri, 13 Mar 2026 19:21:25 -0700
Subject: [PATCH 13/18] Tighten relevance scoring and Polymarket ranking

Score against original user intent on Reddit, remove the artificial low-end relevance floor, and make Polymarket semantics dominate generic market quality signals.

Also apply the relevance filter to Polymarket and update the affected cross-source tests.

Validation: uv run python -m unittest
---
 scripts/last30days.py           |  1 +
 scripts/lib/polymarket.py       | 40 ++++++++-----------
 scripts/lib/reddit.py           |  8 ++--
 scripts/lib/relevance.py        | 70 +++++++++++++++++++++++++++++----
 scripts/lib/score.py            | 12 ++++--
 tests/test_instagram_sc.py      |  4 +-
 tests/test_polymarket.py        | 24 +++++++----
 tests/test_reddit_sc.py         | 20 ++++++++--
 tests/test_relevance.py         | 11 +++++-
 tests/test_scrapecreators_x.py  |  4 +-
 tests/test_tiktok.py            |  4 +-
 tests/test_youtube_relevance.py | 10 ++---
 12 files changed, 147 insertions(+), 61 deletions(-)

diff --git a/scripts/last30days.py b/scripts/last30days.py
index 29365ad..4bdfb02 100644
--- a/scripts/last30days.py
+++ b/scripts/last30days.py
@@ -1838,6 +1838,7 @@ def main():
     deduped_hn = score.relevance_filter(deduped_hn, "HN")
     deduped_bsky = score.relevance_filter(deduped_bsky, "BLUESKY")
     deduped_ts = score.relevance_filter(deduped_ts, "TRUTHSOCIAL")
+    deduped_pm = score.relevance_filter(deduped_pm, "POLYMARKET") if deduped_pm else []
 
     # Cross-source linking: annotate items that discuss the same story
     dedupe.cross_source_link(
diff --git a/scripts/lib/polymarket.py b/scripts/lib/polymarket.py
index 27c93da..f9da705 100644
--- a/scripts/lib/polymarket.py
+++ b/scripts/lib/polymarket.py
@@ -13,6 +13,7 @@ from typing import Any, Dict, List, Optional
 from urllib.parse import quote_plus, urlencode
 
 from . import http
+from .relevance import token_overlap_relevance
 
 GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search"
 
@@ -314,8 +315,8 @@ def _shorten_question(question: str) -> str:
 def _compute_text_similarity(topic: str, title: str, outcomes: List[str] = None) -> float:
     """Score how well the event title (or outcome names) match the search topic.
 
-    Returns 0.0-1.0. Title substring match gets 1.0, outcome match gets 0.85/0.7,
-    title token overlap gets proportional score.
+    Returns 0.0-1.0. Exact title phrase match gets 1.0. Otherwise we reuse the
+    shared query-centric relevance scorer and take the best title/outcome match.
     """
     core = _extract_core_subject(topic).lower()
     title_lower = title.lower()
@@ -326,27 +327,17 @@ def _compute_text_similarity(topic: str, title: str, outcomes: List[str] = None)
     if core in title_lower:
         return 1.0
 
-    # Check if topic appears in any outcome name (bidirectional)
+    best_score = token_overlap_relevance(core, title)
+
     if outcomes:
-        core_tokens = set(core.split())
-        best_outcome_score = 0.0
         for outcome_name in outcomes:
             outcome_lower = outcome_name.lower()
-            # Bidirectional: "arizona" in "arizona basketball" OR "arizona basketball" contains "arizona"
+            outcome_score = token_overlap_relevance(core, outcome_name)
             if core in outcome_lower or outcome_lower in core:
-                best_outcome_score = max(best_outcome_score, 0.85)
-            elif core_tokens & set(outcome_lower.split()):
-                best_outcome_score = max(best_outcome_score, 0.7)
-        if best_outcome_score > 0:
-            return best_outcome_score
+                outcome_score = max(outcome_score, 0.92 if len(outcome_lower.split()) >= 2 else 0.88)
+            best_score = max(best_score, outcome_score)
 
-    # Token overlap fallback against title
-    topic_tokens = set(core.split())
-    title_tokens = set(title_lower.split())
-    if not topic_tokens:
-        return 0.5
-    overlap = len(topic_tokens & title_tokens)
-    return overlap / len(topic_tokens)
+    return round(best_score, 2)
 
 
 def _safe_float(val, default=0.0) -> float:
@@ -484,7 +475,8 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
             except (IndexError, TypeError):
                 end_date = None
 
-        # Quality-signal relevance (replaces position-based decay)
+        # Semantic relevance should dominate. Market quality should refine
+        # relevant matches, not rescue unrelated high-liquidity events.
         text_score = _compute_text_similarity(topic, title, all_outcome_names) if topic else 0.5
 
         # Volume signal: log-scaled monthly volume (most stable signal)
@@ -504,13 +496,13 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
         # Competitive bonus: markets near 50/50 are more interesting
         competitive_score = event_competitive
 
-        relevance = min(1.0, (
-            0.30 * text_score +
-            0.30 * vol_score +
-            0.15 * liq_score +
+        market_quality = (
+            0.50 * vol_score +
+            0.25 * liq_score +
             0.15 * movement_score +
             0.10 * competitive_score
-        ))
+        )
+        relevance = min(1.0, text_score * (0.75 + 0.25 * market_quality))
 
         # Surface the topic-matching outcome to the front before truncating
         if topic and outcome_prices:
diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py
index 8714ed1..6871eb0 100644
--- a/scripts/lib/reddit.py
+++ b/scripts/lib/reddit.py
@@ -108,12 +108,14 @@ def expand_reddit_queries(topic: str, depth: str) -> List[str]:
     if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
         queries.append(original_clean)
 
-    # Add opinion/review variant except for how_to/comparison queries
+    # Opinion/review variants help mostly for product and opinion queries.
+    # They contaminate broader searches like predictions or breaking news.
     qtype = detect_query_type(topic)
-    if depth in ("default", "deep") and qtype not in ("how_to", "comparison"):
+    if depth in ("default", "deep") and qtype in ("product", "opinion"):
         queries.append(f"{core} worth it OR thoughts OR review")
 
-    if depth == "deep":
+    # Problem/bug variants are useful for tool workflows, not generic news.
+    if depth == "deep" and qtype in ("product", "opinion", "how_to"):
         queries.append(f"{core} issues OR problems OR bug OR broken")
 
     return queries
diff --git a/scripts/lib/relevance.py b/scripts/lib/relevance.py
index eb57b61..88c0f0e 100644
--- a/scripts/lib/relevance.py
+++ b/scripts/lib/relevance.py
@@ -1,6 +1,9 @@
 """Shared token-overlap relevance scoring for search result ranking.
 
-Tokenizes text, expands synonyms, and computes query-to-content overlap ratios.
+The score is intentionally query-centric:
+- exact phrase matches should score very high
+- partial matches should pay a meaningful penalty
+- matches on generic words alone ("odds", "review") should not pass as relevant
 """
 
 import re
@@ -36,6 +39,18 @@ SYNONYMS = {
     'vuejs': {'vue'},
 }
 
+# Generic query words that should not carry relevance on their own.
+# They still help when paired with stronger entity/topic matches.
+LOW_SIGNAL_QUERY_TOKENS = frozenset({
+    'advice', 'animation', 'animations', 'best', 'chance', 'chances',
+    'code', 'compare', 'comparison', 'differences', 'explain', 'guide',
+    'guides', 'how', 'latest', 'news', 'odds', 'opinion', 'opinions',
+    'prediction', 'predictions', 'probability', 'probabilities', 'prompt',
+    'prompting', 'prompts', 'rate', 'review', 'reviews', 'thoughts',
+    'tip', 'tips', 'tutorial', 'tutorials', 'update', 'updates', 'use',
+    'using', 'versus', 'vs', 'worth',
+})
+
 
 def tokenize(text: str) -> Set[str]:
     """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.
@@ -51,15 +66,25 @@ def tokenize(text: str) -> Set[str]:
     return expanded
 
 
+def _normalize_phrase(text: str) -> str:
+    """Normalize text for phrase containment checks."""
+    return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split())
+
+
 def token_overlap_relevance(
     query: str,
     text: str,
     hashtags: Optional[List[str]] = None,
 ) -> float:
-    """Compute relevance as ratio of query tokens found in text.
+    """Compute a query-centric relevance score between 0.0 and 1.0.
 
-    Uses ratio overlap (intersection / query_length) so short queries
-    score higher when fully represented in the text. Floors at 0.1.
+    The score combines:
+    - query coverage
+    - informative-token coverage
+    - a small precision term to penalize extra noise
+    - an exact phrase bonus
+
+    Generic tokens alone are capped below the post-retrieval 0.3 threshold.
 
     Args:
         query: Search query
@@ -68,7 +93,7 @@ def token_overlap_relevance(
             hashtags are split to match query tokens (e.g. "claudecode" matches "claude").
 
     Returns:
-        Float between 0.1 and 1.0 (0.5 for empty queries)
+        Float between 0.0 and 1.0 (0.5 for empty queries)
     """
     q_tokens = tokenize(query)
 
@@ -89,6 +114,35 @@ def token_overlap_relevance(
     if not q_tokens:
         return 0.5  # Neutral fallback for empty/stopword-only queries
 
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
+    overlap_tokens = q_tokens & t_tokens
+    overlap = len(overlap_tokens)
+    if overlap == 0:
+        return 0.0
+
+    informative_q_tokens = {t for t in q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
+    if not informative_q_tokens:
+        informative_q_tokens = q_tokens
+
+    coverage = overlap / len(q_tokens)
+    informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens)
+    precision_denominator = min(len(t_tokens), len(q_tokens) + 4) or 1
+    precision = overlap / precision_denominator
+
+    phrase_bonus = 0.0
+    normalized_query = _normalize_phrase(query)
+    normalized_text = _normalize_phrase(combined)
+    if normalized_query and normalized_query in normalized_text:
+        phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
+
+    base = (
+        0.55 * (coverage ** 1.35) +
+        0.25 * informative_overlap +
+        0.20 * precision
+    )
+
+    # If we only matched generic query words, keep the score below the
+    # normal relevance filter threshold so these do not survive by default.
+    if informative_q_tokens and not (informative_q_tokens & t_tokens):
+        return round(min(0.24, base), 2)
+
+    return round(min(1.0, base + phrase_bonus), 2)
diff --git a/scripts/lib/score.py b/scripts/lib/score.py
index 9ac8502..2e0a0ad 100644
--- a/scripts/lib/score.py
+++ b/scripts/lib/score.py
@@ -11,6 +11,12 @@ WEIGHT_RELEVANCE = 0.45
 WEIGHT_RECENCY = 0.25
 WEIGHT_ENGAGEMENT = 0.30
 
+# Polymarket needs stronger semantic weighting because volume/liquidity already
+# show up as engagement and lightly influence parse-time relevance.
+PM_WEIGHT_RELEVANCE = 0.60
+PM_WEIGHT_RECENCY = 0.20
+PM_WEIGHT_ENGAGEMENT = 0.20
+
 # WebSearch weights (no engagement data available)
 WEBSEARCH_WEIGHT_RELEVANCE = 0.55
 WEBSEARCH_WEIGHT_RECENCY = 0.45
@@ -632,9 +638,9 @@ def score_polymarket_items(items: List[schema.PolymarketItem]) -> List[schema.Po
         )
 
         overall = (
-            WEIGHT_RELEVANCE * rel_score +
-            WEIGHT_RECENCY * rec_score +
-            WEIGHT_ENGAGEMENT * eng_score
+            PM_WEIGHT_RELEVANCE * rel_score +
+            PM_WEIGHT_RECENCY * rec_score +
+            PM_WEIGHT_ENGAGEMENT * eng_score
         )
 
         if eng_raw[i] is None:
diff --git a/tests/test_instagram_sc.py b/tests/test_instagram_sc.py
index 59fa3a8..6ca2b14 100644
--- a/tests/test_instagram_sc.py
+++ b/tests/test_instagram_sc.py
@@ -57,9 +57,9 @@ class TestComputeRelevance(unittest.TestCase):
         boosted = instagram._compute_relevance("claude code", "random video about stuff", ["claudecode", "ai"])
         self.assertGreater(boosted, base)
 
-    def test_floor_at_01(self):
+    def test_no_match_returns_zero(self):
         rel = instagram._compute_relevance("quantum physics", "cat dancing video")
-        self.assertGreaterEqual(rel, 0.1)
+        self.assertEqual(rel, 0.0)
 
     def test_empty_query_returns_default(self):
         rel = instagram._compute_relevance("", "Some video title")
diff --git a/tests/test_polymarket.py b/tests/test_polymarket.py
index 8298740..0c4e73b 100644
--- a/tests/test_polymarket.py
+++ b/tests/test_polymarket.py
@@ -569,8 +569,9 @@ class TestTextSimilarity(unittest.TestCase):
 
     def test_partial_token_overlap(self):
         score = polymarket._compute_text_similarity("Arizona Basketball", "Will Arizona win?")
-        # "Arizona" matches, "Basketball" doesn't -> 0.5
-        self.assertAlmostEqual(score, 0.5)
+        # Partial informative match should stay below exact match.
+        self.assertGreater(score, 0.3)
+        self.assertLess(score, 0.6)
 
     def test_no_overlap(self):
         score = polymarket._compute_text_similarity("Arizona Basketball", "Will AI regulation pass?")
@@ -595,7 +596,7 @@ class TestTextSimilarity(unittest.TestCase):
             "Who will be the #1 overall seed?",
             outcomes=["Duke", "Arizona", "Houston"],
         )
-        self.assertEqual(score, 0.85)
+        self.assertEqual(score, 1.0)
 
     def test_outcome_bidirectional_match(self):
         """Topic 'Arizona Basketball' should match outcome 'Arizona' (outcome in core)."""
@@ -604,16 +605,17 @@ class TestTextSimilarity(unittest.TestCase):
             "Who will be the #1 overall seed?",
             outcomes=["Duke", "Arizona", "Houston"],
         )
-        self.assertEqual(score, 0.85)
+        self.assertEqual(score, 0.88)
 
     def test_outcome_token_overlap(self):
-        """Partial token overlap with outcome gets 0.7 when no substring match."""
+        """Partial token overlap with outcome gets a moderate score."""
         score = polymarket._compute_text_similarity(
             "Iran War",
             "Unrelated geopolitics title",
             outcomes=["War continues", "Peace deal"],
         )
-        self.assertEqual(score, 0.7)
+        self.assertGreater(score, 0.3)
+        self.assertLess(score, 0.6)
 
     def test_outcome_no_match(self):
         """No outcome match falls through to title token overlap."""
@@ -632,7 +634,15 @@ class TestTextSimilarity(unittest.TestCase):
             "Unrelated title",
             outcomes=["Arizona"],
         )
-        self.assertEqual(score, 0.85)
+        self.assertEqual(score, 1.0)
+
+    def test_generic_only_odds_match_stays_below_threshold(self):
+        score = polymarket._compute_text_similarity(
+            "Anthropic odds",
+            "Republican 2026 House odds",
+            outcomes=["Yes", "No"],
+        )
+        self.assertLess(score, 0.3)
 
     def test_title_match_still_beats_outcome(self):
         """Title substring match (1.0) takes priority over outcome match (0.85)."""
diff --git a/tests/test_reddit_sc.py b/tests/test_reddit_sc.py
index d621de8..a212055 100644
--- a/tests/test_reddit_sc.py
+++ b/tests/test_reddit_sc.py
@@ -47,16 +47,28 @@ class TestExpandRedditQueries(unittest.TestCase):
         self.assertGreaterEqual(len(queries), 1)
 
     def test_default_includes_review_variant(self):
-        queries = reddit.expand_reddit_queries("cursor IDE", "default")
+        queries = reddit.expand_reddit_queries("cursor IDE pricing", "default")
         self.assertTrue(any("worth it" in q or "review" in q for q in queries))
 
+    def test_default_skips_review_variant_for_prediction(self):
+        queries = reddit.expand_reddit_queries("anthropic odds", "default")
+        self.assertFalse(any("worth it" in q or "review" in q for q in queries))
+
+    def test_default_skips_review_variant_for_breaking_news(self):
+        queries = reddit.expand_reddit_queries("kanye west", "default")
+        self.assertFalse(any("worth it" in q or "review" in q for q in queries))
+
     def test_deep_includes_issues_variant(self):
-        queries = reddit.expand_reddit_queries("cursor IDE", "deep")
+        queries = reddit.expand_reddit_queries("cursor IDE pricing", "deep")
         self.assertTrue(any("issues" in q or "problems" in q for q in queries))
 
+    def test_deep_skips_issues_variant_for_prediction(self):
+        queries = reddit.expand_reddit_queries("anthropic odds", "deep")
+        self.assertFalse(any("issues" in q or "problems" in q for q in queries))
+
     def test_deep_has_more_queries_than_quick(self):
-        quick = reddit.expand_reddit_queries("cursor IDE", "quick")
-        deep = reddit.expand_reddit_queries("cursor IDE", "deep")
+        quick = reddit.expand_reddit_queries("cursor IDE pricing", "quick")
+        deep = reddit.expand_reddit_queries("cursor IDE pricing", "deep")
         self.assertGreater(len(deep), len(quick))
 
 
diff --git a/tests/test_relevance.py b/tests/test_relevance.py
index c641e76..368b1d1 100644
--- a/tests/test_relevance.py
+++ b/tests/test_relevance.py
@@ -75,7 +75,7 @@ class TestTokenOverlapRelevance(unittest.TestCase):
 
     def test_floor_at_0_1(self):
         rel = token_overlap_relevance("quantum physics", "cat dancing video")
-        self.assertGreaterEqual(rel, 0.1)
+        self.assertEqual(rel, 0.0)
 
     def test_full_match_returns_1(self):
         rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners")
@@ -96,6 +96,15 @@ class TestTokenOverlapRelevance(unittest.TestCase):
         rel = token_overlap_relevance("the a is", "some content here")
         self.assertEqual(rel, 0.5)
 
+    def test_generic_only_overlap_stays_below_filter_threshold(self):
+        rel = token_overlap_relevance("anthropic odds", "Republican house odds update")
+        self.assertLess(rel, 0.3)
+
+    def test_informative_partial_match_stays_above_generic_only(self):
+        generic_only = token_overlap_relevance("anthropic odds", "Republican house odds update")
+        informative = token_overlap_relevance("anthropic odds", "Anthropic valuation market")
+        self.assertGreater(informative, generic_only)
+
 
 class TestHashtagRelevance(unittest.TestCase):
     """Tests for hashtag-aware relevance (TikTok/Instagram pattern)."""
diff --git a/tests/test_scrapecreators_x.py b/tests/test_scrapecreators_x.py
index 0ebb429..2359c7a 100644
--- a/tests/test_scrapecreators_x.py
+++ b/tests/test_scrapecreators_x.py
@@ -44,9 +44,9 @@ class TestComputeRelevance(unittest.TestCase):
         score = scrapecreators_x._compute_relevance("", "some text")
         self.assertEqual(score, 0.5)
 
-    def test_floor_at_01(self):
+    def test_no_match_returns_zero(self):
         score = scrapecreators_x._compute_relevance("abcdef ghijkl", "xyz")
-        self.assertGreaterEqual(score, 0.1)
+        self.assertEqual(score, 0.0)
 
 
 class TestExtractCoreSubject(unittest.TestCase):
diff --git a/tests/test_tiktok.py b/tests/test_tiktok.py
index 3307dcc..91e68be 100644
--- a/tests/test_tiktok.py
+++ b/tests/test_tiktok.py
@@ -33,9 +33,9 @@ class TestTikTokRelevance(unittest.TestCase):
         rel = tiktok._compute_relevance("", "Some video title")
         self.assertEqual(rel, 0.5)
 
-    def test_floor(self):
+    def test_no_match_returns_zero(self):
         rel = tiktok._compute_relevance("quantum physics", "cat dancing video")
-        self.assertGreaterEqual(rel, 0.1)
+        self.assertEqual(rel, 0.0)
 
 
 class TestExtractCoreSubject(unittest.TestCase):
diff --git a/tests/test_youtube_relevance.py b/tests/test_youtube_relevance.py
index 432cb69..51e2ceb 100644
--- a/tests/test_youtube_relevance.py
+++ b/tests/test_youtube_relevance.py
@@ -62,7 +62,7 @@ class TestComputeRelevance(unittest.TestCase):
 
     def test_no_match(self):
         result = _compute_relevance("Claude Code", "Python Web Scraping")
-        self.assertEqual(result, 0.1)  # Floor
+        self.assertEqual(result, 0.0)
 
     def test_empty_query_returns_neutral(self):
         result = _compute_relevance("", "Some Video Title")
@@ -74,7 +74,7 @@ class TestComputeRelevance(unittest.TestCase):
 
     def test_empty_title(self):
         result = _compute_relevance("Claude Code", "")
-        self.assertEqual(result, 0.1)  # Floor
+        self.assertEqual(result, 0.0)
 
     def test_case_insensitive(self):
         result = _compute_relevance("claude code", "CLAUDE CODE Tutorial")
@@ -89,9 +89,9 @@ class TestComputeRelevance(unittest.TestCase):
         )
         self.assertEqual(result, 1.0)
 
-    def test_floor_at_0_1(self):
+    def test_no_match_returns_zero(self):
         result = _compute_relevance("quantum computing", "cat videos compilation")
-        self.assertEqual(result, 0.1)
+        self.assertEqual(result, 0.0)
 
     def test_cap_at_1_0(self):
         result = _compute_relevance("AI", "AI AI AI AI AI")
@@ -103,7 +103,7 @@ class TestComputeRelevance(unittest.TestCase):
 
     def test_single_word_no_match(self):
         result = _compute_relevance("Seedance", "Random cooking video")
-        self.assertEqual(result, 0.1)
+        self.assertEqual(result, 0.0)
 
 
 if __name__ == "__main__":

From 8eda5fad5c9fe9dc824c7abd7f6391a8e3711b2b Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Fri, 13 Mar 2026 19:21:33 -0700
Subject: [PATCH 14/18] Add local search quality evaluation harness

Add an optional local evaluator that compares a baseline revision against a candidate checkout, computes deterministic stability metrics, and can call Gemini for judged ranking metrics when configured.

The harness isolates child runs with a temporary HOME and a node-free PATH so historical revisions cannot trigger Bird browser-cookie auth during evaluation.

Validation: uv run python -m unittest and local smoke/full deterministic eval runs.
---
 docs/search-quality-eval.md           |  46 ++
 scripts/evaluate_search_quality.py    | 586 ++++++++++++++++++++++++++
 scripts/lib/env.py                    |   2 +
 tests/test_env_project.py             |  17 +
 tests/test_evaluate_search_quality.py |  81 ++++
 5 files changed, 732 insertions(+)
 create mode 100644 docs/search-quality-eval.md
 create mode 100644 scripts/evaluate_search_quality.py
 create mode 100644 tests/test_evaluate_search_quality.py

diff --git a/docs/search-quality-eval.md b/docs/search-quality-eval.md
new file mode 100644
index 0000000..7a1d91c
--- /dev/null
+++ b/docs/search-quality-eval.md
@@ -0,0 +1,46 @@
+# Search Quality Eval
+
+`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
+
+What it does:
+
+- runs a baseline revision (default `origin/main`) against a candidate checkout
+- evaluates the fixed 5 reviewer topics by default
+- computes deterministic stability metrics:
+  - `Jaccard` overlap vs baseline
+  - retention vs baseline
+  - per-source counts and overlap
+- optionally calls Gemini as a judge for graded relevance labels and then computes:
+  - `Precision@5`
+  - `nDCG@5`
+  - source-coverage recall across the judged union pool
+
+Recommended usage:
+
+```bash
+uv run python scripts/evaluate_search_quality.py
+```
+
+Useful flags:
+
+```bash
+uv run python scripts/evaluate_search_quality.py \
+  --baseline-rev origin/main \
+  --candidate-rev HEAD \
+  --no-default-topics \
+  --topic "cursor IDE pricing" \
+  --per-source-limit 5
+```
+
+Gemini configuration:
+
+- set `GEMINI_API_KEY` to enable LLM judging
+- optional: set `GEMINI_MODEL`
+- default model is `gemini-3-pro-preview` for the direct Gemini API
+
+Notes:
+
+- The script forces a clean env-based auth path when it shells out to `last30days.py`.
+- It passes `XAI_API_KEY`, `OPENAI_API_KEY`, and `SCRAPECREATORS_API_KEY`, but intentionally does not pass browser-cookie X auth. That keeps evaluation runs on the popup-free path.
+- `Jaccard` and retention are regression guards, not truth metrics.
+- `Precision@5` and `nDCG@5` are only as good as the judged pool. They help compare revisions, but they are not a substitute for a larger labeled benchmark.
diff --git a/scripts/evaluate_search_quality.py b/scripts/evaluate_search_quality.py
new file mode 100644
index 0000000..ee8a3bc
--- /dev/null
+++ b/scripts/evaluate_search_quality.py
@@ -0,0 +1,586 @@
+#!/usr/bin/env python3
+"""Run local search-quality evaluations across fixed topics.
+
+This is an optional local gate, not a required CI job. It compares a baseline
+revision against a candidate checkout, computes deterministic regression
+metrics, and optionally calls Gemini as a judge for graded relevance labels.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import textwrap
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+from lib import env as envlib
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_TOPICS: List[Tuple[str, str]] = [
+    ("nano banana pro prompting", "product"),
+    ("codex vs claude code", "comparison"),
+    ("anthropic odds", "prediction"),
+    ("kanye west", "breaking_news"),
+    ("remotion animations for Claude Code", "how_to"),
+]
+DEFAULT_SEARCH = "reddit,x,youtube,hn,polymarket"
+SOURCE_KEYS = [
+    "reddit",
+    "x",
+    "youtube",
+    "tiktok",
+    "instagram",
+    "hackernews",
+    "bluesky",
+    "truthsocial",
+    "polymarket",
+    "websearch",
+]
+DEFAULT_JUDGE_MODEL = "gemini-3-pro-preview"
+GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
+
+
+def slugify(topic: str) -> str:
+    return "".join(c.lower() if c.isalnum() else "-" for c in topic).strip("-")
+
+
+def path_without_node(path_value: str) -> str:
+    parts = []
+    for entry in path_value.split(os.pathsep):
+        if not entry:
+            continue
+        if (Path(entry) / "node").exists():
+            continue
+        parts.append(entry)
+    return os.pathsep.join(parts)
+
+
+def stable_item_key(source: str, item: Dict[str, Any]) -> str:
+    url = str(item.get("url") or "").strip()
+    if url:
+        return url
+    item_id = str(item.get("id") or "").strip()
+    text = item_text(source, item)
+    return f"{source}:{item_id}:{text[:120]}"
+
+
+def item_text(source: str, item: Dict[str, Any]) -> str:
+    if source in {"x", "bluesky", "truthsocial"}:
+        return str(item.get("text") or "").strip()
+    if source == "polymarket":
+        return str(item.get("question") or item.get("title") or "").strip()
+    return str(item.get("title") or "").strip()
+
+
+def build_ranked_items(report: Dict[str, Any], per_source_limit: int) -> List[Dict[str, Any]]:
+    ranked: List[Dict[str, Any]] = []
+    for source in SOURCE_KEYS:
+        items = list(report.get(source) or [])[:per_source_limit]
+        for item in items:
+            ranked.append({
+                "source": source,
+                "key": stable_item_key(source, item),
+                "url": str(item.get("url") or "").strip(),
+                "text": item_text(source, item),
+                "score": float(item.get("score") or 0),
+                "relevance": float(item.get("relevance") or 0),
+                "date": item.get("date"),
+            })
+    ranked.sort(key=lambda item: (-item["score"], item["source"], item["key"]))
+    return ranked
+
+
+def url_sets_by_source(report: Dict[str, Any]) -> Dict[str, set[str]]:
+    result: Dict[str, set[str]] = {}
+    for source in SOURCE_KEYS:
+        items = report.get(source) or []
+        urls = {
+            stable_item_key(source, item)
+            for item in items
+        }
+        result[source] = urls
+    return result
+
+
+def jaccard(left: Iterable[str], right: Iterable[str]) -> float:
+    left_set = set(left)
+    right_set = set(right)
+    if not left_set and not right_set:
+        return 1.0
+    union = left_set | right_set
+    if not union:
+        return 1.0
+    return len(left_set & right_set) / len(union)
+
+
+def retention(left: Iterable[str], right: Iterable[str]) -> float:
+    left_set = set(left)
+    right_set = set(right)
+    if not left_set:
+        return 1.0
+    return len(left_set & right_set) / len(left_set)
+
+
+def precision_at_k(ranking: List[Dict[str, Any]], judgments: Dict[str, int], k: int) -> float:
+    top = ranking[:k]
+    if not top:
+        return 0.0
+    hits = sum(1 for item in top if judgments.get(item["key"], 0) >= 2)
+    return hits / len(top)
+
+
+def ndcg_at_k(ranking: List[Dict[str, Any]], judgments: Dict[str, int], k: int) -> float:
+    top = ranking[:k]
+    if not top:
+        return 0.0
+
+    def dcg(grades: List[int]) -> float:
+        total = 0.0
+        for index, grade in enumerate(grades, start=1):
+            total += (2**grade - 1) / math.log2(index + 1)
+        return total
+
+    actual = [judgments.get(item["key"], 0) for item in top]
+    ideal = sorted(actual, reverse=True)
+    ideal_score = dcg(ideal)
+    if ideal_score == 0:
+        return 0.0
+    return dcg(actual) / ideal_score
+
+
+def source_coverage_recall(
+    ranking: List[Dict[str, Any]],
+    judged_pool: List[Dict[str, Any]],
+    judgments: Dict[str, int],
+) -> float:
+    good_sources = {item["source"] for item in judged_pool if judgments.get(item["key"], 0) >= 2}
+    if not good_sources:
+        return 1.0
+    hit_sources = {
+        item["source"]
+        for item in ranking
+        if judgments.get(item["key"], 0) >= 2
+    }
+    return len(hit_sources & good_sources) / len(good_sources)
+
+
+def create_eval_env(include_web: bool) -> Tuple[Dict[str, str], Path]:
+    config = envlib.get_config()
+    eval_home = Path(tempfile.mkdtemp(prefix="last30days-eval-home-"))
+    (eval_home / ".config").mkdir(parents=True, exist_ok=True)
+    passthrough = {
+        "HOME": str(eval_home),
+        "XDG_CONFIG_HOME": str(eval_home / ".config"),
+        "PATH": path_without_node(os.environ.get("PATH", "")),
+        "LANG": os.environ.get("LANG", "en_US.UTF-8"),
+        "LC_ALL": os.environ.get("LC_ALL", ""),
+        "TMPDIR": os.environ.get("TMPDIR", ""),
+        "PYTHONUTF8": "1",
+        "LAST30DAYS_CONFIG_DIR": "",
+        "BIRD_DISABLE_BROWSER_COOKIES": "1",
+        "LAST30DAYS_DISABLE_BROWSER_COOKIES": "1",
+    }
+    for key in ("OPENAI_API_KEY", "XAI_API_KEY", "SCRAPECREATORS_API_KEY"):
+        value = config.get(key)
+        if value:
+            passthrough[key] = value
+    if include_web:
+        for key in ("PARALLEL_API_KEY", "BRAVE_API_KEY", "OPENROUTER_API_KEY"):
+            value = config.get(key)
+            if value:
+                passthrough[key] = value
+    return passthrough, eval_home
+
+
+def run_last30days(
+    repo_dir: Path,
+    topic: str,
+    *,
+    search: str,
+    timeout_seconds: int,
+    include_web: bool,
+    env: Dict[str, str],
+) -> Tuple[Dict[str, Any], str]:
+    cmd = [
+        sys.executable,
+        "scripts/last30days.py",
+        topic,
+        "--emit",
+        "json",
+        "--search",
+        search,
+        "--timeout",
+        str(timeout_seconds),
+    ]
+    if not include_web:
+        cmd.append("--no-native-web")
+    result = subprocess.run(
+        cmd,
+        cwd=repo_dir,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=timeout_seconds + 30,
+        check=False,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"{repo_dir.name} failed for '{topic}' with exit {result.returncode}\n{result.stderr.strip()}"
+        )
+    return json.loads(result.stdout), result.stderr
+
+
+def create_worktree(rev: str) -> Path:
+    worktree_dir = Path(tempfile.mkdtemp(prefix="last30days-eval-"))
+    subprocess.run(
+        ["git", "worktree", "add", "--detach", str(worktree_dir), rev],
+        cwd=REPO_ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return worktree_dir
+
+
+def remove_worktree(path: Path) -> None:
+    subprocess.run(
+        ["git", "worktree", "remove", "--force", str(path)],
+        cwd=REPO_ROOT,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    shutil.rmtree(path, ignore_errors=True)
+
+
+def extract_gemini_text(payload: Dict[str, Any]) -> str:
+    for candidate in payload.get("candidates", []):
+        content = candidate.get("content") or {}
+        for part in content.get("parts", []):
+            text = part.get("text")
+            if text:
+                return text
+    raise ValueError("Gemini response did not contain text")
+
+
+def call_gemini_judge(api_key: str, model: str, prompt: str) -> Dict[str, Any]:
+    body = {
+        "contents": [{"parts": [{"text": prompt}]}],
+        "generationConfig": {
+            "temperature": 0,
+            "responseMimeType": "application/json",
+        },
+    }
+    url = GEMINI_API_URL.format(model=model, api_key=api_key)
+    request = Request(
+        url,
+        data=json.dumps(body).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urlopen(request, timeout=120) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"Gemini HTTP {exc.code}: {detail}") from exc
+    except URLError as exc:
+        raise RuntimeError(f"Gemini request failed: {exc}") from exc
+    return json.loads(extract_gemini_text(payload))
+
+
+def build_judge_prompt(
+    *,
+    topic: str,
+    query_type: str,
+    items: List[Dict[str, Any]],
+) -> str:
+    item_lines = []
+    for item in items:
+        item_lines.append(
+            "\n".join([
+                f"- id: {item['key']}",
+                f"  source: {item['source']}",
+                f"  title: {item['text'][:220]}",
+                f"  url: {item['url']}",
+                f"  date: {item.get('date') or 'unknown'}",
+            ])
+        )
+    joined = "\n".join(item_lines)
+    return textwrap.dedent(
+        f"""
+        Judge search-result relevance for a last-30-days research tool.
+
+        Topic: {topic}
+        Query type: {query_type}
+
+        Score each item on this 0-3 scale:
+        - 0 = off-topic or clearly bad
+        - 1 = weak or tangential
+        - 2 = relevant and useful
+        - 3 = highly relevant, one of the best results
+
+        Focus on actual user intent, not just token overlap. Penalize items that
+        only match generic words like "odds", "review", or "tips" without
+        matching the real entity or subject. Favor items that would genuinely
+        help answer the topic in the context of recent discussion.
+
+        Return strict JSON with this shape:
+        {{
+          "judgments": [
+            {{"id": "ITEM_ID", "grade": 0, "reason": "short reason"}}
+          ]
+        }}
+
+        Items:
+        {joined}
+        """
+    ).strip()
+
+
+def get_judgments(
+    *,
+    output_dir: Path,
+    slug: str,
+    topic: str,
+    query_type: str,
+    items: List[Dict[str, Any]],
+    judge_model: str,
+    gemini_api_key: Optional[str],
+) -> Dict[str, int]:
+    cache_file = output_dir / "judgments" / f"{slug}.json"
+    cache_file.parent.mkdir(parents=True, exist_ok=True)
+    if cache_file.exists():
+        cached = json.loads(cache_file.read_text())
+        return {entry["id"]: int(entry["grade"]) for entry in cached.get("judgments", [])}
+
+    if not gemini_api_key:
+        return {}
+
+    prompt = build_judge_prompt(topic=topic, query_type=query_type, items=items)
+    payload = call_gemini_judge(gemini_api_key, judge_model, prompt)
+    cache_file.write_text(json.dumps(payload, indent=2))
+    return {entry["id"]: int(entry["grade"]) for entry in payload.get("judgments", [])}
+
+
+def summarize_topic(
+    *,
+    topic: str,
+    query_type: str,
+    baseline_report: Dict[str, Any],
+    candidate_report: Dict[str, Any],
+    judged_pool: List[Dict[str, Any]],
+    judgments: Dict[str, int],
+    per_source_limit: int,
+) -> Dict[str, Any]:
+    baseline_ranked = build_ranked_items(baseline_report, per_source_limit)
+    candidate_ranked = build_ranked_items(candidate_report, per_source_limit)
+
+    baseline_sets = url_sets_by_source(baseline_report)
+    candidate_sets = url_sets_by_source(candidate_report)
+
+    metrics = {
+        "topic": topic,
+        "query_type": query_type,
+        "baseline": {
+            "precision_at_5": precision_at_k(baseline_ranked, judgments, 5),
+            "ndcg_at_5": ndcg_at_k(baseline_ranked, judgments, 5),
+            "source_coverage_recall": source_coverage_recall(baseline_ranked, judged_pool, judgments),
+        },
+        "candidate": {
+            "precision_at_5": precision_at_k(candidate_ranked, judgments, 5),
+            "ndcg_at_5": ndcg_at_k(candidate_ranked, judgments, 5),
+            "source_coverage_recall": source_coverage_recall(candidate_ranked, judged_pool, judgments),
+        },
+        "stability": {
+            "overall_jaccard": jaccard(
+                set().union(*baseline_sets.values()),
+                set().union(*candidate_sets.values()),
+            ),
+            "overall_retention_vs_baseline": retention(
+                set().union(*baseline_sets.values()),
+                set().union(*candidate_sets.values()),
+            ),
+            "per_source": {
+                source: {
+                    "baseline_count": len(baseline_sets[source]),
+                    "candidate_count": len(candidate_sets[source]),
+                    "jaccard": jaccard(baseline_sets[source], candidate_sets[source]),
+                    "retention_vs_baseline": retention(baseline_sets[source], candidate_sets[source]),
+                }
+                for source in SOURCE_KEYS
+            },
+        },
+    }
+    return metrics
+
+
+def write_markdown_summary(
+    output_dir: Path,
+    baseline_label: str,
+    candidate_label: str,
+    topic_summaries: List[Dict[str, Any]],
+) -> None:
+    lines = [
+        f"# Search Quality Evaluation",
+        "",
+        f"- Baseline: `{baseline_label}`",
+        f"- Candidate: `{candidate_label}`",
+        f"- Generated: {datetime.now().isoformat(timespec='seconds')}",
+        "",
+        "## Topic Metrics",
+        "",
+        "| Topic | Base P@5 | Cand P@5 | Base nDCG@5 | Cand nDCG@5 | Jaccard | Retention |",
+        "|---|---:|---:|---:|---:|---:|---:|",
+    ]
+    for summary in topic_summaries:
+        lines.append(
+            "| {topic} | {bp:.2f} | {cp:.2f} | {bn:.2f} | {cn:.2f} | {jac:.2f} | {ret:.2f} |".format(
+                topic=summary["topic"],
+                bp=summary["baseline"]["precision_at_5"],
+                cp=summary["candidate"]["precision_at_5"],
+                bn=summary["baseline"]["ndcg_at_5"],
+                cn=summary["candidate"]["ndcg_at_5"],
+                jac=summary["stability"]["overall_jaccard"],
+                ret=summary["stability"]["overall_retention_vs_baseline"],
+            )
+        )
+    lines.append("")
+    lines.append("## Notes")
+    lines.append("")
+    lines.append("- `Precision@5` and `nDCG@5` depend on the judged union pool, not a full gold corpus.")
+    lines.append("- `Source coverage recall` measures whether a run surfaced at least one judged-good result from the good sources in the judged pool.")
+    lines.append("- `Jaccard` and `retention` are stability guards against baseline drift, not truth metrics.")
+    (output_dir / "summary.md").write_text("\n".join(lines))
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Evaluate last30days search quality locally")
+    parser.add_argument("--baseline-rev", default="origin/main", help="Git revision for the baseline run")
+    parser.add_argument("--candidate-rev", default=None, help="Optional git revision for the candidate run")
+    parser.add_argument("--no-default-topics", action="store_true", help="Do not include the built-in 5-topic suite")
+    parser.add_argument("--topic", action="append", default=[], help="Extra topic to evaluate (repeatable)")
+    parser.add_argument("--search", default=DEFAULT_SEARCH, help="Comma-separated sources passed to --search")
+    parser.add_argument("--timeout", type=int, default=180, help="Per-topic timeout passed to last30days")
+    parser.add_argument("--per-source-limit", type=int, default=5, help="Items per source to judge")
+    parser.add_argument("--include-web", action="store_true", help="Include web-search keys and native web backends")
+    parser.add_argument("--judge-model", default=None, help="Gemini judge model override")
+    parser.add_argument("--judge-provider", choices=["auto", "gemini", "none"], default="auto")
+    parser.add_argument("--keep-worktrees", action="store_true", help="Leave temporary baseline/candidate worktrees on disk")
+    parser.add_argument("--output-dir", default=None, help="Output directory (default: docs/test-results/search-quality-<timestamp>)")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    output_dir = Path(args.output_dir) if args.output_dir else REPO_ROOT / "docs" / "test-results" / f"search-quality-{timestamp}"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    topics = [] if args.no_default_topics else list(DEFAULT_TOPICS)
+    topics.extend((topic, "custom") for topic in args.topic)
+    if not topics:
+        raise SystemExit("No topics configured. Use the default suite or pass --topic.")
+
+    judge_config = envlib.get_config()
+    judge_provider = args.judge_provider
+    gemini_api_key = judge_config.get("GEMINI_API_KEY")
+    judge_model = args.judge_model or judge_config.get("GEMINI_MODEL") or DEFAULT_JUDGE_MODEL
+    if judge_provider == "auto":
+        judge_provider = "gemini" if gemini_api_key else "none"
+    if judge_provider == "none":
+        gemini_api_key = None
+
+    eval_env, eval_home = create_eval_env(include_web=args.include_web)
+    baseline_dir = create_worktree(args.baseline_rev)
+    candidate_dir = create_worktree(args.candidate_rev) if args.candidate_rev else REPO_ROOT
+
+    baseline_label = args.baseline_rev
+    candidate_label = args.candidate_rev or "working-tree"
+    topic_summaries: List[Dict[str, Any]] = []
+
+    try:
+        for topic, query_type in topics:
+            slug = slugify(topic)
+            baseline_report, baseline_stderr = run_last30days(
+                baseline_dir,
+                topic,
+                search=args.search,
+                timeout_seconds=args.timeout,
+                include_web=args.include_web,
+                env=eval_env,
+            )
+            candidate_report, candidate_stderr = run_last30days(
+                candidate_dir,
+                topic,
+                search=args.search,
+                timeout_seconds=args.timeout,
+                include_web=args.include_web,
+                env=eval_env,
+            )
+
+            topic_dir = output_dir / slug
+            topic_dir.mkdir(parents=True, exist_ok=True)
+            (topic_dir / "baseline.json").write_text(json.dumps(baseline_report, indent=2))
+            (topic_dir / "candidate.json").write_text(json.dumps(candidate_report, indent=2))
+            (topic_dir / "baseline.stderr.txt").write_text(baseline_stderr)
+            (topic_dir / "candidate.stderr.txt").write_text(candidate_stderr)
+
+            baseline_ranked = build_ranked_items(baseline_report, args.per_source_limit)
+            candidate_ranked = build_ranked_items(candidate_report, args.per_source_limit)
+            union_map = {item["key"]: item for item in baseline_ranked + candidate_ranked}
+            judgments = get_judgments(
+                output_dir=output_dir,
+                slug=slug,
+                topic=topic,
+                query_type=query_type,
+                items=list(union_map.values()),
+                judge_model=judge_model,
+                gemini_api_key=gemini_api_key,
+            )
+
+            summary = summarize_topic(
+                topic=topic,
+                query_type=query_type,
+                baseline_report=baseline_report,
+                candidate_report=candidate_report,
+                judged_pool=list(union_map.values()),
+                judgments=judgments,
+                per_source_limit=args.per_source_limit,
+            )
+            topic_summaries.append(summary)
+
+        payload = {
+            "baseline": baseline_label,
+            "candidate": candidate_label,
+            "judge_provider": judge_provider,
+            "judge_model": judge_model if gemini_api_key else None,
+            "topics": topic_summaries,
+        }
+        (output_dir / "summary.json").write_text(json.dumps(payload, indent=2))
+        write_markdown_summary(output_dir, baseline_label, candidate_label, topic_summaries)
+        print(output_dir)
+        return 0
+    finally:
+        if not args.keep_worktrees:
+            remove_worktree(baseline_dir)
+            if args.candidate_rev:
+                remove_worktree(candidate_dir)
+        shutil.rmtree(eval_home, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/scripts/lib/env.py b/scripts/lib/env.py
index 87f9b7c..0fd9599 100644
--- a/scripts/lib/env.py
+++ b/scripts/lib/env.py
@@ -243,10 +243,12 @@ def get_config() -> Dict[str, Any]:
 
     keys = [
         ('XAI_API_KEY', None),
+        ('GEMINI_API_KEY', None),
         ('OPENROUTER_API_KEY', None),
         ('PARALLEL_API_KEY', None),
         ('BRAVE_API_KEY', None),
         ('XIAOHONGSHU_API_BASE', None),
+        ('GEMINI_MODEL', None),
         ('OPENAI_MODEL_POLICY', 'auto'),
         ('OPENAI_MODEL_PIN', None),
         ('XAI_MODEL_POLICY', 'latest'),
diff --git a/tests/test_env_project.py b/tests/test_env_project.py
index 9b1f330..c3a4353 100644
--- a/tests/test_env_project.py
+++ b/tests/test_env_project.py
@@ -79,6 +79,23 @@ class TestConfigPrecedence(unittest.TestCase):
                 config = env.get_config()
                 self.assertEqual(config['BRAVE_API_KEY'], 'env-key')
 
+    def test_gemini_keys_load_from_project_env(self):
+        import tempfile
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project_dir = Path(tmpdir) / ".claude"
+            project_dir.mkdir()
+            project_env = project_dir / "last30days.env"
+            project_env.write_text("GEMINI_API_KEY=gem-key\nGEMINI_MODEL=gemini-3-pro-preview\n")
+
+            with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
+                 patch.object(env, 'CONFIG_FILE', None), \
+                 patch.dict(os.environ, {}, clear=False):
+                os.environ.pop('GEMINI_API_KEY', None)
+                os.environ.pop('GEMINI_MODEL', None)
+                config = env.get_config()
+                self.assertEqual(config['GEMINI_API_KEY'], 'gem-key')
+                self.assertEqual(config['GEMINI_MODEL'], 'gemini-3-pro-preview')
+
 
 class TestConfigSource(unittest.TestCase):
     """Tests for _CONFIG_SOURCE tracking."""
diff --git a/tests/test_evaluate_search_quality.py b/tests/test_evaluate_search_quality.py
new file mode 100644
index 0000000..7922f49
--- /dev/null
+++ b/tests/test_evaluate_search_quality.py
@@ -0,0 +1,81 @@
+"""Tests for the local search-quality evaluation harness."""
+
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+import evaluate_search_quality as evalsq
+
+
+class TestMetrics(unittest.TestCase):
+    def test_jaccard(self):
+        self.assertAlmostEqual(evalsq.jaccard({"a", "b"}, {"b", "c"}), 1 / 3)
+
+    def test_retention(self):
+        self.assertAlmostEqual(evalsq.retention({"a", "b"}, {"b", "c"}), 0.5)
+
+    def test_precision_at_k(self):
+        ranking = [
+            {"key": "a", "source": "reddit"},
+            {"key": "b", "source": "x"},
+            {"key": "c", "source": "youtube"},
+        ]
+        judgments = {"a": 3, "b": 1, "c": 2}
+        self.assertAlmostEqual(evalsq.precision_at_k(ranking, judgments, 2), 0.5)
+
+    def test_ndcg_at_k(self):
+        ranking = [
+            {"key": "a", "source": "reddit"},
+            {"key": "b", "source": "x"},
+            {"key": "c", "source": "youtube"},
+        ]
+        judgments = {"a": 3, "b": 0, "c": 2}
+        self.assertGreater(evalsq.ndcg_at_k(ranking, judgments, 3), 0.8)
+
+    def test_source_coverage_recall_uses_union_pool(self):
+        judged_pool = [
+            {"key": "a", "source": "reddit"},
+            {"key": "b", "source": "x"},
+            {"key": "c", "source": "youtube"},
+        ]
+        ranking = [
+            {"key": "a", "source": "reddit"},
+            {"key": "b", "source": "x"},
+        ]
+        judgments = {"a": 3, "b": 0, "c": 2}
+        self.assertAlmostEqual(evalsq.source_coverage_recall(ranking, judged_pool, judgments), 0.5)
+
+
+class TestRankedItems(unittest.TestCase):
+    def test_build_ranked_items_sorts_by_score(self):
+        report = {
+            "reddit": [{"id": "R1", "title": "Low", "url": "r1", "score": 20}],
+            "x": [{"id": "X1", "text": "High", "url": "x1", "score": 90}],
+            "youtube": [],
+            "tiktok": [],
+            "instagram": [],
+            "hackernews": [],
+            "bluesky": [],
+            "truthsocial": [],
+            "polymarket": [],
+            "websearch": [],
+        }
+        ranked = evalsq.build_ranked_items(report, per_source_limit=5)
+        self.assertEqual(ranked[0]["key"], "x1")
+
+
+class TestPathWithoutNode(unittest.TestCase):
+    def test_removes_node_entries(self):
+        path = "/usr/bin:/tmp/node-bin:/opt/homebrew/bin"
+        def fake_exists(path_obj):
+            return str(path_obj).endswith("/tmp/node-bin/node")
+        with patch.object(evalsq.Path, "exists", fake_exists):
+            filtered = evalsq.path_without_node(path)
+        self.assertEqual(filtered, "/usr/bin:/opt/homebrew/bin")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 3a0f3d8b19530ce87e5bed0f5ab838c7619f69a4 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Fri, 13 Mar 2026 19:25:29 -0700
Subject: [PATCH 15/18] Accept GOOGLE_API_KEY for local Gemini eval

This workspace uses GOOGLE_API_KEY as the canonical Google credential. Accept it ahead of the Gemini-specific aliases so the local evaluation harness can run without a separate GEMINI_API_KEY export.

Validation: uv run python -m unittest tests.test_env_project tests.test_evaluate_search_quality and a one-shot keychain-backed resolution check.
---
 docs/search-quality-eval.md           |  3 ++-
 scripts/evaluate_search_quality.py    | 18 +++++++++++++++++-
 scripts/lib/env.py                    |  2 ++
 tests/test_env_project.py             | 15 +++++++++++++++
 tests/test_evaluate_search_quality.py | 20 ++++++++++++++++++++
 5 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/docs/search-quality-eval.md b/docs/search-quality-eval.md
index 7a1d91c..574ae48 100644
--- a/docs/search-quality-eval.md
+++ b/docs/search-quality-eval.md
@@ -34,7 +34,8 @@ uv run python scripts/evaluate_search_quality.py \
 
 Gemini configuration:
 
-- set `GEMINI_API_KEY` to enable LLM judging
+- preferred on this workspace: set `GOOGLE_API_KEY`
+- also accepted: `GEMINI_API_KEY` or `GOOGLE_GENAI_API_KEY`
 - optional: set `GEMINI_MODEL`
 - default model is `gemini-3-pro-preview` for the direct Gemini API
 
diff --git a/scripts/evaluate_search_quality.py b/scripts/evaluate_search_quality.py
index ee8a3bc..fa86393 100644
--- a/scripts/evaluate_search_quality.py
+++ b/scripts/evaluate_search_quality.py
@@ -276,6 +276,22 @@ def extract_gemini_text(payload: Dict[str, Any]) -> str:
     raise ValueError("Gemini response did not contain text")
 
 
+def resolve_google_judge_api_key(config: Dict[str, Any]) -> Optional[str]:
+    """Resolve the local canonical Google API key name.
+
+    This workspace conventionally uses GOOGLE_API_KEY. We also accept the
+    more Gemini-specific aliases for portability.
+    """
+    return (
+        os.environ.get("GOOGLE_API_KEY")
+        or config.get("GOOGLE_API_KEY")
+        or os.environ.get("GEMINI_API_KEY")
+        or config.get("GEMINI_API_KEY")
+        or os.environ.get("GOOGLE_GENAI_API_KEY")
+        or config.get("GOOGLE_GENAI_API_KEY")
+    )
+
+
 def call_gemini_judge(api_key: str, model: str, prompt: str) -> Dict[str, Any]:
     body = {
         "contents": [{"parts": [{"text": prompt}]}],
@@ -497,7 +513,7 @@ def main() -> int:
 
     judge_config = envlib.get_config()
     judge_provider = args.judge_provider
-    gemini_api_key = judge_config.get("GEMINI_API_KEY")
+    gemini_api_key = resolve_google_judge_api_key(judge_config)
     judge_model = args.judge_model or judge_config.get("GEMINI_MODEL") or DEFAULT_JUDGE_MODEL
     if judge_provider == "auto":
         judge_provider = "gemini" if gemini_api_key else "none"
diff --git a/scripts/lib/env.py b/scripts/lib/env.py
index 0fd9599..54812df 100644
--- a/scripts/lib/env.py
+++ b/scripts/lib/env.py
@@ -243,7 +243,9 @@ def get_config() -> Dict[str, Any]:
 
     keys = [
         ('XAI_API_KEY', None),
+        ('GOOGLE_API_KEY', None),
         ('GEMINI_API_KEY', None),
+        ('GOOGLE_GENAI_API_KEY', None),
         ('OPENROUTER_API_KEY', None),
         ('PARALLEL_API_KEY', None),
         ('BRAVE_API_KEY', None),
diff --git a/tests/test_env_project.py b/tests/test_env_project.py
index c3a4353..c9bb85f 100644
--- a/tests/test_env_project.py
+++ b/tests/test_env_project.py
@@ -96,6 +96,21 @@ class TestConfigPrecedence(unittest.TestCase):
                 self.assertEqual(config['GEMINI_API_KEY'], 'gem-key')
                 self.assertEqual(config['GEMINI_MODEL'], 'gemini-3-pro-preview')
 
+    def test_google_api_key_loads_from_project_env(self):
+        import tempfile
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project_dir = Path(tmpdir) / ".claude"
+            project_dir.mkdir()
+            project_env = project_dir / "last30days.env"
+            project_env.write_text("GOOGLE_API_KEY=google-key\n")
+
+            with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
+                 patch.object(env, 'CONFIG_FILE', None), \
+                 patch.dict(os.environ, {}, clear=False):
+                os.environ.pop('GOOGLE_API_KEY', None)
+                config = env.get_config()
+                self.assertEqual(config['GOOGLE_API_KEY'], 'google-key')
+
 
 class TestConfigSource(unittest.TestCase):
     """Tests for _CONFIG_SOURCE tracking."""
diff --git a/tests/test_evaluate_search_quality.py b/tests/test_evaluate_search_quality.py
index 7922f49..7471a52 100644
--- a/tests/test_evaluate_search_quality.py
+++ b/tests/test_evaluate_search_quality.py
@@ -77,5 +77,25 @@ class TestPathWithoutNode(unittest.TestCase):
         self.assertEqual(filtered, "/usr/bin:/opt/homebrew/bin")
 
 
+class TestJudgeKeyResolution(unittest.TestCase):
+    def test_prefers_google_api_key(self):
+        config = {
+            "GOOGLE_API_KEY": "google-key",
+            "GEMINI_API_KEY": "gem-key",
+            "GOOGLE_GENAI_API_KEY": "genai-key",
+        }
+        self.assertEqual(evalsq.resolve_google_judge_api_key(config), "google-key")
+
+    def test_falls_back_to_gemini_aliases(self):
+        self.assertEqual(
+            evalsq.resolve_google_judge_api_key({"GEMINI_API_KEY": "gem-key"}),
+            "gem-key",
+        )
+        self.assertEqual(
+            evalsq.resolve_google_judge_api_key({"GOOGLE_GENAI_API_KEY": "genai-key"}),
+            "genai-key",
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

From 8c1dce95e886bf1f5fecb786b54839561dde880c Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Sat, 14 Mar 2026 00:38:43 -0700
Subject: [PATCH 16/18] Harden local search evaluation harness

Isolate eval subprocesses from local yt-dlp config and fix nDCG normalization against the judged pool.

Validation: uv run python -m unittest tests.test_evaluate_search_quality
---
 docs/search-quality-eval.md           |  1 +
 scripts/evaluate_search_quality.py    | 49 ++++++++++++++++++++++++---
 tests/test_evaluate_search_quality.py | 33 ++++++++++++++++++
 3 files changed, 78 insertions(+), 5 deletions(-)

diff --git a/docs/search-quality-eval.md b/docs/search-quality-eval.md
index 574ae48..5d9cc84 100644
--- a/docs/search-quality-eval.md
+++ b/docs/search-quality-eval.md
@@ -43,5 +43,6 @@ Notes:
 
 - The script forces a clean env-based auth path when it shells out to `last30days.py`.
 - It passes `XAI_API_KEY`, `OPENAI_API_KEY`, and `SCRAPECREATORS_API_KEY`, but intentionally does not pass browser-cookie X auth. That keeps evaluation runs on the popup-free path.
+- It also strips `node` from the eval `PATH` and wraps `yt-dlp` with `--ignore-config`, so older revisions do not inherit local browser-cookie config either.
 - `Jaccard` and retention are regression guards, not truth metrics.
 - `Precision@5` and `nDCG@5` are only as good as the judged pool. They help compare revisions, but they are not a substitute for a larger labeled benchmark.
diff --git a/scripts/evaluate_search_quality.py b/scripts/evaluate_search_quality.py
index fa86393..909678b 100644
--- a/scripts/evaluate_search_quality.py
+++ b/scripts/evaluate_search_quality.py
@@ -12,6 +12,7 @@ import argparse
 import json
 import math
 import os
+import shlex
 import shutil
 import subprocess
 import sys
@@ -68,6 +69,31 @@ def path_without_node(path_value: str) -> str:
     return os.pathsep.join(parts)
 
 
+def write_exec_wrapper(path: Path, target: str, fixed_args: List[str]) -> None:
+    quoted_target = shlex.quote(target)
+    quoted_args = " ".join(shlex.quote(arg) for arg in fixed_args)
+    path.write_text(f"#!/bin/sh\nexec {quoted_target} {quoted_args} \"$@\"\n")
+    path.chmod(0o755)
+
+
+def create_eval_tool_path(eval_home: Path, base_path: str) -> str:
+    """Create safe wrapper binaries for local evaluation subprocesses."""
+    bin_dir = eval_home / "bin"
+    bin_dir.mkdir(parents=True, exist_ok=True)
+
+    real_ytdlp = shutil.which("yt-dlp")
+    if real_ytdlp:
+        write_exec_wrapper(
+            bin_dir / "yt-dlp",
+            real_ytdlp,
+            ["--ignore-config", "--no-cookies-from-browser"],
+        )
+
+    if not base_path:
+        return str(bin_dir)
+    return os.pathsep.join([str(bin_dir), base_path])
+
+
 def stable_item_key(source: str, item: Dict[str, Any]) -> str:
     url = str(item.get("url") or "").strip()
     if url:
@@ -142,7 +168,12 @@ def precision_at_k(ranking: List[Dict[str, Any]], judgments: Dict[str, int], k:
     return hits / len(top)
 
 
-def ndcg_at_k(ranking: List[Dict[str, Any]], judgments: Dict[str, int], k: int) -> float:
+def ndcg_at_k(
+    ranking: List[Dict[str, Any]],
+    judgments: Dict[str, int],
+    k: int,
+    judged_pool: Optional[List[Dict[str, Any]]] = None,
+) -> float:
     top = ranking[:k]
     if not top:
         return 0.0
@@ -154,7 +185,11 @@ def ndcg_at_k(ranking: List[Dict[str, Any]], judgments: Dict[str, int], k: int)
         return total
 
     actual = [judgments.get(item["key"], 0) for item in top]
-    ideal = sorted(actual, reverse=True)
+    ideal_candidates = judged_pool or ranking
+    ideal = sorted(
+        (judgments.get(item["key"], 0) for item in ideal_candidates),
+        reverse=True,
+    )[:len(top)]
     ideal_score = dcg(ideal)
     if ideal_score == 0:
         return 0.0
@@ -181,10 +216,14 @@ def create_eval_env(include_web: bool) -> Tuple[Dict[str, str], Path]:
     config = envlib.get_config()
     eval_home = Path(tempfile.mkdtemp(prefix="last30days-eval-home-"))
     (eval_home / ".config").mkdir(parents=True, exist_ok=True)
+    safe_path = create_eval_tool_path(
+        eval_home,
+        path_without_node(os.environ.get("PATH", "")),
+    )
     passthrough = {
         "HOME": str(eval_home),
         "XDG_CONFIG_HOME": str(eval_home / ".config"),
-        "PATH": path_without_node(os.environ.get("PATH", "")),
+        "PATH": safe_path,
         "LANG": os.environ.get("LANG", "en_US.UTF-8"),
         "LC_ALL": os.environ.get("LC_ALL", ""),
         "TMPDIR": os.environ.get("TMPDIR", ""),
@@ -413,12 +452,12 @@ def summarize_topic(
         "query_type": query_type,
         "baseline": {
             "precision_at_5": precision_at_k(baseline_ranked, judgments, 5),
-            "ndcg_at_5": ndcg_at_k(baseline_ranked, judgments, 5),
+            "ndcg_at_5": ndcg_at_k(baseline_ranked, judgments, 5, judged_pool),
             "source_coverage_recall": source_coverage_recall(baseline_ranked, judged_pool, judgments),
         },
         "candidate": {
             "precision_at_5": precision_at_k(candidate_ranked, judgments, 5),
-            "ndcg_at_5": ndcg_at_k(candidate_ranked, judgments, 5),
+            "ndcg_at_5": ndcg_at_k(candidate_ranked, judgments, 5, judged_pool),
             "source_coverage_recall": source_coverage_recall(candidate_ranked, judged_pool, judgments),
         },
         "stability": {
diff --git a/tests/test_evaluate_search_quality.py b/tests/test_evaluate_search_quality.py
index 7471a52..1b94ffa 100644
--- a/tests/test_evaluate_search_quality.py
+++ b/tests/test_evaluate_search_quality.py
@@ -1,6 +1,7 @@
 """Tests for the local search-quality evaluation harness."""
 
 import sys
+import tempfile
 import unittest
 from pathlib import Path
 from unittest.mock import patch
@@ -35,6 +36,22 @@ class TestMetrics(unittest.TestCase):
         judgments = {"a": 3, "b": 0, "c": 2}
         self.assertGreater(evalsq.ndcg_at_k(ranking, judgments, 3), 0.8)
 
+    def test_ndcg_at_k_uses_best_items_from_judged_pool(self):
+        ranking = [
+            {"key": "a", "source": "reddit"},
+            {"key": "b", "source": "x"},
+            {"key": "c", "source": "youtube"},
+        ]
+        judged_pool = ranking + [
+            {"key": "d", "source": "reddit"},
+            {"key": "e", "source": "x"},
+        ]
+        judgments = {"a": 3, "b": 0, "c": 0, "d": 3, "e": 2}
+        self.assertLess(
+            evalsq.ndcg_at_k(ranking, judgments, 3, judged_pool),
+            1.0,
+        )
+
     def test_source_coverage_recall_uses_union_pool(self):
         judged_pool = [
             {"key": "a", "source": "reddit"},
@@ -70,13 +87,29 @@ class TestRankedItems(unittest.TestCase):
 class TestPathWithoutNode(unittest.TestCase):
     def test_removes_node_entries(self):
         path = "/usr/bin:/tmp/node-bin:/opt/homebrew/bin"
+
         def fake_exists(path_obj):
             return str(path_obj).endswith("/tmp/node-bin/node")
+
         with patch.object(evalsq.Path, "exists", fake_exists):
             filtered = evalsq.path_without_node(path)
         self.assertEqual(filtered, "/usr/bin:/opt/homebrew/bin")
 
 
+class TestEvalToolPath(unittest.TestCase):
+    def test_wraps_ytdlp_with_ignore_config(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            eval_home = Path(tmpdir)
+            with patch.object(evalsq.shutil, "which", return_value="/opt/homebrew/bin/yt-dlp"):
+                path_value = evalsq.create_eval_tool_path(eval_home, "/usr/bin")
+            wrapper = eval_home / "bin" / "yt-dlp"
+            self.assertTrue(wrapper.exists())
+            text = wrapper.read_text()
+            self.assertIn("--ignore-config", text)
+            self.assertIn("--no-cookies-from-browser", text)
+            self.assertEqual(path_value, f"{eval_home / 'bin'}:/usr/bin")
+
+
 class TestJudgeKeyResolution(unittest.TestCase):
     def test_prefers_google_api_key(self):
         config = {

From c711e443fe1053f314827560181c36475fd0bb92 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Sat, 14 Mar 2026 00:38:52 -0700
Subject: [PATCH 17/18] Reduce Reddit and Polymarket false positives

Weight Reddit relevance toward titles, stop Polymarket from expanding low-signal standalone terms, and prevent short binary outcomes from matching unrelated queries.

Validation: uv run python -m unittest tests.test_reddit_sc tests.test_polymarket
---
 scripts/lib/polymarket.py | 37 ++++++++++++++++++++++---
 scripts/lib/reddit.py     | 21 +++++++++++++--
 tests/test_polymarket.py  | 57 ++++++++++++++++++++++++++++++++-------
 tests/test_reddit_sc.py   | 19 +++++++++++++
 4 files changed, 118 insertions(+), 16 deletions(-)

diff --git a/scripts/lib/polymarket.py b/scripts/lib/polymarket.py
index f9da705..de0ee14 100644
--- a/scripts/lib/polymarket.py
+++ b/scripts/lib/polymarket.py
@@ -13,7 +13,8 @@ from typing import Any, Dict, List, Optional
 from urllib.parse import quote_plus, urlencode
 
 from . import http
-from .relevance import token_overlap_relevance
+from .query_type import detect_query_type
+from .relevance import LOW_SIGNAL_QUERY_TOKENS, token_overlap_relevance
 
 GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search"
 
@@ -74,7 +75,7 @@ def _expand_queries(topic: str) -> List[str]:
     words = core.split()
     if len(words) >= 2:
         for word in words:
-            if len(word) > 1:  # skip single-char words
+            if len(word) > 1 and word.lower() not in LOW_SIGNAL_QUERY_TOKENS:
                 queries.append(word)
 
     # Add the full topic if different from core
@@ -327,19 +328,47 @@ def _compute_text_similarity(topic: str, title: str, outcomes: List[str] = None)
     if core in title_lower:
         return 1.0
 
-    best_score = token_overlap_relevance(core, title)
+    query_type = detect_query_type(topic)
+    title_score = token_overlap_relevance(core, title)
+    best_score = title_score
 
     if outcomes:
         for outcome_name in outcomes:
             outcome_lower = outcome_name.lower()
             outcome_score = token_overlap_relevance(core, outcome_name)
-            if core in outcome_lower or outcome_lower in core:
+            if _strong_phrase_match(core, outcome_lower):
                 outcome_score = max(outcome_score, 0.92 if len(outcome_lower.split()) >= 2 else 0.88)
+            if title_score < 0.3:
+                outcome_cap = 0.55 if query_type == "prediction" else 0.24
+                outcome_score = min(outcome_cap, outcome_score)
+            else:
+                outcome_score = max(title_score, 0.75 * title_score + 0.25 * outcome_score)
             best_score = max(best_score, outcome_score)
 
     return round(best_score, 2)
 
 
+def _strong_phrase_match(core: str, candidate: str) -> bool:
+    """Require real token matches, not accidental short substrings.
+
+    This prevents binary outcomes like "No" from matching "nano" or similar
+    short-string accidents.
+    """
+    candidate = " ".join(re.sub(r"[^\w\s]", " ", candidate.lower()).split())
+    core = " ".join(re.sub(r"[^\w\s]", " ", core.lower()).split())
+    if not candidate or not core:
+        return False
+
+    candidate_tokens = candidate.split()
+    core_tokens = set(core.split())
+
+    if len(candidate_tokens) >= 2:
+        return candidate in core or core in candidate
+
+    token = candidate_tokens[0]
+    return len(token) > 2 and token in core_tokens
+
+
 def _safe_float(val, default=0.0) -> float:
     """Safely convert a value to float."""
     try:
diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py
index 6871eb0..19d3edc 100644
--- a/scripts/lib/reddit.py
+++ b/scripts/lib/reddit.py
@@ -202,8 +202,9 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global"
     title = str(post.get("title", "")).strip()
     selftext = str(post.get("selftext", ""))
 
-    # Compute relevance from query-to-content overlap (or default 0.7)
-    relevance = token_overlap_relevance(query, title + " " + selftext) if query else 0.7
+    # Score the title first, then let the body provide limited support.
+    # This keeps long selftexts from overpowering the visible topic signal.
+    relevance = _compute_post_relevance(query, title, selftext) if query else 0.7
 
     return {
         "id": f"R{idx}",
@@ -223,6 +224,22 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global"
     }
 
 
+def _compute_post_relevance(query: str, title: str, selftext: str) -> float:
+    """Compute Reddit relevance with title-first weighting.
+
+    Title should carry most of the weight because it is the visible summary the
+    user sees. Selftext can lift a marginal match, but it should not rescue a
+    weak or ambiguous title into the top ranks.
+    """
+    title_score = token_overlap_relevance(query, title)
+    if not selftext.strip():
+        return title_score
+
+    body_score = token_overlap_relevance(query, selftext)
+    support_score = max(title_score, body_score)
+    return round(0.75 * title_score + 0.25 * support_score, 2)
+
+
 def _global_search(
     query: str,
     token: str,
diff --git a/tests/test_polymarket.py b/tests/test_polymarket.py
index 0c4e73b..7689e9d 100644
--- a/tests/test_polymarket.py
+++ b/tests/test_polymarket.py
@@ -78,6 +78,12 @@ class TestExpandQueries(unittest.TestCase):
         self.assertIn("new", queries)
         self.assertIn("idea", queries)
 
+    def test_low_signal_words_not_expanded_standalone(self):
+        queries = polymarket._expand_queries("anthropic odds")
+        self.assertIn("anthropic odds", queries)
+        self.assertIn("anthropic", queries)
+        self.assertNotIn("odds", queries)
+
 
 class TestExtractDomainQueries(unittest.TestCase):
     def _make_tag(self, label):
@@ -195,6 +201,37 @@ class TestFormatPriceMovement(unittest.TestCase):
         self.assertIsNone(result)
 
 
+class TestTextSimilarity(unittest.TestCase):
+    def test_short_binary_outcome_does_not_match_substring(self):
+        score = polymarket._compute_text_similarity(
+            "nano banana pro prompting",
+            "NATO x Russia military clash by...?",
+            ["No", "Yes"],
+        )
+        self.assertLess(score, 0.3)
+
+    def test_outcome_only_match_is_capped_for_non_prediction_queries(self):
+        score = polymarket._compute_text_similarity(
+            "kanye west",
+            "Top Spotify artist in March?",
+            ["Kanye West", "Taylor Swift"],
+        )
+        self.assertLess(score, 0.3)
+
+    def test_direct_title_match_beats_outcome_only_prediction_market(self):
+        direct = polymarket._compute_text_similarity(
+            "anthropic odds",
+            "Will Anthropic or OpenAI IPO first?",
+            [],
+        )
+        generic = polymarket._compute_text_similarity(
+            "anthropic odds",
+            "Which company will have the best AI model for coding on March 31",
+            ["Anthropic", "OpenAI", "Google"],
+        )
+        self.assertGreater(direct, generic)
+
+
 class TestParseOutcomePrices(unittest.TestCase):
     def test_binary_market_json_strings(self):
         market = {
@@ -590,27 +627,27 @@ class TestTextSimilarity(unittest.TestCase):
         self.assertEqual(score, 1.0)
 
     def test_outcome_substring_match(self):
-        """Topic 'Arizona' should match outcome 'Arizona' even when title has no overlap."""
+        """Prediction queries can still use outcome-only entity matches."""
         score = polymarket._compute_text_similarity(
-            "Arizona",
+            "Arizona odds",
             "Who will be the #1 overall seed?",
             outcomes=["Duke", "Arizona", "Houston"],
         )
-        self.assertEqual(score, 1.0)
+        self.assertEqual(score, 0.55)
 
     def test_outcome_bidirectional_match(self):
-        """Topic 'Arizona Basketball' should match outcome 'Arizona' (outcome in core)."""
+        """Longer prediction topics keep the same moderated outcome-only cap."""
         score = polymarket._compute_text_similarity(
-            "Arizona Basketball",
+            "Arizona Basketball odds",
             "Who will be the #1 overall seed?",
             outcomes=["Duke", "Arizona", "Houston"],
         )
-        self.assertEqual(score, 0.88)
+        self.assertEqual(score, 0.55)
 
     def test_outcome_token_overlap(self):
-        """Partial token overlap with outcome gets a moderate score."""
+        """Outcome-only prediction matches stay moderate, not dominant."""
         score = polymarket._compute_text_similarity(
-            "Iran War",
+            "Iran War odds",
             "Unrelated geopolitics title",
             outcomes=["War continues", "Peace deal"],
         )
@@ -630,11 +667,11 @@ class TestTextSimilarity(unittest.TestCase):
         """Outcomes with price <= 1% should be filtered by the caller, not this function."""
         # This function doesn't filter - it trusts the caller to pass only relevant outcomes
         score = polymarket._compute_text_similarity(
-            "Arizona",
+            "Arizona odds",
             "Unrelated title",
             outcomes=["Arizona"],
         )
-        self.assertEqual(score, 1.0)
+        self.assertEqual(score, 0.55)
 
     def test_generic_only_odds_match_stays_below_threshold(self):
         score = polymarket._compute_text_similarity(
diff --git a/tests/test_reddit_sc.py b/tests/test_reddit_sc.py
index a212055..9d38943 100644
--- a/tests/test_reddit_sc.py
+++ b/tests/test_reddit_sc.py
@@ -158,5 +158,24 @@ class TestDepthConfig(unittest.TestCase):
         )
 
 
+class TestPostRelevance(unittest.TestCase):
+    def test_body_cannot_rescue_weak_title_too_far(self):
+        score = reddit._compute_post_relevance(
+            "anthropic odds",
+            "President Trump orders agencies to stop using Anthropic technology",
+            "Long body text eventually mentions odds and other tangential details.",
+        )
+        self.assertLess(score, 0.7)
+        self.assertGreaterEqual(score, 0.5)
+
+    def test_exact_title_match_stays_high(self):
+        score = reddit._compute_post_relevance(
+            "claude code tips",
+            "Claude Code tips for faster workflows",
+            "",
+        )
+        self.assertGreater(score, 0.7)
+
+
 if __name__ == "__main__":
     unittest.main()

From 058c4e1899c171a1516cef6956a9e516378bb9b6 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Sat, 14 Mar 2026 00:38:59 -0700
Subject: [PATCH 18/18] Classify prompt and animation queries earlier

Map prompt-oriented product searches and animation-oriented build searches away from the breaking-news default so source tiering and tiebreakers align with the benchmark topics.

Validation: uv run python -m unittest tests.test_query_type
---
 scripts/lib/query_type.py | 4 ++--
 tests/test_query_type.py  | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/lib/query_type.py b/scripts/lib/query_type.py
index 36d325d..4e10dff 100644
--- a/scripts/lib/query_type.py
+++ b/scripts/lib/query_type.py
@@ -7,7 +7,7 @@ QueryType = Literal["product", "concept", "opinion", "how_to", "comparison", "br
 
 # Pattern-based classification (no LLM, no external deps)
 _PRODUCT_PATTERNS = re.compile(
-    r"\b(price|pricing|cost|buy|purchase|deal|discount|subscription|plan|tier|free tier|alternative)\b", re.I
+    r"\b(price|pricing|cost|buy|purchase|deal|discount|subscription|plan|tier|free tier|alternative|prompt|prompts|prompting|template|templates)\b", re.I
 )
 _CONCEPT_PATTERNS = re.compile(
     r"\b(what is|what are|explain|definition|how does|how do|overview|introduction|guide to|primer)\b", re.I
@@ -16,7 +16,7 @@ _OPINION_PATTERNS = re.compile(
     r"\b(worth it|thoughts on|opinion|review|experience with|recommend|should i|pros and cons|good or bad)\b", re.I
 )
 _HOWTO_PATTERNS = re.compile(
-    r"\b(how to|tutorial|step by step|setup|install|configure|deploy|migrate|implement|build a|create a)\b", re.I
+    r"\b(how to|tutorial|step by step|setup|install|configure|deploy|migrate|implement|build a|create a|animation|animations|video workflow|render pipeline)\b", re.I
 )
 _COMPARISON_PATTERNS = re.compile(
     r"\b(vs\.?|versus|compared to|comparison|better than|difference between|switch from)\b", re.I
diff --git a/tests/test_query_type.py b/tests/test_query_type.py
index c07f343..12838ec 100644
--- a/tests/test_query_type.py
+++ b/tests/test_query_type.py
@@ -20,6 +20,7 @@ class TestDetectQueryType(unittest.TestCase):
         self.assertEqual(detect_query_type("cursor IDE pricing"), "product")
         self.assertEqual(detect_query_type("is Claude Pro worth the cost"), "product")
         self.assertEqual(detect_query_type("best free tier LLM API"), "product")
+        self.assertEqual(detect_query_type("nano banana pro prompting"), "product")
 
     def test_concept_queries(self):
         self.assertEqual(detect_query_type("what is WebTransport"), "concept")
@@ -35,6 +36,7 @@ class TestDetectQueryType(unittest.TestCase):
         self.assertEqual(detect_query_type("how to deploy on Vercel"), "how_to")
         self.assertEqual(detect_query_type("tutorial for building MCP servers"), "how_to")
         self.assertEqual(detect_query_type("step by step Kubernetes setup"), "how_to")
+        self.assertEqual(detect_query_type("remotion animations for Claude Code"), "how_to")
 
     def test_comparison_queries(self):
         self.assertEqual(detect_query_type("cursor vs windsurf"), "comparison")