Add shared query.py and relevance.py modules

Consolidate duplicated _extract_core_subject() (7 copies across bird_x,
reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into
query.extract_core_subject() with parameterized noise set, max_words,
and suffix stripping.

Consolidate duplicated _tokenize/_compute_relevance/STOPWORDS/SYNONYMS
(4 copies across youtube_yt, tiktok, instagram, scrapecreators_x) into
relevance.token_overlap_relevance() with hashtag-aware matching.

Integration into per-module imports follows in next commits.

scripts/lib/query.py

@@ -0,0 +1,98 @@
+"""Shared query utilities for /last30days search modules.
+
+Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
+youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
+parameterized function. Each platform calls with its own overrides.
+"""
+
+from typing import FrozenSet, List, Optional, Set
+
+# Common multi-word prefixes stripped from all queries (identical across modules)
+PREFIXES = [
+    'what are the best', 'what is the best', 'what are the latest',
+    'what are people saying about', 'what do people think about',
+    'how do i use', 'how to use', 'how to',
+    'what are', 'what is', 'tips for', 'best practices for',
+]
+
+# Multi-word suffixes (used by bird_x)
+SUFFIXES = [
+    'best practices', 'use cases', 'prompt techniques',
+    'prompting techniques', 'prompting tips',
+]
+
+# Base noise words shared across most modules
+NOISE_WORDS = frozenset({
+    # Articles/prepositions/conjunctions
+    'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
+    'of', 'in', 'on', 'for', 'with', 'about', 'to',
+    # Question words
+    'how', 'what', 'which', 'who', 'why', 'when', 'where',
+    'does', 'should', 'could', 'would',
+    # Research/meta descriptors
+    'best', 'top', 'good', 'great', 'awesome', 'killer',
+    'latest', 'new', 'news', 'update', 'updates',
+    'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
+    'practices', 'features', 'guide', 'tutorial',
+    'recommendations', 'advice', 'review', 'reviews',
+    'usecases', 'examples', 'comparison', 'versus', 'vs',
+    'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
+    # Prompting meta words
+    'prompt', 'prompts', 'prompting', 'techniques', 'tips',
+    'tricks', 'methods', 'strategies', 'approaches',
+    # Action words
+    'using', 'uses', 'use',
+    # Misc filler
+    'people', 'saying', 'think', 'said', 'lately',
+})
+
+
+def extract_core_subject(
+    topic: str,
+    *,
+    noise: Optional[FrozenSet[str]] = None,
+    max_words: Optional[int] = None,
+    strip_suffixes: bool = False,
+) -> str:
+    """Extract core subject from a verbose search query.
+
+    Strips common question/meta prefixes and noise words to produce a
+    compact search-friendly query. Platforms customize via parameters.
+
+    Args:
+        topic: Raw user query
+        noise: Override noise word set (default: NOISE_WORDS)
+        max_words: Cap result to N words (default: no cap)
+        strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this)
+
+    Returns:
+        Cleaned query string
+    """
+    text = topic.lower().strip()
+    if not text:
+        return text
+
+    # Phase 1: Strip multi-word prefixes (longest first, stop after first match)
+    for p in PREFIXES:
+        if text.startswith(p + ' '):
+            text = text[len(p):].strip()
+            break
+
+    # Phase 2: Strip multi-word suffixes (opt-in)
+    if strip_suffixes:
+        for s in SUFFIXES:
+            if text.endswith(' ' + s):
+                text = text[:-len(s)].strip()
+                break
+
+    # Phase 3: Filter individual noise words
+    noise_set = noise if noise is not None else NOISE_WORDS
+    words = text.split()
+    filtered = [w for w in words if w not in noise_set]
+
+    # Apply word cap if requested
+    if max_words is not None and filtered:
+        filtered = filtered[:max_words]
+
+    result = ' '.join(filtered) if filtered else text
+    return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())

scripts/lib/relevance.py

@@ -0,0 +1,95 @@
+"""Shared relevance scoring for /last30days search modules.
+
+Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS
+from youtube_yt, tiktok, instagram, and scrapecreators_x into one module.
+"""
+
+import re
+from typing import List, Optional, Set
+
+# Stopwords for relevance computation (common English words that dilute token overlap)
+STOPWORDS = frozenset({
+    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
+    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
+    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
+    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
+    'all', 'just', 'get', 'has', 'have', 'was', 'will',
+})
+
+# Synonym groups for relevance scoring (bidirectional expansion)
+# Superset of all platform-specific synonym dicts
+SYNONYMS = {
+    'hip': {'rap', 'hiphop'},
+    'hop': {'rap', 'hiphop'},
+    'rap': {'hip', 'hop', 'hiphop'},
+    'hiphop': {'rap', 'hip', 'hop'},
+    'js': {'javascript'},
+    'javascript': {'js'},
+    'ts': {'typescript'},
+    'typescript': {'ts'},
+    'ai': {'artificial', 'intelligence'},
+    'ml': {'machine', 'learning'},
+    'react': {'reactjs'},
+    'reactjs': {'react'},
+    'svelte': {'sveltejs'},
+    'sveltejs': {'svelte'},
+    'vue': {'vuejs'},
+    'vuejs': {'vue'},
+}
+
+
+def tokenize(text: str) -> Set[str]:
+    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.
+
+    Expands tokens with synonyms for better cross-domain matching.
+    """
+    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
+    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
+    expanded = set(tokens)
+    for t in tokens:
+        if t in SYNONYMS:
+            expanded.update(SYNONYMS[t])
+    return expanded
+
+
+def token_overlap_relevance(
+    query: str,
+    text: str,
+    hashtags: Optional[List[str]] = None,
+) -> float:
+    """Compute relevance as ratio of query tokens found in text.
+
+    Uses ratio overlap (intersection / query_length) so short queries
+    score higher when fully represented in the text. Floors at 0.1.
+
+    Args:
+        query: Search query
+        text: Content text to match against
+        hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated
+            hashtags are split to match query tokens (e.g. "claudecode" matches "claude").
+
+    Returns:
+        Float between 0.1 and 1.0 (0.5 for empty queries)
+    """
+    q_tokens = tokenize(query)
+
+    # Combine text and hashtags for matching
+    combined = text
+    if hashtags:
+        combined = f"{text} {' '.join(hashtags)}"
+    t_tokens = tokenize(combined)
+
+    # Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code")
+    if hashtags:
+        for tag in hashtags:
+            tag_lower = tag.lower()
+            for qt in q_tokens:
+                if qt in tag_lower and qt != tag_lower:
+                    t_tokens.add(qt)
+
+    if not q_tokens:
+        return 0.5  # Neutral fallback for empty/stopword-only queries
+
+    overlap = len(q_tokens & t_tokens)
+    ratio = overlap / len(q_tokens)
+    return max(0.1, min(1.0, ratio))

tests/test_query.py

@@ -0,0 +1,130 @@
+"""Tests for query.py — shared query utilities."""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.query import NOISE_WORDS, extract_core_subject
+
+
+class TestExtractCoreSubject(unittest.TestCase):
+    """Tests for extract_core_subject() with default noise set."""
+
+    def test_strips_what_are_prefix(self):
+        self.assertEqual(extract_core_subject("what are the best AI tools"), "ai")
+
+    def test_strips_how_to_prefix(self):
+        self.assertEqual(extract_core_subject("how to use cursor IDE"), "cursor ide")
+
+    def test_strips_what_do_people_think(self):
+        result = extract_core_subject("what do people think about React Server Components")
+        self.assertEqual(result, "react server components")
+
+    def test_preserves_product_name(self):
+        self.assertEqual(extract_core_subject("cursor IDE"), "cursor ide")
+
+    def test_strips_trailing_punctuation(self):
+        result = extract_core_subject("what is Claude?")
+        self.assertFalse(result.endswith("?"))
+
+    def test_empty_string(self):
+        self.assertEqual(extract_core_subject(""), "")
+
+    def test_all_noise_returns_original(self):
+        # When all words are noise, fall back to original text
+        result = extract_core_subject("best latest new")
+        self.assertTrue(len(result) > 0)
+
+    def test_only_first_prefix_stripped(self):
+        # "how to" should match, stripping once, not recursively
+        result = extract_core_subject("how to use how to debug")
+        self.assertIn("debug", result)
+
+
+class TestMaxWords(unittest.TestCase):
+    """Tests for max_words parameter."""
+
+    def test_max_words_caps_output(self):
+        result = extract_core_subject(
+            "multi agent reinforcement learning framework",
+            max_words=5,
+        )
+        self.assertLessEqual(len(result.split()), 5)
+
+    def test_max_words_none_no_cap(self):
+        result = extract_core_subject("cursor IDE react native components")
+        # Without max_words, no cap applied
+        self.assertGreaterEqual(len(result.split()), 3)
+
+    def test_max_words_fallback_on_empty(self):
+        # All words filtered + max_words should fall back to original
+        result = extract_core_subject("best top latest", max_words=3)
+        self.assertTrue(len(result) > 0)
+
+
+class TestStripSuffixes(unittest.TestCase):
+    """Tests for strip_suffixes parameter."""
+
+    def test_strips_best_practices(self):
+        result = extract_core_subject(
+            "claude code best practices",
+            strip_suffixes=True,
+        )
+        self.assertNotIn("practices", result)
+
+    def test_strips_use_cases(self):
+        result = extract_core_subject(
+            "react hooks use cases",
+            strip_suffixes=True,
+        )
+        self.assertNotIn("cases", result)
+
+    def test_no_strip_without_flag(self):
+        result = extract_core_subject("claude code best practices")
+        # "best" and "practices" are noise words so they get filtered anyway
+        # but the suffix phase doesn't run
+        self.assertIn("claude", result)
+
+
+class TestCustomNoise(unittest.TestCase):
+    """Tests for noise override parameter."""
+
+    def test_custom_noise_keeps_tips(self):
+        # YouTube keeps tips/tricks/tutorial — pass a noise set without them
+        youtube_noise = frozenset({
+            'best', 'top', 'good', 'great', 'awesome', 'killer',
+            'latest', 'new', 'news', 'update', 'updates',
+            'trending', 'hottest', 'popular', 'viral',
+            'practices', 'features',
+            'recommendations', 'advice',
+            'prompt', 'prompts', 'prompting',
+            'methods', 'strategies', 'approaches',
+        })
+        result = extract_core_subject("best react tips", noise=youtube_noise)
+        self.assertIn("tips", result)
+
+    def test_default_noise_removes_tips(self):
+        result = extract_core_subject("best react tips")
+        self.assertNotIn("tips", result)
+
+
+class TestNoiseWordsCompleteness(unittest.TestCase):
+    """Verify NOISE_WORDS superset covers all platform sets."""
+
+    def test_question_words_present(self):
+        for w in ('who', 'why', 'when', 'where', 'does', 'should', 'could', 'would'):
+            self.assertIn(w, NOISE_WORDS, f"Missing question word: {w}")
+
+    def test_core_filler_present(self):
+        for w in ('the', 'a', 'an', 'is', 'are', 'for', 'with', 'about'):
+            self.assertIn(w, NOISE_WORDS)
+
+    def test_research_meta_present(self):
+        for w in ('best', 'top', 'latest', 'trending', 'popular'):
+            self.assertIn(w, NOISE_WORDS)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_relevance.py

@@ -0,0 +1,122 @@
+"""Tests for relevance.py — shared relevance scoring.
+
+Migrated from test_youtube_relevance.py + new hashtag/synonym tests.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.relevance import STOPWORDS, SYNONYMS, token_overlap_relevance, tokenize
+
+
+class TestTokenize(unittest.TestCase):
+    """Tests for tokenize()."""
+
+    def test_removes_stopwords(self):
+        tokens = tokenize("how to use the AI tools")
+        self.assertNotIn("how", tokens)
+        self.assertNotIn("to", tokens)
+        self.assertNotIn("the", tokens)
+        self.assertIn("ai", tokens)
+        self.assertIn("tools", tokens)
+
+    def test_lowercases(self):
+        tokens = tokenize("Python REACT")
+        self.assertIn("python", tokens)
+        self.assertIn("react", tokens)
+
+    def test_strips_punctuation(self):
+        tokens = tokenize("hello, world!")
+        self.assertIn("hello", tokens)
+        self.assertIn("world", tokens)
+
+    def test_drops_single_char(self):
+        tokens = tokenize("a b c python")
+        self.assertNotIn("a", tokens)
+        self.assertNotIn("b", tokens)
+        self.assertNotIn("c", tokens)
+        self.assertIn("python", tokens)
+
+    def test_expands_synonyms(self):
+        tokens = tokenize("ai tools")
+        self.assertIn("artificial", tokens)
+        self.assertIn("intelligence", tokens)
+
+    def test_expands_js_synonym(self):
+        tokens = tokenize("js framework")
+        self.assertIn("javascript", tokens)
+
+    def test_expands_svelte(self):
+        tokens = tokenize("svelte app")
+        self.assertIn("sveltejs", tokens)
+
+    def test_expands_vue(self):
+        tokens = tokenize("vue components")
+        self.assertIn("vuejs", tokens)
+
+
+class TestTokenOverlapRelevance(unittest.TestCase):
+    """Tests for token_overlap_relevance()."""
+
+    def test_high_relevance_exact_match(self):
+        rel = token_overlap_relevance("claude code", "Claude Code tricks and tips")
+        self.assertGreater(rel, 0.7)
+
+    def test_low_relevance_no_match(self):
+        rel = token_overlap_relevance("claude code tips", "Best AI tools for coding")
+        self.assertLess(rel, 0.5)
+
+    def test_empty_query_returns_neutral(self):
+        rel = token_overlap_relevance("", "Some video title")
+        self.assertEqual(rel, 0.5)
+
+    def test_floor_at_0_1(self):
+        rel = token_overlap_relevance("quantum physics", "cat dancing video")
+        self.assertGreaterEqual(rel, 0.1)
+
+    def test_full_match_returns_1(self):
+        rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners")
+        self.assertEqual(rel, 1.0)
+
+    def test_partial_match(self):
+        rel = token_overlap_relevance("react native tutorial", "React Native Guide")
+        self.assertGreater(rel, 0.3)
+        self.assertLess(rel, 1.0)
+
+    def test_synonym_boosts_relevance(self):
+        # "js" should match "javascript" via synonym expansion
+        rel_with_syn = token_overlap_relevance("js framework", "javascript framework comparison")
+        rel_without = token_overlap_relevance("python framework", "javascript framework comparison")
+        self.assertGreater(rel_with_syn, rel_without)
+
+    def test_stopword_only_query(self):
+        rel = token_overlap_relevance("the a is", "some content here")
+        self.assertEqual(rel, 0.5)
+
+
+class TestHashtagRelevance(unittest.TestCase):
+    """Tests for hashtag-aware relevance (TikTok/Instagram pattern)."""
+
+    def test_hashtag_boost(self):
+        rel_no_hash = token_overlap_relevance("claude code", "random video about stuff")
+        rel_with_hash = token_overlap_relevance(
+            "claude code", "random video about stuff", ["claudecode", "ai"]
+        )
+        self.assertGreater(rel_with_hash, rel_no_hash)
+
+    def test_concatenated_hashtag_splitting(self):
+        # "claudecode" should match "claude" from query via substring check
+        rel = token_overlap_relevance("claude", "video", ["claudecode"])
+        self.assertGreater(rel, 0.5)
+
+    def test_none_hashtags_same_as_no_hashtags(self):
+        rel1 = token_overlap_relevance("test query", "test content", None)
+        rel2 = token_overlap_relevance("test query", "test content")
+        self.assertEqual(rel1, rel2)
+
+
+if __name__ == "__main__":
+    unittest.main()