Add shared query.py and relevance.py modules

Consolidate duplicated _extract_core_subject() (7 copies across bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into query.extract_core_subject() with parameterized noise set, max_words, and suffix stripping. Consolidate duplicated _tokenize/_compute_relevance/STOPWORDS/SYNONYMS (4 copies across youtube_yt, tiktok, instagram, scrapecreators_x) into relevance.token_overlap_relevance() with hashtag-aware matching. Integration into per-module imports follows in next commits.
2026-03-11 15:07:50 -07:00
parent ce8e289692
commit d667586597
4 changed files with 445 additions and 0 deletions
@@ -0,0 +1,130 @@
+"""Tests for query.py — shared query utilities."""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.query import NOISE_WORDS, extract_core_subject
+
+
+class TestExtractCoreSubject(unittest.TestCase):
+    """Tests for extract_core_subject() with default noise set."""
+
+    def test_strips_what_are_prefix(self):
+        self.assertEqual(extract_core_subject("what are the best AI tools"), "ai")
+
+    def test_strips_how_to_prefix(self):
+        self.assertEqual(extract_core_subject("how to use cursor IDE"), "cursor ide")
+
+    def test_strips_what_do_people_think(self):
+        result = extract_core_subject("what do people think about React Server Components")
+        self.assertEqual(result, "react server components")
+
+    def test_preserves_product_name(self):
+        self.assertEqual(extract_core_subject("cursor IDE"), "cursor ide")
+
+    def test_strips_trailing_punctuation(self):
+        result = extract_core_subject("what is Claude?")
+        self.assertFalse(result.endswith("?"))
+
+    def test_empty_string(self):
+        self.assertEqual(extract_core_subject(""), "")
+
+    def test_all_noise_returns_original(self):
+        # When all words are noise, fall back to original text
+        result = extract_core_subject("best latest new")
+        self.assertTrue(len(result) > 0)
+
+    def test_only_first_prefix_stripped(self):
+        # "how to" should match, stripping once, not recursively
+        result = extract_core_subject("how to use how to debug")
+        self.assertIn("debug", result)
+
+
+class TestMaxWords(unittest.TestCase):
+    """Tests for max_words parameter."""
+
+    def test_max_words_caps_output(self):
+        result = extract_core_subject(
+            "multi agent reinforcement learning framework",
+            max_words=5,
+        )
+        self.assertLessEqual(len(result.split()), 5)
+
+    def test_max_words_none_no_cap(self):
+        result = extract_core_subject("cursor IDE react native components")
+        # Without max_words, no cap applied
+        self.assertGreaterEqual(len(result.split()), 3)
+
+    def test_max_words_fallback_on_empty(self):
+        # All words filtered + max_words should fall back to original
+        result = extract_core_subject("best top latest", max_words=3)
+        self.assertTrue(len(result) > 0)
+
+
+class TestStripSuffixes(unittest.TestCase):
+    """Tests for strip_suffixes parameter."""
+
+    def test_strips_best_practices(self):
+        result = extract_core_subject(
+            "claude code best practices",
+            strip_suffixes=True,
+        )
+        self.assertNotIn("practices", result)
+
+    def test_strips_use_cases(self):
+        result = extract_core_subject(
+            "react hooks use cases",
+            strip_suffixes=True,
+        )
+        self.assertNotIn("cases", result)
+
+    def test_no_strip_without_flag(self):
+        result = extract_core_subject("claude code best practices")
+        # "best" and "practices" are noise words so they get filtered anyway
+        # but the suffix phase doesn't run
+        self.assertIn("claude", result)
+
+
+class TestCustomNoise(unittest.TestCase):
+    """Tests for noise override parameter."""
+
+    def test_custom_noise_keeps_tips(self):
+        # YouTube keeps tips/tricks/tutorial — pass a noise set without them
+        youtube_noise = frozenset({
+            'best', 'top', 'good', 'great', 'awesome', 'killer',
+            'latest', 'new', 'news', 'update', 'updates',
+            'trending', 'hottest', 'popular', 'viral',
+            'practices', 'features',
+            'recommendations', 'advice',
+            'prompt', 'prompts', 'prompting',
+            'methods', 'strategies', 'approaches',
+        })
+        result = extract_core_subject("best react tips", noise=youtube_noise)
+        self.assertIn("tips", result)
+
+    def test_default_noise_removes_tips(self):
+        result = extract_core_subject("best react tips")
+        self.assertNotIn("tips", result)
+
+
+class TestNoiseWordsCompleteness(unittest.TestCase):
+    """Verify NOISE_WORDS superset covers all platform sets."""
+
+    def test_question_words_present(self):
+        for w in ('who', 'why', 'when', 'where', 'does', 'should', 'could', 'would'):
+            self.assertIn(w, NOISE_WORDS, f"Missing question word: {w}")
+
+    def test_core_filler_present(self):
+        for w in ('the', 'a', 'an', 'is', 'are', 'for', 'with', 'about'):
+            self.assertIn(w, NOISE_WORDS)
+
+    def test_research_meta_present(self):
+        for w in ('best', 'top', 'latest', 'trending', 'popular'):
+            self.assertIn(w, NOISE_WORDS)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,122 @@
+"""Tests for relevance.py — shared relevance scoring.
+
+Migrated from test_youtube_relevance.py + new hashtag/synonym tests.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+from lib.relevance import STOPWORDS, SYNONYMS, token_overlap_relevance, tokenize
+
+
+class TestTokenize(unittest.TestCase):
+    """Tests for tokenize()."""
+
+    def test_removes_stopwords(self):
+        tokens = tokenize("how to use the AI tools")
+        self.assertNotIn("how", tokens)
+        self.assertNotIn("to", tokens)
+        self.assertNotIn("the", tokens)
+        self.assertIn("ai", tokens)
+        self.assertIn("tools", tokens)
+
+    def test_lowercases(self):
+        tokens = tokenize("Python REACT")
+        self.assertIn("python", tokens)
+        self.assertIn("react", tokens)
+
+    def test_strips_punctuation(self):
+        tokens = tokenize("hello, world!")
+        self.assertIn("hello", tokens)
+        self.assertIn("world", tokens)
+
+    def test_drops_single_char(self):
+        tokens = tokenize("a b c python")
+        self.assertNotIn("a", tokens)
+        self.assertNotIn("b", tokens)
+        self.assertNotIn("c", tokens)
+        self.assertIn("python", tokens)
+
+    def test_expands_synonyms(self):
+        tokens = tokenize("ai tools")
+        self.assertIn("artificial", tokens)
+        self.assertIn("intelligence", tokens)
+
+    def test_expands_js_synonym(self):
+        tokens = tokenize("js framework")
+        self.assertIn("javascript", tokens)
+
+    def test_expands_svelte(self):
+        tokens = tokenize("svelte app")
+        self.assertIn("sveltejs", tokens)
+
+    def test_expands_vue(self):
+        tokens = tokenize("vue components")
+        self.assertIn("vuejs", tokens)
+
+
+class TestTokenOverlapRelevance(unittest.TestCase):
+    """Tests for token_overlap_relevance()."""
+
+    def test_high_relevance_exact_match(self):
+        rel = token_overlap_relevance("claude code", "Claude Code tricks and tips")
+        self.assertGreater(rel, 0.7)
+
+    def test_low_relevance_no_match(self):
+        rel = token_overlap_relevance("claude code tips", "Best AI tools for coding")
+        self.assertLess(rel, 0.5)
+
+    def test_empty_query_returns_neutral(self):
+        rel = token_overlap_relevance("", "Some video title")
+        self.assertEqual(rel, 0.5)
+
+    def test_floor_at_0_1(self):
+        rel = token_overlap_relevance("quantum physics", "cat dancing video")
+        self.assertGreaterEqual(rel, 0.1)
+
+    def test_full_match_returns_1(self):
+        rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners")
+        self.assertEqual(rel, 1.0)
+
+    def test_partial_match(self):
+        rel = token_overlap_relevance("react native tutorial", "React Native Guide")
+        self.assertGreater(rel, 0.3)
+        self.assertLess(rel, 1.0)
+
+    def test_synonym_boosts_relevance(self):
+        # "js" should match "javascript" via synonym expansion
+        rel_with_syn = token_overlap_relevance("js framework", "javascript framework comparison")
+        rel_without = token_overlap_relevance("python framework", "javascript framework comparison")
+        self.assertGreater(rel_with_syn, rel_without)
+
+    def test_stopword_only_query(self):
+        rel = token_overlap_relevance("the a is", "some content here")
+        self.assertEqual(rel, 0.5)
+
+
+class TestHashtagRelevance(unittest.TestCase):
+    """Tests for hashtag-aware relevance (TikTok/Instagram pattern)."""
+
+    def test_hashtag_boost(self):
+        rel_no_hash = token_overlap_relevance("claude code", "random video about stuff")
+        rel_with_hash = token_overlap_relevance(
+            "claude code", "random video about stuff", ["claudecode", "ai"]
+        )
+        self.assertGreater(rel_with_hash, rel_no_hash)
+
+    def test_concatenated_hashtag_splitting(self):
+        # "claudecode" should match "claude" from query via substring check
+        rel = token_overlap_relevance("claude", "video", ["claudecode"])
+        self.assertGreater(rel, 0.5)
+
+    def test_none_hashtags_same_as_no_hashtags(self):
+        rel1 = token_overlap_relevance("test query", "test content", None)
+        rel2 = token_overlap_relevance("test query", "test content")
+        self.assertEqual(rel1, rel2)
+
+
+if __name__ == "__main__":
+    unittest.main()