Add shared query.py and relevance.py modules
Consolidate duplicated _extract_core_subject() (7 copies across bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into query.extract_core_subject() with parameterized noise set, max_words, and suffix stripping. Consolidate duplicated _tokenize/_compute_relevance/STOPWORDS/SYNONYMS (4 copies across youtube_yt, tiktok, instagram, scrapecreators_x) into relevance.token_overlap_relevance() with hashtag-aware matching. Integration into per-module imports follows in next commits.
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
"""Tests for relevance.py — shared relevance scoring.
|
||||
|
||||
Migrated from test_youtube_relevance.py + new hashtag/synonym tests.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||
|
||||
from lib.relevance import STOPWORDS, SYNONYMS, token_overlap_relevance, tokenize
|
||||
|
||||
|
||||
class TestTokenize(unittest.TestCase):
|
||||
"""Tests for tokenize()."""
|
||||
|
||||
def test_removes_stopwords(self):
|
||||
tokens = tokenize("how to use the AI tools")
|
||||
self.assertNotIn("how", tokens)
|
||||
self.assertNotIn("to", tokens)
|
||||
self.assertNotIn("the", tokens)
|
||||
self.assertIn("ai", tokens)
|
||||
self.assertIn("tools", tokens)
|
||||
|
||||
def test_lowercases(self):
|
||||
tokens = tokenize("Python REACT")
|
||||
self.assertIn("python", tokens)
|
||||
self.assertIn("react", tokens)
|
||||
|
||||
def test_strips_punctuation(self):
|
||||
tokens = tokenize("hello, world!")
|
||||
self.assertIn("hello", tokens)
|
||||
self.assertIn("world", tokens)
|
||||
|
||||
def test_drops_single_char(self):
|
||||
tokens = tokenize("a b c python")
|
||||
self.assertNotIn("a", tokens)
|
||||
self.assertNotIn("b", tokens)
|
||||
self.assertNotIn("c", tokens)
|
||||
self.assertIn("python", tokens)
|
||||
|
||||
def test_expands_synonyms(self):
|
||||
tokens = tokenize("ai tools")
|
||||
self.assertIn("artificial", tokens)
|
||||
self.assertIn("intelligence", tokens)
|
||||
|
||||
def test_expands_js_synonym(self):
|
||||
tokens = tokenize("js framework")
|
||||
self.assertIn("javascript", tokens)
|
||||
|
||||
def test_expands_svelte(self):
|
||||
tokens = tokenize("svelte app")
|
||||
self.assertIn("sveltejs", tokens)
|
||||
|
||||
def test_expands_vue(self):
|
||||
tokens = tokenize("vue components")
|
||||
self.assertIn("vuejs", tokens)
|
||||
|
||||
|
||||
class TestTokenOverlapRelevance(unittest.TestCase):
|
||||
"""Tests for token_overlap_relevance()."""
|
||||
|
||||
def test_high_relevance_exact_match(self):
|
||||
rel = token_overlap_relevance("claude code", "Claude Code tricks and tips")
|
||||
self.assertGreater(rel, 0.7)
|
||||
|
||||
def test_low_relevance_no_match(self):
|
||||
rel = token_overlap_relevance("claude code tips", "Best AI tools for coding")
|
||||
self.assertLess(rel, 0.5)
|
||||
|
||||
def test_empty_query_returns_neutral(self):
|
||||
rel = token_overlap_relevance("", "Some video title")
|
||||
self.assertEqual(rel, 0.5)
|
||||
|
||||
def test_floor_at_0_1(self):
|
||||
rel = token_overlap_relevance("quantum physics", "cat dancing video")
|
||||
self.assertGreaterEqual(rel, 0.1)
|
||||
|
||||
def test_full_match_returns_1(self):
|
||||
rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners")
|
||||
self.assertEqual(rel, 1.0)
|
||||
|
||||
def test_partial_match(self):
|
||||
rel = token_overlap_relevance("react native tutorial", "React Native Guide")
|
||||
self.assertGreater(rel, 0.3)
|
||||
self.assertLess(rel, 1.0)
|
||||
|
||||
def test_synonym_boosts_relevance(self):
|
||||
# "js" should match "javascript" via synonym expansion
|
||||
rel_with_syn = token_overlap_relevance("js framework", "javascript framework comparison")
|
||||
rel_without = token_overlap_relevance("python framework", "javascript framework comparison")
|
||||
self.assertGreater(rel_with_syn, rel_without)
|
||||
|
||||
def test_stopword_only_query(self):
|
||||
rel = token_overlap_relevance("the a is", "some content here")
|
||||
self.assertEqual(rel, 0.5)
|
||||
|
||||
|
||||
class TestHashtagRelevance(unittest.TestCase):
|
||||
"""Tests for hashtag-aware relevance (TikTok/Instagram pattern)."""
|
||||
|
||||
def test_hashtag_boost(self):
|
||||
rel_no_hash = token_overlap_relevance("claude code", "random video about stuff")
|
||||
rel_with_hash = token_overlap_relevance(
|
||||
"claude code", "random video about stuff", ["claudecode", "ai"]
|
||||
)
|
||||
self.assertGreater(rel_with_hash, rel_no_hash)
|
||||
|
||||
def test_concatenated_hashtag_splitting(self):
|
||||
# "claudecode" should match "claude" from query via substring check
|
||||
rel = token_overlap_relevance("claude", "video", ["claudecode"])
|
||||
self.assertGreater(rel, 0.5)
|
||||
|
||||
def test_none_hashtags_same_as_no_hashtags(self):
|
||||
rel1 = token_overlap_relevance("test query", "test content", None)
|
||||
rel2 = token_overlap_relevance("test query", "test content")
|
||||
self.assertEqual(rel1, rel2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user