From d66758659703ffd65a659f67271ce2d2e7f12c5a Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 15:07:50 -0700 Subject: [PATCH] Add shared query.py and relevance.py modules Consolidate duplicated _extract_core_subject() (7 copies across bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into query.extract_core_subject() with parameterized noise set, max_words, and suffix stripping. Consolidate duplicated _tokenize/_compute_relevance/STOPWORDS/SYNONYMS (4 copies across youtube_yt, tiktok, instagram, scrapecreators_x) into relevance.token_overlap_relevance() with hashtag-aware matching. Integration into per-module imports follows in next commits. --- scripts/lib/query.py | 98 +++++++++++++++++++++++++++++ scripts/lib/relevance.py | 95 ++++++++++++++++++++++++++++ tests/test_query.py | 130 +++++++++++++++++++++++++++++++++++++++ tests/test_relevance.py | 122 ++++++++++++++++++++++++++++++++++++ 4 files changed, 445 insertions(+) create mode 100644 scripts/lib/query.py create mode 100644 scripts/lib/relevance.py create mode 100644 tests/test_query.py create mode 100644 tests/test_relevance.py diff --git a/scripts/lib/query.py b/scripts/lib/query.py new file mode 100644 index 0000000..1d48875 --- /dev/null +++ b/scripts/lib/query.py @@ -0,0 +1,98 @@ +"""Shared query utilities for /last30days search modules. + +Consolidates duplicated _extract_core_subject() logic from bird_x, reddit, +youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one +parameterized function. Each platform calls with its own overrides. +""" + +from typing import FrozenSet, List, Optional, Set + +# Common multi-word prefixes stripped from all queries (identical across modules) +PREFIXES = [ + 'what are the best', 'what is the best', 'what are the latest', + 'what are people saying about', 'what do people think about', + 'how do i use', 'how to use', 'how to', + 'what are', 'what is', 'tips for', 'best practices for', +] + +# Multi-word suffixes (used by bird_x) +SUFFIXES = [ + 'best practices', 'use cases', 'prompt techniques', + 'prompting techniques', 'prompting tips', +] + +# Base noise words shared across most modules +NOISE_WORDS = frozenset({ + # Articles/prepositions/conjunctions + 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or', + 'of', 'in', 'on', 'for', 'with', 'about', 'to', + # Question words + 'how', 'what', 'which', 'who', 'why', 'when', 'where', + 'does', 'should', 'could', 'would', + # Research/meta descriptors + 'best', 'top', 'good', 'great', 'awesome', 'killer', + 'latest', 'new', 'news', 'update', 'updates', + 'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral', + 'practices', 'features', 'guide', 'tutorial', + 'recommendations', 'advice', 'review', 'reviews', + 'usecases', 'examples', 'comparison', 'versus', 'vs', + 'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools', + # Prompting meta words + 'prompt', 'prompts', 'prompting', 'techniques', 'tips', + 'tricks', 'methods', 'strategies', 'approaches', + # Action words + 'using', 'uses', 'use', + # Misc filler + 'people', 'saying', 'think', 'said', 'lately', +}) + + +def extract_core_subject( + topic: str, + *, + noise: Optional[FrozenSet[str]] = None, + max_words: Optional[int] = None, + strip_suffixes: bool = False, +) -> str: + """Extract core subject from a verbose search query. + + Strips common question/meta prefixes and noise words to produce a + compact search-friendly query. Platforms customize via parameters. + + Args: + topic: Raw user query + noise: Override noise word set (default: NOISE_WORDS) + max_words: Cap result to N words (default: no cap) + strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this) + + Returns: + Cleaned query string + """ + text = topic.lower().strip() + if not text: + return text + + # Phase 1: Strip multi-word prefixes (longest first, stop after first match) + for p in PREFIXES: + if text.startswith(p + ' '): + text = text[len(p):].strip() + break + + # Phase 2: Strip multi-word suffixes (opt-in) + if strip_suffixes: + for s in SUFFIXES: + if text.endswith(' ' + s): + text = text[:-len(s)].strip() + break + + # Phase 3: Filter individual noise words + noise_set = noise if noise is not None else NOISE_WORDS + words = text.split() + filtered = [w for w in words if w not in noise_set] + + # Apply word cap if requested + if max_words is not None and filtered: + filtered = filtered[:max_words] + + result = ' '.join(filtered) if filtered else text + return result.rstrip('?!.') if not max_words else (result or topic.lower().strip()) diff --git a/scripts/lib/relevance.py b/scripts/lib/relevance.py new file mode 100644 index 0000000..f936b70 --- /dev/null +++ b/scripts/lib/relevance.py @@ -0,0 +1,95 @@ +"""Shared relevance scoring for /last30days search modules. + +Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS +from youtube_yt, tiktok, instagram, and scrapecreators_x into one module. +""" + +import re +from typing import List, Optional, Set + +# Stopwords for relevance computation (common English words that dilute token overlap) +STOPWORDS = frozenset({ + 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on', + 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my', + 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can', + 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about', + 'all', 'just', 'get', 'has', 'have', 'was', 'will', +}) + +# Synonym groups for relevance scoring (bidirectional expansion) +# Superset of all platform-specific synonym dicts +SYNONYMS = { + 'hip': {'rap', 'hiphop'}, + 'hop': {'rap', 'hiphop'}, + 'rap': {'hip', 'hop', 'hiphop'}, + 'hiphop': {'rap', 'hip', 'hop'}, + 'js': {'javascript'}, + 'javascript': {'js'}, + 'ts': {'typescript'}, + 'typescript': {'ts'}, + 'ai': {'artificial', 'intelligence'}, + 'ml': {'machine', 'learning'}, + 'react': {'reactjs'}, + 'reactjs': {'react'}, + 'svelte': {'sveltejs'}, + 'sveltejs': {'svelte'}, + 'vue': {'vuejs'}, + 'vuejs': {'vue'}, +} + + +def tokenize(text: str) -> Set[str]: + """Lowercase, strip punctuation, remove stopwords, drop single-char tokens. + + Expands tokens with synonyms for better cross-domain matching. + """ + words = re.sub(r'[^\w\s]', ' ', text.lower()).split() + tokens = {w for w in words if w not in STOPWORDS and len(w) > 1} + expanded = set(tokens) + for t in tokens: + if t in SYNONYMS: + expanded.update(SYNONYMS[t]) + return expanded + + +def token_overlap_relevance( + query: str, + text: str, + hashtags: Optional[List[str]] = None, +) -> float: + """Compute relevance as ratio of query tokens found in text. + + Uses ratio overlap (intersection / query_length) so short queries + score higher when fully represented in the text. Floors at 0.1. + + Args: + query: Search query + text: Content text to match against + hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated + hashtags are split to match query tokens (e.g. "claudecode" matches "claude"). + + Returns: + Float between 0.1 and 1.0 (0.5 for empty queries) + """ + q_tokens = tokenize(query) + + # Combine text and hashtags for matching + combined = text + if hashtags: + combined = f"{text} {' '.join(hashtags)}" + t_tokens = tokenize(combined) + + # Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code") + if hashtags: + for tag in hashtags: + tag_lower = tag.lower() + for qt in q_tokens: + if qt in tag_lower and qt != tag_lower: + t_tokens.add(qt) + + if not q_tokens: + return 0.5 # Neutral fallback for empty/stopword-only queries + + overlap = len(q_tokens & t_tokens) + ratio = overlap / len(q_tokens) + return max(0.1, min(1.0, ratio)) diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..a081f4f --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,130 @@ +"""Tests for query.py — shared query utilities.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib.query import NOISE_WORDS, extract_core_subject + + +class TestExtractCoreSubject(unittest.TestCase): + """Tests for extract_core_subject() with default noise set.""" + + def test_strips_what_are_prefix(self): + self.assertEqual(extract_core_subject("what are the best AI tools"), "ai") + + def test_strips_how_to_prefix(self): + self.assertEqual(extract_core_subject("how to use cursor IDE"), "cursor ide") + + def test_strips_what_do_people_think(self): + result = extract_core_subject("what do people think about React Server Components") + self.assertEqual(result, "react server components") + + def test_preserves_product_name(self): + self.assertEqual(extract_core_subject("cursor IDE"), "cursor ide") + + def test_strips_trailing_punctuation(self): + result = extract_core_subject("what is Claude?") + self.assertFalse(result.endswith("?")) + + def test_empty_string(self): + self.assertEqual(extract_core_subject(""), "") + + def test_all_noise_returns_original(self): + # When all words are noise, fall back to original text + result = extract_core_subject("best latest new") + self.assertTrue(len(result) > 0) + + def test_only_first_prefix_stripped(self): + # "how to" should match, stripping once, not recursively + result = extract_core_subject("how to use how to debug") + self.assertIn("debug", result) + + +class TestMaxWords(unittest.TestCase): + """Tests for max_words parameter.""" + + def test_max_words_caps_output(self): + result = extract_core_subject( + "multi agent reinforcement learning framework", + max_words=5, + ) + self.assertLessEqual(len(result.split()), 5) + + def test_max_words_none_no_cap(self): + result = extract_core_subject("cursor IDE react native components") + # Without max_words, no cap applied + self.assertGreaterEqual(len(result.split()), 3) + + def test_max_words_fallback_on_empty(self): + # All words filtered + max_words should fall back to original + result = extract_core_subject("best top latest", max_words=3) + self.assertTrue(len(result) > 0) + + +class TestStripSuffixes(unittest.TestCase): + """Tests for strip_suffixes parameter.""" + + def test_strips_best_practices(self): + result = extract_core_subject( + "claude code best practices", + strip_suffixes=True, + ) + self.assertNotIn("practices", result) + + def test_strips_use_cases(self): + result = extract_core_subject( + "react hooks use cases", + strip_suffixes=True, + ) + self.assertNotIn("cases", result) + + def test_no_strip_without_flag(self): + result = extract_core_subject("claude code best practices") + # "best" and "practices" are noise words so they get filtered anyway + # but the suffix phase doesn't run + self.assertIn("claude", result) + + +class TestCustomNoise(unittest.TestCase): + """Tests for noise override parameter.""" + + def test_custom_noise_keeps_tips(self): + # YouTube keeps tips/tricks/tutorial — pass a noise set without them + youtube_noise = frozenset({ + 'best', 'top', 'good', 'great', 'awesome', 'killer', + 'latest', 'new', 'news', 'update', 'updates', + 'trending', 'hottest', 'popular', 'viral', + 'practices', 'features', + 'recommendations', 'advice', + 'prompt', 'prompts', 'prompting', + 'methods', 'strategies', 'approaches', + }) + result = extract_core_subject("best react tips", noise=youtube_noise) + self.assertIn("tips", result) + + def test_default_noise_removes_tips(self): + result = extract_core_subject("best react tips") + self.assertNotIn("tips", result) + + +class TestNoiseWordsCompleteness(unittest.TestCase): + """Verify NOISE_WORDS superset covers all platform sets.""" + + def test_question_words_present(self): + for w in ('who', 'why', 'when', 'where', 'does', 'should', 'could', 'would'): + self.assertIn(w, NOISE_WORDS, f"Missing question word: {w}") + + def test_core_filler_present(self): + for w in ('the', 'a', 'an', 'is', 'are', 'for', 'with', 'about'): + self.assertIn(w, NOISE_WORDS) + + def test_research_meta_present(self): + for w in ('best', 'top', 'latest', 'trending', 'popular'): + self.assertIn(w, NOISE_WORDS) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_relevance.py b/tests/test_relevance.py new file mode 100644 index 0000000..c641e76 --- /dev/null +++ b/tests/test_relevance.py @@ -0,0 +1,122 @@ +"""Tests for relevance.py — shared relevance scoring. + +Migrated from test_youtube_relevance.py + new hashtag/synonym tests. +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib.relevance import STOPWORDS, SYNONYMS, token_overlap_relevance, tokenize + + +class TestTokenize(unittest.TestCase): + """Tests for tokenize().""" + + def test_removes_stopwords(self): + tokens = tokenize("how to use the AI tools") + self.assertNotIn("how", tokens) + self.assertNotIn("to", tokens) + self.assertNotIn("the", tokens) + self.assertIn("ai", tokens) + self.assertIn("tools", tokens) + + def test_lowercases(self): + tokens = tokenize("Python REACT") + self.assertIn("python", tokens) + self.assertIn("react", tokens) + + def test_strips_punctuation(self): + tokens = tokenize("hello, world!") + self.assertIn("hello", tokens) + self.assertIn("world", tokens) + + def test_drops_single_char(self): + tokens = tokenize("a b c python") + self.assertNotIn("a", tokens) + self.assertNotIn("b", tokens) + self.assertNotIn("c", tokens) + self.assertIn("python", tokens) + + def test_expands_synonyms(self): + tokens = tokenize("ai tools") + self.assertIn("artificial", tokens) + self.assertIn("intelligence", tokens) + + def test_expands_js_synonym(self): + tokens = tokenize("js framework") + self.assertIn("javascript", tokens) + + def test_expands_svelte(self): + tokens = tokenize("svelte app") + self.assertIn("sveltejs", tokens) + + def test_expands_vue(self): + tokens = tokenize("vue components") + self.assertIn("vuejs", tokens) + + +class TestTokenOverlapRelevance(unittest.TestCase): + """Tests for token_overlap_relevance().""" + + def test_high_relevance_exact_match(self): + rel = token_overlap_relevance("claude code", "Claude Code tricks and tips") + self.assertGreater(rel, 0.7) + + def test_low_relevance_no_match(self): + rel = token_overlap_relevance("claude code tips", "Best AI tools for coding") + self.assertLess(rel, 0.5) + + def test_empty_query_returns_neutral(self): + rel = token_overlap_relevance("", "Some video title") + self.assertEqual(rel, 0.5) + + def test_floor_at_0_1(self): + rel = token_overlap_relevance("quantum physics", "cat dancing video") + self.assertGreaterEqual(rel, 0.1) + + def test_full_match_returns_1(self): + rel = token_overlap_relevance("python tutorial", "Python Tutorial for Beginners") + self.assertEqual(rel, 1.0) + + def test_partial_match(self): + rel = token_overlap_relevance("react native tutorial", "React Native Guide") + self.assertGreater(rel, 0.3) + self.assertLess(rel, 1.0) + + def test_synonym_boosts_relevance(self): + # "js" should match "javascript" via synonym expansion + rel_with_syn = token_overlap_relevance("js framework", "javascript framework comparison") + rel_without = token_overlap_relevance("python framework", "javascript framework comparison") + self.assertGreater(rel_with_syn, rel_without) + + def test_stopword_only_query(self): + rel = token_overlap_relevance("the a is", "some content here") + self.assertEqual(rel, 0.5) + + +class TestHashtagRelevance(unittest.TestCase): + """Tests for hashtag-aware relevance (TikTok/Instagram pattern).""" + + def test_hashtag_boost(self): + rel_no_hash = token_overlap_relevance("claude code", "random video about stuff") + rel_with_hash = token_overlap_relevance( + "claude code", "random video about stuff", ["claudecode", "ai"] + ) + self.assertGreater(rel_with_hash, rel_no_hash) + + def test_concatenated_hashtag_splitting(self): + # "claudecode" should match "claude" from query via substring check + rel = token_overlap_relevance("claude", "video", ["claudecode"]) + self.assertGreater(rel, 0.5) + + def test_none_hashtags_same_as_no_hashtags(self): + rel1 = token_overlap_relevance("test query", "test content", None) + rel2 = token_overlap_relevance("test query", "test content") + self.assertEqual(rel1, rel2) + + +if __name__ == "__main__": + unittest.main()