Deduplicate relevance code across youtube/tiktok/instagram/scrapecreators_x

Replace duplicated STOPWORDS, SYNONYMS, _tokenize, and _compute_relevance
in four modules with imports from the shared relevance.py module.

Existing tests pass unchanged since modules re-export the functions
under the same names via import aliases.

scripts/lib/instagram.py

@@ -31,71 +31,12 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-# Stopwords for relevance computation (shared with tiktok.py pattern)
+from .relevance import (
-STOPWORDS = frozenset({
+    STOPWORDS,
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
+    SYNONYMS,
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
+    token_overlap_relevance as _compute_relevance,
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
+    tokenize as _tokenize,
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
+)
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
-})
-
-# Synonym groups for relevance scoring
-SYNONYMS = {
-    'hip': {'rap', 'hiphop'},
-    'hop': {'rap', 'hiphop'},
-    'rap': {'hip', 'hop', 'hiphop'},
-    'hiphop': {'rap', 'hip', 'hop'},
-    'js': {'javascript'},
-    'javascript': {'js'},
-    'ts': {'typescript'},
-    'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'},
-    'reactjs': {'react'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> float:
-    """Compute relevance as ratio of query tokens found in text + hashtags.
-
-    Uses ratio overlap (intersection / query_length). Hashtags provide
-    an Instagram-specific relevance boost. Floors at 0.1.
-    """
-    q_tokens = _tokenize(query)
-
-    # Combine text and hashtags for matching
-    combined = text
-    if hashtags:
-        combined = f"{text} {' '.join(hashtags)}"
-    t_tokens = _tokenize(combined)
-
-    # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code")
-    if hashtags:
-        for tag in hashtags:
-            tag_lower = tag.lower()
-            for qt in q_tokens:
-                if qt in tag_lower and qt != tag_lower:
-                    t_tokens.add(qt)
-
-    if not q_tokens:
-        return 0.5  # Neutral fallback
-
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
 
 
 def _extract_core_subject(topic: str) -> str:

scripts/lib/scrapecreators_x.py

@@ -7,10 +7,9 @@ Requires SCRAPECREATORS_API_KEY in config.
 API docs: https://scrapecreators.com/docs
 """
 
-import re
 import sys
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, List, Optional
 
 try:
     import requests as _requests
@@ -25,43 +24,12 @@ DEPTH_CONFIG = {
     "deep":    {"results_per_page": 40},
 }
 
-STOPWORDS = frozenset({
+from .relevance import (
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
+    STOPWORDS,
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
+    SYNONYMS,
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
+    token_overlap_relevance as _compute_relevance,
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
+    tokenize as _tokenize,
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
+)
-})
-
-SYNONYMS = {
-    'js': {'javascript'}, 'javascript': {'js'},
-    'ts': {'typescript'}, 'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'}, 'reactjs': {'react'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, text: str) -> float:
-    """Compute relevance as ratio of query tokens found in text. Floors at 0.1."""
-    q_tokens = _tokenize(query)
-    t_tokens = _tokenize(text)
-    if not q_tokens:
-        return 0.5
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
 
 
 def _extract_core_subject(topic: str) -> str:

scripts/lib/tiktok.py

@@ -31,71 +31,12 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-# Stopwords for relevance computation (shared with youtube_yt.py pattern)
+from .relevance import (
-STOPWORDS = frozenset({
+    STOPWORDS,
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
+    SYNONYMS,
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
+    token_overlap_relevance as _compute_relevance,
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
+    tokenize as _tokenize,
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
+)
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
-})
-
-# Synonym groups for relevance scoring
-SYNONYMS = {
-    'hip': {'rap', 'hiphop'},
-    'hop': {'rap', 'hiphop'},
-    'rap': {'hip', 'hop', 'hiphop'},
-    'hiphop': {'rap', 'hip', 'hop'},
-    'js': {'javascript'},
-    'javascript': {'js'},
-    'ts': {'typescript'},
-    'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'},
-    'reactjs': {'react'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> float:
-    """Compute relevance as ratio of query tokens found in text + hashtags.
-
-    Uses ratio overlap (intersection / query_length). Hashtags provide
-    a TikTok-specific relevance boost. Floors at 0.1.
-    """
-    q_tokens = _tokenize(query)
-
-    # Combine text and hashtags for matching
-    combined = text
-    if hashtags:
-        combined = f"{text} {' '.join(hashtags)}"
-    t_tokens = _tokenize(combined)
-
-    # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code")
-    if hashtags:
-        for tag in hashtags:
-            tag_lower = tag.lower()
-            for qt in q_tokens:
-                if qt in tag_lower and qt != tag_lower:
-                    t_tokens.add(qt)
-
-    if not q_tokens:
-        return 0.5  # Neutral fallback
-
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
 
 
 def _extract_core_subject(topic: str) -> str:

scripts/lib/youtube_yt.py

@@ -35,65 +35,12 @@ TRANSCRIPT_LIMITS = {
 # Max words to keep from each transcript
 TRANSCRIPT_MAX_WORDS = 500
 
-# Stopwords for relevance computation (common English words that dilute token overlap)
+from .relevance import (
-STOPWORDS = frozenset({
+    STOPWORDS,
-    'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
+    SYNONYMS,
-    'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
+    token_overlap_relevance as _compute_relevance,
-    'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
+    tokenize as _tokenize,
-    'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
+)
-    'all', 'just', 'get', 'has', 'have', 'was', 'will',
-})
-
-
-# Synonym groups for relevance scoring (bidirectional expansion)
-SYNONYMS = {
-    'hip': {'rap', 'hiphop'},
-    'hop': {'rap', 'hiphop'},
-    'rap': {'hip', 'hop', 'hiphop'},
-    'hiphop': {'rap', 'hip', 'hop'},
-    'js': {'javascript'},
-    'javascript': {'js'},
-    'ts': {'typescript'},
-    'typescript': {'ts'},
-    'ai': {'artificial', 'intelligence'},
-    'ml': {'machine', 'learning'},
-    'react': {'reactjs'},
-    'reactjs': {'react'},
-    'svelte': {'sveltejs'},
-    'sveltejs': {'svelte'},
-    'vue': {'vuejs'},
-    'vuejs': {'vue'},
-}
-
-
-def _tokenize(text: str) -> Set[str]:
-    """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.
-    Expands tokens with synonyms for better cross-domain matching."""
-    words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
-    tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
-    # Expand synonyms
-    expanded = set(tokens)
-    for t in tokens:
-        if t in SYNONYMS:
-            expanded.update(SYNONYMS[t])
-    return expanded
-
-
-def _compute_relevance(query: str, title: str) -> float:
-    """Compute relevance as ratio of query tokens found in title.
-
-    Uses ratio overlap (intersection / query_length) so short queries
-    score higher when fully represented in the title. Floors at 0.1.
-    """
-    q_tokens = _tokenize(query)
-    t_tokens = _tokenize(title)
-
-    if not q_tokens:
-        return 0.5  # Neutral fallback for empty/stopword-only queries
-
-    overlap = len(q_tokens & t_tokens)
-    ratio = overlap / len(q_tokens)
-    return max(0.1, min(1.0, ratio))
 
 
 def _log(msg: str):