From 96948cc7c0480dfcea48ae5a2be6b479cb84034c Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 15:14:39 -0700 Subject: [PATCH] Deduplicate relevance code across youtube/tiktok/instagram/scrapecreators_x Replace duplicated STOPWORDS, SYNONYMS, _tokenize, and _compute_relevance in four modules with imports from the shared relevance.py module. Existing tests pass unchanged since modules re-export the functions under the same names via import aliases. --- scripts/lib/instagram.py | 71 +++------------------------------ scripts/lib/scrapecreators_x.py | 46 ++++----------------- scripts/lib/tiktok.py | 71 +++------------------------------ scripts/lib/youtube_yt.py | 65 +++--------------------------- 4 files changed, 25 insertions(+), 228 deletions(-) diff --git a/scripts/lib/instagram.py b/scripts/lib/instagram.py index 0f39fa1..cb166fd 100644 --- a/scripts/lib/instagram.py +++ b/scripts/lib/instagram.py @@ -31,71 +31,12 @@ DEPTH_CONFIG = { # Max words to keep from each caption CAPTION_MAX_WORDS = 500 -# Stopwords for relevance computation (shared with tiktok.py pattern) -STOPWORDS = frozenset({ - 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on', - 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my', - 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can', - 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about', - 'all', 'just', 'get', 'has', 'have', 'was', 'will', -}) - -# Synonym groups for relevance scoring -SYNONYMS = { - 'hip': {'rap', 'hiphop'}, - 'hop': {'rap', 'hiphop'}, - 'rap': {'hip', 'hop', 'hiphop'}, - 'hiphop': {'rap', 'hip', 'hop'}, - 'js': {'javascript'}, - 'javascript': {'js'}, - 'ts': {'typescript'}, - 'typescript': {'ts'}, - 'ai': {'artificial', 'intelligence'}, - 'ml': {'machine', 'learning'}, - 'react': {'reactjs'}, - 'reactjs': {'react'}, -} - - -def _tokenize(text: str) -> Set[str]: - """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.""" - words = re.sub(r'[^\w\s]', ' ', text.lower()).split() - tokens = {w for w in words if w not in STOPWORDS and len(w) > 1} - expanded = set(tokens) - for t in tokens: - if t in SYNONYMS: - expanded.update(SYNONYMS[t]) - return expanded - - -def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> float: - """Compute relevance as ratio of query tokens found in text + hashtags. - - Uses ratio overlap (intersection / query_length). Hashtags provide - an Instagram-specific relevance boost. Floors at 0.1. - """ - q_tokens = _tokenize(query) - - # Combine text and hashtags for matching - combined = text - if hashtags: - combined = f"{text} {' '.join(hashtags)}" - t_tokens = _tokenize(combined) - - # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code") - if hashtags: - for tag in hashtags: - tag_lower = tag.lower() - for qt in q_tokens: - if qt in tag_lower and qt != tag_lower: - t_tokens.add(qt) - - if not q_tokens: - return 0.5 # Neutral fallback - - overlap = len(q_tokens & t_tokens) - ratio = overlap / len(q_tokens) - return max(0.1, min(1.0, ratio)) +from .relevance import ( + STOPWORDS, + SYNONYMS, + token_overlap_relevance as _compute_relevance, + tokenize as _tokenize, +) def _extract_core_subject(topic: str) -> str: diff --git a/scripts/lib/scrapecreators_x.py b/scripts/lib/scrapecreators_x.py index 727a6db..27bbb89 100644 --- a/scripts/lib/scrapecreators_x.py +++ b/scripts/lib/scrapecreators_x.py @@ -7,10 +7,9 @@ Requires SCRAPECREATORS_API_KEY in config. API docs: https://scrapecreators.com/docs """ -import re import sys from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional try: import requests as _requests @@ -25,43 +24,12 @@ DEPTH_CONFIG = { "deep": {"results_per_page": 40}, } -STOPWORDS = frozenset({ - 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on', - 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my', - 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can', - 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about', - 'all', 'just', 'get', 'has', 'have', 'was', 'will', -}) - -SYNONYMS = { - 'js': {'javascript'}, 'javascript': {'js'}, - 'ts': {'typescript'}, 'typescript': {'ts'}, - 'ai': {'artificial', 'intelligence'}, - 'ml': {'machine', 'learning'}, - 'react': {'reactjs'}, 'reactjs': {'react'}, -} - - -def _tokenize(text: str) -> Set[str]: - """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.""" - words = re.sub(r'[^\w\s]', ' ', text.lower()).split() - tokens = {w for w in words if w not in STOPWORDS and len(w) > 1} - expanded = set(tokens) - for t in tokens: - if t in SYNONYMS: - expanded.update(SYNONYMS[t]) - return expanded - - -def _compute_relevance(query: str, text: str) -> float: - """Compute relevance as ratio of query tokens found in text. Floors at 0.1.""" - q_tokens = _tokenize(query) - t_tokens = _tokenize(text) - if not q_tokens: - return 0.5 - overlap = len(q_tokens & t_tokens) - ratio = overlap / len(q_tokens) - return max(0.1, min(1.0, ratio)) +from .relevance import ( + STOPWORDS, + SYNONYMS, + token_overlap_relevance as _compute_relevance, + tokenize as _tokenize, +) def _extract_core_subject(topic: str) -> str: diff --git a/scripts/lib/tiktok.py b/scripts/lib/tiktok.py index 27c4ec3..56473f6 100644 --- a/scripts/lib/tiktok.py +++ b/scripts/lib/tiktok.py @@ -31,71 +31,12 @@ DEPTH_CONFIG = { # Max words to keep from each caption CAPTION_MAX_WORDS = 500 -# Stopwords for relevance computation (shared with youtube_yt.py pattern) -STOPWORDS = frozenset({ - 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on', - 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my', - 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can', - 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about', - 'all', 'just', 'get', 'has', 'have', 'was', 'will', -}) - -# Synonym groups for relevance scoring -SYNONYMS = { - 'hip': {'rap', 'hiphop'}, - 'hop': {'rap', 'hiphop'}, - 'rap': {'hip', 'hop', 'hiphop'}, - 'hiphop': {'rap', 'hip', 'hop'}, - 'js': {'javascript'}, - 'javascript': {'js'}, - 'ts': {'typescript'}, - 'typescript': {'ts'}, - 'ai': {'artificial', 'intelligence'}, - 'ml': {'machine', 'learning'}, - 'react': {'reactjs'}, - 'reactjs': {'react'}, -} - - -def _tokenize(text: str) -> Set[str]: - """Lowercase, strip punctuation, remove stopwords, drop single-char tokens.""" - words = re.sub(r'[^\w\s]', ' ', text.lower()).split() - tokens = {w for w in words if w not in STOPWORDS and len(w) > 1} - expanded = set(tokens) - for t in tokens: - if t in SYNONYMS: - expanded.update(SYNONYMS[t]) - return expanded - - -def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> float: - """Compute relevance as ratio of query tokens found in text + hashtags. - - Uses ratio overlap (intersection / query_length). Hashtags provide - a TikTok-specific relevance boost. Floors at 0.1. - """ - q_tokens = _tokenize(query) - - # Combine text and hashtags for matching - combined = text - if hashtags: - combined = f"{text} {' '.join(hashtags)}" - t_tokens = _tokenize(combined) - - # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code") - if hashtags: - for tag in hashtags: - tag_lower = tag.lower() - for qt in q_tokens: - if qt in tag_lower and qt != tag_lower: - t_tokens.add(qt) - - if not q_tokens: - return 0.5 # Neutral fallback - - overlap = len(q_tokens & t_tokens) - ratio = overlap / len(q_tokens) - return max(0.1, min(1.0, ratio)) +from .relevance import ( + STOPWORDS, + SYNONYMS, + token_overlap_relevance as _compute_relevance, + tokenize as _tokenize, +) def _extract_core_subject(topic: str) -> str: diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py index b171d41..680de91 100644 --- a/scripts/lib/youtube_yt.py +++ b/scripts/lib/youtube_yt.py @@ -35,65 +35,12 @@ TRANSCRIPT_LIMITS = { # Max words to keep from each transcript TRANSCRIPT_MAX_WORDS = 500 -# Stopwords for relevance computation (common English words that dilute token overlap) -STOPWORDS = frozenset({ - 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on', - 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my', - 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can', - 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about', - 'all', 'just', 'get', 'has', 'have', 'was', 'will', -}) - - -# Synonym groups for relevance scoring (bidirectional expansion) -SYNONYMS = { - 'hip': {'rap', 'hiphop'}, - 'hop': {'rap', 'hiphop'}, - 'rap': {'hip', 'hop', 'hiphop'}, - 'hiphop': {'rap', 'hip', 'hop'}, - 'js': {'javascript'}, - 'javascript': {'js'}, - 'ts': {'typescript'}, - 'typescript': {'ts'}, - 'ai': {'artificial', 'intelligence'}, - 'ml': {'machine', 'learning'}, - 'react': {'reactjs'}, - 'reactjs': {'react'}, - 'svelte': {'sveltejs'}, - 'sveltejs': {'svelte'}, - 'vue': {'vuejs'}, - 'vuejs': {'vue'}, -} - - -def _tokenize(text: str) -> Set[str]: - """Lowercase, strip punctuation, remove stopwords, drop single-char tokens. - Expands tokens with synonyms for better cross-domain matching.""" - words = re.sub(r'[^\w\s]', ' ', text.lower()).split() - tokens = {w for w in words if w not in STOPWORDS and len(w) > 1} - # Expand synonyms - expanded = set(tokens) - for t in tokens: - if t in SYNONYMS: - expanded.update(SYNONYMS[t]) - return expanded - - -def _compute_relevance(query: str, title: str) -> float: - """Compute relevance as ratio of query tokens found in title. - - Uses ratio overlap (intersection / query_length) so short queries - score higher when fully represented in the title. Floors at 0.1. - """ - q_tokens = _tokenize(query) - t_tokens = _tokenize(title) - - if not q_tokens: - return 0.5 # Neutral fallback for empty/stopword-only queries - - overlap = len(q_tokens & t_tokens) - ratio = overlap / len(q_tokens) - return max(0.1, min(1.0, ratio)) +from .relevance import ( + STOPWORDS, + SYNONYMS, + token_overlap_relevance as _compute_relevance, + tokenize as _tokenize, +) def _log(msg: str):