Add shared query.py and relevance.py modules
Consolidate duplicated _extract_core_subject() (7 copies across bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into query.extract_core_subject() with parameterized noise set, max_words, and suffix stripping. Consolidate duplicated _tokenize/_compute_relevance/STOPWORDS/SYNONYMS (4 copies across youtube_yt, tiktok, instagram, scrapecreators_x) into relevance.token_overlap_relevance() with hashtag-aware matching. Integration into per-module imports follows in next commits.
This commit is contained in:
Jeffrey Sperling
@@ -0,0 +1,98 @@
|
||||
"""Shared query utilities for /last30days search modules.
|
||||
|
||||
Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
|
||||
youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
|
||||
parameterized function. Each platform calls with its own overrides.
|
||||
"""
|
||||
|
||||
from typing import FrozenSet, List, Optional, Set
|
||||
|
||||
# Common multi-word prefixes stripped from all queries (identical across modules)
|
||||
PREFIXES = [
|
||||
'what are the best', 'what is the best', 'what are the latest',
|
||||
'what are people saying about', 'what do people think about',
|
||||
'how do i use', 'how to use', 'how to',
|
||||
'what are', 'what is', 'tips for', 'best practices for',
|
||||
]
|
||||
|
||||
# Multi-word suffixes (used by bird_x)
|
||||
SUFFIXES = [
|
||||
'best practices', 'use cases', 'prompt techniques',
|
||||
'prompting techniques', 'prompting tips',
|
||||
]
|
||||
|
||||
# Base noise words shared across most modules
|
||||
NOISE_WORDS = frozenset({
|
||||
# Articles/prepositions/conjunctions
|
||||
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
|
||||
'of', 'in', 'on', 'for', 'with', 'about', 'to',
|
||||
# Question words
|
||||
'how', 'what', 'which', 'who', 'why', 'when', 'where',
|
||||
'does', 'should', 'could', 'would',
|
||||
# Research/meta descriptors
|
||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
||||
'latest', 'new', 'news', 'update', 'updates',
|
||||
'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
|
||||
'practices', 'features', 'guide', 'tutorial',
|
||||
'recommendations', 'advice', 'review', 'reviews',
|
||||
'usecases', 'examples', 'comparison', 'versus', 'vs',
|
||||
'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
|
||||
# Prompting meta words
|
||||
'prompt', 'prompts', 'prompting', 'techniques', 'tips',
|
||||
'tricks', 'methods', 'strategies', 'approaches',
|
||||
# Action words
|
||||
'using', 'uses', 'use',
|
||||
# Misc filler
|
||||
'people', 'saying', 'think', 'said', 'lately',
|
||||
})
|
||||
|
||||
|
||||
def extract_core_subject(
|
||||
topic: str,
|
||||
*,
|
||||
noise: Optional[FrozenSet[str]] = None,
|
||||
max_words: Optional[int] = None,
|
||||
strip_suffixes: bool = False,
|
||||
) -> str:
|
||||
"""Extract core subject from a verbose search query.
|
||||
|
||||
Strips common question/meta prefixes and noise words to produce a
|
||||
compact search-friendly query. Platforms customize via parameters.
|
||||
|
||||
Args:
|
||||
topic: Raw user query
|
||||
noise: Override noise word set (default: NOISE_WORDS)
|
||||
max_words: Cap result to N words (default: no cap)
|
||||
strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this)
|
||||
|
||||
Returns:
|
||||
Cleaned query string
|
||||
"""
|
||||
text = topic.lower().strip()
|
||||
if not text:
|
||||
return text
|
||||
|
||||
# Phase 1: Strip multi-word prefixes (longest first, stop after first match)
|
||||
for p in PREFIXES:
|
||||
if text.startswith(p + ' '):
|
||||
text = text[len(p):].strip()
|
||||
break
|
||||
|
||||
# Phase 2: Strip multi-word suffixes (opt-in)
|
||||
if strip_suffixes:
|
||||
for s in SUFFIXES:
|
||||
if text.endswith(' ' + s):
|
||||
text = text[:-len(s)].strip()
|
||||
break
|
||||
|
||||
# Phase 3: Filter individual noise words
|
||||
noise_set = noise if noise is not None else NOISE_WORDS
|
||||
words = text.split()
|
||||
filtered = [w for w in words if w not in noise_set]
|
||||
|
||||
# Apply word cap if requested
|
||||
if max_words is not None and filtered:
|
||||
filtered = filtered[:max_words]
|
||||
|
||||
result = ' '.join(filtered) if filtered else text
|
||||
return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Shared relevance scoring for /last30days search modules.
|
||||
|
||||
Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS
|
||||
from youtube_yt, tiktok, instagram, and scrapecreators_x into one module.
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import List, Optional, Set
|
||||
|
||||
# Stopwords for relevance computation (common English words that dilute token overlap)
|
||||
STOPWORDS = frozenset({
|
||||
'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
|
||||
'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
|
||||
'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
|
||||
'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
|
||||
'all', 'just', 'get', 'has', 'have', 'was', 'will',
|
||||
})
|
||||
|
||||
# Synonym groups for relevance scoring (bidirectional expansion)
|
||||
# Superset of all platform-specific synonym dicts
|
||||
SYNONYMS = {
|
||||
'hip': {'rap', 'hiphop'},
|
||||
'hop': {'rap', 'hiphop'},
|
||||
'rap': {'hip', 'hop', 'hiphop'},
|
||||
'hiphop': {'rap', 'hip', 'hop'},
|
||||
'js': {'javascript'},
|
||||
'javascript': {'js'},
|
||||
'ts': {'typescript'},
|
||||
'typescript': {'ts'},
|
||||
'ai': {'artificial', 'intelligence'},
|
||||
'ml': {'machine', 'learning'},
|
||||
'react': {'reactjs'},
|
||||
'reactjs': {'react'},
|
||||
'svelte': {'sveltejs'},
|
||||
'sveltejs': {'svelte'},
|
||||
'vue': {'vuejs'},
|
||||
'vuejs': {'vue'},
|
||||
}
|
||||
|
||||
|
||||
def tokenize(text: str) -> Set[str]:
|
||||
"""Lowercase, strip punctuation, remove stopwords, drop single-char tokens.
|
||||
|
||||
Expands tokens with synonyms for better cross-domain matching.
|
||||
"""
|
||||
words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
|
||||
tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
|
||||
expanded = set(tokens)
|
||||
for t in tokens:
|
||||
if t in SYNONYMS:
|
||||
expanded.update(SYNONYMS[t])
|
||||
return expanded
|
||||
|
||||
|
||||
def token_overlap_relevance(
|
||||
query: str,
|
||||
text: str,
|
||||
hashtags: Optional[List[str]] = None,
|
||||
) -> float:
|
||||
"""Compute relevance as ratio of query tokens found in text.
|
||||
|
||||
Uses ratio overlap (intersection / query_length) so short queries
|
||||
score higher when fully represented in the text. Floors at 0.1.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
text: Content text to match against
|
||||
hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated
|
||||
hashtags are split to match query tokens (e.g. "claudecode" matches "claude").
|
||||
|
||||
Returns:
|
||||
Float between 0.1 and 1.0 (0.5 for empty queries)
|
||||
"""
|
||||
q_tokens = tokenize(query)
|
||||
|
||||
# Combine text and hashtags for matching
|
||||
combined = text
|
||||
if hashtags:
|
||||
combined = f"{text} {' '.join(hashtags)}"
|
||||
t_tokens = tokenize(combined)
|
||||
|
||||
# Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code")
|
||||
if hashtags:
|
||||
for tag in hashtags:
|
||||
tag_lower = tag.lower()
|
||||
for qt in q_tokens:
|
||||
if qt in tag_lower and qt != tag_lower:
|
||||
t_tokens.add(qt)
|
||||
|
||||
if not q_tokens:
|
||||
return 0.5 # Neutral fallback for empty/stopword-only queries
|
||||
|
||||
overlap = len(q_tokens & t_tokens)
|
||||
ratio = overlap / len(q_tokens)
|
||||
return max(0.1, min(1.0, ratio))
|
||||
Reference in New Issue
Block a user