Jeffrey Sperling
d667586597
Consolidate duplicated _extract_core_subject() (7 copies across bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, scrapecreators_x) into query.extract_core_subject() with parameterized noise set, max_words, and suffix stripping. Consolidate duplicated _tokenize/_compute_relevance/STOPWORDS/SYNONYMS (4 copies across youtube_yt, tiktok, instagram, scrapecreators_x) into relevance.token_overlap_relevance() with hashtag-aware matching. Integration into per-module imports follows in next commits.
99 lines
3.4 KiB
Python
99 lines
3.4 KiB
Python
"""Shared query utilities for /last30days search modules.
|
|
|
|
Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
|
|
youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
|
|
parameterized function. Each platform calls with its own overrides.
|
|
"""
|
|
|
|
from typing import FrozenSet, List, Optional, Set
|
|
|
|
# Common multi-word prefixes stripped from all queries (identical across modules)
|
|
PREFIXES = [
|
|
'what are the best', 'what is the best', 'what are the latest',
|
|
'what are people saying about', 'what do people think about',
|
|
'how do i use', 'how to use', 'how to',
|
|
'what are', 'what is', 'tips for', 'best practices for',
|
|
]
|
|
|
|
# Multi-word suffixes (used by bird_x)
|
|
SUFFIXES = [
|
|
'best practices', 'use cases', 'prompt techniques',
|
|
'prompting techniques', 'prompting tips',
|
|
]
|
|
|
|
# Base noise words shared across most modules
|
|
NOISE_WORDS = frozenset({
|
|
# Articles/prepositions/conjunctions
|
|
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
|
|
'of', 'in', 'on', 'for', 'with', 'about', 'to',
|
|
# Question words
|
|
'how', 'what', 'which', 'who', 'why', 'when', 'where',
|
|
'does', 'should', 'could', 'would',
|
|
# Research/meta descriptors
|
|
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
|
'latest', 'new', 'news', 'update', 'updates',
|
|
'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
|
|
'practices', 'features', 'guide', 'tutorial',
|
|
'recommendations', 'advice', 'review', 'reviews',
|
|
'usecases', 'examples', 'comparison', 'versus', 'vs',
|
|
'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
|
|
# Prompting meta words
|
|
'prompt', 'prompts', 'prompting', 'techniques', 'tips',
|
|
'tricks', 'methods', 'strategies', 'approaches',
|
|
# Action words
|
|
'using', 'uses', 'use',
|
|
# Misc filler
|
|
'people', 'saying', 'think', 'said', 'lately',
|
|
})
|
|
|
|
|
|
def extract_core_subject(
|
|
topic: str,
|
|
*,
|
|
noise: Optional[FrozenSet[str]] = None,
|
|
max_words: Optional[int] = None,
|
|
strip_suffixes: bool = False,
|
|
) -> str:
|
|
"""Extract core subject from a verbose search query.
|
|
|
|
Strips common question/meta prefixes and noise words to produce a
|
|
compact search-friendly query. Platforms customize via parameters.
|
|
|
|
Args:
|
|
topic: Raw user query
|
|
noise: Override noise word set (default: NOISE_WORDS)
|
|
max_words: Cap result to N words (default: no cap)
|
|
strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this)
|
|
|
|
Returns:
|
|
Cleaned query string
|
|
"""
|
|
text = topic.lower().strip()
|
|
if not text:
|
|
return text
|
|
|
|
# Phase 1: Strip multi-word prefixes (longest first, stop after first match)
|
|
for p in PREFIXES:
|
|
if text.startswith(p + ' '):
|
|
text = text[len(p):].strip()
|
|
break
|
|
|
|
# Phase 2: Strip multi-word suffixes (opt-in)
|
|
if strip_suffixes:
|
|
for s in SUFFIXES:
|
|
if text.endswith(' ' + s):
|
|
text = text[:-len(s)].strip()
|
|
break
|
|
|
|
# Phase 3: Filter individual noise words
|
|
noise_set = noise if noise is not None else NOISE_WORDS
|
|
words = text.split()
|
|
filtered = [w for w in words if w not in noise_set]
|
|
|
|
# Apply word cap if requested
|
|
if max_words is not None and filtered:
|
|
filtered = filtered[:max_words]
|
|
|
|
result = ' '.join(filtered) if filtered else text
|
|
return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
|