Integrate shared query.py into per-source modules

Replace duplicated _extract_core_subject() in bird_x, reddit, youtube_yt,
tiktok, instagram, bluesky, and scrapecreators_x with thin wrappers that
delegate to query.extract_core_subject() with platform-specific noise sets.

Each module preserves its current behavior exactly:
- bird_x: max_words=5, strip_suffixes=True, full noise set
- youtube_yt: keeps tips/tricks/tutorial/guide/review (content types)
- reddit: preserves original smaller noise set
- tiktok/instagram: same small noise set
- bluesky/scrapecreators_x: minimal noise set

Existing tests pass without modification since _extract_core_subject()
still exists as a callable on each module.
This commit is contained in:
Jeffrey Sperling
2026-03-11 15:12:46 -07:00
parent fa42a5d031
commit dc88c215be
7 changed files with 32 additions and 175 deletions
+3 -49
View File
@@ -54,56 +54,10 @@ def _extract_core_subject(topic: str) -> str:
X search is literal keyword AND matching — all words must appear. X search is literal keyword AND matching — all words must appear.
Aggressively strip question/meta/research words to keep only the Aggressively strip question/meta/research words to keep only the
core product/concept name (2-3 words max). core product/concept name (max 5 words).
""" """
text = topic.lower().strip() from .query import NOISE_WORDS, extract_core_subject
return extract_core_subject(topic, max_words=5, strip_suffixes=True)
# Phase 1: Strip multi-word prefixes (longest first)
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
break
# Phase 2: Strip multi-word suffixes
suffixes = [
'best practices', 'use cases', 'prompt techniques',
'prompting techniques', 'prompting tips',
]
for s in suffixes:
if text.endswith(' ' + s):
text = text[:-len(s)].strip()
break
# Phase 3: Filter individual noise words
_noise = {
# Question/filler words
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
'of', 'in', 'on', 'for', 'with', 'about', 'to',
'people', 'saying', 'think', 'said', 'lately',
# Research/meta descriptors
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
'practices', 'features', 'guide', 'tutorial',
'recommendations', 'advice', 'review', 'reviews',
'usecases', 'examples', 'comparison', 'versus', 'vs',
'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
# Prompting meta words
'prompt', 'prompts', 'prompting', 'techniques', 'tips',
'tricks', 'methods', 'strategies', 'approaches',
# Action words
'using', 'uses', 'use',
}
words = text.split()
result = [w for w in words if w not in _noise]
return ' '.join(result[:3]) or topic.lower().strip() # Max 3 words
def is_bird_installed() -> bool: def is_bird_installed() -> bool:
+4 -16
View File
@@ -67,26 +67,14 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
def _extract_core_subject(topic: str) -> str: def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Bluesky search.""" """Extract core subject from verbose query for Bluesky search."""
text = topic.lower().strip() from .query import extract_core_subject
prefixes = [ _BSKY_NOISE = frozenset({
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
noise = {
'best', 'top', 'good', 'great', 'awesome', 'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates', 'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral', 'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice', 'practices', 'features', 'recommendations', 'advice',
} })
words = text.split() return extract_core_subject(topic, noise=_BSKY_NOISE)
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
def _parse_date(item: Dict[str, Any]) -> Optional[str]: def _parse_date(item: Dict[str, Any]) -> Optional[str]:
+5 -25
View File
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
def _extract_core_subject(topic: str) -> str: def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Instagram search. """Extract core subject from verbose query for Instagram search."""
from .query import extract_core_subject
Strips meta/research words to keep only the core product/concept name. _INSTAGRAM_NOISE = frozenset({
"""
text = topic.lower().strip()
# Strip multi-word prefixes
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
# Strip individual noise words
noise = {
'best', 'top', 'good', 'great', 'awesome', 'killer', 'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates', 'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral', 'trending', 'hottest', 'popular', 'viral',
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
'recommendations', 'advice', 'recommendations', 'advice',
'prompt', 'prompts', 'prompting', 'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches', 'methods', 'strategies', 'approaches',
} })
words = text.split() return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
def _log(msg: str): def _log(msg: str):
+4 -19
View File
@@ -48,7 +48,9 @@ DEPTH_CONFIG = {
}, },
} }
# Stopwords for query extraction from .query import extract_core_subject as _query_extract
# Reddit-specific noise words (preserves original smaller set)
NOISE_WORDS = frozenset({ NOISE_WORDS = frozenset({
'best', 'top', 'good', 'great', 'awesome', 'killer', 'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates', 'latest', 'new', 'news', 'update', 'updates',
@@ -82,24 +84,7 @@ def _extract_core_subject(topic: str) -> str:
Strips meta/research words to keep only the core product/concept name. Strips meta/research words to keep only the core product/concept name.
""" """
text = topic.lower().strip() return _query_extract(topic, noise=NOISE_WORDS)
# Strip multi-word prefixes
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
words = text.split()
filtered = [w for w in words if w not in NOISE_WORDS]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
def expand_reddit_queries(topic: str, depth: str) -> List[str]: def expand_reddit_queries(topic: str, depth: str) -> List[str]:
+4 -16
View File
@@ -66,26 +66,14 @@ def _compute_relevance(query: str, text: str) -> float:
def _extract_core_subject(topic: str) -> str: def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Twitter search.""" """Extract core subject from verbose query for Twitter search."""
text = topic.lower().strip() from .query import extract_core_subject
prefixes = [ _SC_X_NOISE = frozenset({
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
noise = {
'best', 'top', 'good', 'great', 'awesome', 'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates', 'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral', 'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice', 'practices', 'features', 'recommendations', 'advice',
} })
words = text.split() return extract_core_subject(topic, noise=_SC_X_NOISE)
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
def _log(msg: str): def _log(msg: str):
+5 -25
View File
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
def _extract_core_subject(topic: str) -> str: def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for TikTok search. """Extract core subject from verbose query for TikTok search."""
from .query import extract_core_subject
Strips meta/research words to keep only the core product/concept name. _TIKTOK_NOISE = frozenset({
"""
text = topic.lower().strip()
# Strip multi-word prefixes
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
# Strip individual noise words
noise = {
'best', 'top', 'good', 'great', 'awesome', 'killer', 'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates', 'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral', 'trending', 'hottest', 'popular', 'viral',
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
'recommendations', 'advice', 'recommendations', 'advice',
'prompt', 'prompts', 'prompting', 'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches', 'methods', 'strategies', 'approaches',
} })
words = text.split() return extract_core_subject(topic, noise=_TIKTOK_NOISE)
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
def _log(msg: str): def _log(msg: str):
+7 -25
View File
@@ -110,26 +110,12 @@ def is_ytdlp_installed() -> bool:
def _extract_core_subject(topic: str) -> str: def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for YouTube search. """Extract core subject from verbose query for YouTube search.
Strips meta/research words to keep only the core product/concept name, NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
similar to bird_x.py's approach. are intentionally KEPT — they're YouTube content types that improve search.
""" """
text = topic.lower().strip() from .query import extract_core_subject
# YouTube-specific noise set: smaller than default, keeps content-type words
# Strip multi-word prefixes _YT_NOISE = frozenset({
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
# Strip individual noise words
# NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
# are intentionally KEPT — they're YouTube content types that improve search
noise = {
'best', 'top', 'good', 'great', 'awesome', 'killer', 'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates', 'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral', 'trending', 'hottest', 'popular', 'viral',
@@ -137,12 +123,8 @@ def _extract_core_subject(topic: str) -> str:
'recommendations', 'advice', 'recommendations', 'advice',
'prompt', 'prompts', 'prompting', 'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches', 'methods', 'strategies', 'approaches',
} })
words = text.split() return extract_core_subject(topic, noise=_YT_NOISE)
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
def search_youtube( def search_youtube(