sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate shared query.py into per-source modules

Browse Source 
Replace duplicated _extract_core_subject() in bird_x, reddit, youtube_yt,
tiktok, instagram, bluesky, and scrapecreators_x with thin wrappers that
delegate to query.extract_core_subject() with platform-specific noise sets.

Each module preserves its current behavior exactly:
- bird_x: max_words=5, strip_suffixes=True, full noise set
- youtube_yt: keeps tips/tricks/tutorial/guide/review (content types)
- reddit: preserves original smaller noise set
- tiktok/instagram: same small noise set
- bluesky/scrapecreators_x: minimal noise set

Existing tests pass without modification since _extract_core_subject()
still exists as a callable on each module.
This commit is contained in:
Jeffrey Sperling
2026-03-11 15:12:46 -07:00
parent fa42a5d031
commit dc88c215be
7 changed files with 32 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lib/bird_x.py
+3 -49
View File
@@ -54,56 +54,10 @@ def _extract_core_subject(topic: str) -> str:
X search is literal keyword AND matching — all words must appear.
Aggressively strip question/meta/research words to keep only the
core product/concept name (2-3 words max).
core product/concept name (max 5 words).
"""
text = topic.lower().strip()
# Phase 1: Strip multi-word prefixes (longest first)
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
break
# Phase 2: Strip multi-word suffixes
suffixes = [
'best practices', 'use cases', 'prompt techniques',
'prompting techniques', 'prompting tips',
]
for s in suffixes:
if text.endswith(' ' + s):
text = text[:-len(s)].strip()
break
# Phase 3: Filter individual noise words
_noise = {
# Question/filler words
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
'of', 'in', 'on', 'for', 'with', 'about', 'to',
'people', 'saying', 'think', 'said', 'lately',
# Research/meta descriptors
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
'practices', 'features', 'guide', 'tutorial',
'recommendations', 'advice', 'review', 'reviews',
'usecases', 'examples', 'comparison', 'versus', 'vs',
'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
# Prompting meta words
'prompt', 'prompts', 'prompting', 'techniques', 'tips',
'tricks', 'methods', 'strategies', 'approaches',
# Action words
'using', 'uses', 'use',
}
words = text.split()
result = [w for w in words if w not in _noise]
return ' '.join(result[:3]) or topic.lower().strip() # Max 3 words
from .query import NOISE_WORDS, extract_core_subject
return extract_core_subject(topic, max_words=5, strip_suffixes=True)
def is_bird_installed() -> bool:
scripts/lib/bluesky.py
+4 -16
View File
@@ -67,26 +67,14 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Bluesky search."""
text = topic.lower().strip()
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
noise = {
from .query import extract_core_subject
_BSKY_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice',
}
words = text.split()
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
})
return extract_core_subject(topic, noise=_BSKY_NOISE)
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
scripts/lib/instagram.py
+5 -25
View File
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Instagram search.
Strips meta/research words to keep only the core product/concept name.
"""
text = topic.lower().strip()
# Strip multi-word prefixes
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
# Strip individual noise words
noise = {
"""Extract core subject from verbose query for Instagram search."""
from .query import extract_core_subject
_INSTAGRAM_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
'recommendations', 'advice',
'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches',
}
words = text.split()
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
})
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
def _log(msg: str):
scripts/lib/reddit.py
+4 -19
View File
@@ -48,7 +48,9 @@ DEPTH_CONFIG = {
},
}
# Stopwords for query extraction
from .query import extract_core_subject as _query_extract
# Reddit-specific noise words (preserves original smaller set)
NOISE_WORDS = frozenset({
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
@@ -82,24 +84,7 @@ def _extract_core_subject(topic: str) -> str:
Strips meta/research words to keep only the core product/concept name.
"""
text = topic.lower().strip()
# Strip multi-word prefixes
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
words = text.split()
filtered = [w for w in words if w not in NOISE_WORDS]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
return _query_extract(topic, noise=NOISE_WORDS)
def expand_reddit_queries(topic: str, depth: str) -> List[str]:
scripts/lib/scrapecreators_x.py
+4 -16
View File
@@ -66,26 +66,14 @@ def _compute_relevance(query: str, text: str) -> float:
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Twitter search."""
text = topic.lower().strip()
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
noise = {
from .query import extract_core_subject
_SC_X_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice',
}
words = text.split()
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
})
return extract_core_subject(topic, noise=_SC_X_NOISE)
def _log(msg: str):
scripts/lib/tiktok.py
+5 -25
View File
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for TikTok search.
Strips meta/research words to keep only the core product/concept name.
"""
text = topic.lower().strip()
# Strip multi-word prefixes
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
# Strip individual noise words
noise = {
"""Extract core subject from verbose query for TikTok search."""
from .query import extract_core_subject
_TIKTOK_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
'recommendations', 'advice',
'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches',
}
words = text.split()
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
})
return extract_core_subject(topic, noise=_TIKTOK_NOISE)
def _log(msg: str):
scripts/lib/youtube_yt.py
+7 -25
View File
@@ -110,26 +110,12 @@ def is_ytdlp_installed() -> bool:
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for YouTube search.
Strips meta/research words to keep only the core product/concept name,
similar to bird_x.py's approach.
NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
are intentionally KEPT — they're YouTube content types that improve search.
"""
text = topic.lower().strip()
# Strip multi-word prefixes
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
# Strip individual noise words
# NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
# are intentionally KEPT — they're YouTube content types that improve search
noise = {
from .query import extract_core_subject
# YouTube-specific noise set: smaller than default, keeps content-type words
_YT_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
@@ -137,12 +123,8 @@ def _extract_core_subject(topic: str) -> str:
'recommendations', 'advice',
'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches',
}
words = text.split()
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
})
return extract_core_subject(topic, noise=_YT_NOISE)
def search_youtube(