Integrate shared query.py into per-source modules
Replace duplicated _extract_core_subject() in bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x with thin wrappers that delegate to query.extract_core_subject() with platform-specific noise sets. Each module preserves its current behavior exactly: - bird_x: max_words=5, strip_suffixes=True, full noise set - youtube_yt: keeps tips/tricks/tutorial/guide/review (content types) - reddit: preserves original smaller noise set - tiktok/instagram: same small noise set - bluesky/scrapecreators_x: minimal noise set Existing tests pass without modification since _extract_core_subject() still exists as a callable on each module.
This commit is contained in:
+3
-49
@@ -54,56 +54,10 @@ def _extract_core_subject(topic: str) -> str:
|
|||||||
|
|
||||||
X search is literal keyword AND matching — all words must appear.
|
X search is literal keyword AND matching — all words must appear.
|
||||||
Aggressively strip question/meta/research words to keep only the
|
Aggressively strip question/meta/research words to keep only the
|
||||||
core product/concept name (2-3 words max).
|
core product/concept name (max 5 words).
|
||||||
"""
|
"""
|
||||||
text = topic.lower().strip()
|
from .query import NOISE_WORDS, extract_core_subject
|
||||||
|
return extract_core_subject(topic, max_words=5, strip_suffixes=True)
|
||||||
# Phase 1: Strip multi-word prefixes (longest first)
|
|
||||||
prefixes = [
|
|
||||||
'what are the best', 'what is the best', 'what are the latest',
|
|
||||||
'what are people saying about', 'what do people think about',
|
|
||||||
'how do i use', 'how to use', 'how to',
|
|
||||||
'what are', 'what is', 'tips for', 'best practices for',
|
|
||||||
]
|
|
||||||
for p in prefixes:
|
|
||||||
if text.startswith(p + ' '):
|
|
||||||
text = text[len(p):].strip()
|
|
||||||
break
|
|
||||||
|
|
||||||
# Phase 2: Strip multi-word suffixes
|
|
||||||
suffixes = [
|
|
||||||
'best practices', 'use cases', 'prompt techniques',
|
|
||||||
'prompting techniques', 'prompting tips',
|
|
||||||
]
|
|
||||||
for s in suffixes:
|
|
||||||
if text.endswith(' ' + s):
|
|
||||||
text = text[:-len(s)].strip()
|
|
||||||
break
|
|
||||||
|
|
||||||
# Phase 3: Filter individual noise words
|
|
||||||
_noise = {
|
|
||||||
# Question/filler words
|
|
||||||
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
|
|
||||||
'of', 'in', 'on', 'for', 'with', 'about', 'to',
|
|
||||||
'people', 'saying', 'think', 'said', 'lately',
|
|
||||||
# Research/meta descriptors
|
|
||||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
|
||||||
'latest', 'new', 'news', 'update', 'updates',
|
|
||||||
'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
|
|
||||||
'practices', 'features', 'guide', 'tutorial',
|
|
||||||
'recommendations', 'advice', 'review', 'reviews',
|
|
||||||
'usecases', 'examples', 'comparison', 'versus', 'vs',
|
|
||||||
'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
|
|
||||||
# Prompting meta words
|
|
||||||
'prompt', 'prompts', 'prompting', 'techniques', 'tips',
|
|
||||||
'tricks', 'methods', 'strategies', 'approaches',
|
|
||||||
# Action words
|
|
||||||
'using', 'uses', 'use',
|
|
||||||
}
|
|
||||||
words = text.split()
|
|
||||||
result = [w for w in words if w not in _noise]
|
|
||||||
|
|
||||||
return ' '.join(result[:3]) or topic.lower().strip() # Max 3 words
|
|
||||||
|
|
||||||
|
|
||||||
def is_bird_installed() -> bool:
|
def is_bird_installed() -> bool:
|
||||||
|
|||||||
+4
-16
@@ -67,26 +67,14 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
|
|||||||
|
|
||||||
def _extract_core_subject(topic: str) -> str:
|
def _extract_core_subject(topic: str) -> str:
|
||||||
"""Extract core subject from verbose query for Bluesky search."""
|
"""Extract core subject from verbose query for Bluesky search."""
|
||||||
text = topic.lower().strip()
|
from .query import extract_core_subject
|
||||||
prefixes = [
|
_BSKY_NOISE = frozenset({
|
||||||
'what are the best', 'what is the best', 'what are the latest',
|
|
||||||
'what are people saying about', 'what do people think about',
|
|
||||||
'how do i use', 'how to use', 'how to',
|
|
||||||
'what are', 'what is', 'tips for', 'best practices for',
|
|
||||||
]
|
|
||||||
for p in prefixes:
|
|
||||||
if text.startswith(p + ' '):
|
|
||||||
text = text[len(p):].strip()
|
|
||||||
noise = {
|
|
||||||
'best', 'top', 'good', 'great', 'awesome',
|
'best', 'top', 'good', 'great', 'awesome',
|
||||||
'latest', 'new', 'news', 'update', 'updates',
|
'latest', 'new', 'news', 'update', 'updates',
|
||||||
'trending', 'hottest', 'popular', 'viral',
|
'trending', 'hottest', 'popular', 'viral',
|
||||||
'practices', 'features', 'recommendations', 'advice',
|
'practices', 'features', 'recommendations', 'advice',
|
||||||
}
|
})
|
||||||
words = text.split()
|
return extract_core_subject(topic, noise=_BSKY_NOISE)
|
||||||
filtered = [w for w in words if w not in noise]
|
|
||||||
result = ' '.join(filtered) if filtered else text
|
|
||||||
return result.rstrip('?!.')
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
||||||
|
|||||||
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
|
|||||||
|
|
||||||
|
|
||||||
def _extract_core_subject(topic: str) -> str:
|
def _extract_core_subject(topic: str) -> str:
|
||||||
"""Extract core subject from verbose query for Instagram search.
|
"""Extract core subject from verbose query for Instagram search."""
|
||||||
|
from .query import extract_core_subject
|
||||||
Strips meta/research words to keep only the core product/concept name.
|
_INSTAGRAM_NOISE = frozenset({
|
||||||
"""
|
|
||||||
text = topic.lower().strip()
|
|
||||||
|
|
||||||
# Strip multi-word prefixes
|
|
||||||
prefixes = [
|
|
||||||
'what are the best', 'what is the best', 'what are the latest',
|
|
||||||
'what are people saying about', 'what do people think about',
|
|
||||||
'how do i use', 'how to use', 'how to',
|
|
||||||
'what are', 'what is', 'tips for', 'best practices for',
|
|
||||||
]
|
|
||||||
for p in prefixes:
|
|
||||||
if text.startswith(p + ' '):
|
|
||||||
text = text[len(p):].strip()
|
|
||||||
|
|
||||||
# Strip individual noise words
|
|
||||||
noise = {
|
|
||||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
||||||
'latest', 'new', 'news', 'update', 'updates',
|
'latest', 'new', 'news', 'update', 'updates',
|
||||||
'trending', 'hottest', 'popular', 'viral',
|
'trending', 'hottest', 'popular', 'viral',
|
||||||
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
|
|||||||
'recommendations', 'advice',
|
'recommendations', 'advice',
|
||||||
'prompt', 'prompts', 'prompting',
|
'prompt', 'prompts', 'prompting',
|
||||||
'methods', 'strategies', 'approaches',
|
'methods', 'strategies', 'approaches',
|
||||||
}
|
})
|
||||||
words = text.split()
|
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
||||||
filtered = [w for w in words if w not in noise]
|
|
||||||
|
|
||||||
result = ' '.join(filtered) if filtered else text
|
|
||||||
return result.rstrip('?!.')
|
|
||||||
|
|
||||||
|
|
||||||
def _log(msg: str):
|
def _log(msg: str):
|
||||||
|
|||||||
+4
-19
@@ -48,7 +48,9 @@ DEPTH_CONFIG = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Stopwords for query extraction
|
from .query import extract_core_subject as _query_extract
|
||||||
|
|
||||||
|
# Reddit-specific noise words (preserves original smaller set)
|
||||||
NOISE_WORDS = frozenset({
|
NOISE_WORDS = frozenset({
|
||||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
||||||
'latest', 'new', 'news', 'update', 'updates',
|
'latest', 'new', 'news', 'update', 'updates',
|
||||||
@@ -82,24 +84,7 @@ def _extract_core_subject(topic: str) -> str:
|
|||||||
|
|
||||||
Strips meta/research words to keep only the core product/concept name.
|
Strips meta/research words to keep only the core product/concept name.
|
||||||
"""
|
"""
|
||||||
text = topic.lower().strip()
|
return _query_extract(topic, noise=NOISE_WORDS)
|
||||||
|
|
||||||
# Strip multi-word prefixes
|
|
||||||
prefixes = [
|
|
||||||
'what are the best', 'what is the best', 'what are the latest',
|
|
||||||
'what are people saying about', 'what do people think about',
|
|
||||||
'how do i use', 'how to use', 'how to',
|
|
||||||
'what are', 'what is', 'tips for', 'best practices for',
|
|
||||||
]
|
|
||||||
for p in prefixes:
|
|
||||||
if text.startswith(p + ' '):
|
|
||||||
text = text[len(p):].strip()
|
|
||||||
|
|
||||||
words = text.split()
|
|
||||||
filtered = [w for w in words if w not in NOISE_WORDS]
|
|
||||||
|
|
||||||
result = ' '.join(filtered) if filtered else text
|
|
||||||
return result.rstrip('?!.')
|
|
||||||
|
|
||||||
|
|
||||||
def expand_reddit_queries(topic: str, depth: str) -> List[str]:
|
def expand_reddit_queries(topic: str, depth: str) -> List[str]:
|
||||||
|
|||||||
@@ -66,26 +66,14 @@ def _compute_relevance(query: str, text: str) -> float:
|
|||||||
|
|
||||||
def _extract_core_subject(topic: str) -> str:
|
def _extract_core_subject(topic: str) -> str:
|
||||||
"""Extract core subject from verbose query for Twitter search."""
|
"""Extract core subject from verbose query for Twitter search."""
|
||||||
text = topic.lower().strip()
|
from .query import extract_core_subject
|
||||||
prefixes = [
|
_SC_X_NOISE = frozenset({
|
||||||
'what are the best', 'what is the best', 'what are the latest',
|
|
||||||
'what are people saying about', 'what do people think about',
|
|
||||||
'how do i use', 'how to use', 'how to',
|
|
||||||
'what are', 'what is', 'tips for', 'best practices for',
|
|
||||||
]
|
|
||||||
for p in prefixes:
|
|
||||||
if text.startswith(p + ' '):
|
|
||||||
text = text[len(p):].strip()
|
|
||||||
noise = {
|
|
||||||
'best', 'top', 'good', 'great', 'awesome',
|
'best', 'top', 'good', 'great', 'awesome',
|
||||||
'latest', 'new', 'news', 'update', 'updates',
|
'latest', 'new', 'news', 'update', 'updates',
|
||||||
'trending', 'hottest', 'popular', 'viral',
|
'trending', 'hottest', 'popular', 'viral',
|
||||||
'practices', 'features', 'recommendations', 'advice',
|
'practices', 'features', 'recommendations', 'advice',
|
||||||
}
|
})
|
||||||
words = text.split()
|
return extract_core_subject(topic, noise=_SC_X_NOISE)
|
||||||
filtered = [w for w in words if w not in noise]
|
|
||||||
result = ' '.join(filtered) if filtered else text
|
|
||||||
return result.rstrip('?!.')
|
|
||||||
|
|
||||||
|
|
||||||
def _log(msg: str):
|
def _log(msg: str):
|
||||||
|
|||||||
+5
-25
@@ -99,25 +99,9 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
|
|||||||
|
|
||||||
|
|
||||||
def _extract_core_subject(topic: str) -> str:
|
def _extract_core_subject(topic: str) -> str:
|
||||||
"""Extract core subject from verbose query for TikTok search.
|
"""Extract core subject from verbose query for TikTok search."""
|
||||||
|
from .query import extract_core_subject
|
||||||
Strips meta/research words to keep only the core product/concept name.
|
_TIKTOK_NOISE = frozenset({
|
||||||
"""
|
|
||||||
text = topic.lower().strip()
|
|
||||||
|
|
||||||
# Strip multi-word prefixes
|
|
||||||
prefixes = [
|
|
||||||
'what are the best', 'what is the best', 'what are the latest',
|
|
||||||
'what are people saying about', 'what do people think about',
|
|
||||||
'how do i use', 'how to use', 'how to',
|
|
||||||
'what are', 'what is', 'tips for', 'best practices for',
|
|
||||||
]
|
|
||||||
for p in prefixes:
|
|
||||||
if text.startswith(p + ' '):
|
|
||||||
text = text[len(p):].strip()
|
|
||||||
|
|
||||||
# Strip individual noise words
|
|
||||||
noise = {
|
|
||||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
||||||
'latest', 'new', 'news', 'update', 'updates',
|
'latest', 'new', 'news', 'update', 'updates',
|
||||||
'trending', 'hottest', 'popular', 'viral',
|
'trending', 'hottest', 'popular', 'viral',
|
||||||
@@ -125,12 +109,8 @@ def _extract_core_subject(topic: str) -> str:
|
|||||||
'recommendations', 'advice',
|
'recommendations', 'advice',
|
||||||
'prompt', 'prompts', 'prompting',
|
'prompt', 'prompts', 'prompting',
|
||||||
'methods', 'strategies', 'approaches',
|
'methods', 'strategies', 'approaches',
|
||||||
}
|
})
|
||||||
words = text.split()
|
return extract_core_subject(topic, noise=_TIKTOK_NOISE)
|
||||||
filtered = [w for w in words if w not in noise]
|
|
||||||
|
|
||||||
result = ' '.join(filtered) if filtered else text
|
|
||||||
return result.rstrip('?!.')
|
|
||||||
|
|
||||||
|
|
||||||
def _log(msg: str):
|
def _log(msg: str):
|
||||||
|
|||||||
@@ -110,26 +110,12 @@ def is_ytdlp_installed() -> bool:
|
|||||||
def _extract_core_subject(topic: str) -> str:
|
def _extract_core_subject(topic: str) -> str:
|
||||||
"""Extract core subject from verbose query for YouTube search.
|
"""Extract core subject from verbose query for YouTube search.
|
||||||
|
|
||||||
Strips meta/research words to keep only the core product/concept name,
|
NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
|
||||||
similar to bird_x.py's approach.
|
are intentionally KEPT — they're YouTube content types that improve search.
|
||||||
"""
|
"""
|
||||||
text = topic.lower().strip()
|
from .query import extract_core_subject
|
||||||
|
# YouTube-specific noise set: smaller than default, keeps content-type words
|
||||||
# Strip multi-word prefixes
|
_YT_NOISE = frozenset({
|
||||||
prefixes = [
|
|
||||||
'what are the best', 'what is the best', 'what are the latest',
|
|
||||||
'what are people saying about', 'what do people think about',
|
|
||||||
'how do i use', 'how to use', 'how to',
|
|
||||||
'what are', 'what is', 'tips for', 'best practices for',
|
|
||||||
]
|
|
||||||
for p in prefixes:
|
|
||||||
if text.startswith(p + ' '):
|
|
||||||
text = text[len(p):].strip()
|
|
||||||
|
|
||||||
# Strip individual noise words
|
|
||||||
# NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
|
|
||||||
# are intentionally KEPT — they're YouTube content types that improve search
|
|
||||||
noise = {
|
|
||||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
||||||
'latest', 'new', 'news', 'update', 'updates',
|
'latest', 'new', 'news', 'update', 'updates',
|
||||||
'trending', 'hottest', 'popular', 'viral',
|
'trending', 'hottest', 'popular', 'viral',
|
||||||
@@ -137,12 +123,8 @@ def _extract_core_subject(topic: str) -> str:
|
|||||||
'recommendations', 'advice',
|
'recommendations', 'advice',
|
||||||
'prompt', 'prompts', 'prompting',
|
'prompt', 'prompts', 'prompting',
|
||||||
'methods', 'strategies', 'approaches',
|
'methods', 'strategies', 'approaches',
|
||||||
}
|
})
|
||||||
words = text.split()
|
return extract_core_subject(topic, noise=_YT_NOISE)
|
||||||
filtered = [w for w in words if w not in noise]
|
|
||||||
|
|
||||||
result = ' '.join(filtered) if filtered else text
|
|
||||||
return result.rstrip('?!.')
|
|
||||||
|
|
||||||
|
|
||||||
def search_youtube(
|
def search_youtube(
|
||||||
|
|||||||
Reference in New Issue
Block a user