Integrate shared query.py into per-source modules
Replace duplicated _extract_core_subject() in bird_x, reddit, youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x with thin wrappers that delegate to query.extract_core_subject() with platform-specific noise sets. Each module preserves its current behavior exactly: - bird_x: max_words=5, strip_suffixes=True, full noise set - youtube_yt: keeps tips/tricks/tutorial/guide/review (content types) - reddit: preserves original smaller noise set - tiktok/instagram: same small noise set - bluesky/scrapecreators_x: minimal noise set Existing tests pass without modification since _extract_core_subject() still exists as a callable on each module.
This commit is contained in:
+4
-16
@@ -67,26 +67,14 @@ def _create_session(handle: str, app_password: str) -> Optional[str]:
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for Bluesky search."""
|
||||
text = topic.lower().strip()
|
||||
prefixes = [
|
||||
'what are the best', 'what is the best', 'what are the latest',
|
||||
'what are people saying about', 'what do people think about',
|
||||
'how do i use', 'how to use', 'how to',
|
||||
'what are', 'what is', 'tips for', 'best practices for',
|
||||
]
|
||||
for p in prefixes:
|
||||
if text.startswith(p + ' '):
|
||||
text = text[len(p):].strip()
|
||||
noise = {
|
||||
from .query import extract_core_subject
|
||||
_BSKY_NOISE = frozenset({
|
||||
'best', 'top', 'good', 'great', 'awesome',
|
||||
'latest', 'new', 'news', 'update', 'updates',
|
||||
'trending', 'hottest', 'popular', 'viral',
|
||||
'practices', 'features', 'recommendations', 'advice',
|
||||
}
|
||||
words = text.split()
|
||||
filtered = [w for w in words if w not in noise]
|
||||
result = ' '.join(filtered) if filtered else text
|
||||
return result.rstrip('?!.')
|
||||
})
|
||||
return extract_core_subject(topic, noise=_BSKY_NOISE)
|
||||
|
||||
|
||||
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
||||
|
||||
Reference in New Issue
Block a user