Add platform-specific query optimizations
- hackernews: use extract_core_subject instead of raw topic, add points>5 filter and restrictSearchableAttributes=title to reduce noise from URL-match and low-signal posts - youtube: add --dateafter parameter to yt-dlp for server-side date filtering (Python soft filter still handles fallback) - reddit: skip opinion/review query variant for how_to/comparison queries where it adds noise - bird_x: add OR-group retry with compound terms before falling back to word-dropping (uses X OR operator for multi-concept queries) - query.py: add detect_query_type() and extract_compound_terms()
This commit is contained in:
@@ -5,6 +5,7 @@ youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
|
||||
parameterized function. Each platform calls with its own overrides.
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import FrozenSet, List, Optional, Set
|
||||
|
||||
# Common multi-word prefixes stripped from all queries (identical across modules)
|
||||
@@ -96,3 +97,68 @@ def extract_core_subject(
|
||||
|
||||
result = ' '.join(filtered) if filtered else text
|
||||
return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
|
||||
|
||||
|
||||
# ---- Query type detection (heuristic, no LLM) ----
|
||||
|
||||
_OPINION_SIGNALS = frozenset({
|
||||
'worth', 'thoughts', 'opinion', 'opinions', 'review', 'reviews',
|
||||
'recommend', 'recommendation', 'recommendations', 'should',
|
||||
'anyone', 'anybody', 'experience', 'experiences',
|
||||
})
|
||||
|
||||
_HOW_TO_SIGNALS = frozenset({
|
||||
'how', 'setup', 'configure', 'install', 'tutorial', 'guide',
|
||||
'step', 'steps', 'instructions',
|
||||
})
|
||||
|
||||
_COMPARISON_SIGNALS = frozenset({
|
||||
'vs', 'versus', 'compared', 'comparison', 'better', 'alternative',
|
||||
'alternatives', 'difference', 'differences',
|
||||
})
|
||||
|
||||
_PRODUCT_SIGNALS = frozenset({
|
||||
'pricing', 'price', 'cost', 'plan', 'plans', 'tier', 'tiers',
|
||||
'buy', 'purchase', 'subscription', 'trial', 'free',
|
||||
})
|
||||
|
||||
|
||||
def detect_query_type(topic: str) -> str:
|
||||
"""Classify query intent without an LLM.
|
||||
|
||||
Returns one of: "product", "concept", "opinion", "how_to", "comparison".
|
||||
Used to adapt per-platform query construction.
|
||||
"""
|
||||
words = set(topic.lower().split())
|
||||
|
||||
if words & _COMPARISON_SIGNALS:
|
||||
return "comparison"
|
||||
if words & _HOW_TO_SIGNALS or topic.lower().startswith("how "):
|
||||
return "how_to"
|
||||
if words & _OPINION_SIGNALS:
|
||||
return "opinion"
|
||||
if words & _PRODUCT_SIGNALS:
|
||||
return "product"
|
||||
return "concept"
|
||||
|
||||
|
||||
def extract_compound_terms(topic: str) -> List[str]:
|
||||
"""Detect multi-word terms that should be quoted in search queries.
|
||||
|
||||
Identifies:
|
||||
- Hyphenated terms: "multi-agent", "vc-backed"
|
||||
- Title-cased multi-word names: "Claude Code", "React Native"
|
||||
|
||||
Returns list of terms suitable for quoting (e.g., '"multi-agent"').
|
||||
"""
|
||||
terms: List[str] = []
|
||||
|
||||
# Hyphenated terms
|
||||
for match in re.finditer(r'\b\w+-\w+(?:-\w+)*\b', topic):
|
||||
terms.append(match.group())
|
||||
|
||||
# Title-cased sequences (2+ capitalized words in a row)
|
||||
for match in re.finditer(r'(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+', topic):
|
||||
terms.append(match.group())
|
||||
|
||||
return terms
|
||||
|
||||
Reference in New Issue
Block a user