Add platform-specific query optimizations

- hackernews: use extract_core_subject instead of raw topic, add
  points>5 filter and restrictSearchableAttributes=title to reduce
  noise from URL-match and low-signal posts
- youtube: add --dateafter parameter to yt-dlp for server-side date
  filtering (Python soft filter still handles fallback)
- reddit: skip opinion/review query variant for how_to/comparison
  queries where it adds noise
- bird_x: add OR-group retry with compound terms before falling back
  to word-dropping (uses X OR operator for multi-concept queries)
- query.py: add detect_query_type() and extract_compound_terms()
This commit is contained in:
Jeffrey Sperling
2026-03-11 15:24:44 -07:00
parent c5be117701
commit 1002f1f020
6 changed files with 140 additions and 11 deletions
+66
View File
@@ -5,6 +5,7 @@ youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
parameterized function. Each platform calls with its own overrides.
"""
import re
from typing import FrozenSet, List, Optional, Set
# Common multi-word prefixes stripped from all queries (identical across modules)
@@ -96,3 +97,68 @@ def extract_core_subject(
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
# ---- Query type detection (heuristic, no LLM) ----
_OPINION_SIGNALS = frozenset({
'worth', 'thoughts', 'opinion', 'opinions', 'review', 'reviews',
'recommend', 'recommendation', 'recommendations', 'should',
'anyone', 'anybody', 'experience', 'experiences',
})
_HOW_TO_SIGNALS = frozenset({
'how', 'setup', 'configure', 'install', 'tutorial', 'guide',
'step', 'steps', 'instructions',
})
_COMPARISON_SIGNALS = frozenset({
'vs', 'versus', 'compared', 'comparison', 'better', 'alternative',
'alternatives', 'difference', 'differences',
})
_PRODUCT_SIGNALS = frozenset({
'pricing', 'price', 'cost', 'plan', 'plans', 'tier', 'tiers',
'buy', 'purchase', 'subscription', 'trial', 'free',
})
def detect_query_type(topic: str) -> str:
"""Classify query intent without an LLM.
Returns one of: "product", "concept", "opinion", "how_to", "comparison".
Used to adapt per-platform query construction.
"""
words = set(topic.lower().split())
if words & _COMPARISON_SIGNALS:
return "comparison"
if words & _HOW_TO_SIGNALS or topic.lower().startswith("how "):
return "how_to"
if words & _OPINION_SIGNALS:
return "opinion"
if words & _PRODUCT_SIGNALS:
return "product"
return "concept"
def extract_compound_terms(topic: str) -> List[str]:
"""Detect multi-word terms that should be quoted in search queries.
Identifies:
- Hyphenated terms: "multi-agent", "vc-backed"
- Title-cased multi-word names: "Claude Code", "React Native"
Returns list of terms suitable for quoting (e.g., '"multi-agent"').
"""
terms: List[str] = []
# Hyphenated terms
for match in re.finditer(r'\b\w+-\w+(?:-\w+)*\b', topic):
terms.append(match.group())
# Title-cased sequences (2+ capitalized words in a row)
for match in re.finditer(r'(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+', topic):
terms.append(match.group())
return terms