From 1002f1f0205e1d94f0fa03cf9ffc5ea1d9cb9199 Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 15:24:44 -0700 Subject: [PATCH] Add platform-specific query optimizations - hackernews: use extract_core_subject instead of raw topic, add points>5 filter and restrictSearchableAttributes=title to reduce noise from URL-match and low-signal posts - youtube: add --dateafter parameter to yt-dlp for server-side date filtering (Python soft filter still handles fallback) - reddit: skip opinion/review query variant for how_to/comparison queries where it adds noise - bird_x: add OR-group retry with compound terms before falling back to word-dropping (uses X OR operator for multi-concept queries) - query.py: add detect_query_type() and extract_compound_terms() --- scripts/lib/bird_x.py | 14 ++++++++- scripts/lib/hackernews.py | 12 ++++--- scripts/lib/query.py | 66 +++++++++++++++++++++++++++++++++++++++ scripts/lib/reddit.py | 6 ++-- scripts/lib/youtube_yt.py | 7 +++-- tests/test_query.py | 46 ++++++++++++++++++++++++++- 6 files changed, 140 insertions(+), 11 deletions(-) diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py index 0270a8c..dd9fa9f 100644 --- a/scripts/lib/bird_x.py +++ b/scripts/lib/bird_x.py @@ -237,8 +237,20 @@ def search_x( # Check if we got results items = parse_bird_response(response, query=core_topic) - # Retry with fewer keywords if 0 results and query has 3+ words + # Retry with OR groups for multi-word queries (X supports OR operator) core_words = core_topic.split() + if not items and len(core_words) >= 2: + from .query import extract_compound_terms + compounds = extract_compound_terms(topic) + if compounds: + # Build OR-group query: ("multi-agent" OR "agent simulation") since:DATE + or_parts = ' OR '.join(f'"{t}"' for t in compounds[:3]) + _log(f"0 results for '{core_topic}', retrying with OR groups: {or_parts}") + query = f"({or_parts}) since:{from_date}" + response = _run_bird_search(query, count, timeout) + items = parse_bird_response(response, query=core_topic) + + # Retry with fewer keywords if still 0 results and query has 3+ words if not items and len(core_words) > 2: shorter = ' '.join(core_words[:2]) _log(f"0 results for '{core_topic}', retrying with '{shorter}'") diff --git a/scripts/lib/hackernews.py b/scripts/lib/hackernews.py index 01f544a..5b6fab1 100644 --- a/scripts/lib/hackernews.py +++ b/scripts/lib/hackernews.py @@ -12,6 +12,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Dict, List, Optional from . import http +from .query import extract_core_subject from .relevance import token_overlap_relevance ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search" @@ -85,14 +86,17 @@ def search_hackernews( from_ts = _date_to_unix(from_date) to_ts = _date_to_unix(to_date) + 86400 # Include the end date - _log(f"Searching for '{topic}' (since {from_date}, count={count})") + # Use extracted core subject instead of raw topic for cleaner Algolia matching + core = extract_core_subject(topic) + _log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})") - # Use relevance-sorted search (better for topic matching) + # Use relevance-sorted search with minimum engagement filter params = { - "query": topic, + "query": core, "tags": "story", - "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts}", + "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>5", "hitsPerPage": str(count), + "restrictSearchableAttributes": "title", } from urllib.parse import urlencode diff --git a/scripts/lib/query.py b/scripts/lib/query.py index 1d48875..e7bda9a 100644 --- a/scripts/lib/query.py +++ b/scripts/lib/query.py @@ -5,6 +5,7 @@ youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one parameterized function. Each platform calls with its own overrides. """ +import re from typing import FrozenSet, List, Optional, Set # Common multi-word prefixes stripped from all queries (identical across modules) @@ -96,3 +97,68 @@ def extract_core_subject( result = ' '.join(filtered) if filtered else text return result.rstrip('?!.') if not max_words else (result or topic.lower().strip()) + + +# ---- Query type detection (heuristic, no LLM) ---- + +_OPINION_SIGNALS = frozenset({ + 'worth', 'thoughts', 'opinion', 'opinions', 'review', 'reviews', + 'recommend', 'recommendation', 'recommendations', 'should', + 'anyone', 'anybody', 'experience', 'experiences', +}) + +_HOW_TO_SIGNALS = frozenset({ + 'how', 'setup', 'configure', 'install', 'tutorial', 'guide', + 'step', 'steps', 'instructions', +}) + +_COMPARISON_SIGNALS = frozenset({ + 'vs', 'versus', 'compared', 'comparison', 'better', 'alternative', + 'alternatives', 'difference', 'differences', +}) + +_PRODUCT_SIGNALS = frozenset({ + 'pricing', 'price', 'cost', 'plan', 'plans', 'tier', 'tiers', + 'buy', 'purchase', 'subscription', 'trial', 'free', +}) + + +def detect_query_type(topic: str) -> str: + """Classify query intent without an LLM. + + Returns one of: "product", "concept", "opinion", "how_to", "comparison". + Used to adapt per-platform query construction. + """ + words = set(topic.lower().split()) + + if words & _COMPARISON_SIGNALS: + return "comparison" + if words & _HOW_TO_SIGNALS or topic.lower().startswith("how "): + return "how_to" + if words & _OPINION_SIGNALS: + return "opinion" + if words & _PRODUCT_SIGNALS: + return "product" + return "concept" + + +def extract_compound_terms(topic: str) -> List[str]: + """Detect multi-word terms that should be quoted in search queries. + + Identifies: + - Hyphenated terms: "multi-agent", "vc-backed" + - Title-cased multi-word names: "Claude Code", "React Native" + + Returns list of terms suitable for quoting (e.g., '"multi-agent"'). + """ + terms: List[str] = [] + + # Hyphenated terms + for match in re.finditer(r'\b\w+-\w+(?:-\w+)*\b', topic): + terms.append(match.group()) + + # Title-cased sequences (2+ capitalized words in a row) + for match in re.finditer(r'(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+', topic): + terms.append(match.group()) + + return terms diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py index 8957464..bf0695d 100644 --- a/scripts/lib/reddit.py +++ b/scripts/lib/reddit.py @@ -48,7 +48,7 @@ DEPTH_CONFIG = { }, } -from .query import extract_core_subject as _query_extract +from .query import detect_query_type, extract_core_subject as _query_extract from .relevance import token_overlap_relevance # Reddit-specific noise words (preserves original smaller set) @@ -107,7 +107,9 @@ def expand_reddit_queries(topic: str, depth: str) -> List[str]: if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8: queries.append(original_clean) - if depth in ("default", "deep"): + # Add opinion/review variant except for how_to/comparison queries + qtype = detect_query_type(topic) + if depth in ("default", "deep") and qtype not in ("how_to", "comparison"): queries.append(f"{core} worth it OR thoughts OR review") if depth == "deep": diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py index 680de91..e6b23ba 100644 --- a/scripts/lib/youtube_yt.py +++ b/scripts/lib/youtube_yt.py @@ -100,15 +100,16 @@ def search_youtube( _log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})") # yt-dlp search with full metadata (no --flat-playlist so dates are real). - # No --dateafter — we filter by date in Python with a soft fallback, - # because YouTube search returns relevance-sorted results and strict date - # filtering returns 0 for evergreen topics like "thumbnail tips". + # --dateafter helps yt-dlp filter server-side, but Python soft filter + # (below) handles the fallback for evergreen topics with 0 recent results. + dateafter = from_date.replace("-", "") # YYYYMMDD format for yt-dlp cmd = [ "yt-dlp", f"ytsearch{count}:{core_topic}", "--dump-json", "--no-warnings", "--no-download", + "--dateafter", dateafter, ] preexec = os.setsid if hasattr(os, 'setsid') else None diff --git a/tests/test_query.py b/tests/test_query.py index a081f4f..1cf8b49 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -6,7 +6,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) -from lib.query import NOISE_WORDS, extract_core_subject +from lib.query import NOISE_WORDS, detect_query_type, extract_compound_terms, extract_core_subject class TestExtractCoreSubject(unittest.TestCase): @@ -126,5 +126,49 @@ class TestNoiseWordsCompleteness(unittest.TestCase): self.assertIn(w, NOISE_WORDS) +class TestDetectQueryType(unittest.TestCase): + """Tests for detect_query_type().""" + + def test_comparison(self): + self.assertEqual(detect_query_type("React vs Vue"), "comparison") + + def test_how_to(self): + self.assertEqual(detect_query_type("how to deploy on Vercel"), "how_to") + + def test_opinion(self): + self.assertEqual(detect_query_type("cursor IDE worth it"), "opinion") + + def test_product(self): + self.assertEqual(detect_query_type("cursor IDE pricing"), "product") + + def test_concept_default(self): + self.assertEqual(detect_query_type("multi-agent reinforcement learning"), "concept") + + def test_how_prefix(self): + self.assertEqual(detect_query_type("how does Claude work"), "how_to") + + +class TestExtractCompoundTerms(unittest.TestCase): + """Tests for extract_compound_terms().""" + + def test_hyphenated(self): + terms = extract_compound_terms("multi-agent reinforcement learning") + self.assertIn("multi-agent", terms) + + def test_title_case(self): + terms = extract_compound_terms("Claude Code and React Native") + self.assertTrue(any("Claude Code" in t for t in terms)) + self.assertTrue(any("React Native" in t for t in terms)) + + def test_no_compounds(self): + terms = extract_compound_terms("python tutorial") + self.assertEqual(len(terms), 0) + + def test_multiple_hyphens(self): + terms = extract_compound_terms("vc-backed start-up") + self.assertIn("vc-backed", terms) + self.assertIn("start-up", terms) + + if __name__ == "__main__": unittest.main()