Address review feedback: deduplicate query_type, clean unused imports, fix defaults

- Remove duplicate detect_query_type from query.py (divergent 5-type version);
  canonical 7-type version lives in query_type.py
- Fix reddit.py import to use query_type.detect_query_type
- Clean unused STOPWORDS/SYNONYMS/tokenize imports from youtube_yt, instagram,
  tiktok, scrapecreators_x, bird_x after relevance consolidation
- Fix _relevance_filter default from 0.7 to 0.0 (items without relevance
  should not silently pass the filter)
- Remove --dateafter from yt-dlp (returns 0 results for evergreen topics)
- Remove restrictSearchableAttributes from HN search (misses Ask/Show HN)
- Lower HN points filter from >5 to >2 (avoids filtering niche posts)
- Add error logging to select_openai_model HTTP failures
- Remove mise.toml and internal planning doc from repo
- Update module docstrings to describe current purpose, not migration history
- Update tests to import from canonical relevance module
This commit is contained in:
Jeffrey Sperling
2026-03-11 18:40:07 -07:00
parent 6c402f66b7
commit 036bcd2ae3
18 changed files with 44 additions and 207 deletions
+2 -49
View File
@@ -1,9 +1,5 @@
"""Shared query utilities for /last30days search modules.
Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
parameterized function. Each platform calls with its own overrides.
"""
"""Shared query preprocessing utilities: noise-word stripping, core subject
extraction, and compound term detection. Used by all search modules."""
import re
from typing import FrozenSet, List, Optional, Set
@@ -99,49 +95,6 @@ def extract_core_subject(
return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
# ---- Query type detection (heuristic, no LLM) ----
_OPINION_SIGNALS = frozenset({
'worth', 'thoughts', 'opinion', 'opinions', 'review', 'reviews',
'recommend', 'recommendation', 'recommendations', 'should',
'anyone', 'anybody', 'experience', 'experiences',
})
_HOW_TO_SIGNALS = frozenset({
'how', 'setup', 'configure', 'install', 'tutorial', 'guide',
'step', 'steps', 'instructions',
})
_COMPARISON_SIGNALS = frozenset({
'vs', 'versus', 'compared', 'comparison', 'better', 'alternative',
'alternatives', 'difference', 'differences',
})
_PRODUCT_SIGNALS = frozenset({
'pricing', 'price', 'cost', 'plan', 'plans', 'tier', 'tiers',
'buy', 'purchase', 'subscription', 'trial', 'free',
})
def detect_query_type(topic: str) -> str:
"""Classify query intent without an LLM.
Returns one of: "product", "concept", "opinion", "how_to", "comparison".
Used to adapt per-platform query construction.
"""
words = set(topic.lower().split())
if words & _COMPARISON_SIGNALS:
return "comparison"
if words & _HOW_TO_SIGNALS or topic.lower().startswith("how "):
return "how_to"
if words & _OPINION_SIGNALS:
return "opinion"
if words & _PRODUCT_SIGNALS:
return "product"
return "concept"
def extract_compound_terms(topic: str) -> List[str]:
"""Detect multi-word terms that should be quoted in search queries.