Address review feedback: deduplicate query_type, clean unused imports, fix defaults

- Remove duplicate detect_query_type from query.py (divergent 5-type version);
  canonical 7-type version lives in query_type.py
- Fix reddit.py import to use query_type.detect_query_type
- Clean unused STOPWORDS/SYNONYMS/tokenize imports from youtube_yt, instagram,
  tiktok, scrapecreators_x, bird_x after relevance consolidation
- Fix _relevance_filter default from 0.7 to 0.0 (items without relevance
  should not silently pass the filter)
- Remove --dateafter from yt-dlp (returns 0 results for evergreen topics)
- Remove restrictSearchableAttributes from HN search (misses Ask/Show HN)
- Lower HN points filter from >5 to >2 (avoids filtering niche posts)
- Add error logging to select_openai_model HTTP failures
- Remove mise.toml and internal planning doc from repo
- Update module docstrings to describe current purpose, not migration history
- Update tests to import from canonical relevance module

scripts/last30days.py

@@ -1835,11 +1835,11 @@ def main():
         """Filter items below relevance threshold with minimum-result guarantee."""
         if len(items) <= 3:
             return items
-        passed = [i for i in items if getattr(i, 'relevance', 0.7) >= threshold]
+        passed = [i for i in items if getattr(i, 'relevance', 0.0) >= threshold]
         if not passed:
             # Keep top 3 by relevance if all filtered
             print(f"[{source_name} WARNING] All results below relevance {threshold}, keeping top 3", file=sys.stderr)
-            by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.7), reverse=True)
+            by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.0), reverse=True)
             return by_rel[:3]
         return passed
 

scripts/lib/bird_x.py

@@ -58,7 +58,7 @@ def _extract_core_subject(topic: str) -> str:
     Aggressively strip question/meta/research words to keep only the
     core product/concept name (max 5 words).
     """
-    from .query import NOISE_WORDS, extract_core_subject
+    from .query import extract_core_subject
     return extract_core_subject(topic, max_words=5, strip_suffixes=True)
 
 

scripts/lib/hackernews.py

@@ -90,13 +90,14 @@ def search_hackernews(
     core = extract_core_subject(topic)
     _log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
 
-    # Use relevance-sorted search with minimum engagement filter
+    # Use relevance-sorted search with minimum engagement filter.
+    # NOTE: restrictSearchableAttributes=title omitted intentionally — it would
+    # miss Ask HN/Show HN threads where the topic appears in the body.
     params = {
         "query": core,
         "tags": "story",
-        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>5",
+        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
         "hitsPerPage": str(count),
-        "restrictSearchableAttributes": "title",
     }
 
     from urllib.parse import urlencode

scripts/lib/instagram.py

@@ -31,12 +31,7 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _extract_core_subject(topic: str) -> str:

scripts/lib/models.py

@@ -53,6 +53,10 @@ def is_search_capable_model(model_id: str) -> bool:
     Includes mini variants (same structured extraction quality, lower cost).
     Excludes: nano (no web_search), gpt-4o-mini (no domain filtering),
     chat/codex/pro/preview/turbo/search (specialized variants).
+
+    Note: gpt-5 with reasoning effort="minimal" does NOT support web_search
+    (per OpenAI docs). We never set reasoning params — our usage is pure
+    tool invocation + JSON extraction — so gpt-5 is safe to include here.
     """
     model_lower = model_id.lower()
 

scripts/lib/query.py

@@ -1,9 +1,5 @@
-"""Shared query utilities for /last30days search modules.
-
-Consolidates duplicated _extract_core_subject() logic from bird_x, reddit,
-youtube_yt, tiktok, instagram, bluesky, and scrapecreators_x into one
-parameterized function. Each platform calls with its own overrides.
-"""
+"""Shared query preprocessing utilities: noise-word stripping, core subject
+extraction, and compound term detection. Used by all search modules."""
 
 import re
 from typing import FrozenSet, List, Optional, Set
@@ -99,49 +95,6 @@ def extract_core_subject(
     return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
 
 
-# ---- Query type detection (heuristic, no LLM) ----
-
-_OPINION_SIGNALS = frozenset({
-    'worth', 'thoughts', 'opinion', 'opinions', 'review', 'reviews',
-    'recommend', 'recommendation', 'recommendations', 'should',
-    'anyone', 'anybody', 'experience', 'experiences',
-})
-
-_HOW_TO_SIGNALS = frozenset({
-    'how', 'setup', 'configure', 'install', 'tutorial', 'guide',
-    'step', 'steps', 'instructions',
-})
-
-_COMPARISON_SIGNALS = frozenset({
-    'vs', 'versus', 'compared', 'comparison', 'better', 'alternative',
-    'alternatives', 'difference', 'differences',
-})
-
-_PRODUCT_SIGNALS = frozenset({
-    'pricing', 'price', 'cost', 'plan', 'plans', 'tier', 'tiers',
-    'buy', 'purchase', 'subscription', 'trial', 'free',
-})
-
-
-def detect_query_type(topic: str) -> str:
-    """Classify query intent without an LLM.
-
-    Returns one of: "product", "concept", "opinion", "how_to", "comparison".
-    Used to adapt per-platform query construction.
-    """
-    words = set(topic.lower().split())
-
-    if words & _COMPARISON_SIGNALS:
-        return "comparison"
-    if words & _HOW_TO_SIGNALS or topic.lower().startswith("how "):
-        return "how_to"
-    if words & _OPINION_SIGNALS:
-        return "opinion"
-    if words & _PRODUCT_SIGNALS:
-        return "product"
-    return "concept"
-
-
 def extract_compound_terms(topic: str) -> List[str]:
     """Detect multi-word terms that should be quoted in search queries.
 

scripts/lib/reddit.py

@@ -48,7 +48,8 @@ DEPTH_CONFIG = {
     },
 }
 
-from .query import detect_query_type, extract_core_subject as _query_extract
+from .query import extract_core_subject as _query_extract
+from .query_type import detect_query_type
 from .relevance import token_overlap_relevance
 
 # Reddit-specific noise words (preserves original smaller set)

scripts/lib/relevance.py

@@ -1,7 +1,6 @@
-"""Shared relevance scoring for /last30days search modules.
+"""Shared token-overlap relevance scoring for search result ranking.
 
-Consolidates duplicated _tokenize, _compute_relevance, STOPWORDS, and SYNONYMS
-from youtube_yt, tiktok, instagram, and scrapecreators_x into one module.
+Tokenizes text, expands synonyms, and computes query-to-content overlap ratios.
 """
 
 import re

scripts/lib/scrapecreators_x.py

@@ -24,12 +24,7 @@ DEPTH_CONFIG = {
     "deep":    {"results_per_page": 40},
 }
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _extract_core_subject(topic: str) -> str:

scripts/lib/tiktok.py

@@ -31,12 +31,7 @@ DEPTH_CONFIG = {
 # Max words to keep from each caption
 CAPTION_MAX_WORDS = 500
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _extract_core_subject(topic: str) -> str:

scripts/lib/youtube_yt.py

@@ -35,12 +35,7 @@ TRANSCRIPT_LIMITS = {
 # Max words to keep from each transcript
 TRANSCRIPT_MAX_WORDS = 500
 
-from .relevance import (
-    STOPWORDS,
-    SYNONYMS,
-    token_overlap_relevance as _compute_relevance,
-    tokenize as _tokenize,
-)
+from .relevance import token_overlap_relevance as _compute_relevance
 
 
 def _log(msg: str):
@@ -100,16 +95,15 @@ def search_youtube(
     _log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})")
 
     # yt-dlp search with full metadata (no --flat-playlist so dates are real).
-    # --dateafter helps yt-dlp filter server-side, but Python soft filter
-    # (below) handles the fallback for evergreen topics with 0 recent results.
-    dateafter = from_date.replace("-", "")  # YYYYMMDD format for yt-dlp
+    # NOTE: --dateafter intentionally omitted — YouTube search returns
+    # relevance-sorted results and strict date filtering returns 0 for
+    # evergreen topics. Python soft filter (below) handles date filtering.
     cmd = [
         "yt-dlp",
         f"ytsearch{count}:{core_topic}",
         "--dump-json",
         "--no-warnings",
         "--no-download",
-        "--dateafter", dateafter,
     ]
 
     preexec = os.setsid if hasattr(os, 'setsid') else None