Address review feedback: deduplicate query_type, clean unused imports, fix defaults
- Remove duplicate detect_query_type from query.py (divergent 5-type version); canonical 7-type version lives in query_type.py - Fix reddit.py import to use query_type.detect_query_type - Clean unused STOPWORDS/SYNONYMS/tokenize imports from youtube_yt, instagram, tiktok, scrapecreators_x, bird_x after relevance consolidation - Fix _relevance_filter default from 0.7 to 0.0 (items without relevance should not silently pass the filter) - Remove --dateafter from yt-dlp (returns 0 results for evergreen topics) - Remove restrictSearchableAttributes from HN search (misses Ask/Show HN) - Lower HN points filter from >5 to >2 (avoids filtering niche posts) - Add error logging to select_openai_model HTTP failures - Remove mise.toml and internal planning doc from repo - Update module docstrings to describe current purpose, not migration history - Update tests to import from canonical relevance module
This commit is contained in:
@@ -35,12 +35,7 @@ TRANSCRIPT_LIMITS = {
|
||||
# Max words to keep from each transcript
|
||||
TRANSCRIPT_MAX_WORDS = 500
|
||||
|
||||
from .relevance import (
|
||||
STOPWORDS,
|
||||
SYNONYMS,
|
||||
token_overlap_relevance as _compute_relevance,
|
||||
tokenize as _tokenize,
|
||||
)
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
@@ -100,16 +95,15 @@ def search_youtube(
|
||||
_log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})")
|
||||
|
||||
# yt-dlp search with full metadata (no --flat-playlist so dates are real).
|
||||
# --dateafter helps yt-dlp filter server-side, but Python soft filter
|
||||
# (below) handles the fallback for evergreen topics with 0 recent results.
|
||||
dateafter = from_date.replace("-", "") # YYYYMMDD format for yt-dlp
|
||||
# NOTE: --dateafter intentionally omitted — YouTube search returns
|
||||
# relevance-sorted results and strict date filtering returns 0 for
|
||||
# evergreen topics. Python soft filter (below) handles date filtering.
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
f"ytsearch{count}:{core_topic}",
|
||||
"--dump-json",
|
||||
"--no-warnings",
|
||||
"--no-download",
|
||||
"--dateafter", dateafter,
|
||||
]
|
||||
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
|
||||
Reference in New Issue
Block a user