Address review feedback: deduplicate query_type, clean unused imports, fix defaults

- Remove duplicate detect_query_type from query.py (divergent 5-type version);
  canonical 7-type version lives in query_type.py
- Fix reddit.py import to use query_type.detect_query_type
- Clean unused STOPWORDS/SYNONYMS/tokenize imports from youtube_yt, instagram,
  tiktok, scrapecreators_x, bird_x after relevance consolidation
- Fix _relevance_filter default from 0.7 to 0.0 (items without relevance
  should not silently pass the filter)
- Remove --dateafter from yt-dlp (returns 0 results for evergreen topics)
- Remove restrictSearchableAttributes from HN search (misses Ask/Show HN)
- Lower HN points filter from >5 to >2 (avoids filtering niche posts)
- Add error logging to select_openai_model HTTP failures
- Remove mise.toml and internal planning doc from repo
- Update module docstrings to describe current purpose, not migration history
- Update tests to import from canonical relevance module
This commit is contained in:
Jeffrey Sperling
2026-03-11 18:40:07 -07:00
parent 6c402f66b7
commit 036bcd2ae3
18 changed files with 44 additions and 207 deletions
+5 -4
View File
@@ -6,26 +6,27 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
from lib import scrapecreators_x
from lib.relevance import tokenize as _tokenize
class TestTokenize(unittest.TestCase):
def test_lowercases(self):
tokens = scrapecreators_x._tokenize("Claude AI")
tokens = _tokenize("Claude AI")
self.assertIn("claude", tokens)
def test_strips_stopwords(self):
tokens = scrapecreators_x._tokenize("the best AI tool")
tokens = _tokenize("the best AI tool")
self.assertNotIn("the", tokens)
self.assertIn("best", tokens) # 'best' is not a stopword in tokenizer
def test_removes_single_char(self):
tokens = scrapecreators_x._tokenize("a b cd ef")
tokens = _tokenize("a b cd ef")
self.assertNotIn("a", tokens)
self.assertNotIn("b", tokens)
self.assertIn("cd", tokens)
def test_expands_synonyms(self):
tokens = scrapecreators_x._tokenize("ai research")
tokens = _tokenize("ai research")
self.assertIn("artificial", tokens)
self.assertIn("intelligence", tokens)