From edea402b7c55fbee8cfd726689a6c1570a60364f Mon Sep 17 00:00:00 2001 From: Brad Ferguson <67075438+bradferguson@users.noreply.github.com> Date: Sat, 16 May 2026 19:37:16 -0700 Subject: [PATCH] fix(sources): unblock SC YouTube + multi-token HN searches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related fixes that surface when running last30days with multi-keyword themed queries (e.g. "claude, personal agents, agentic infra"). Both bugs caused entire sources to silently return zero items. YouTube (ScrapeCreators) SC's /v1/youtube/search rejects ?keyword= with HTTP 400: {"error":"missing_parameter","message":"You must provide a query"} The canonical SC parameter for that endpoint is `query`. Other SC endpoints we use (Reddit, TikTok, Instagram) happened to work because they use their own per-endpoint parameter names — YouTube was the lone outlier. Hacker News (Algolia) Multi-keyword theme queries returned zero hits across every theme. Algolia treats query= as strict AND across tokens, so a 4-5 word query like "claude, personal agents, agentic infra" matches no stories. Three changes in hackernews.py: 1. Hoist comma/hyphen flattening into _flatten_query_for_algolia() so search_hackernews and _title_matches_query normalize the query the same way — addresses Greptile P2 #2 about the two callsites needing to stay in sync. 2. Pass `optionalWords` for all-but-the-first token so Algolia ranks by token-overlap instead of requiring every token. 3. Relax _title_matches_query from all-words to any-word, *but match on word boundaries (\b\b) rather than naive substring* — addresses Greptile P2 #1, which flagged that the previous any-word relaxation would let "ai" falsely match "email" or "rail". Token-overlap relevance scoring at parse time already demotes weak matches, so word-boundary any-word matching is safe. Tests: added coverage for no-token-in-title rejection, word-boundary vs substring, and hyphen/comma flattening alignment between the search parameter and the post-filter. Co-authored-by: Trevin Chow --- skills/last30days/scripts/lib/hackernews.py | 71 ++++++++++++++++----- skills/last30days/scripts/lib/youtube_yt.py | 5 +- tests/test_hackernews.py | 40 ++++++++++-- 3 files changed, 94 insertions(+), 22 deletions(-) diff --git a/skills/last30days/scripts/lib/hackernews.py b/skills/last30days/scripts/lib/hackernews.py index e052b26..51e56cb 100644 --- a/skills/last30days/scripts/lib/hackernews.py +++ b/skills/last30days/scripts/lib/hackernews.py @@ -88,17 +88,26 @@ def search_hackernews( # Use extracted core subject instead of raw topic for cleaner Algolia matching core = extract_core_subject(topic) - _log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})") + # Hyphens and commas tokenize awkwardly in Algolia; flatten them so themed + # queries like "ts-bun-node" or "claude, personal agents" become plain words. + core_flat = _flatten_query_for_algolia(core) + _log(f"Searching for '{core_flat}' (raw: '{topic}', since {from_date}, count={count})") # Use relevance-sorted search with minimum engagement filter. # NOTE: restrictSearchableAttributes=title omitted intentionally — it would # miss Ask HN/Show HN threads where the topic appears in the body. params = { - "query": core, + "query": core_flat, "tags": "story", "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2", "hitsPerPage": str(count), } + # Algolia defaults to AND across query tokens, so a 4-5 word theme query + # matches no stories. Mark all-but-the-first token as optional so Algolia + # ranks by how many tokens match instead of requiring every one. + tokens = core_flat.split() + if len(tokens) > 1: + params["optionalWords"] = " ".join(tokens[1:]) from urllib.parse import urlencode url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}" @@ -117,28 +126,56 @@ def search_hackernews( return response -def _title_matches_query(title: str, query: str, author: str = "") -> bool: - """Check if the query term appears in the title content, not just an HN prefix or author. +_WORD_BOUNDARY_RE_CACHE: Dict[str, "re.Pattern[str]"] = {} - Returns True if the query (or any multi-word token) appears in the title - after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes - and ignoring the author name. Returns True when query is empty (no filter). + +def _flatten_query_for_algolia(text: str) -> str: + """Normalise query for Algolia + post-filter comparison. + + Multi-keyword theme queries frequently contain commas (delimiters) or + hyphens (compound terms like ``ts-bun-node``); both tokenize awkwardly. + Flatten them to spaces and collapse runs of whitespace so the search + parameter and the post-filter operate on the same shape. + """ + return " ".join(text.replace(",", " ").replace("-", " ").split()) + + +def _title_matches_query(title: str, query: str, author: str = "") -> bool: + """Check if any query token appears as a whole word in the title. + + Returns True when the query is empty (no filter), or when at least one + query token matches as a whole word in the title after stripping + "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes. + + We previously required *every* token to appear (all-words), which killed + every Algolia hit on multi-keyword themes like "claude, personal agents, + agentic infra" because real HN titles never contain all five tokens + verbatim. Relaxing to any-word matches Algolia's `optionalWords` behaviour + in `search_hackernews`. Token-overlap relevance scoring at parse time + demotes hits where only one weak token matched, so the loosened gate + won't surface noise to the top of the ranking. + + Word-boundary matching (rather than naive substring) prevents short + tokens like ``ai`` or ``ts`` from matching unrelated words like + ``email`` or ``artists``. """ if not query: return True stripped = _HN_PREFIXES.sub("", title).strip() - # Also check that the match isn't solely in the author's username check_text = stripped.lower() - query_lower = query.lower() - # Check each word of the query independently; all must appear somewhere - # in the stripped title (not just the prefix). - query_words = query_lower.split() + # Normalise the query the same way search_hackernews does so post-filter + # tokens line up with what Algolia actually saw. + query_words = [w for w in _flatten_query_for_algolia(query.lower()).split() if w] + if not query_words: + return True for word in query_words: - if word in check_text: - continue - # Word not found in stripped title — reject - return False - return True + pattern = _WORD_BOUNDARY_RE_CACHE.get(word) + if pattern is None: + pattern = re.compile(rf"\b{re.escape(word)}\b") + _WORD_BOUNDARY_RE_CACHE[word] = pattern + if pattern.search(check_text): + return True + return False def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]: diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index e3dc427..2a907b5 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -866,9 +866,12 @@ def _sc_youtube_search(keyword: str, token: str) -> List[Dict[str, Any]]: List of raw video dicts from the API. """ try: + # SC's /v1/youtube/search rejects ?keyword= with HTTP 400; the canonical + # parameter for that endpoint is `query`. Other SC endpoints use their + # own per-endpoint param names so this was the lone outlier. data = http.get( f"{SCRAPECREATORS_YT_BASE}/search", - params={"keyword": keyword}, + params={"query": keyword}, headers=http.scrapecreators_headers(token), timeout=30, retries=2, diff --git a/tests/test_hackernews.py b/tests/test_hackernews.py index c8d1579..29a991c 100644 --- a/tests/test_hackernews.py +++ b/tests/test_hackernews.py @@ -160,12 +160,44 @@ def test_title_matches_query_empty_query(): def test_title_matches_query_partial_match(): - """Test that all query words must match.""" + """Any-word matching: at least one query token in title is enough. + + Previously required *all* tokens, which killed every hit on multi-keyword + theme queries like 'claude, personal agents, agentic infra' since no real + HN title contains all 5 tokens verbatim. Token-overlap relevance at parse + time still demotes weak matches, so the loosened gate is safe. + """ title = "New AI framework" query = "AI blockchain" - - # "blockchain" is not in title, so should fail - assert hackernews._title_matches_query(title, query) is False + + # "AI" matches as a whole word, even though "blockchain" doesn't appear + assert hackernews._title_matches_query(title, query) is True + + +def test_title_matches_query_no_token_in_title(): + """If no query token appears in the title at all, reject.""" + assert hackernews._title_matches_query("New rust compiler", "AI blockchain") is False + + +def test_title_matches_query_word_boundary_not_substring(): + """Short tokens must match on word boundaries, not as substrings. + + Without word-boundary matching, 'ai' would falsely match 'email', + 'rail', 'artists', etc. + """ + # 'ai' as a substring of 'email' must not match + assert hackernews._title_matches_query("New email service", "ai blockchain") is False + # 'ai' as a whole word does match + assert hackernews._title_matches_query("Cool AI tool launched", "ai blockchain") is True + + +def test_title_matches_query_flattens_hyphens_and_commas(): + """Query tokens split on hyphens/commas the same way search_hackernews + flattens them, so the post-filter stays aligned with what Algolia saw.""" + # query 'ts-bun-node' flattens to ['ts', 'bun', 'node']; title contains 'bun' + assert hackernews._title_matches_query("Bun 1.2 released", "ts-bun-node") is True + # query 'rust, go, zig' flattens; title contains 'go' + assert hackernews._title_matches_query("Go 1.24 generics update", "rust, go, zig") is True # === Tests for search_hackernews() ===