From a7d6ef051a8c095743b3a5ce350721529d8d0a5f Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 19 Apr 2026 10:28:36 -0700 Subject: [PATCH] fix: expand entity-grounding haystack to transcripts + top comments PR #285's entity grounding checked only title + snippet. That missed: - YouTube videos where the entity is mentioned in transcript but not in title (false demotion of on-topic content) - Reddit posts where the entity is in top comments but not in title (false demotion of on-topic discussion) And it also wasn't strong enough to reliably demote items like the 2026-04-19 Nate Herk "Managed Agents" video - which had no Hermes anywhere - because the -25 penalty on rerank_score composed to only -15 on final_score via the 0.60 weight, and engagement bonus partially offset that. Two fixes: 1. _candidate_haystack() now joins title + snippet + metadata[transcript_snippet] + metadata[transcript_highlights] + metadata[top_comments][*].excerpt/text + metadata[comment_insights]. Catches entity mentions wherever they actually live. Guarded with isinstance checks so malformed metadata doesn't raise. 2. ENTITY_MISS_FINAL_PENALTY (20.0) applied directly in _final_score when candidate.explanation contains "entity-miss". This lands the full penalty weight on the composite signal that cluster-scoring consumes, instead of being diluted by the rerank_score weight. Combined effect: entity-miss gap grows from ~15 to ~35 points. Tests: 8 new scenarios covering transcript match, transcript highlight match, top-comment match, comment-insight match, empty-text skip, no-primary-entity no-op, and the dual-penalty composition check. --- scripts/lib/rerank.py | 65 ++++++++++++++++++-- tests/test_rerank_v3.py | 128 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 6 deletions(-) diff --git a/scripts/lib/rerank.py b/scripts/lib/rerank.py index 9954613..fa07fd3 100644 --- a/scripts/lib/rerank.py +++ b/scripts/lib/rerank.py @@ -214,6 +214,39 @@ def _apply_fallback_scores(candidates: list[schema.Candidate], *, primary_entity candidate.final_score = _final_score(candidate) +def _candidate_haystack(candidate: schema.Candidate) -> str: + """Build the lowercase text blob against which entity-grounding is checked. + + Expanded 2026-04-19 to include transcript snippets, transcript highlights, + and top-comment text. The prior `title + snippet` check missed YouTube + videos whose entity mentions live in transcript content and Reddit posts + whose mentions are in top comments. Now checks all text surfaces a human + would see. + """ + parts: list[str] = [candidate.title or "", candidate.snippet or ""] + metadata = candidate.metadata or {} + + transcript_snippet = metadata.get("transcript_snippet") or "" + if isinstance(transcript_snippet, str): + parts.append(transcript_snippet) + + for hl in metadata.get("transcript_highlights") or []: + if isinstance(hl, str): + parts.append(hl) + + for tc in metadata.get("top_comments") or []: + if isinstance(tc, dict): + parts.append(str(tc.get("excerpt", "") or tc.get("text", "") or "")) + elif isinstance(tc, str): + parts.append(tc) + + for insight in metadata.get("comment_insights") or []: + if isinstance(insight, str): + parts.append(insight) + + return " ".join(parts).lower() + + def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]: score = ( (candidate.local_relevance * 100.0 * 0.7) @@ -222,12 +255,16 @@ def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> ) reason = "fallback-local-score" # Entity-grounding demotion: if the primary entity (topic minus intent - # modifier) is not present in the candidate's title or snippet, subtract - # ENTITY_MISS_PENALTY. Skip for candidates with no text at all (e.g., - # image-only TikToks) to avoid penalizing thin-text sources unfairly. - if primary_entity and (candidate.title or candidate.snippet): - haystack = f"{candidate.title} {candidate.snippet}".lower() - if primary_entity.lower() not in haystack: + # modifier) is not present anywhere in the candidate's text surfaces + # (title, snippet, transcript, transcript highlights, top comments, + # insights), subtract ENTITY_MISS_PENALTY. Skip for candidates with + # NO text anywhere (e.g., image-only TikToks) to avoid penalizing + # thin-text sources unfairly. 2026-04-19 Nate Herk "Managed Agents" + # video ranked #2 on a Hermes query despite zero Hermes mentions + # because the old haystack only checked title + snippet. + if primary_entity: + haystack = _candidate_haystack(candidate) + if haystack.strip() and primary_entity.lower() not in haystack: score -= ENTITY_MISS_PENALTY reason = "fallback-local-score (entity-miss demotion)" return max(0.0, min(100.0, score)), reason @@ -247,6 +284,17 @@ def _primary_entity(topic: str) -> str: return stripped +#: Secondary entity-miss penalty applied directly to final_score (not just +#: rerank_score). The -25 on rerank_score composes to only -15 on final_score +#: via the 0.60 weight, which engagement bonus partially offsets on +#: high-view YouTube items. This secondary penalty lands the full weight on +#: the composite signal the cluster-scoring layer consumes. 2026-04-19 +#: Nate Herk "Managed Agents" video ranked at cluster #2 with score 51 +#: despite the rerank_score demotion because engagement + freshness drowned +#: the dilute penalty. This backstop makes the demotion actually decisive. +ENTITY_MISS_FINAL_PENALTY = 20.0 + + def _final_score(candidate: schema.Candidate) -> float: normalized_rrf = _normalized_rrf(candidate.rrf_score) rerank_score = candidate.rerank_score or 0.0 @@ -265,6 +313,11 @@ def _final_score(candidate: schema.Candidate) -> float: ) if candidate.rerank_score is not None and candidate.rerank_score < 20.0: base *= 0.3 + # Secondary entity-grounding penalty: when the fallback path flagged + # entity-miss via candidate.explanation, apply an additional penalty + # at final_score level so engagement signal can't mask the demotion. + if candidate.explanation and "entity-miss" in candidate.explanation: + base = max(0.0, base - ENTITY_MISS_FINAL_PENALTY) return base diff --git a/tests/test_rerank_v3.py b/tests/test_rerank_v3.py index 861575e..852c262 100644 --- a/tests/test_rerank_v3.py +++ b/tests/test_rerank_v3.py @@ -256,5 +256,133 @@ class EntityGroundingTests(unittest.TestCase): self.assertNotIn("Primary entity grounding", prompt) +class ExpandedHaystackTests(unittest.TestCase): + """Unit 3: Entity-grounding haystack covers transcript snippets, + transcript highlights, top comments, and comment insights - not + just title + snippet. + """ + + def _youtube_candidate(self, title: str, transcript_snippet: str = "", + transcript_highlights: list[str] | None = None) -> schema.Candidate: + c = schema.Candidate( + candidate_id=f"c-{title[:10]}", + item_id="i1", + source="youtube", + title=title, + url="https://youtube.com/watch?v=x", + snippet="", + subquery_labels=["primary"], + native_ranks={"primary:youtube": 1}, + local_relevance=0.8, + freshness=80, + engagement=50, + source_quality=0.7, + rrf_score=0.02, + ) + c.metadata = {} + if transcript_snippet: + c.metadata["transcript_snippet"] = transcript_snippet + if transcript_highlights: + c.metadata["transcript_highlights"] = transcript_highlights + return c + + def test_entity_found_in_transcript_snippet_avoids_demotion(self): + # Title + snippet miss the entity, but the transcript contains it. + c = self._youtube_candidate( + "Weekly roundup", + transcript_snippet="In this video I walk through using Hermes Agent in production.", + ) + rerank._apply_fallback_scores([c], primary_entity="Hermes Agent") + self.assertEqual("fallback-local-score", c.explanation) + + def test_entity_found_in_transcript_highlights_avoids_demotion(self): + c = self._youtube_candidate( + "Some review", + transcript_highlights=[ + "Today we're talking about Hermes Agent", + "Let's compare it to the alternatives", + ], + ) + rerank._apply_fallback_scores([c], primary_entity="Hermes Agent") + self.assertEqual("fallback-local-score", c.explanation) + + def test_entity_missing_everywhere_still_demoted_for_video(self): + # Nate Herk "Managed Agents" case: no Hermes in title, snippet, + # or transcript - demotion fires. + c = self._youtube_candidate( + "I Tested Claude's New Managed Agents", + transcript_snippet="Managed agents are Anthropic's new product with ClickUp and cron...", + ) + rerank._apply_fallback_scores([c], primary_entity="Hermes Agent") + self.assertIn("entity-miss", c.explanation) + + def test_entity_found_in_reddit_top_comments_avoids_demotion(self): + c = schema.Candidate( + candidate_id="r1", + item_id="i1", + source="reddit", + title="Best agent framework?", + url="https://reddit.com/r/x", + snippet="", + subquery_labels=["primary"], + native_ranks={"primary:reddit": 1}, + local_relevance=0.8, freshness=80, engagement=50, + source_quality=0.7, rrf_score=0.02, + ) + c.metadata = { + "top_comments": [ + {"excerpt": "I've been using Hermes Agent for a month and it's great"}, + {"text": "another comment"}, + ], + } + rerank._apply_fallback_scores([c], primary_entity="Hermes Agent") + self.assertEqual("fallback-local-score", c.explanation) + + def test_entity_found_in_comment_insights_avoids_demotion(self): + c = schema.Candidate( + candidate_id="r2", item_id="i1", source="reddit", + title="AI tools", url="https://reddit.com/r/x", snippet="", + subquery_labels=["primary"], + native_ranks={"primary:reddit": 1}, + local_relevance=0.8, freshness=80, engagement=50, + source_quality=0.7, rrf_score=0.02, + ) + c.metadata = { + "comment_insights": ["Consensus: Hermes Agent handles long sessions best"], + } + rerank._apply_fallback_scores([c], primary_entity="Hermes Agent") + self.assertEqual("fallback-local-score", c.explanation) + + def test_truly_empty_candidate_still_skipped(self): + # Image-only TikTok with no text anywhere - do not penalize. + c = self._youtube_candidate("") # empty title + rerank._apply_fallback_scores([c], primary_entity="Hermes Agent") + self.assertEqual("fallback-local-score", c.explanation) + + def test_final_score_secondary_penalty_applied_on_entity_miss(self): + # When fallback flags entity-miss, final_score gets an ADDITIONAL + # -20 penalty beyond the rerank_score reduction. Verify by + # comparing final_score for a demoted candidate vs an identical + # candidate that matched the entity. + off_topic = self._youtube_candidate("Managed Agents from Anthropic") + on_topic = self._youtube_candidate( + "Hermes Agent walkthrough", + transcript_snippet="Hermes Agent review", + ) + rerank._apply_fallback_scores([off_topic, on_topic], primary_entity="Hermes Agent") + # Gap should be well above the rerank_score-only path's 0.60 * 25 = 15; + # with the secondary penalty it's 15 + 20 = 35 points. + gap = on_topic.final_score - off_topic.final_score + self.assertGreater(gap, 25.0, + f"entity-miss demotion gap only {gap:.1f}; secondary penalty may not be firing") + + def test_secondary_penalty_not_applied_when_entity_match(self): + on_topic = self._youtube_candidate("Hermes Agent: use cases") + rerank._apply_fallback_scores([on_topic], primary_entity="Hermes Agent") + # Explanation does NOT contain entity-miss, so secondary penalty + # should not fire; final_score reflects only base signal. + self.assertNotIn("entity-miss", on_topic.explanation or "") + + if __name__ == "__main__": unittest.main()