fix: expand entity-grounding haystack to transcripts + top comments

PR #285's entity grounding checked only title + snippet. That missed:

- YouTube videos where the entity is mentioned in transcript but not
  in title (false demotion of on-topic content)
- Reddit posts where the entity is in top comments but not in title
  (false demotion of on-topic discussion)

And it also wasn't strong enough to reliably demote items like the
2026-04-19 Nate Herk "Managed Agents" video - which had no Hermes
anywhere - because the -25 penalty on rerank_score composed to only
-15 on final_score via the 0.60 weight, and engagement bonus partially
offset that.

Two fixes:

1. _candidate_haystack() now joins title + snippet +
   metadata[transcript_snippet] + metadata[transcript_highlights] +
   metadata[top_comments][*].excerpt/text + metadata[comment_insights].
   Catches entity mentions wherever they actually live. Guarded with
   isinstance checks so malformed metadata doesn't raise.
2. ENTITY_MISS_FINAL_PENALTY (20.0) applied directly in _final_score
   when candidate.explanation contains "entity-miss". This lands the
   full penalty weight on the composite signal that cluster-scoring
   consumes, instead of being diluted by the rerank_score weight.
   Combined effect: entity-miss gap grows from ~15 to ~35 points.

Tests: 8 new scenarios covering transcript match, transcript highlight
match, top-comment match, comment-insight match, empty-text skip,
no-primary-entity no-op, and the dual-penalty composition check.

scripts/lib/rerank.py

@@ -214,6 +214,39 @@ def _apply_fallback_scores(candidates: list[schema.Candidate], *, primary_entity
         candidate.final_score = _final_score(candidate)
 
 
+def _candidate_haystack(candidate: schema.Candidate) -> str:
+    """Build the lowercase text blob against which entity-grounding is checked.
+
+    Expanded 2026-04-19 to include transcript snippets, transcript highlights,
+    and top-comment text. The prior `title + snippet` check missed YouTube
+    videos whose entity mentions live in transcript content and Reddit posts
+    whose mentions are in top comments. Now checks all text surfaces a human
+    would see.
+    """
+    parts: list[str] = [candidate.title or "", candidate.snippet or ""]
+    metadata = candidate.metadata or {}
+
+    transcript_snippet = metadata.get("transcript_snippet") or ""
+    if isinstance(transcript_snippet, str):
+        parts.append(transcript_snippet)
+
+    for hl in metadata.get("transcript_highlights") or []:
+        if isinstance(hl, str):
+            parts.append(hl)
+
+    for tc in metadata.get("top_comments") or []:
+        if isinstance(tc, dict):
+            parts.append(str(tc.get("excerpt", "") or tc.get("text", "") or ""))
+        elif isinstance(tc, str):
+            parts.append(tc)
+
+    for insight in metadata.get("comment_insights") or []:
+        if isinstance(insight, str):
+            parts.append(insight)
+
+    return " ".join(parts).lower()
+
+
 def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]:
     score = (
         (candidate.local_relevance * 100.0 * 0.7)
@@ -222,12 +255,16 @@ def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") ->
     )
     reason = "fallback-local-score"
     # Entity-grounding demotion: if the primary entity (topic minus intent
-    # modifier) is not present in the candidate's title or snippet, subtract
-    # ENTITY_MISS_PENALTY. Skip for candidates with no text at all (e.g.,
-    # image-only TikToks) to avoid penalizing thin-text sources unfairly.
-    if primary_entity and (candidate.title or candidate.snippet):
-        haystack = f"{candidate.title} {candidate.snippet}".lower()
-        if primary_entity.lower() not in haystack:
+    # modifier) is not present anywhere in the candidate's text surfaces
+    # (title, snippet, transcript, transcript highlights, top comments,
+    # insights), subtract ENTITY_MISS_PENALTY. Skip for candidates with
+    # NO text anywhere (e.g., image-only TikToks) to avoid penalizing
+    # thin-text sources unfairly. 2026-04-19 Nate Herk "Managed Agents"
+    # video ranked #2 on a Hermes query despite zero Hermes mentions
+    # because the old haystack only checked title + snippet.
+    if primary_entity:
+        haystack = _candidate_haystack(candidate)
+        if haystack.strip() and primary_entity.lower() not in haystack:
             score -= ENTITY_MISS_PENALTY
             reason = "fallback-local-score (entity-miss demotion)"
     return max(0.0, min(100.0, score)), reason
@@ -247,6 +284,17 @@ def _primary_entity(topic: str) -> str:
     return stripped
 
 
+#: Secondary entity-miss penalty applied directly to final_score (not just
+#: rerank_score). The -25 on rerank_score composes to only -15 on final_score
+#: via the 0.60 weight, which engagement bonus partially offsets on
+#: high-view YouTube items. This secondary penalty lands the full weight on
+#: the composite signal the cluster-scoring layer consumes. 2026-04-19
+#: Nate Herk "Managed Agents" video ranked at cluster #2 with score 51
+#: despite the rerank_score demotion because engagement + freshness drowned
+#: the dilute penalty. This backstop makes the demotion actually decisive.
+ENTITY_MISS_FINAL_PENALTY = 20.0
+
+
 def _final_score(candidate: schema.Candidate) -> float:
     normalized_rrf = _normalized_rrf(candidate.rrf_score)
     rerank_score = candidate.rerank_score or 0.0
@@ -265,6 +313,11 @@ def _final_score(candidate: schema.Candidate) -> float:
     )
     if candidate.rerank_score is not None and candidate.rerank_score < 20.0:
         base *= 0.3
+    # Secondary entity-grounding penalty: when the fallback path flagged
+    # entity-miss via candidate.explanation, apply an additional penalty
+    # at final_score level so engagement signal can't mask the demotion.
+    if candidate.explanation and "entity-miss" in candidate.explanation:
+        base = max(0.0, base - ENTITY_MISS_FINAL_PENALTY)
     return base
 
 

tests/test_rerank_v3.py

@@ -256,5 +256,133 @@ class EntityGroundingTests(unittest.TestCase):
         self.assertNotIn("Primary entity grounding", prompt)
 
 
+class ExpandedHaystackTests(unittest.TestCase):
+    """Unit 3: Entity-grounding haystack covers transcript snippets,
+    transcript highlights, top comments, and comment insights - not
+    just title + snippet.
+    """
+
+    def _youtube_candidate(self, title: str, transcript_snippet: str = "",
+                           transcript_highlights: list[str] | None = None) -> schema.Candidate:
+        c = schema.Candidate(
+            candidate_id=f"c-{title[:10]}",
+            item_id="i1",
+            source="youtube",
+            title=title,
+            url="https://youtube.com/watch?v=x",
+            snippet="",
+            subquery_labels=["primary"],
+            native_ranks={"primary:youtube": 1},
+            local_relevance=0.8,
+            freshness=80,
+            engagement=50,
+            source_quality=0.7,
+            rrf_score=0.02,
+        )
+        c.metadata = {}
+        if transcript_snippet:
+            c.metadata["transcript_snippet"] = transcript_snippet
+        if transcript_highlights:
+            c.metadata["transcript_highlights"] = transcript_highlights
+        return c
+
+    def test_entity_found_in_transcript_snippet_avoids_demotion(self):
+        # Title + snippet miss the entity, but the transcript contains it.
+        c = self._youtube_candidate(
+            "Weekly roundup",
+            transcript_snippet="In this video I walk through using Hermes Agent in production.",
+        )
+        rerank._apply_fallback_scores([c], primary_entity="Hermes Agent")
+        self.assertEqual("fallback-local-score", c.explanation)
+
+    def test_entity_found_in_transcript_highlights_avoids_demotion(self):
+        c = self._youtube_candidate(
+            "Some review",
+            transcript_highlights=[
+                "Today we're talking about Hermes Agent",
+                "Let's compare it to the alternatives",
+            ],
+        )
+        rerank._apply_fallback_scores([c], primary_entity="Hermes Agent")
+        self.assertEqual("fallback-local-score", c.explanation)
+
+    def test_entity_missing_everywhere_still_demoted_for_video(self):
+        # Nate Herk "Managed Agents" case: no Hermes in title, snippet,
+        # or transcript - demotion fires.
+        c = self._youtube_candidate(
+            "I Tested Claude's New Managed Agents",
+            transcript_snippet="Managed agents are Anthropic's new product with ClickUp and cron...",
+        )
+        rerank._apply_fallback_scores([c], primary_entity="Hermes Agent")
+        self.assertIn("entity-miss", c.explanation)
+
+    def test_entity_found_in_reddit_top_comments_avoids_demotion(self):
+        c = schema.Candidate(
+            candidate_id="r1",
+            item_id="i1",
+            source="reddit",
+            title="Best agent framework?",
+            url="https://reddit.com/r/x",
+            snippet="",
+            subquery_labels=["primary"],
+            native_ranks={"primary:reddit": 1},
+            local_relevance=0.8, freshness=80, engagement=50,
+            source_quality=0.7, rrf_score=0.02,
+        )
+        c.metadata = {
+            "top_comments": [
+                {"excerpt": "I've been using Hermes Agent for a month and it's great"},
+                {"text": "another comment"},
+            ],
+        }
+        rerank._apply_fallback_scores([c], primary_entity="Hermes Agent")
+        self.assertEqual("fallback-local-score", c.explanation)
+
+    def test_entity_found_in_comment_insights_avoids_demotion(self):
+        c = schema.Candidate(
+            candidate_id="r2", item_id="i1", source="reddit",
+            title="AI tools", url="https://reddit.com/r/x", snippet="",
+            subquery_labels=["primary"],
+            native_ranks={"primary:reddit": 1},
+            local_relevance=0.8, freshness=80, engagement=50,
+            source_quality=0.7, rrf_score=0.02,
+        )
+        c.metadata = {
+            "comment_insights": ["Consensus: Hermes Agent handles long sessions best"],
+        }
+        rerank._apply_fallback_scores([c], primary_entity="Hermes Agent")
+        self.assertEqual("fallback-local-score", c.explanation)
+
+    def test_truly_empty_candidate_still_skipped(self):
+        # Image-only TikTok with no text anywhere - do not penalize.
+        c = self._youtube_candidate("")  # empty title
+        rerank._apply_fallback_scores([c], primary_entity="Hermes Agent")
+        self.assertEqual("fallback-local-score", c.explanation)
+
+    def test_final_score_secondary_penalty_applied_on_entity_miss(self):
+        # When fallback flags entity-miss, final_score gets an ADDITIONAL
+        # -20 penalty beyond the rerank_score reduction. Verify by
+        # comparing final_score for a demoted candidate vs an identical
+        # candidate that matched the entity.
+        off_topic = self._youtube_candidate("Managed Agents from Anthropic")
+        on_topic = self._youtube_candidate(
+            "Hermes Agent walkthrough",
+            transcript_snippet="Hermes Agent review",
+        )
+        rerank._apply_fallback_scores([off_topic, on_topic], primary_entity="Hermes Agent")
+        # Gap should be well above the rerank_score-only path's 0.60 * 25 = 15;
+        # with the secondary penalty it's 15 + 20 = 35 points.
+        gap = on_topic.final_score - off_topic.final_score
+        self.assertGreater(gap, 25.0,
+            f"entity-miss demotion gap only {gap:.1f}; secondary penalty may not be firing")
+
+    def test_secondary_penalty_not_applied_when_entity_match(self):
+        on_topic = self._youtube_candidate("Hermes Agent: use cases")
+        rerank._apply_fallback_scores([on_topic], primary_entity="Hermes Agent")
+        # Explanation does NOT contain entity-miss, so secondary penalty
+        # should not fire; final_score reflects only base signal.
+        self.assertNotIn("entity-miss", on_topic.explanation or "")
+
+
 if __name__ == "__main__":
     unittest.main()