From 6a92f63a56baee209715786e9fbc6da45f08f291 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Tue, 9 Jun 2026 16:24:16 -0700 Subject: [PATCH] fix(rerank): ground entity-miss demotion on head token, not full phrase The entity-grounding demotion required the full multi-word primary entity as a contiguous substring, so on-entity items missing a trailing search descriptor were buried: a 323-pt HN thread "Stripe is friendly to 'friendly fraud'" scored 0 on a "Stripe payments" query. New _entity_grounded helper keys on the brand head token; items that never name the brand still miss it and stay demoted. reddit_keyless _slot_priority, which had re-implemented the old check while claiming to mirror rerank's signal, now calls the shared helper so the two paths cannot diverge. --- .../last30days/scripts/lib/reddit_keyless.py | 10 +++-- skills/last30days/scripts/lib/rerank.py | 37 +++++++++++++++---- tests/test_reddit_keyless.py | 24 +++++++----- tests/test_rerank_v3.py | 22 +++++++++++ 4 files changed, 71 insertions(+), 22 deletions(-) diff --git a/skills/last30days/scripts/lib/reddit_keyless.py b/skills/last30days/scripts/lib/reddit_keyless.py index 9879c10..e8daa86 100644 --- a/skills/last30days/scripts/lib/reddit_keyless.py +++ b/skills/last30days/scripts/lib/reddit_keyless.py @@ -170,9 +170,11 @@ def _slot_priority(topic: str, posts: List[Dict[str, Any]]) -> List[Dict[str, An posts that rerank later demotes as entity misses starves the on-topic posts the user actually sees (2026-06-06 "OpenClaw vs Hermes" run: 2,000+ upvote Gemma/GPU threads took every slot, then were demoted to - zero). Mirror rerank's demotion signal — the topic's stripped primary - entity contained in the post text — so slots go to posts likely to - survive final ranking. Falls back to token-overlap relevance when the + zero). Mirror rerank's demotion signal via the shared `_entity_grounded` + check (head token of the topic's stripped primary entity present in the + post text) so slots go to posts likely to survive final ranking — keying + on the same head token keeps the two paths from diverging. Falls back to + token-overlap relevance when the topic yields no usable primary entity. Within each tier the incoming (score-first) order is preserved. Never raises; on any failure the incoming order is returned unchanged. @@ -186,7 +188,7 @@ def _slot_priority(topic: str, posts: List[Dict[str, Any]]) -> List[Dict[str, An entity = rerank._primary_entity(topic).lower() if entity: def _matches(post: Dict[str, Any]) -> bool: - return entity in _post_text(post).lower() + return rerank._entity_grounded(_post_text(post).lower(), entity) else: prepared = relevance.PreparedQuery(topic) diff --git a/skills/last30days/scripts/lib/rerank.py b/skills/last30days/scripts/lib/rerank.py index fa07fd3..2a39c71 100644 --- a/skills/last30days/scripts/lib/rerank.py +++ b/skills/last30days/scripts/lib/rerank.py @@ -247,6 +247,29 @@ def _candidate_haystack(candidate: schema.Candidate) -> str: return " ".join(parts).lower() +def _entity_grounded(haystack: str, primary_entity: str) -> bool: + """True if the candidate text plausibly mentions the primary entity. + + Grounds on the HEAD token of the primary entity (the brand / proper-noun + core), not the full multi-word phrase. Trailing tokens are usually category + descriptors the user/planner appended for search ("Stripe payments"), not + part of the entity, so requiring the whole phrase over-demotes on-entity + items that omit the descriptor. Items that never name the brand at all still + miss the head token and stay demoted. + + Trade-off: a proper noun with a generic head ("New York Times" -> "new") + under-demotes rather than over-demotes - the safe direction, since the + observed harm was burying real high-engagement signal. Substring (not + word-boundary) matching is likewise deliberate: it catches plurals and + compounds ("stripes"), and vacuous matches from very short heads ("X", + "Go") merely disable the penalty rather than burying good items. + """ + tokens = primary_entity.lower().split() + if not tokens: + return True + return tokens[0] in haystack + + def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]: score = ( (candidate.local_relevance * 100.0 * 0.7) @@ -254,17 +277,15 @@ def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> + (candidate.source_quality * 100.0 * 0.1) ) reason = "fallback-local-score" - # Entity-grounding demotion: if the primary entity (topic minus intent - # modifier) is not present anywhere in the candidate's text surfaces + # Entity-grounding demotion: subtract ENTITY_MISS_PENALTY when the candidate + # never mentions the primary entity's head token, across all text surfaces # (title, snippet, transcript, transcript highlights, top comments, - # insights), subtract ENTITY_MISS_PENALTY. Skip for candidates with - # NO text anywhere (e.g., image-only TikToks) to avoid penalizing - # thin-text sources unfairly. 2026-04-19 Nate Herk "Managed Agents" - # video ranked #2 on a Hermes query despite zero Hermes mentions - # because the old haystack only checked title + snippet. + # insights). Skip for candidates with NO text anywhere (e.g. image-only + # TikToks) so thin-text sources aren't penalized unfairly. See + # _entity_grounded for why grounding keys on the head token, not the phrase. if primary_entity: haystack = _candidate_haystack(candidate) - if haystack.strip() and primary_entity.lower() not in haystack: + if haystack.strip() and not _entity_grounded(haystack, primary_entity): score -= ENTITY_MISS_PENALTY reason = "fallback-local-score (entity-miss demotion)" return max(0.0, min(100.0, score)), reason diff --git a/tests/test_reddit_keyless.py b/tests/test_reddit_keyless.py index 0a36155..1ffb52b 100644 --- a/tests/test_reddit_keyless.py +++ b/tests/test_reddit_keyless.py @@ -204,18 +204,22 @@ class TestSlotPriority: assert posts[4]["url"] in enriched_urls assert len(enriched_urls) == reddit_keyless.ENRICH_LIMITS["quick"] - def test_multiword_topic_uses_substring_not_token_overlap(self): - # "claude tips" clears token overlap for "Claude Code" but rerank - # demotes it; the partition must mirror rerank's substring test. - token_only = self._titled(1, "claude tips", score=500) - full_entity = self._titled(2, "Claude Code best setup", score=5) - out = reddit_keyless._slot_priority("Claude Code", [token_only, full_entity]) - assert out[0] is full_entity - assert out[1] is token_only + def test_slot_priority_grounds_on_head_token_not_full_phrase(self): + # Mirrors rerank's head-token grounding: a post naming the brand head + # ("Stripe") lands in the match tier even without the trailing search + # descriptor ("payments"), so it is not buried under an unrelated + # high-upvote post that never names the brand. + head_only = self._titled(1, "Stripe is friendly to 'friendly fraud'", score=5) + off_topic = self._titled(2, "PayPal raises dispute fees again", score=900) + out = reddit_keyless._slot_priority("Stripe payments", [off_topic, head_only]) + assert out[0] is head_only + assert out[1] is off_topic - def test_intent_modifier_stripped_from_topic(self): + def test_intent_modifier_topic_prioritizes_head_token_match(self): + # Intent-modifier topics still partition by the brand head token: the + # on-entity post wins over a high-upvote post that never names the brand. on_topic = self._titled(1, "Hermes Agent v0.13 is great", score=1) - off_topic = self._titled(2, "Hermes Birkin unboxing", score=900) + off_topic = self._titled(2, "LangGraph tutorial walkthrough", score=900) out = reddit_keyless._slot_priority("Hermes Agent review", [off_topic, on_topic]) assert out[0] is on_topic diff --git a/tests/test_rerank_v3.py b/tests/test_rerank_v3.py index ccb8efa..8a02032 100644 --- a/tests/test_rerank_v3.py +++ b/tests/test_rerank_v3.py @@ -221,6 +221,28 @@ class EntityGroundingTests(unittest.TestCase): self.assertIn("entity-miss", off_topic.explanation or "") self.assertEqual(on_topic.explanation, "fallback-local-score") + def test_fallback_grounds_on_head_token_not_full_phrase(self): + # Regression: a 323-pt HN thread titled "Stripe is friendly to + # 'friendly fraud'" was demoted to score 0 on a "Stripe payments" + # query because it lacked the trailing word "payments". The brand + # token alone must ground the item - trailing descriptors are search + # hints, not part of the entity. + brand_only = self._candidate( + "Stripe is friendly to 'friendly fraud'", "discussion of chargebacks and disputes" + ) + rerank._apply_fallback_scores([brand_only], primary_entity="Stripe payments") + self.assertEqual("fallback-local-score", brand_only.explanation) + self.assertNotIn("entity-miss", brand_only.explanation or "") + + def test_fallback_still_demotes_when_head_token_absent_on_multiword_topic(self): + # The fix must not neuter the demotion: an item that never names the + # brand head token stays demoted even on a multi-word topic. + off_topic = self._candidate( + "PayPal raises dispute fees again", "merchants react to the new pricing" + ) + rerank._apply_fallback_scores([off_topic], primary_entity="Stripe payments") + self.assertIn("entity-miss", off_topic.explanation or "") + def test_fallback_match_is_case_insensitive(self): on_topic = self._candidate("HERMES agent rocks", "some text") rerank._apply_fallback_scores([on_topic], primary_entity="Hermes Agent")