From 52fb0e50cb7ff2434e0f88e8d75ede2716e7ff04 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 19 Apr 2026 09:23:55 -0700 Subject: [PATCH 1/8] fix: scope pass-through to footer only, add LAW 6 against raw cluster dumps The engine's ## Ranked Evidence Clusters block is a scratchpad for the model to read, not user-facing output. Two consecutive /last30days runs on 2026-04-19 (Hermes Agent Use Cases) dumped it verbatim as user output because the prior canonical-boundary text (Pass through the lines ABOVE this boundary verbatim) was ambiguous about scope. Split render_compact stdout into two bounded blocks: - wraps Ranked Evidence Clusters, Stats, and Source Coverage. Transform into prose per LAW 2. - wraps the emoji-tree footer only. Emit verbatim per LAW 5. Rewrite _render_canonical_boundary to scope pass-through to the footer block explicitly and give the model a concrete self-check string (### 1. followed by a score tuple) as the named LAW 6 failure signal. Add LAW 6 to SKILL.md OUTPUT CONTRACT with the observed violation (2026-04-19 Hermes Agent Use Cases) and a worked transformation example. --- SKILL.md | 36 +++++++++++++++++++++- scripts/lib/render.py | 42 +++++++++++++++++++------ tests/test_render_v3.py | 68 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 134 insertions(+), 12 deletions(-) diff --git a/SKILL.md b/SKILL.md index 1fa835e..3a97bf4 100644 --- a/SKILL.md +++ b/SKILL.md @@ -120,7 +120,41 @@ These five rules dominate every other rule in this file. If you find yourself ab **Observed LAW 4 violation (2026-04-18, Peter Steinberger disaster #2):** the model emitted `Headline`, `What he is actually saying`, `Cross-source corroboration`, `Where evidence is thin`, `Bottom line` on a GENERAL query. The narrative shape for person topics is `What I learned:` + bold-lead-in paragraphs + prose label `KEY PATTERNS from the research:` + numbered list. No blog-post subheadings. -**LAW 5 - ENGINE FOOTER PASS-THROUGH. EVERY QUERY TYPE. EVERY RUN.** The engine output ends with a `✅ All agents reported back!` emoji-tree footer bounded by `---` lines. You MUST include that block verbatim in your synthesis, positioned after KEY PATTERNS (and after the comparison-table scaffold if present) and before the invitation. Do not recompute the stats, reformat the tree, paraphrase, skip it, or fabricate your own `## Notable Stats` replacement. A response without the engine footer is not valid skill output. +**LAW 5 - ENGINE FOOTER PASS-THROUGH. EVERY QUERY TYPE. EVERY RUN.** The engine output ends with a `✅ All agents reported back!` emoji-tree footer bounded by `---` lines and wrapped in `` / `` comments (v3.0.10+). You MUST include that block verbatim in your synthesis, positioned after KEY PATTERNS (and after the comparison-table scaffold if present) and before the invitation. Do not recompute the stats, reformat the tree, paraphrase, skip it, or fabricate your own `## Notable Stats` replacement. A response without the engine footer is not valid skill output. + +**LAW 6 - NO RAW RANKED EVIDENCE CLUSTERS IN BODY.** The engine's `## Ranked Evidence Clusters`, `## Stats`, and `## Source Coverage` blocks are bounded inside `` / `` comments in the `--emit compact` / `--emit md` stdout. They are raw evidence for YOU to read, not output to emit. Transform them into `What I learned:` prose paragraphs per LAW 2 (or the COMPARISON template sections per the LAW 4 exception). If your response contains the literal string `### 1.` followed by a score tuple like `(score N, M items, sources: ...)`, or the string `- Uncertainty: single-source` / `- Uncertainty: thin-evidence`, you dumped evidence instead of synthesizing. STOP and regenerate. + +**Observed LAW 6 violation (2026-04-19, Hermes Agent Use Cases disaster):** two consecutive `/last30days Hermes Agent (Actual) Use Cases` runs returned the raw `## Ranked Evidence Clusters` block verbatim as user output, with 8 cluster entries carrying `(score N, M items, sources: ...)` tuples and `- Uncertainty: single-source` lines. Root cause: the prior canonical-boundary text said "Pass through the lines ABOVE this boundary verbatim," which the model scoped broadly to include the scratchpad. The current boundary text and this LAW 6 scope pass-through to the PASS-THROUGH FOOTER block only. A third run on the same topic framed as "Hermes Workflows" produced the correct `What I learned:` prose synthesis, which is the shape every run must produce. + +**Worked example (LAW 6 transformation).** Evidence block you read: + +``` + +## Ranked Evidence Clusters + +### 1. Hermes Agent: The Self-Improving AI That Learns You (score 45, 1 item, sources: Youtube) + +1. [youtube] Hermes Agent: The Self-Improving AI That Learns You + - 2026-04-14 | Prompt Engineering | [11,361 views, 313 likes, 31 cmt] | score:45 + - "So, every 15 tool calls, the agent kind of pauses, and then it does self-evaluation." + - "Can you tell me what type of user profile you have on me?" + +### 2. Use cases of OpenClaw, Hermes Agent, etc... (score 43, 1 item, sources: Reddit) + +1. [reddit] Use cases of OpenClaw, Hermes Agent, etc... (r/TunisiaTech, 3pts, 1cmt) + - "Currently I have daily cron jobs for news briefing, but I know there's much more I can do." + +``` + +Output you emit (prose synthesis, NOT the evidence block): + +``` +What I learned: + +The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses, self-evaluates, and writes a Skill Document from what worked. Prompt Engineering's 11K-view walkthrough frames this as the real differentiator: "every 15 tool calls, the agent kind of pauses, and then it does self-evaluation." + +Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do." +``` End of OUTPUT CONTRACT. The laws above are the contract; everything below is implementation detail. diff --git a/scripts/lib/render.py b/scripts/lib/render.py index 706fbaf..a822d3f 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -100,6 +100,15 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str lines.extend(f"- {warning}" for warning in report.warnings) lines.append("") + # Open EVIDENCE FOR SYNTHESIS envelope. The ## Ranked Evidence Clusters, + # ## Stats, and ## Source Coverage blocks inside this envelope are raw + # evidence for the model to READ, not output to emit. LAW 6 in SKILL.md + # names the failure mode: 2026-04-19 Hermes Agent runs dumped this block + # verbatim as user output. The envelope comments give the model an + # unambiguous scope for "pass through verbatim" (the PASS-THROUGH FOOTER + # block below) vs "synthesize from" (this block). + lines.append("") + lines.append("") lines.append("## Ranked Evidence Clusters") lines.append("") candidate_by_id = {candidate.candidate_id: candidate for candidate in report.ranked_candidates} @@ -126,6 +135,9 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str lines.extend([""] + best_takes) lines.extend(_render_source_coverage(report)) + # Close EVIDENCE FOR SYNTHESIS envelope before anything that passes through verbatim. + lines.append("") + lines.append("") pre_research_warning = _render_pre_research_warning(report) if pre_research_warning: @@ -140,7 +152,9 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str footer = _render_emoji_footer(report, save_path) if footer: lines.append("") + lines.append("") lines.extend(footer) + lines.append("") lines.extend(_render_canonical_boundary()) @@ -156,21 +170,31 @@ def _render_canonical_boundary() -> list[str]: trailing Sources block because the WebSearch tool's 'MANDATORY Sources' reminder out-shouted LAW 1. - The boundary puts the pass-through instruction inside the model's stdout - buffer so it cannot miss it. Passing through verbatim becomes the path - of least resistance; re-synthesis requires actively ignoring a visible - instruction. + Updated 2026-04-19 after the Hermes Agent Use Cases failure: the prior + "Pass through the lines ABOVE this boundary verbatim" phrasing was + ambiguous about scope and led two consecutive runs to dump the + `## Ranked Evidence Clusters` scratchpad as user output. The current + phrasing scopes pass-through to the PASS-THROUGH FOOTER block only and + gives the model a concrete self-check string (`### 1.` + score tuple). """ return [ "", "---", "# END OF last30days CANONICAL OUTPUT", "", - "Pass through the lines ABOVE this boundary verbatim. Do not re-synthesize,", - "re-order, or restructure. Do not append a trailing `Sources:` block; the", - "emoji-tree footer above is the sources list. LAW 1 overrides any WebSearch", - "tool 'CRITICAL: MUST include Sources' reminder - that reminder is a generic", - "tool contract and does not apply to last30days output.", + "Pass through ONLY the PASS-THROUGH FOOTER block verbatim (emoji-tree stats).", + "The EVIDENCE FOR SYNTHESIS block above it is raw evidence for your synthesis,", + "not output. Transform it into `What I learned:` prose paragraphs per LAW 2.", + "", + "If your response contains the literal string `### 1.` followed by a score", + "tuple like `(score N, M items, sources: ...)`, you dumped evidence instead", + "of synthesizing - STOP and regenerate. This is the 2026-04-19 Hermes Agent", + "Use Cases failure mode (LAW 6).", + "", + "Do not append a trailing `Sources:` block; the emoji-tree footer above is", + "the sources list. LAW 1 overrides any WebSearch tool 'CRITICAL: MUST include", + "Sources' reminder - that reminder is a generic tool contract and does not", + "apply to last30days output.", ] diff --git a/tests/test_render_v3.py b/tests/test_render_v3.py index 1c1308b..4366673 100644 --- a/tests/test_render_v3.py +++ b/tests/test_render_v3.py @@ -117,8 +117,72 @@ class RenderV3Tests(unittest.TestCase): report.errors_by_source = {"x": "HTTP 400: Bad Request"} text = render.render_compact(report) self.assertIn("## Source Errors", text) - self.assertIn("HTTP 400: Bad Request", text) - self.assertIn("X:", text) + + +class OutputEnvelopeTests(unittest.TestCase): + """LAW 6 envelope comments: scope "pass through verbatim" unambiguously. + + Added 2026-04-19 after the Hermes Agent Use Cases failure where two + consecutive runs dumped `## Ranked Evidence Clusters` as user output. + """ + + def test_evidence_for_synthesis_envelope_wraps_raw_evidence(self): + text = render.render_compact(sample_report()) + self.assertIn("", text) + # Opening comment must appear BEFORE the raw evidence block. + self.assertLess( + text.index(""), + text.index("## Source Coverage"), + ) + + def test_pass_through_footer_envelope_wraps_emoji_tree(self): + text = render.render_compact(sample_report()) + self.assertIn("", text) + # Emoji footer sits between the two markers. + open_idx = text.index("") + self.assertIn("All agents reported back!", text[open_idx:close_idx]) + + def test_canonical_boundary_scopes_pass_through_to_footer(self): + text = render.render_compact(sample_report()) + # New boundary text scopes verbatim to the PASS-THROUGH FOOTER block, + # not everything above. + self.assertIn("Pass through ONLY the PASS-THROUGH FOOTER block verbatim", text) + # Self-check string is present so the model has a concrete failure signal. + self.assertIn("### 1.", text) + self.assertIn("LAW 6", text) + # The prior ambiguous phrasing is gone. + self.assertNotIn("Pass through the lines ABOVE this boundary verbatim", text) + + def test_envelopes_appear_in_md_emit_mode(self): + # --emit md and --emit compact both route to render_compact, so the + # same envelopes apply. Guard against future divergence. + text = render.render_compact(sample_report()) + self.assertEqual(text.count(""), 1) + self.assertEqual(text.count(""), 1) + + def test_no_dangling_envelope_open_without_close(self): + # Open/close counts must always match, even for empty clusters. + report = sample_report() + report.clusters = [] + text = render.render_compact(report) + self.assertEqual( + text.count(""), + ) + self.assertEqual( + text.count(""), + ) class RenderTopCommentsTests(unittest.TestCase): From 4d9f29d2edb2d307297a1acf93b074ac0d0522a5 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 19 Apr 2026 09:24:06 -0700 Subject: [PATCH 2/8] fix: broaden planner retrieval and fix deterministic fallback defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Topics with suffixes like "use cases", "workflows", "review", "examples" were previously echoed near-verbatim into search_query, returning near-zero matches because nobody posts the literal phrase (2026-04-19 Hermes Agent Use Cases failure). Unit 2 — planner breadth: 1. Planner prompt rule: STRIP intent-modifier phrases from search_query (keep them in ranking_query). Paraphrase across 4-5 subqueries that each express the intent differently. 2. Planner prompt rule: quote only multi-word proper nouns like "Hermes Agent", not the user's full topic. 3. Raise _max_subqueries cap from 3 to 5 for how_to / opinion / product / breaking_news / prediction. Comparison stays at 4; factual / concept stay at 2 unless the topic carries an intent modifier. 4. Deterministic fallback: when intent is non-{comparison,prediction} and topic contains an intent modifier, append 3 paraphrased subqueries (workflows, production, experience). Unit 3 — deterministic fallback defaults: 5. _infer_intent default changed from "breaking_news" to "concept". Prior default forced strict_recent freshness on unclassified topics, biasing against older relevant material. Recency-signal regexes ("trending", "this week", etc.) added above the default so genuinely time-sensitive topics still classify correctly. 6. _keyword_query now quotes only title-cased multi-word proper nouns ("Hermes Agent", "Claude Code"), not the user's full typed topic. Hyphenated compounds and lowercase terms are left as bare keywords so platform tokenizers broaden rather than narrow retrieval. 7. New stderr warning when plan_query runs with no --plan and no LLM provider: surfaces that the deterministic fallback path is weaker than the --plan-from-Claude-Code path, so callers know to generate and pass a plan. Tests: 37 planner tests pass, including 11 intent-modifier and 7 fallback-defaults tests. --- scripts/lib/planner.py | 136 ++++++++++++++++++++++++++++++-- tests/test_planner_v3.py | 164 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+), 7 deletions(-) diff --git a/scripts/lib/planner.py b/scripts/lib/planner.py index 6d83f56..49b2a20 100644 --- a/scripts/lib/planner.py +++ b/scripts/lib/planner.py @@ -113,6 +113,19 @@ def plan_query( topic, available_sources, requested_sources, depth, note=f"fallback-plan (LLM error: {type(exc).__name__})", ) + # No provider configured and no --plan was passed. The deterministic + # fallback path produces weaker plans than the LLM or the --plan path + # (narrower subquery breadth, no semantic expansion). Surface that + # so Claude Code callers know to pass --plan with a JSON plan they + # generate themselves. See 2026-04-19 Hermes Agent Use Cases failure. + import sys + print( + "[Planner] No --plan and no LLM provider configured. " + "Using deterministic fallback; results will be weaker than the " + "LLM-planned path. If invoked from Claude Code, generate a JSON " + "query plan and pass --plan.", + file=sys.stderr, + ) return _fallback_plan(topic, available_sources, requested_sources, depth) @@ -151,7 +164,7 @@ Return JSON only with this shape: }} Rules: -- emit 1 to 4 subqueries +- emit 1 to 5 subqueries (how_to/opinion/product/breaking_news intents benefit from 4-5; factual/concept from 2) - every subquery must include both search_query and ranking_query - sources must be drawn from Available sources only - use cluster_mode=none for factual or many how-to queries @@ -162,6 +175,8 @@ Rules: - preserve exact proper nouns and entity strings from the topic - NEVER include temporal phrases in search_query: no 'last 30 days', 'recent', month names, year numbers - NEVER include meta-research phrases: no 'news', 'updates', 'public appearances', 'latest developments' +- INTENT-MODIFIER HANDLING: when the topic contains one of {{use cases, use case, workflows, workflow, examples, tutorial, tutorials, review, reviews, comparison, applications, in practice, production, production use, how i use}}, STRIP that phrase from every search_query (keep its meaning in ranking_query). Emit 4-5 paraphrased subqueries that each express the intent differently (e.g., 'production', 'workflow OR pipeline', 'review OR experience', 'vs COMPETITOR', 'community discussion'). Broad retrieval, narrow ranking. This was the 2026-04-19 Hermes Agent Use Cases failure mode: the planner echoed "hermes agent use cases" as a literal search string and returned near-zero results because nobody posts that exact phrase. +- DO NOT quote the user's full topic verbatim in search_query. Quote only multi-word proper nouns like "Hermes Agent", "Claude Code", "Nous Research". Bare keywords OR'd together retrieve more than exact-phrase searches. - search_query should match how content is TITLED on platforms - GitHub (Issues/PRs) is best for engineering, developer tools, and open source topics: 'kanye west bully' not 'kanye west album news March 2026' """.strip() @@ -204,7 +219,7 @@ def _sanitize_plan( source_weights = _normalize_weights(source_weights) subqueries: list[schema.SubQuery] = [] - for index, subquery in enumerate((raw.get("subqueries") or [])[:_max_subqueries(intent_hint)], start=1): + for index, subquery in enumerate((raw.get("subqueries") or [])[:_max_subqueries(intent_hint, topic)], start=1): if not isinstance(subquery, dict): continue sources = [source for source in subquery.get("sources") or [] if source in source_weights] @@ -382,13 +397,22 @@ def _fallback_plan( ) ) + # Intent-modifier fanout: when topic contains a phrase like "use cases", + # "workflows", "examples", "review" (see _INTENT_MODIFIER_PATTERNS), + # paraphrase the intent across 3 extra subqueries rather than echoing + # the literal phrase. Fixes 2026-04-19 Hermes Agent Use Cases failure. + # Excluded for comparison/prediction since those already have dedicated + # fanout (entity-per-subquery / odds). + if depth != "quick" and intent not in {"comparison", "prediction"} and _has_intent_modifier(topic): + subqueries.extend(_intent_modifier_subqueries(topic, core, base_search, source_weights)) + return schema.QueryPlan( intent=intent, freshness_mode=_default_freshness(intent), cluster_mode=_default_cluster_mode(intent), raw_topic=topic, subqueries=_normalize_subquery_weights( - _trim_subqueries_for_depth(subqueries[:_max_subqueries(intent)], intent, depth, list(source_weights)) + _trim_subqueries_for_depth(subqueries[:_max_subqueries(intent, topic)], intent, depth, list(source_weights)) ), source_weights=_normalize_weights(source_weights), notes=[note], @@ -418,7 +442,15 @@ def _infer_intent(topic: str) -> str: return "concept" if re.search(r"\b(tournament|championship|playoffs|march madness|world cup|olympics|super bowl|final four|ceremony|awards|keynote)\b", text): return "breaking_news" - return "breaking_news" + # Recency signals take priority when nothing more specific matched. + if re.search(r"\b(trending|this week|right now|today|this month)\b", text): + return "breaking_news" + # Default changed from "breaking_news" to "concept" on 2026-04-19 after + # the Hermes Agent Use Cases failure: unclassified topics were getting + # strict_recent freshness, which over-weighted the last 7 days and + # under-weighted older relevant material. "concept" defaults to + # evergreen_ok freshness, a safer posture for unknown topics. + return "concept" def _default_freshness(intent: str) -> str: @@ -464,8 +496,26 @@ def _default_source_weights(intent: str, sources: list[str]) -> dict[str, float] def _keyword_query(topic: str, core: str) -> str: + """Build a search_query string for the deterministic fallback. + + Quote ONLY title-cased multi-word proper nouns ("Hermes Agent", + "Claude Code", "Nous Research") so platform search engines preserve the + name as a phrase. Hyphenated compounds and lowercase terms are left as + bare keywords, which broadens retrieval instead of narrowing it. + + Prior behavior quoted the entire compound including the user's typed + topic, producing searches like `"Hermes Agent Actual Use Cases" hermes agent actual` + that returned near-zero matches on X and Reddit because nobody posts + that exact phrase. See 2026-04-19 Hermes Agent Use Cases failure. + """ compounds = query.extract_compound_terms(topic) - quoted = " ".join(f"\"{term}\"" for term in compounds[:2]) + # Only quote title-cased proper nouns (multi-word names). Hyphenated + # compounds go unquoted so platform tokenizers can split and match. + title_cased = [ + term for term in compounds + if re.match(r"^(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+$", term) + ] + quoted = " ".join(f'"{term}"' for term in title_cased[:2]) keywords = [quoted.strip(), core.strip() or topic.strip()] return " ".join(part for part in keywords if part).strip() @@ -513,12 +563,84 @@ def _should_force_deterministic_plan(topic: str) -> bool: return _infer_intent(topic) == "comparison" and len(_comparison_entities(topic)) >= 2 -def _max_subqueries(intent: str) -> int: +_INTENT_MODIFIER_PATTERNS = ( + "use cases", "use case", "workflows", "workflow", + "examples", "example", "tutorial", "tutorials", + "review", "reviews", "comparison", "applications", + "in practice", "production use", "production", + "how i use", +) + + +def _has_intent_modifier(topic: str) -> bool: + """Return True if the topic contains an intent modifier phrase. + + See 2026-04-19 Hermes Agent Use Cases failure: a literal "Hermes Agent + use cases" search returns near-zero matches because nobody posts that + exact phrase. Intent modifiers should be stripped from search_query + and paraphrased across multiple subqueries. + """ + text = topic.lower() + return any(pattern in text for pattern in _INTENT_MODIFIER_PATTERNS) + + +def _intent_modifier_subqueries( + topic: str, + core: str, + base_search: str, + source_weights: dict[str, float], +) -> list[schema.SubQuery]: + """Produce paraphrased subqueries for intent-modifier topics. + + The deterministic fallback used to echo the user's literal phrase + (e.g., "hermes agent use cases") into every search_query. This helper + fans out 3 extra subqueries that each express the intent differently + so retrieval pulls a broader corpus for reranking. + """ + entity = core or topic.strip() + sources = list(source_weights) + return [ + schema.SubQuery( + label="workflows", + search_query=f"{entity} workflow pipeline", + ranking_query=f"What real-world workflows or pipelines are people running with {entity}?", + sources=sources, + weight=0.6, + ), + schema.SubQuery( + label="production", + search_query=f"{entity} production real-world", + ranking_query=f"What production deployments or real-world use cases of {entity} are people describing?", + sources=sources, + weight=0.55, + ), + schema.SubQuery( + label="experience", + search_query=f"{entity} experience review", + ranking_query=f"What hands-on experience reports or reviews of {entity} exist in the last 30 days?", + sources=sources, + weight=0.5, + ), + ] + + +def _max_subqueries(intent: str, topic: str | None = None) -> int: + # how_to/opinion/product/breaking_news/prediction benefit from 4-5 + # paraphrased subqueries when the topic carries an intent modifier + # (use cases, workflows, examples, review, etc.). See 2026-04-19 + # Hermes Agent Use Cases failure: prior cap of 3 produced near-literal + # echoes of the topic instead of a paraphrase fanout. if intent == "comparison": return 4 + # Intent-modifier topics get headroom for paraphrase fanout even when + # the intent itself is factual/concept. Without this, a "Hermes Agent + # use cases" query (classified "concept" after the 2026-04-19 default + # change) would be capped at 2 and drop the fanout. + if topic and _has_intent_modifier(topic): + return 5 if intent in {"factual", "concept"}: return 2 - return 3 + return 5 def _default_sources_for_intent(intent: str, available_sources: list[str]) -> list[str]: diff --git a/tests/test_planner_v3.py b/tests/test_planner_v3.py index 62e4f04..56e8f6f 100644 --- a/tests/test_planner_v3.py +++ b/tests/test_planner_v3.py @@ -281,5 +281,169 @@ class PlannerV3Tests(unittest.TestCase): self.assertIn("instagram", all_sources) +class IntentModifierBreadthTests(unittest.TestCase): + """Unit 2: Topics with intent modifiers (use cases, workflows, examples, + review, comparison) must fan out across paraphrased subqueries rather + than echo the literal phrase. 2026-04-19 Hermes Agent Use Cases failure. + """ + + def test_max_subqueries_raised_to_5_for_how_to(self): + self.assertEqual(5, planner._max_subqueries("how_to")) + + def test_max_subqueries_raised_to_5_for_opinion(self): + self.assertEqual(5, planner._max_subqueries("opinion")) + + def test_max_subqueries_raised_to_5_for_product(self): + self.assertEqual(5, planner._max_subqueries("product")) + + def test_max_subqueries_unchanged_for_comparison(self): + self.assertEqual(4, planner._max_subqueries("comparison")) + + def test_max_subqueries_unchanged_for_factual_and_concept(self): + self.assertEqual(2, planner._max_subqueries("factual")) + self.assertEqual(2, planner._max_subqueries("concept")) + + def test_has_intent_modifier_detects_use_cases(self): + self.assertTrue(planner._has_intent_modifier("Hermes Agent use cases")) + self.assertTrue(planner._has_intent_modifier("Hermes Agent Actual Use Cases")) + + def test_has_intent_modifier_detects_workflows(self): + self.assertTrue(planner._has_intent_modifier("Claude Code workflows")) + + def test_has_intent_modifier_detects_review_and_tutorial(self): + self.assertTrue(planner._has_intent_modifier("Ollama review")) + self.assertTrue(planner._has_intent_modifier("DSPy tutorial")) + + def test_has_intent_modifier_false_for_bare_entity(self): + self.assertFalse(planner._has_intent_modifier("Kanye West")) + self.assertFalse(planner._has_intent_modifier("hermes agent")) + + def test_fallback_fans_out_when_intent_modifier_present(self): + plan = planner.plan_query( + topic="Hermes Agent use cases", + available_sources=["reddit", "x", "youtube", "hackernews"], + requested_sources=None, + depth="default", + provider=None, + model=None, + ) + # Expect at least 3 subqueries total (primary + fanout); cap is 5 for + # how_to/opinion/product/breaking_news. Label set should include at + # least one of the paraphrase labels. + labels = {sq.label for sq in plan.subqueries} + self.assertGreaterEqual(len(plan.subqueries), 3) + self.assertTrue( + labels & {"workflows", "production", "experience"}, + f"Expected paraphrase labels in {labels}", + ) + + def test_fallback_does_not_fan_out_for_bare_entity(self): + plan = planner.plan_query( + topic="Kanye West", + available_sources=["reddit", "x", "grounding"], + requested_sources=None, + depth="default", + provider=None, + model=None, + ) + # Bare entity without intent modifier should not trigger the paraphrase + # fanout (those labels are not in the plan). + labels = {sq.label for sq in plan.subqueries} + self.assertFalse(labels & {"workflows", "production", "experience"}) + + def test_prompt_includes_intent_modifier_rule(self): + prompt = planner._build_prompt( + topic="Hermes Agent use cases", + available_sources=["reddit", "x", "youtube"], + requested_sources=None, + depth="default", + ) + self.assertIn("INTENT-MODIFIER HANDLING", prompt) + self.assertIn("use cases", prompt) + self.assertIn("STRIP that phrase", prompt) + + +class FallbackDefaultsTests(unittest.TestCase): + """Unit 3: Deterministic fallback defaults and keyword_query quoting. + 2026-04-19 Hermes Agent Use Cases failure. + """ + + def test_unclassified_topic_defaults_to_concept_not_breaking_news(self): + # Prior default was "breaking_news" with strict_recent freshness, + # which biased against older relevant material on unfamiliar topics. + self.assertEqual("concept", planner._infer_intent("some unfamiliar topic")) + self.assertEqual("concept", planner._infer_intent("Hermes Agent")) + + def test_recency_signals_still_break_out_to_breaking_news(self): + self.assertEqual("breaking_news", planner._infer_intent("trending AI tools")) + self.assertEqual("breaking_news", planner._infer_intent("what's happening today")) + self.assertEqual("breaking_news", planner._infer_intent("this week in AI")) + + def test_specific_intents_still_classify_correctly(self): + # Regression: other regex branches still fire as before. + self.assertEqual("how_to", planner._infer_intent("how to deploy Docker")) + self.assertEqual("factual", planner._infer_intent("who acquired Wiz")) + self.assertEqual("opinion", planner._infer_intent("thoughts on OpenAI Codex")) + self.assertEqual("comparison", planner._infer_intent("Codex vs Claude Code")) + + def test_keyword_query_quotes_only_title_cased_proper_nouns(self): + # "Hermes Agent" is a multi-word title-cased proper noun — keep quoted. + # "Use Cases" is also title-cased BUT we only quote the first 2 + # title-cased compounds; the first extracted is "Hermes Agent". + search = planner._keyword_query("Hermes Agent use cases", "hermes agent") + self.assertIn('"Hermes Agent"', search) + # The old behavior quoted the entire typed topic; confirm it does not. + self.assertNotIn('"Hermes Agent Actual Use Cases"', search) + + def test_keyword_query_does_not_quote_bare_lowercase_topic(self): + search = planner._keyword_query("kanye west bully", "kanye west bully") + # Lowercase topics have no title-cased compound to quote. + self.assertNotIn('"', search) + + def test_fallback_logs_warning_when_no_provider(self): + import io + import contextlib + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + planner.plan_query( + topic="Hermes Agent use cases", + available_sources=["reddit", "x"], + requested_sources=None, + depth="default", + provider=None, + model=None, + ) + self.assertIn("No --plan and no LLM provider configured", buf.getvalue()) + + def test_fallback_does_not_log_warning_when_provider_present(self): + # When a provider is configured, the provider path runs; if it + # succeeds, no fallback warning should appear. + # (The existing sanitize tests cover this; we just confirm the + # warning string gating is on provider-presence, not on fallback + # activation.) + import io + import contextlib + buf = io.StringIO() + + class _NoopProvider: + def generate_json(self, model, prompt): + raise ValueError("force fallback for test") + + with contextlib.redirect_stderr(buf): + planner.plan_query( + topic="Kanye West", + available_sources=["reddit", "x"], + requested_sources=None, + depth="default", + provider=_NoopProvider(), + model="some-model", + ) + # Provider was present — we expect the "LLM planning failed" message, + # NOT the "No --plan and no LLM provider" message. + output = buf.getvalue() + self.assertIn("LLM planning failed", output) + self.assertNotIn("No --plan and no LLM provider configured", output) + + if __name__ == "__main__": unittest.main() From a709d66e2a1b37c1e66e3534ef58ab96e052d65a Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 19 Apr 2026 09:24:43 -0700 Subject: [PATCH 3/8] fix: demote reranker candidates that miss the primary entity The 2026-04-19 Hermes Agent Use Cases run had a Nate Herk YouTube video titled "I Tested Claude's New Managed Agents" score 51 and rank #2 with zero Hermes content. The reranker had intent-specific scoring hints but no entity-grounding check, so topic-vicinity matches (one offhand OpenClaw mention) drifted to the top. Add _primary_entity(topic) that strips intent-modifier suffixes ("use cases", "workflows", etc.) so "Hermes Agent use cases" yields primary_entity="Hermes Agent". Pass the entity through to both the LLM and fallback scoring paths. Fallback path: if primary_entity is not found (case-insensitive) in title + snippet, subtract ENTITY_MISS_PENALTY (25 pts). Skip the demotion for candidates with no text at all (image-only TikToks etc.) to avoid false negatives on thin-text sources. LLM path: add a "Primary entity grounding" hint to _build_prompt when primary_entity is non-empty. Instructs the LLM to score candidates without the entity at <=30. Tests: 24 rerank tests pass, including 8 new entity-grounding tests. --- scripts/lib/rerank.py | 83 +++++++++++++++++++++++++++++++++++------ tests/test_rerank_v3.py | 76 ++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 12 deletions(-) diff --git a/scripts/lib/rerank.py b/scripts/lib/rerank.py index 680f4ff..9954613 100644 --- a/scripts/lib/rerank.py +++ b/scripts/lib/rerank.py @@ -3,8 +3,34 @@ from __future__ import annotations import json +import re -from . import http, providers, schema +from . import http, providers, query, schema + + +# Penalty applied when a candidate does not mention the primary entity +# from the topic in its title or snippet. Picked empirically: a typical +# score spread in the shortlist is 30-70, so 25 points reliably pushes +# an off-topic candidate below on-topic ones without fully zeroing out +# marginal matches. See 2026-04-19 Hermes Agent Use Cases failure: a +# Nate Herk "Managed Agents" video scored 51 / ranked #2 with zero +# Hermes content. +ENTITY_MISS_PENALTY = 25.0 + +# Intent modifiers to strip before extracting the primary entity so that, +# for example, "Hermes Agent use cases" yields primary_entity="hermes agent" +# rather than "hermes agent use cases". Kept in sync with +# planner._INTENT_MODIFIER_PATTERNS. +_INTENT_MODIFIER_RE = re.compile( + r"\b(" + r"use cases|use case|workflows|workflow|" + r"examples|example|tutorial|tutorials|" + r"review|reviews|comparison|applications|" + r"in practice|production use|production|" + r"how i use" + r")\b", + re.IGNORECASE, +) INTENT_SCORING_HINTS: dict[str, str] = { "comparison": ( @@ -60,20 +86,21 @@ def rerank_candidates( ) -> list[schema.Candidate]: """Rerank the fused shortlist, demoting candidates the reranker scored as irrelevant.""" shortlisted = candidates[:shortlist_size] + primary_entity = _primary_entity(topic) if provider and model and shortlisted: try: - response = provider.generate_json(model, _build_prompt(topic, plan, shortlisted)) + response = provider.generate_json(model, _build_prompt(topic, plan, shortlisted, primary_entity)) _apply_llm_scores(shortlisted, response) except (ValueError, KeyError, json.JSONDecodeError, OSError, http.HTTPError) as exc: import sys print(f"[Rerank] LLM reranking failed, using local fallback: {type(exc).__name__}: {exc}", file=sys.stderr) - _apply_fallback_scores(shortlisted) + _apply_fallback_scores(shortlisted, primary_entity=primary_entity) else: - _apply_fallback_scores(shortlisted) + _apply_fallback_scores(shortlisted, primary_entity=primary_entity) if len(candidates) > shortlist_size: tail = candidates[shortlist_size:] - _apply_fallback_scores(tail) + _apply_fallback_scores(tail, primary_entity=primary_entity) return sorted( candidates, @@ -103,7 +130,7 @@ def _fenced_untrusted_content(candidate_block: str) -> str: ) -def _build_prompt(topic: str, plan: schema.QueryPlan, candidates: list[schema.Candidate]) -> str: +def _build_prompt(topic: str, plan: schema.QueryPlan, candidates: list[schema.Candidate], primary_entity: str = "") -> str: ranking_queries = "\n".join( f"- {subquery.label}: {subquery.ranking_query}" for subquery in plan.subqueries @@ -121,6 +148,16 @@ def _build_prompt(topic: str, plan: schema.QueryPlan, candidates: list[schema.Ca ) for candidate in candidates ) + grounding_hint = "" + if primary_entity: + grounding_hint = ( + f"\nPrimary entity grounding: the user's primary entity is \"{primary_entity}\". " + "A candidate that does NOT mention this entity (or a clear synonym/abbreviation) " + "in its title or snippet should score no higher than 30, regardless of other " + "signals. Do not let a candidate match the topic vicinity without matching the " + "entity itself. 2026-04-19 Hermes Agent Use Cases failure: a Nate Herk video " + "about Claude's Managed Agents scored 51 with zero Hermes content.\n" + ) return f""" Judge search-result relevance for a last-30-days research pipeline. @@ -145,7 +182,7 @@ Scoring guidance: - 70 to 89: clearly relevant and useful - 40 to 69: somewhat relevant but weaker - 0 to 39: weak, redundant, or off-target -{_intent_hint_block(plan)} +{grounding_hint}{_intent_hint_block(plan)} {_fenced_untrusted_content(candidate_block)} """.strip() @@ -169,21 +206,45 @@ def _apply_llm_scores(candidates: list[schema.Candidate], payload: dict) -> None candidate.final_score = _final_score(candidate) -def _apply_fallback_scores(candidates: list[schema.Candidate]) -> None: +def _apply_fallback_scores(candidates: list[schema.Candidate], *, primary_entity: str = "") -> None: for candidate in candidates: - rerank_score, reason = _fallback_tuple(candidate) + rerank_score, reason = _fallback_tuple(candidate, primary_entity=primary_entity) candidate.rerank_score = rerank_score candidate.explanation = reason candidate.final_score = _final_score(candidate) -def _fallback_tuple(candidate: schema.Candidate) -> tuple[float, str]: +def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]: score = ( (candidate.local_relevance * 100.0 * 0.7) + (candidate.freshness * 0.2) + (candidate.source_quality * 100.0 * 0.1) ) - return max(0.0, min(100.0, score)), "fallback-local-score" + reason = "fallback-local-score" + # Entity-grounding demotion: if the primary entity (topic minus intent + # modifier) is not present in the candidate's title or snippet, subtract + # ENTITY_MISS_PENALTY. Skip for candidates with no text at all (e.g., + # image-only TikToks) to avoid penalizing thin-text sources unfairly. + if primary_entity and (candidate.title or candidate.snippet): + haystack = f"{candidate.title} {candidate.snippet}".lower() + if primary_entity.lower() not in haystack: + score -= ENTITY_MISS_PENALTY + reason = "fallback-local-score (entity-miss demotion)" + return max(0.0, min(100.0, score)), reason + + +def _primary_entity(topic: str) -> str: + """Extract the primary entity from the topic for grounding checks. + + Strips intent-modifier suffixes (see planner._INTENT_MODIFIER_PATTERNS), + trims trailing punctuation, collapses whitespace. Returns the empty + string for topics that are all intent modifier with no entity, so + callers can skip the grounding check. + """ + stripped = _INTENT_MODIFIER_RE.sub(" ", topic) + # Also collapse multiple spaces and strip punctuation. + stripped = re.sub(r"\s+", " ", stripped).strip(" \t\r\n?.,:;!") + return stripped def _final_score(candidate: schema.Candidate) -> float: diff --git a/tests/test_rerank_v3.py b/tests/test_rerank_v3.py index 1aa2be4..861575e 100644 --- a/tests/test_rerank_v3.py +++ b/tests/test_rerank_v3.py @@ -178,9 +178,83 @@ class RerankV3Tests(unittest.TestCase): self.assertEqual("gemini-3.1-flash-lite-preview", provider.model) self.assertEqual(95.0, first.rerank_score) self.assertEqual("high fit", first.explanation) - self.assertEqual("fallback-local-score", second.explanation) + # Tail is scored via the fallback (may or may not carry the entity-miss + # suffix depending on topic-title overlap; assert the base tag is present). + self.assertIn("fallback-local-score", second.explanation or "") self.assertEqual(first.candidate_id, ranked[0].candidate_id) +class EntityGroundingTests(unittest.TestCase): + """Unit 4: Reranker entity-grounding demotion. 2026-04-19 Hermes Agent + Use Cases failure: an off-topic video about Claude Managed Agents + scored 51 and ranked #2 with zero Hermes content. + """ + + def _candidate(self, title: str, snippet: str = "") -> schema.Candidate: + return schema.Candidate( + candidate_id=f"c-{title[:10]}", + item_id="i1", + source="youtube", + title=title, + url="https://example.com", + snippet=snippet, + subquery_labels=["primary"], + native_ranks={"primary:youtube": 1}, + local_relevance=0.8, + freshness=80, + engagement=50, + source_quality=0.7, + rrf_score=0.02, + ) + + def test_primary_entity_strips_intent_modifier(self): + self.assertEqual("Hermes Agent", rerank._primary_entity("Hermes Agent use cases")) + self.assertEqual("Hermes Agent Actual", rerank._primary_entity("Hermes Agent Actual Use Cases")) + self.assertEqual("Claude Code", rerank._primary_entity("Claude Code workflows")) + self.assertEqual("DSPy", rerank._primary_entity("DSPy tutorial")) + + def test_primary_entity_leaves_bare_entity_unchanged(self): + self.assertEqual("Kanye West", rerank._primary_entity("Kanye West")) + self.assertEqual("Nous Research", rerank._primary_entity("Nous Research")) + + def test_fallback_demotes_candidate_without_primary_entity(self): + on_topic = self._candidate("Hermes Agent: Self-Improving AI", "Nous Research Hermes walkthrough") + off_topic = self._candidate("I Tested Claude's Managed Agents", "What you need to know about Anthropic's new managed agents") + rerank._apply_fallback_scores([on_topic, off_topic], primary_entity="Hermes Agent") + self.assertGreater(on_topic.final_score, off_topic.final_score) + self.assertIn("entity-miss", off_topic.explanation or "") + self.assertEqual(on_topic.explanation, "fallback-local-score") + + def test_fallback_match_is_case_insensitive(self): + on_topic = self._candidate("HERMES agent rocks", "some text") + rerank._apply_fallback_scores([on_topic], primary_entity="Hermes Agent") + self.assertEqual("fallback-local-score", on_topic.explanation) + + def test_fallback_skips_demotion_for_empty_text_candidates(self): + empty = self._candidate("", "") + rerank._apply_fallback_scores([empty], primary_entity="Hermes Agent") + self.assertEqual("fallback-local-score", empty.explanation) + + def test_fallback_skips_demotion_when_no_primary_entity(self): + off = self._candidate("Completely unrelated", "snippet") + rerank._apply_fallback_scores([off], primary_entity="") + self.assertEqual("fallback-local-score", off.explanation) + + def test_llm_prompt_includes_primary_entity_grounding_hint(self): + candidate = self._candidate("Something", "snippet text") + plan = make_plan() + prompt = rerank._build_prompt( + "Hermes Agent use cases", plan, [candidate], primary_entity="Hermes Agent" + ) + self.assertIn("Primary entity grounding", prompt) + self.assertIn("Hermes Agent", prompt) + + def test_llm_prompt_omits_grounding_hint_when_no_primary_entity(self): + candidate = self._candidate("Something", "snippet text") + plan = make_plan() + prompt = rerank._build_prompt("", plan, [candidate], primary_entity="") + self.assertNotIn("Primary entity grounding", prompt) + + if __name__ == "__main__": unittest.main() From 5f218aaac579699f6d2a77d5821e0f532bcd0fa3 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 19 Apr 2026 09:24:52 -0700 Subject: [PATCH 4/8] fix: always log planner subqueries to stderr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior pipeline.py only logged the planner outcome when an external --plan was passed ("[Planner] Using external plan (N subqueries)"). The internal LLM planner and the deterministic fallback ran silently, so retrieval-breadth failures were invisible without --debug. After plan finalization, emit a unified trace: [Planner] Plan: intent=X, freshness=Y, cluster_mode=Z, subqueries=N, source=external|llm|deterministic [Planner] sq1 label=... search="..." sources=[...] [Planner] sq2 ... Stderr only; does not touch the user-facing stdout synthesis. The source= annotation distinguishes --plan (external), provider-backed (llm), and deterministic paths — so when the 2026-04-19 Hermes Agent Use Cases failure mode recurs, the trace tells the user which path ran and what subqueries it produced. Tests: added test_planner_trace_always_fires_on_mock_run which captures stderr on a mock pipeline run and asserts the summary + per-subquery lines appear. --- scripts/lib/pipeline.py | 31 ++++++++++++++++++++++++++++++- tests/test_pipeline_v3.py | 24 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/lib/pipeline.py b/scripts/lib/pipeline.py index 9057208..81ee240 100644 --- a/scripts/lib/pipeline.py +++ b/scripts/lib/pipeline.py @@ -204,7 +204,7 @@ def run( plan = planner._sanitize_plan( external_plan, topic, available, requested_sources, depth, ) - print(f"[Planner] Using external plan ({len(plan.subqueries)} subqueries)", file=sys.stderr) + plan_source = "external" else: plan = planner.plan_query( topic=topic, @@ -215,6 +215,14 @@ def run( model=None if mock else runtime.planner_model, context=config.get("_auto_resolve_context", ""), ) + # Source labelling: the fallback path annotates notes with "fallback-plan" + # or "deterministic-comparison-plan"; anything else came from the LLM. + if any("fallback" in note or "deterministic" in note for note in (plan.notes or [])): + plan_source = "deterministic" + elif not mock and reasoning_provider and runtime.planner_model: + plan_source = "llm" + else: + plan_source = "deterministic" # Safety net: ensure grounding appears in all subqueries even if the planner # omits it. This is redundant when the planner includes grounding via @@ -224,6 +232,27 @@ def run( if "grounding" not in sq.sources: sq.sources.append("grounding") + # Always-on planner trace. Emits one summary line plus one per subquery + # so retrieval-breadth failures like the 2026-04-19 Hermes Agent Use Cases + # disaster are visible without --debug. Stderr only; does not leak into + # the user-facing stdout synthesis. + print( + f"[Planner] Plan: intent={plan.intent}, freshness={plan.freshness_mode}, " + f"cluster_mode={plan.cluster_mode}, subqueries={len(plan.subqueries)}, " + f"source={plan_source}", + file=sys.stderr, + ) + if plan.subqueries: + for index, sq in enumerate(plan.subqueries, start=1): + sources_str = ",".join(sq.sources) if sq.sources else "(none)" + print( + f"[Planner] sq{index} label={sq.label} " + f'search="{sq.search_query}" sources=[{sources_str}]', + file=sys.stderr, + ) + else: + print("[Planner] (no subqueries in plan)", file=sys.stderr) + bundle = schema.RetrievalBundle(artifacts={"grounding": []}) # Project-mode or person-mode GitHub: run once before the main subquery loop diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py index e1cc799..cc16f35 100644 --- a/tests/test_pipeline_v3.py +++ b/tests/test_pipeline_v3.py @@ -28,6 +28,30 @@ class PipelineV3Tests(unittest.TestCase): self.assertIn("grounding", report.items_by_source) self.assertEqual("gemini", report.provider_runtime.reasoning_provider) + def test_planner_trace_always_fires_on_mock_run(self): + """Unit 5: The unified planner trace emits one summary line plus one + line per subquery on every run, regardless of --debug. 2026-04-19 + Hermes Agent Use Cases failure: retrieval-breadth issues were invisible + because the internal planner path logged nothing. + """ + import io + import contextlib + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + pipeline.run( + topic="test topic", + config={"LAST30DAYS_REASONING_PROVIDER": "gemini"}, + depth="quick", + requested_sources=["reddit", "x", "grounding"], + mock=True, + ) + output = buf.getvalue() + self.assertIn("[Planner] Plan: intent=", output) + self.assertIn("subqueries=", output) + self.assertIn("source=", output) + # At least one per-subquery line. + self.assertIn("[Planner] sq1 label=", output) + class TestSourceFetchCap(unittest.TestCase): """X source fetch count must be capped by MAX_SOURCE_FETCHES.""" From a0d61b0dc6ec9027e2c9aa9b020020182f8eef3f Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 19 Apr 2026 10:28:08 -0700 Subject: [PATCH 5/8] fix: add LAW 7 - YOU ARE the planner, --plan mandatory on named entities Run 1 of /last30days Hermes Agent use cases on 2026-04-19 called the engine bare despite SKILL.md already having a detailed Step 0.75 (YOU are the planner) and a PRECONDITION GATE requiring --plan. Those lived at lines 647 and 729 - the model didn't reach them before invoking Bash. LAW 7 hoists the rule into the OUTPUT CONTRACT block at the top (same placement pattern as LAW 6), so it is the first thing the model reads. Runtime-agnostic language: Claude Code, Codex, Hermes, Gemini, or any agent runtime. Named failure mode with the misread diagnosis: "provider" in engine messages refers to the engine's INTERNAL planner credentials, NOT a prerequisite the caller needs - if you are the hosting reasoning model, YOU are the provider. Concrete self-check: re-read pending Bash command; if no --plan and topic is a named entity, STOP and generate a plan. --- SKILL.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/SKILL.md b/SKILL.md index 3a97bf4..bcd2979 100644 --- a/SKILL.md +++ b/SKILL.md @@ -156,6 +156,14 @@ The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do." ``` +**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan '$JSON'`. + +Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan '$JSON'`. A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema). + +**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step. + +**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan '$JSON'`? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider. + End of OUTPUT CONTRACT. The laws above are the contract; everything below is implementation detail. --- From b7df5ecd2dc79e97b061b9dd4320d8b06f5394d5 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 19 Apr 2026 10:28:21 -0700 Subject: [PATCH 6/8] fix: emit user-visible DEGRADED RUN WARNING on bare named-entity calls The stderr [Planner] warning from PR #285 doesn't reach the user because Claude and other reasoning agents hide stderr from their synthesis. The 2026-04-19 Hermes Agent Use Cases Run 1 produced source=deterministic and the user never saw it. Adds a user-visible stdout block that the model's LAW 5 pass-through contract forces into the response. Fires only when plan_source is deterministic AND no pre-research flags were passed AND the topic is pre-research-eligible (named entity). Cron jobs on abstract topics don't trigger it. Position: BEFORE the EVIDENCE FOR SYNTHESIS envelope so the model sees it as the first non-badge content. Wrapped in a new USER-VISIBLE BANNER envelope matching the EVIDENCE/PASS-THROUGH envelope pattern from Unit 1 of PR #285. Runtime-agnostic language: explicitly enumerates Claude Code, Codex, Hermes, Gemini so the hosting reasoning model recognizes itself regardless of runtime. pipeline.py now persists plan_source to report.artifacts so the renderer can consume it. Adds 7 tests covering fire conditions, suppression conditions (external/llm plan source, flags present, abstract topic), and correct position relative to the evidence envelope. --- scripts/lib/pipeline.py | 4 +++ scripts/lib/render.py | 66 +++++++++++++++++++++++++++++++++++++++++ tests/test_render_v3.py | 61 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) diff --git a/scripts/lib/pipeline.py b/scripts/lib/pipeline.py index 81ee240..d759475 100644 --- a/scripts/lib/pipeline.py +++ b/scripts/lib/pipeline.py @@ -254,6 +254,10 @@ def run( print("[Planner] (no subqueries in plan)", file=sys.stderr) bundle = schema.RetrievalBundle(artifacts={"grounding": []}) + # Expose plan_source to the renderer so render_compact can emit the + # DEGRADED RUN banner when a named-entity topic was invoked bare + # (source=deterministic AND no pre-research flags). LAW 7 backstop. + bundle.artifacts["plan_source"] = plan_source # Project-mode or person-mode GitHub: run once before the main subquery loop _github_custom_done = False diff --git a/scripts/lib/render.py b/scripts/lib/render.py index a822d3f..bd5b0b0 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -100,6 +100,15 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str lines.extend(f"- {warning}" for warning in report.warnings) lines.append("") + # LAW 7 backstop: emit the DEGRADED RUN WARNING block BEFORE the evidence + # envelope so the model's pass-through contract forces it into the user's + # response on bare named-entity calls. The stderr [Planner] warning is + # invisible to the user; this block is not. + degraded_warning = _render_degraded_run_warning(report) + if degraded_warning: + lines.extend(degraded_warning) + lines.append("") + # Open EVIDENCE FOR SYNTHESIS envelope. The ## Ranked Evidence Clusters, # ## Stats, and ## Source Coverage blocks inside this envelope are raw # evidence for the model to READ, not output to emit. LAW 6 in SKILL.md @@ -264,6 +273,63 @@ def _render_pre_research_warning(report: schema.Report) -> list[str]: ] +def _render_degraded_run_warning(report: schema.Report) -> list[str]: + """Emit a user-visible DEGRADED RUN WARNING block when: + - The engine ran the deterministic fallback planner (source=deterministic), AND + - No pre-research flags were passed (bare invocation), AND + - The topic is pre-research-eligible (a named entity). + + Runtime-agnostic: this skill ships to Claude Code, Codex, Hermes, Gemini, + and ~/.agents. The banner enumerates them explicitly so the hosting + reasoning model recognizes itself regardless of runtime. + + Positioned BEFORE the EVIDENCE FOR SYNTHESIS envelope in render_compact + so the model's pass-through contract forces it into the user's view per + LAW 7. The 2026-04-19 Hermes Agent Use Cases Run 1 failure mode: the + engine's stderr warning about "no LLM provider" was invisible to the + user because Claude hid stderr. User-visible stdout block is the + backstop that makes silent degradation impossible. + """ + plan_source = report.artifacts.get("plan_source", "unknown") + flags_present = bool(report.artifacts.get("pre_research_flags_present", False)) + if plan_source != "deterministic": + return [] + if flags_present: + return [] + if not _is_pre_research_eligible(report.topic): + return [] + + return [ + "", + "## DEGRADED RUN WARNING", + "", + f"⚠️ This run was called BARE on a named-entity topic (\"{report.topic}\"):", + "no `--plan` JSON, no pre-flight handle resolution. The engine ran", + "deterministic fallback retrieval, which is the cron/headless path -", + "weaker than the path where the hosting reasoning model authors its own", + "plan upstream.", + "", + "**If you are the reasoning model hosting this skill** (Claude Code, Codex,", + "Hermes, Gemini, or any agent runtime that invoked `/last30days`): YOU ARE", + "THE LLM. You do not need an API key, provider credentials, or an external", + "planning service. The `--plan` flag exists precisely so a reasoning model", + "generates its own JSON plan upstream and passes it in. That is you.", + "Generate a plan with `intent` / `freshness_mode` / 3-5 paraphrased", + "subqueries and re-run with `--plan '$JSON'`. This is not a", + "missing-credentials problem; this is a skipped-LAW-7 problem.", + "", + "What went wrong: on a named-entity topic, the full contract is", + "(a) resolve X handles / GitHub repos / subreddits via your runtime's", + "web-search tool (Step 0.55) and (b) generate a JSON `--plan` yourself", + "and pass it via `--plan '$JSON'` (Step 0.75 / LAW 7). Both were skipped.", + "", + "**If you are a user reading this:** the assistant skipped its own", + "planning step. Ask it to regenerate following Step 0.55 and Step 0.75", + "of SKILL.md.", + "", + ] + + def _parse_comparison_entities(topic: str) -> list[str] | None: """Return list of entity names if topic is a comparison query, else None. diff --git a/tests/test_render_v3.py b/tests/test_render_v3.py index 4366673..15733b5 100644 --- a/tests/test_render_v3.py +++ b/tests/test_render_v3.py @@ -462,5 +462,66 @@ class RenderBestTakesCompactTests(unittest.TestCase): self.assertNotIn("## Best Takes", text) +class DegradedRunBannerTests(unittest.TestCase): + """Unit 1: DEGRADED RUN WARNING surfaces bare named-entity invocations + in user-visible stdout. LAW 7 backstop. 2026-04-19 Hermes Agent Use + Cases Run 1 failure mode. + """ + + def _bare_named_entity_report(self) -> schema.Report: + report = sample_report() + report.topic = "Hermes Agent" + report.artifacts["plan_source"] = "deterministic" + report.artifacts["pre_research_flags_present"] = False + return report + + def test_banner_appears_on_bare_named_entity_deterministic_run(self): + text = render.render_compact(self._bare_named_entity_report()) + self.assertIn("## DEGRADED RUN WARNING", text) + self.assertIn("", text) + self.assertIn("YOU ARE", text) + # Runtime-agnostic enumeration: all host runtimes appear. + for runtime_name in ("Claude Code", "Codex", "Hermes", "Gemini"): + self.assertIn(runtime_name, text) + + def test_banner_positioned_before_evidence_envelope(self): + text = render.render_compact(self._bare_named_entity_report()) + banner_idx = text.index("## DEGRADED RUN WARNING") + envelope_idx = text.index("