fix: emit user-visible DEGRADED RUN WARNING on bare named-entity calls
The stderr [Planner] warning from PR #285 doesn't reach the user because Claude and other reasoning agents hide stderr from their synthesis. The 2026-04-19 Hermes Agent Use Cases Run 1 produced source=deterministic and the user never saw it. Adds a user-visible stdout block that the model's LAW 5 pass-through contract forces into the response. Fires only when plan_source is deterministic AND no pre-research flags were passed AND the topic is pre-research-eligible (named entity). Cron jobs on abstract topics don't trigger it. Position: BEFORE the EVIDENCE FOR SYNTHESIS envelope so the model sees it as the first non-badge content. Wrapped in a new USER-VISIBLE BANNER envelope matching the EVIDENCE/PASS-THROUGH envelope pattern from Unit 1 of PR #285. Runtime-agnostic language: explicitly enumerates Claude Code, Codex, Hermes, Gemini so the hosting reasoning model recognizes itself regardless of runtime. pipeline.py now persists plan_source to report.artifacts so the renderer can consume it. Adds 7 tests covering fire conditions, suppression conditions (external/llm plan source, flags present, abstract topic), and correct position relative to the evidence envelope.
This commit is contained in:
@@ -254,6 +254,10 @@ def run(
|
|||||||
print("[Planner] (no subqueries in plan)", file=sys.stderr)
|
print("[Planner] (no subqueries in plan)", file=sys.stderr)
|
||||||
|
|
||||||
bundle = schema.RetrievalBundle(artifacts={"grounding": []})
|
bundle = schema.RetrievalBundle(artifacts={"grounding": []})
|
||||||
|
# Expose plan_source to the renderer so render_compact can emit the
|
||||||
|
# DEGRADED RUN banner when a named-entity topic was invoked bare
|
||||||
|
# (source=deterministic AND no pre-research flags). LAW 7 backstop.
|
||||||
|
bundle.artifacts["plan_source"] = plan_source
|
||||||
|
|
||||||
# Project-mode or person-mode GitHub: run once before the main subquery loop
|
# Project-mode or person-mode GitHub: run once before the main subquery loop
|
||||||
_github_custom_done = False
|
_github_custom_done = False
|
||||||
|
|||||||
@@ -100,6 +100,15 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
|||||||
lines.extend(f"- {warning}" for warning in report.warnings)
|
lines.extend(f"- {warning}" for warning in report.warnings)
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
|
# LAW 7 backstop: emit the DEGRADED RUN WARNING block BEFORE the evidence
|
||||||
|
# envelope so the model's pass-through contract forces it into the user's
|
||||||
|
# response on bare named-entity calls. The stderr [Planner] warning is
|
||||||
|
# invisible to the user; this block is not.
|
||||||
|
degraded_warning = _render_degraded_run_warning(report)
|
||||||
|
if degraded_warning:
|
||||||
|
lines.extend(degraded_warning)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
# Open EVIDENCE FOR SYNTHESIS envelope. The ## Ranked Evidence Clusters,
|
# Open EVIDENCE FOR SYNTHESIS envelope. The ## Ranked Evidence Clusters,
|
||||||
# ## Stats, and ## Source Coverage blocks inside this envelope are raw
|
# ## Stats, and ## Source Coverage blocks inside this envelope are raw
|
||||||
# evidence for the model to READ, not output to emit. LAW 6 in SKILL.md
|
# evidence for the model to READ, not output to emit. LAW 6 in SKILL.md
|
||||||
@@ -264,6 +273,63 @@ def _render_pre_research_warning(report: schema.Report) -> list[str]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _render_degraded_run_warning(report: schema.Report) -> list[str]:
|
||||||
|
"""Emit a user-visible DEGRADED RUN WARNING block when:
|
||||||
|
- The engine ran the deterministic fallback planner (source=deterministic), AND
|
||||||
|
- No pre-research flags were passed (bare invocation), AND
|
||||||
|
- The topic is pre-research-eligible (a named entity).
|
||||||
|
|
||||||
|
Runtime-agnostic: this skill ships to Claude Code, Codex, Hermes, Gemini,
|
||||||
|
and ~/.agents. The banner enumerates them explicitly so the hosting
|
||||||
|
reasoning model recognizes itself regardless of runtime.
|
||||||
|
|
||||||
|
Positioned BEFORE the EVIDENCE FOR SYNTHESIS envelope in render_compact
|
||||||
|
so the model's pass-through contract forces it into the user's view per
|
||||||
|
LAW 7. The 2026-04-19 Hermes Agent Use Cases Run 1 failure mode: the
|
||||||
|
engine's stderr warning about "no LLM provider" was invisible to the
|
||||||
|
user because Claude hid stderr. User-visible stdout block is the
|
||||||
|
backstop that makes silent degradation impossible.
|
||||||
|
"""
|
||||||
|
plan_source = report.artifacts.get("plan_source", "unknown")
|
||||||
|
flags_present = bool(report.artifacts.get("pre_research_flags_present", False))
|
||||||
|
if plan_source != "deterministic":
|
||||||
|
return []
|
||||||
|
if flags_present:
|
||||||
|
return []
|
||||||
|
if not _is_pre_research_eligible(report.topic):
|
||||||
|
return []
|
||||||
|
|
||||||
|
return [
|
||||||
|
"<!-- USER-VISIBLE BANNER: emit verbatim before synthesis per LAW 5 / LAW 7. -->",
|
||||||
|
"## DEGRADED RUN WARNING",
|
||||||
|
"",
|
||||||
|
f"⚠️ This run was called BARE on a named-entity topic (\"{report.topic}\"):",
|
||||||
|
"no `--plan` JSON, no pre-flight handle resolution. The engine ran",
|
||||||
|
"deterministic fallback retrieval, which is the cron/headless path -",
|
||||||
|
"weaker than the path where the hosting reasoning model authors its own",
|
||||||
|
"plan upstream.",
|
||||||
|
"",
|
||||||
|
"**If you are the reasoning model hosting this skill** (Claude Code, Codex,",
|
||||||
|
"Hermes, Gemini, or any agent runtime that invoked `/last30days`): YOU ARE",
|
||||||
|
"THE LLM. You do not need an API key, provider credentials, or an external",
|
||||||
|
"planning service. The `--plan` flag exists precisely so a reasoning model",
|
||||||
|
"generates its own JSON plan upstream and passes it in. That is you.",
|
||||||
|
"Generate a plan with `intent` / `freshness_mode` / 3-5 paraphrased",
|
||||||
|
"subqueries and re-run with `--plan '$JSON'`. This is not a",
|
||||||
|
"missing-credentials problem; this is a skipped-LAW-7 problem.",
|
||||||
|
"",
|
||||||
|
"What went wrong: on a named-entity topic, the full contract is",
|
||||||
|
"(a) resolve X handles / GitHub repos / subreddits via your runtime's",
|
||||||
|
"web-search tool (Step 0.55) and (b) generate a JSON `--plan` yourself",
|
||||||
|
"and pass it via `--plan '$JSON'` (Step 0.75 / LAW 7). Both were skipped.",
|
||||||
|
"",
|
||||||
|
"**If you are a user reading this:** the assistant skipped its own",
|
||||||
|
"planning step. Ask it to regenerate following Step 0.55 and Step 0.75",
|
||||||
|
"of SKILL.md.",
|
||||||
|
"<!-- END USER-VISIBLE BANNER -->",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _parse_comparison_entities(topic: str) -> list[str] | None:
|
def _parse_comparison_entities(topic: str) -> list[str] | None:
|
||||||
"""Return list of entity names if topic is a comparison query, else None.
|
"""Return list of entity names if topic is a comparison query, else None.
|
||||||
|
|
||||||
|
|||||||
@@ -462,5 +462,66 @@ class RenderBestTakesCompactTests(unittest.TestCase):
|
|||||||
self.assertNotIn("## Best Takes", text)
|
self.assertNotIn("## Best Takes", text)
|
||||||
|
|
||||||
|
|
||||||
|
class DegradedRunBannerTests(unittest.TestCase):
|
||||||
|
"""Unit 1: DEGRADED RUN WARNING surfaces bare named-entity invocations
|
||||||
|
in user-visible stdout. LAW 7 backstop. 2026-04-19 Hermes Agent Use
|
||||||
|
Cases Run 1 failure mode.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _bare_named_entity_report(self) -> schema.Report:
|
||||||
|
report = sample_report()
|
||||||
|
report.topic = "Hermes Agent"
|
||||||
|
report.artifacts["plan_source"] = "deterministic"
|
||||||
|
report.artifacts["pre_research_flags_present"] = False
|
||||||
|
return report
|
||||||
|
|
||||||
|
def test_banner_appears_on_bare_named_entity_deterministic_run(self):
|
||||||
|
text = render.render_compact(self._bare_named_entity_report())
|
||||||
|
self.assertIn("## DEGRADED RUN WARNING", text)
|
||||||
|
self.assertIn("<!-- USER-VISIBLE BANNER:", text)
|
||||||
|
self.assertIn("<!-- END USER-VISIBLE BANNER -->", text)
|
||||||
|
self.assertIn("YOU ARE", text)
|
||||||
|
# Runtime-agnostic enumeration: all host runtimes appear.
|
||||||
|
for runtime_name in ("Claude Code", "Codex", "Hermes", "Gemini"):
|
||||||
|
self.assertIn(runtime_name, text)
|
||||||
|
|
||||||
|
def test_banner_positioned_before_evidence_envelope(self):
|
||||||
|
text = render.render_compact(self._bare_named_entity_report())
|
||||||
|
banner_idx = text.index("## DEGRADED RUN WARNING")
|
||||||
|
envelope_idx = text.index("<!-- EVIDENCE FOR SYNTHESIS:")
|
||||||
|
self.assertLess(banner_idx, envelope_idx,
|
||||||
|
"DEGRADED RUN banner must appear BEFORE evidence envelope so pass-through catches it.")
|
||||||
|
|
||||||
|
def test_banner_suppressed_when_plan_source_external(self):
|
||||||
|
report = self._bare_named_entity_report()
|
||||||
|
report.artifacts["plan_source"] = "external"
|
||||||
|
text = render.render_compact(report)
|
||||||
|
self.assertNotIn("## DEGRADED RUN WARNING", text)
|
||||||
|
|
||||||
|
def test_banner_suppressed_when_plan_source_llm(self):
|
||||||
|
report = self._bare_named_entity_report()
|
||||||
|
report.artifacts["plan_source"] = "llm"
|
||||||
|
text = render.render_compact(report)
|
||||||
|
self.assertNotIn("## DEGRADED RUN WARNING", text)
|
||||||
|
|
||||||
|
def test_banner_suppressed_when_pre_research_flags_present(self):
|
||||||
|
report = self._bare_named_entity_report()
|
||||||
|
report.artifacts["pre_research_flags_present"] = True
|
||||||
|
text = render.render_compact(report)
|
||||||
|
self.assertNotIn("## DEGRADED RUN WARNING", text)
|
||||||
|
|
||||||
|
def test_banner_suppressed_on_non_eligible_abstract_topic(self):
|
||||||
|
report = self._bare_named_entity_report()
|
||||||
|
# Multi-word lowercase abstract phrase is NOT pre-research-eligible.
|
||||||
|
report.topic = "how to deploy containers in the cloud"
|
||||||
|
text = render.render_compact(report)
|
||||||
|
self.assertNotIn("## DEGRADED RUN WARNING", text)
|
||||||
|
|
||||||
|
def test_banner_mentions_law_7_and_plan_flag(self):
|
||||||
|
text = render.render_compact(self._bare_named_entity_report())
|
||||||
|
self.assertIn("LAW 7", text)
|
||||||
|
self.assertIn("--plan", text)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user