Matt Van Horn
e8105df4fd
Five Opus 4.7 self-debugs on v3.0.8 (3 passing, 2 failing runs) converged
on four fixes:
1. Engine refuses Class 1 demographic-shopping queries at main() front-door.
Birthday-gift failure mode becomes structurally impossible - the pipeline
never runs on a doomed query. Exit code 2 with a REFUSE message on stderr
pointing the model to ask for hobbies/relationship/budget. Escape hatch:
LAST30DAYS_SKIP_PREFLIGHT=1 for "just run it" overrides.
2. Delete stale `.agents/skills/last30days/SKILL.md` (1382 lines, April 13
snapshot) and `.hermes-plugin/SKILL.md` (269 lines, April 13 snapshot).
Peter Steinberger's self-debug named the first file as the one it read
instead of the real SKILL.md. One SKILL.md per plugin, at the plugin root.
Sync script simplified: Hermes now always uses main SKILL.md.
3. render_compact() appends an explicit END-OF-CANONICAL-OUTPUT boundary
with pass-through instruction. The model had the canonical body in its
buffer on the Peter run and discarded it; the boundary makes pass-through
the path of least resistance.
4. LAW 1 gains a verbatim-pattern override clause naming the exact WebSearch
tool-result reminder ("CRITICAL REQUIREMENT: MUST include Sources:
section") that caused Peter's trailing Sources leak. No more ambiguity
at synthesis time.
Tests: tests/test_preflight.py, 29 scenarios covering Class 1 matches
(birthday gift, best-for-demographic, what-to-buy-relationship), qualifier
skips (budget, hobbies, activity after year-old), and the REFUSE message
shape.
Validation gate before merging to main: re-run the 5 debug topics
(Peter Steinberger, birthday gift for 40 year old, Kanye West, Garry Tan,
OpenClaw vs Paperclip vs Hermes) on v3.0.9 and confirm 5/5 canonical
compliance. Rollback to v3.0.8 if any previously-passing topic regresses.
Plan: docs/plans/2026-04-18-015-fix-engine-refuse-keyword-traps-delete-stale-skillmd-files-plan.md
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
129 lines
4.8 KiB
Python
129 lines
4.8 KiB
Python
"""Tests for scripts/lib/preflight.py Class 1 keyword-trap refuse-gate.
|
|
|
|
Class 1 (demographic shopping) is the one failure class that shipped to
|
|
public v3.0.8 and still returned junk for queries like 'birthday gift for
|
|
40 year old'. This module is the engine's structural refusal, so the model
|
|
cannot bypass by skipping SKILL.md.
|
|
"""
|
|
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
|
|
|
from lib import preflight
|
|
|
|
|
|
class TestClass1Match(unittest.TestCase):
|
|
"""Queries that MUST trigger the refuse-gate."""
|
|
|
|
def test_birthday_gift_for_age(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("birthday gift for 40 year old"))
|
|
|
|
def test_gift_for_age(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("gift for 42 year old"))
|
|
|
|
def test_gift_for_age_relationship(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("gift for my 42 year old husband"))
|
|
|
|
def test_gift_ideas_for_age(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("gift ideas for 30 year old"))
|
|
|
|
def test_present_for_age(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("present for a 50 year old"))
|
|
|
|
def test_hyphenated_year_old(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("gift for 40-year-old"))
|
|
|
|
def test_best_for_men(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("best running shoes for men"))
|
|
|
|
def test_best_for_women(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("best gifts for women"))
|
|
|
|
def test_best_for_kids(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("best toys for kids"))
|
|
|
|
def test_what_to_buy_husband(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("what to buy my husband"))
|
|
|
|
def test_what_to_get_boss(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("what to get my boss"))
|
|
|
|
def test_what_to_gift_age(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("what to gift a 35 year old"))
|
|
|
|
def test_gifts_for_husband(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("gifts for my husband"))
|
|
|
|
def test_case_insensitive(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap("Birthday Gift For 40 Year Old"))
|
|
|
|
def test_leading_whitespace(self):
|
|
self.assertIsNotNone(preflight.check_class_1_trap(" gift for 40 year old "))
|
|
|
|
|
|
class TestClass1Skip(unittest.TestCase):
|
|
"""Queries that MUST NOT trigger the refuse-gate (qualifier present or not shopping)."""
|
|
|
|
def test_named_person(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("Peter Steinberger"))
|
|
|
|
def test_comparison(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("OpenClaw vs Paperclip"))
|
|
|
|
def test_entity_query(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("Kanye West"))
|
|
|
|
def test_general_concept(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("vibe coding"))
|
|
|
|
def test_budget_qualifier(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("gift for my husband, $200 budget"))
|
|
|
|
def test_hobby_qualifier(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("gift for my cooking-obsessed husband"))
|
|
|
|
def test_loves_qualifier(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("gift for my dad who loves golf"))
|
|
|
|
def test_is_into_qualifier(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("gift for my brother who is into woodworking"))
|
|
|
|
def test_specific_interest_in_query(self):
|
|
self.assertIsNone(preflight.check_class_1_trap("birthday gift for 40 year old runner"))
|
|
|
|
|
|
class TestRefuseMessage(unittest.TestCase):
|
|
"""The REFUSE message must contain the diagnostic content the model needs."""
|
|
|
|
def test_refuse_mentions_class_1(self):
|
|
msg = preflight.check_class_1_trap("birthday gift for 40 year old")
|
|
assert msg is not None
|
|
self.assertIn("Class 1", msg)
|
|
|
|
def test_refuse_asks_for_hobbies(self):
|
|
msg = preflight.check_class_1_trap("gift for 40 year old")
|
|
assert msg is not None
|
|
self.assertIn("hobbies", msg.lower())
|
|
|
|
def test_refuse_asks_for_relationship(self):
|
|
msg = preflight.check_class_1_trap("gift for 40 year old")
|
|
assert msg is not None
|
|
self.assertIn("relationship", msg.lower())
|
|
|
|
def test_refuse_asks_for_budget(self):
|
|
msg = preflight.check_class_1_trap("gift for 40 year old")
|
|
assert msg is not None
|
|
self.assertIn("budget", msg.lower())
|
|
|
|
def test_refuse_echoes_topic(self):
|
|
msg = preflight.check_class_1_trap("birthday gift for 40 year old")
|
|
assert msg is not None
|
|
self.assertIn("birthday gift for 40 year old", msg)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|