Files
last30days-skill/tests/test_categories.py
T
Matt Van Horn 4e91f4e754 fix: Step 0.55 category-peer subreddit expansion (#305)
* feat(resolve): category-peer subreddit map for Step 0.55

Introduces scripts/lib/categories.py with a curated category->peer-subs
map and wires scripts/lib/resolve.py auto_resolve() to merge peers into
the WebSearch-extracted subreddit list. Named 2026-04-22 failure mode:
a "Prompting GPT Image 2" run resolved only r/OpenAI + r/ChatGPT and
missed r/StableDiffusion, r/midjourney, r/dalle2, r/aiArt where
prompting techniques actually live.

Map is static, curated, ~11 categories (ai_image_generation,
ai_video_generation, ai_music_generation, ai_coding_agent,
ai_agent_framework, ai_chat_model, saas_screen_recording,
saas_productivity, prediction_markets, crypto_defi, dev_tool_cli).
First-match-wins ordering from most-specific to least-specific.
Compound-term patterns only (no bare common nouns like "image", "ai").

auto_resolve now:
- calls detect_category(topic) after _extract_subreddits
- merges peer_subs case-insensitively, caps at MAX_SUBS (10)
- preserves every WebSearch-returned sub (freshest signal)
- emits [Resolve] Matched category=<id>, adding peers: <list> on stderr
  only when peers were actually added
- returns new "category" key in the result dict for observability
- wraps classifier in try/except so failures degrade to unwidened list

Includes drive-by: test_full_resolve / test_partial_failure
searches_run expectations bumped from 3->4 / 2->3 to match the current
queries dict (subreddit + news + x_handle + github).

* feat(skill): Step 0.55 category-peer expansion and self-check

Adds Section 2a (category-peer expansion, MANDATORY for product topics)
and the Step 0.55 self-check checkpoint that fires immediately before
the Resolved block displays. Structural mirror of the engine-side
categories.py map: same categories, same peer subs, same priority
order.

The model-side path now:
- Applies category-peer expansion to the WebSearch-resolved subs on
  every product-in-a-known-category run.
- Emits the (+ <category_id> peers) annotation on the Reddit line of
  the Resolved block as the observable contract. Absence on a
  product-in-a-known-category topic is a Step 0.55 regression.
- Runs a self-check before emitting Resolved: "does the resolved list
  include at least 2 peer subs for the matched category? if not,
  widen NOW and do not run the engine yet."

Mirror of the Python map lives inside Step 0.55 as a table for the
model to pattern-match against; extrapolation to unlisted categories
is explicitly allowed. Worked example (the exact failing query)
appears below the table so reviewers can see before/after at a glance.

Both changes land inside the existing Step 0.55 block. No new
top-level section, no new LAW. LAWs 1-6 wording unchanged.

* test: end-to-end regression for GPT Image 2 failure mode

Stubs grounding.web_search to return the OpenAI-only subs that caused
the 2026-04-22 failure, then asserts that auto_resolve widens to
include the image-gen peers and emits the [Resolve] Matched
category=ai_image_generation stderr line. Covers the cap boundary
and the uncategorized-topic no-op path.

Fixture tests/fixtures/prompting-gpt-image-2-resolved-block.md is
documentation-grade (not parsed by tests) and shows the pre-fix vs
post-fix Resolved block shape so reviewers can evaluate future
categories.py edits against the original bug.

---------

Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
2026-04-22 14:31:39 -07:00

155 lines
5.7 KiB
Python

"""Unit tests for scripts/lib/categories.py — the Step 0.55 category-peer map.
Guards the 2026-04-22 `Prompting GPT Image 2` failure mode: the original bug
was that Step 0.55 resolved only brand-adjacent subs (r/OpenAI, r/ChatGPT)
and missed the category peers (r/StableDiffusion, r/midjourney, r/dalle2)
where prompting techniques actually live.
"""
import re
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
from lib import categories
from lib.categories import CATEGORY_PEERS, detect_category, peer_subs_for
class DetectCategoryHappyPath(unittest.TestCase):
def test_prompting_gpt_image_2_matches_image_generation(self):
self.assertEqual(
detect_category("Prompting GPT Image 2"),
"ai_image_generation",
)
def test_claude_code_matches_coding_agent(self):
self.assertEqual(
detect_category("Claude Code skills"),
"ai_coding_agent",
)
def test_suno_matches_music_generation(self):
self.assertEqual(detect_category("Suno v4 review"), "ai_music_generation")
def test_polymarket_matches_prediction_markets(self):
self.assertEqual(
detect_category("Polymarket election odds"),
"prediction_markets",
)
def test_sora_matches_video_generation(self):
self.assertEqual(detect_category("Sora 2 prompts"), "ai_video_generation")
class PeerSubsForHappyPath(unittest.TestCase):
def test_image_generation_peer_subs_priority_order(self):
subs = peer_subs_for("ai_image_generation")
self.assertIn("StableDiffusion", subs)
self.assertIn("midjourney", subs)
self.assertIn("dalle2", subs)
self.assertLess(subs.index("StableDiffusion"), subs.index("midjourney"))
self.assertLess(subs.index("midjourney"), subs.index("dalle2"))
def test_unknown_category_returns_empty_list(self):
self.assertEqual(peer_subs_for("unknown_category"), [])
def test_none_category_returns_empty_list(self):
self.assertEqual(peer_subs_for(None), [])
def test_returned_list_is_fresh_copy(self):
first = peer_subs_for("ai_image_generation")
first.append("MutatedSub")
second = peer_subs_for("ai_image_generation")
self.assertNotIn("MutatedSub", second)
class DetectCategoryEdgeCases(unittest.TestCase):
def test_case_insensitive_match(self):
self.assertEqual(
detect_category("STABLE DIFFUSION walkthrough"),
"ai_image_generation",
)
def test_non_category_topic_returns_none(self):
self.assertIsNone(detect_category("Kanye West"))
def test_bare_image_word_does_not_trigger_image_generation(self):
# Compound-term guard: "image" alone is not a pattern; only
# multi-word compounds or domain-specific brand names match.
self.assertIsNone(detect_category("image editing on my phone"))
def test_bare_ai_word_does_not_trigger_any_category(self):
self.assertIsNone(detect_category("ai news today"))
def test_empty_topic_returns_none(self):
self.assertIsNone(detect_category(""))
def test_none_topic_returns_none(self):
self.assertIsNone(detect_category(None))
def test_first_match_wins_image_gen_before_chat_model(self):
# "gpt image 2" contains "gpt image" (ai_image_generation) and the
# substring "gpt" could resemble gpt-N chat-model patterns. The
# narrower category wins because it is declared earlier.
self.assertEqual(
detect_category("gpt image 2 review"),
"ai_image_generation",
)
class CategoryMapInvariants(unittest.TestCase):
"""Regression guards on the map itself — catch accidental bare-word patterns."""
# Common nouns that would produce false positives if used as bare patterns.
FORBIDDEN_BARE_PATTERNS = frozenset({
"image", "video", "music", "ai", "model", "agent", "chat",
"code", "cli", "app", "tool", "defi",
})
def test_no_category_has_a_bare_common_noun_pattern(self):
offenders = []
for category_id, entry in CATEGORY_PEERS.items():
for pattern in entry["patterns"]:
if pattern.strip() in self.FORBIDDEN_BARE_PATTERNS:
offenders.append((category_id, pattern))
self.assertEqual(
offenders,
[],
msg=(
"Bare common-noun patterns cause false positives. "
f"Offenders: {offenders}. Patterns must be compound "
"(e.g. 'image generation') or domain-specific "
"(e.g. 'midjourney')."
),
)
def test_every_category_has_at_least_one_compound_or_brand_pattern(self):
multi_word_or_brand = re.compile(r"(\s|-|\.)|^[a-z][a-z0-9]{3,}$")
for category_id, entry in CATEGORY_PEERS.items():
patterns = entry["patterns"]
self.assertTrue(patterns, f"{category_id} has no patterns")
has_strong = any(multi_word_or_brand.search(p) for p in patterns)
self.assertTrue(
has_strong,
f"{category_id} needs at least one multi-word or brand pattern",
)
def test_every_category_has_at_least_two_peer_subs(self):
for category_id, entry in CATEGORY_PEERS.items():
self.assertGreaterEqual(
len(entry["peer_subs"]),
2,
f"{category_id} should list at least 2 peer subs",
)
def test_category_count_is_in_expected_range(self):
# Sanity check: the map is intentionally small and curated.
self.assertGreaterEqual(len(CATEGORY_PEERS), 8)
self.assertLessEqual(len(CATEGORY_PEERS), 20)
if __name__ == "__main__":
unittest.main()