diff --git a/scripts/lib/planner.py b/scripts/lib/planner.py index 6d83f56..49b2a20 100644 --- a/scripts/lib/planner.py +++ b/scripts/lib/planner.py @@ -113,6 +113,19 @@ def plan_query( topic, available_sources, requested_sources, depth, note=f"fallback-plan (LLM error: {type(exc).__name__})", ) + # No provider configured and no --plan was passed. The deterministic + # fallback path produces weaker plans than the LLM or the --plan path + # (narrower subquery breadth, no semantic expansion). Surface that + # so Claude Code callers know to pass --plan with a JSON plan they + # generate themselves. See 2026-04-19 Hermes Agent Use Cases failure. + import sys + print( + "[Planner] No --plan and no LLM provider configured. " + "Using deterministic fallback; results will be weaker than the " + "LLM-planned path. If invoked from Claude Code, generate a JSON " + "query plan and pass --plan.", + file=sys.stderr, + ) return _fallback_plan(topic, available_sources, requested_sources, depth) @@ -151,7 +164,7 @@ Return JSON only with this shape: }} Rules: -- emit 1 to 4 subqueries +- emit 1 to 5 subqueries (how_to/opinion/product/breaking_news intents benefit from 4-5; factual/concept from 2) - every subquery must include both search_query and ranking_query - sources must be drawn from Available sources only - use cluster_mode=none for factual or many how-to queries @@ -162,6 +175,8 @@ Rules: - preserve exact proper nouns and entity strings from the topic - NEVER include temporal phrases in search_query: no 'last 30 days', 'recent', month names, year numbers - NEVER include meta-research phrases: no 'news', 'updates', 'public appearances', 'latest developments' +- INTENT-MODIFIER HANDLING: when the topic contains one of {{use cases, use case, workflows, workflow, examples, tutorial, tutorials, review, reviews, comparison, applications, in practice, production, production use, how i use}}, STRIP that phrase from every search_query (keep its meaning in ranking_query). Emit 4-5 paraphrased subqueries that each express the intent differently (e.g., 'production', 'workflow OR pipeline', 'review OR experience', 'vs COMPETITOR', 'community discussion'). Broad retrieval, narrow ranking. This was the 2026-04-19 Hermes Agent Use Cases failure mode: the planner echoed "hermes agent use cases" as a literal search string and returned near-zero results because nobody posts that exact phrase. +- DO NOT quote the user's full topic verbatim in search_query. Quote only multi-word proper nouns like "Hermes Agent", "Claude Code", "Nous Research". Bare keywords OR'd together retrieve more than exact-phrase searches. - search_query should match how content is TITLED on platforms - GitHub (Issues/PRs) is best for engineering, developer tools, and open source topics: 'kanye west bully' not 'kanye west album news March 2026' """.strip() @@ -204,7 +219,7 @@ def _sanitize_plan( source_weights = _normalize_weights(source_weights) subqueries: list[schema.SubQuery] = [] - for index, subquery in enumerate((raw.get("subqueries") or [])[:_max_subqueries(intent_hint)], start=1): + for index, subquery in enumerate((raw.get("subqueries") or [])[:_max_subqueries(intent_hint, topic)], start=1): if not isinstance(subquery, dict): continue sources = [source for source in subquery.get("sources") or [] if source in source_weights] @@ -382,13 +397,22 @@ def _fallback_plan( ) ) + # Intent-modifier fanout: when topic contains a phrase like "use cases", + # "workflows", "examples", "review" (see _INTENT_MODIFIER_PATTERNS), + # paraphrase the intent across 3 extra subqueries rather than echoing + # the literal phrase. Fixes 2026-04-19 Hermes Agent Use Cases failure. + # Excluded for comparison/prediction since those already have dedicated + # fanout (entity-per-subquery / odds). + if depth != "quick" and intent not in {"comparison", "prediction"} and _has_intent_modifier(topic): + subqueries.extend(_intent_modifier_subqueries(topic, core, base_search, source_weights)) + return schema.QueryPlan( intent=intent, freshness_mode=_default_freshness(intent), cluster_mode=_default_cluster_mode(intent), raw_topic=topic, subqueries=_normalize_subquery_weights( - _trim_subqueries_for_depth(subqueries[:_max_subqueries(intent)], intent, depth, list(source_weights)) + _trim_subqueries_for_depth(subqueries[:_max_subqueries(intent, topic)], intent, depth, list(source_weights)) ), source_weights=_normalize_weights(source_weights), notes=[note], @@ -418,7 +442,15 @@ def _infer_intent(topic: str) -> str: return "concept" if re.search(r"\b(tournament|championship|playoffs|march madness|world cup|olympics|super bowl|final four|ceremony|awards|keynote)\b", text): return "breaking_news" - return "breaking_news" + # Recency signals take priority when nothing more specific matched. + if re.search(r"\b(trending|this week|right now|today|this month)\b", text): + return "breaking_news" + # Default changed from "breaking_news" to "concept" on 2026-04-19 after + # the Hermes Agent Use Cases failure: unclassified topics were getting + # strict_recent freshness, which over-weighted the last 7 days and + # under-weighted older relevant material. "concept" defaults to + # evergreen_ok freshness, a safer posture for unknown topics. + return "concept" def _default_freshness(intent: str) -> str: @@ -464,8 +496,26 @@ def _default_source_weights(intent: str, sources: list[str]) -> dict[str, float] def _keyword_query(topic: str, core: str) -> str: + """Build a search_query string for the deterministic fallback. + + Quote ONLY title-cased multi-word proper nouns ("Hermes Agent", + "Claude Code", "Nous Research") so platform search engines preserve the + name as a phrase. Hyphenated compounds and lowercase terms are left as + bare keywords, which broadens retrieval instead of narrowing it. + + Prior behavior quoted the entire compound including the user's typed + topic, producing searches like `"Hermes Agent Actual Use Cases" hermes agent actual` + that returned near-zero matches on X and Reddit because nobody posts + that exact phrase. See 2026-04-19 Hermes Agent Use Cases failure. + """ compounds = query.extract_compound_terms(topic) - quoted = " ".join(f"\"{term}\"" for term in compounds[:2]) + # Only quote title-cased proper nouns (multi-word names). Hyphenated + # compounds go unquoted so platform tokenizers can split and match. + title_cased = [ + term for term in compounds + if re.match(r"^(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+$", term) + ] + quoted = " ".join(f'"{term}"' for term in title_cased[:2]) keywords = [quoted.strip(), core.strip() or topic.strip()] return " ".join(part for part in keywords if part).strip() @@ -513,12 +563,84 @@ def _should_force_deterministic_plan(topic: str) -> bool: return _infer_intent(topic) == "comparison" and len(_comparison_entities(topic)) >= 2 -def _max_subqueries(intent: str) -> int: +_INTENT_MODIFIER_PATTERNS = ( + "use cases", "use case", "workflows", "workflow", + "examples", "example", "tutorial", "tutorials", + "review", "reviews", "comparison", "applications", + "in practice", "production use", "production", + "how i use", +) + + +def _has_intent_modifier(topic: str) -> bool: + """Return True if the topic contains an intent modifier phrase. + + See 2026-04-19 Hermes Agent Use Cases failure: a literal "Hermes Agent + use cases" search returns near-zero matches because nobody posts that + exact phrase. Intent modifiers should be stripped from search_query + and paraphrased across multiple subqueries. + """ + text = topic.lower() + return any(pattern in text for pattern in _INTENT_MODIFIER_PATTERNS) + + +def _intent_modifier_subqueries( + topic: str, + core: str, + base_search: str, + source_weights: dict[str, float], +) -> list[schema.SubQuery]: + """Produce paraphrased subqueries for intent-modifier topics. + + The deterministic fallback used to echo the user's literal phrase + (e.g., "hermes agent use cases") into every search_query. This helper + fans out 3 extra subqueries that each express the intent differently + so retrieval pulls a broader corpus for reranking. + """ + entity = core or topic.strip() + sources = list(source_weights) + return [ + schema.SubQuery( + label="workflows", + search_query=f"{entity} workflow pipeline", + ranking_query=f"What real-world workflows or pipelines are people running with {entity}?", + sources=sources, + weight=0.6, + ), + schema.SubQuery( + label="production", + search_query=f"{entity} production real-world", + ranking_query=f"What production deployments or real-world use cases of {entity} are people describing?", + sources=sources, + weight=0.55, + ), + schema.SubQuery( + label="experience", + search_query=f"{entity} experience review", + ranking_query=f"What hands-on experience reports or reviews of {entity} exist in the last 30 days?", + sources=sources, + weight=0.5, + ), + ] + + +def _max_subqueries(intent: str, topic: str | None = None) -> int: + # how_to/opinion/product/breaking_news/prediction benefit from 4-5 + # paraphrased subqueries when the topic carries an intent modifier + # (use cases, workflows, examples, review, etc.). See 2026-04-19 + # Hermes Agent Use Cases failure: prior cap of 3 produced near-literal + # echoes of the topic instead of a paraphrase fanout. if intent == "comparison": return 4 + # Intent-modifier topics get headroom for paraphrase fanout even when + # the intent itself is factual/concept. Without this, a "Hermes Agent + # use cases" query (classified "concept" after the 2026-04-19 default + # change) would be capped at 2 and drop the fanout. + if topic and _has_intent_modifier(topic): + return 5 if intent in {"factual", "concept"}: return 2 - return 3 + return 5 def _default_sources_for_intent(intent: str, available_sources: list[str]) -> list[str]: diff --git a/tests/test_planner_v3.py b/tests/test_planner_v3.py index 62e4f04..56e8f6f 100644 --- a/tests/test_planner_v3.py +++ b/tests/test_planner_v3.py @@ -281,5 +281,169 @@ class PlannerV3Tests(unittest.TestCase): self.assertIn("instagram", all_sources) +class IntentModifierBreadthTests(unittest.TestCase): + """Unit 2: Topics with intent modifiers (use cases, workflows, examples, + review, comparison) must fan out across paraphrased subqueries rather + than echo the literal phrase. 2026-04-19 Hermes Agent Use Cases failure. + """ + + def test_max_subqueries_raised_to_5_for_how_to(self): + self.assertEqual(5, planner._max_subqueries("how_to")) + + def test_max_subqueries_raised_to_5_for_opinion(self): + self.assertEqual(5, planner._max_subqueries("opinion")) + + def test_max_subqueries_raised_to_5_for_product(self): + self.assertEqual(5, planner._max_subqueries("product")) + + def test_max_subqueries_unchanged_for_comparison(self): + self.assertEqual(4, planner._max_subqueries("comparison")) + + def test_max_subqueries_unchanged_for_factual_and_concept(self): + self.assertEqual(2, planner._max_subqueries("factual")) + self.assertEqual(2, planner._max_subqueries("concept")) + + def test_has_intent_modifier_detects_use_cases(self): + self.assertTrue(planner._has_intent_modifier("Hermes Agent use cases")) + self.assertTrue(planner._has_intent_modifier("Hermes Agent Actual Use Cases")) + + def test_has_intent_modifier_detects_workflows(self): + self.assertTrue(planner._has_intent_modifier("Claude Code workflows")) + + def test_has_intent_modifier_detects_review_and_tutorial(self): + self.assertTrue(planner._has_intent_modifier("Ollama review")) + self.assertTrue(planner._has_intent_modifier("DSPy tutorial")) + + def test_has_intent_modifier_false_for_bare_entity(self): + self.assertFalse(planner._has_intent_modifier("Kanye West")) + self.assertFalse(planner._has_intent_modifier("hermes agent")) + + def test_fallback_fans_out_when_intent_modifier_present(self): + plan = planner.plan_query( + topic="Hermes Agent use cases", + available_sources=["reddit", "x", "youtube", "hackernews"], + requested_sources=None, + depth="default", + provider=None, + model=None, + ) + # Expect at least 3 subqueries total (primary + fanout); cap is 5 for + # how_to/opinion/product/breaking_news. Label set should include at + # least one of the paraphrase labels. + labels = {sq.label for sq in plan.subqueries} + self.assertGreaterEqual(len(plan.subqueries), 3) + self.assertTrue( + labels & {"workflows", "production", "experience"}, + f"Expected paraphrase labels in {labels}", + ) + + def test_fallback_does_not_fan_out_for_bare_entity(self): + plan = planner.plan_query( + topic="Kanye West", + available_sources=["reddit", "x", "grounding"], + requested_sources=None, + depth="default", + provider=None, + model=None, + ) + # Bare entity without intent modifier should not trigger the paraphrase + # fanout (those labels are not in the plan). + labels = {sq.label for sq in plan.subqueries} + self.assertFalse(labels & {"workflows", "production", "experience"}) + + def test_prompt_includes_intent_modifier_rule(self): + prompt = planner._build_prompt( + topic="Hermes Agent use cases", + available_sources=["reddit", "x", "youtube"], + requested_sources=None, + depth="default", + ) + self.assertIn("INTENT-MODIFIER HANDLING", prompt) + self.assertIn("use cases", prompt) + self.assertIn("STRIP that phrase", prompt) + + +class FallbackDefaultsTests(unittest.TestCase): + """Unit 3: Deterministic fallback defaults and keyword_query quoting. + 2026-04-19 Hermes Agent Use Cases failure. + """ + + def test_unclassified_topic_defaults_to_concept_not_breaking_news(self): + # Prior default was "breaking_news" with strict_recent freshness, + # which biased against older relevant material on unfamiliar topics. + self.assertEqual("concept", planner._infer_intent("some unfamiliar topic")) + self.assertEqual("concept", planner._infer_intent("Hermes Agent")) + + def test_recency_signals_still_break_out_to_breaking_news(self): + self.assertEqual("breaking_news", planner._infer_intent("trending AI tools")) + self.assertEqual("breaking_news", planner._infer_intent("what's happening today")) + self.assertEqual("breaking_news", planner._infer_intent("this week in AI")) + + def test_specific_intents_still_classify_correctly(self): + # Regression: other regex branches still fire as before. + self.assertEqual("how_to", planner._infer_intent("how to deploy Docker")) + self.assertEqual("factual", planner._infer_intent("who acquired Wiz")) + self.assertEqual("opinion", planner._infer_intent("thoughts on OpenAI Codex")) + self.assertEqual("comparison", planner._infer_intent("Codex vs Claude Code")) + + def test_keyword_query_quotes_only_title_cased_proper_nouns(self): + # "Hermes Agent" is a multi-word title-cased proper noun — keep quoted. + # "Use Cases" is also title-cased BUT we only quote the first 2 + # title-cased compounds; the first extracted is "Hermes Agent". + search = planner._keyword_query("Hermes Agent use cases", "hermes agent") + self.assertIn('"Hermes Agent"', search) + # The old behavior quoted the entire typed topic; confirm it does not. + self.assertNotIn('"Hermes Agent Actual Use Cases"', search) + + def test_keyword_query_does_not_quote_bare_lowercase_topic(self): + search = planner._keyword_query("kanye west bully", "kanye west bully") + # Lowercase topics have no title-cased compound to quote. + self.assertNotIn('"', search) + + def test_fallback_logs_warning_when_no_provider(self): + import io + import contextlib + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + planner.plan_query( + topic="Hermes Agent use cases", + available_sources=["reddit", "x"], + requested_sources=None, + depth="default", + provider=None, + model=None, + ) + self.assertIn("No --plan and no LLM provider configured", buf.getvalue()) + + def test_fallback_does_not_log_warning_when_provider_present(self): + # When a provider is configured, the provider path runs; if it + # succeeds, no fallback warning should appear. + # (The existing sanitize tests cover this; we just confirm the + # warning string gating is on provider-presence, not on fallback + # activation.) + import io + import contextlib + buf = io.StringIO() + + class _NoopProvider: + def generate_json(self, model, prompt): + raise ValueError("force fallback for test") + + with contextlib.redirect_stderr(buf): + planner.plan_query( + topic="Kanye West", + available_sources=["reddit", "x"], + requested_sources=None, + depth="default", + provider=_NoopProvider(), + model="some-model", + ) + # Provider was present — we expect the "LLM planning failed" message, + # NOT the "No --plan and no LLM provider" message. + output = buf.getvalue() + self.assertIn("LLM planning failed", output) + self.assertNotIn("No --plan and no LLM provider configured", output) + + if __name__ == "__main__": unittest.main()