diff --git a/SKILL.md b/SKILL.md index b0950bd..bd174de 100644 --- a/SKILL.md +++ b/SKILL.md @@ -157,27 +157,28 @@ Notable mentions: [other specific things with 1-2 mentions] **If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns: -**CRITICAL: Every insight MUST cite at least one source.** Use @handle for X posts, r/subreddit for Reddit. This proves you're using real research, not making things up. +CITATION RULE: Cite sources sparingly to prove research is real. +- In the "What I learned" intro: cite 1-2 top sources total, not every sentence +- In KEY PATTERNS: cite 1 source per pattern, short format: "per @handle" or "per r/sub" +- Do NOT include engagement metrics in citations (likes, upvotes) - save those for stats box +- Do NOT chain multiple citations: "per @x, @y, @z" is too much. Pick the strongest one. -**BAD (no attribution):** -``` -His 12th studio album is now set for March 20, 2026 via a new deal with Gamma. -``` - -**GOOD (cites sources):** -``` -His 12th studio album BULLY is set for March 20, 2026 via Gamma (per @XXX, 15 likes; r/kanye thread with 200 upvotes). -``` +**BAD:** "His album is set for March 20 (per @cocoabutterbf; Rolling Stone; HotNewHipHop; Complex)." +**GOOD:** "His album BULLY is set for March 20 via Gamma, per Rolling Stone." ``` What I learned: -[2-4 sentences synthesizing key insights. EVERY claim cites @handle or r/subreddit.] +**{Topic 1}** — [1-2 sentences about this storyline, per source] + +**{Topic 2}** — [1-2 sentences, per source] + +**{Topic 3}** — [1-2 sentences, per source] KEY PATTERNS from the research: -1. [Pattern from research] - per @handle, r/sub -2. [Pattern from research] - per @handle -3. [Pattern from research] - per r/sub +1. [Pattern] — per @handle +2. [Pattern] — per r/sub +3. [Pattern] — per source ``` **THEN - Stats (right before invitation):** @@ -187,34 +188,21 @@ KEY PATTERNS from the research: - Sum engagement: parse `[Xlikes, Yrt]` from each X post, `[Xpts, Ycmt]` from Reddit - Identify top voices: highest-engagement @handles from X, most active subreddits -**You MUST use this EXACT format with these EXACT emoji characters. Do NOT use markdown tables. Do NOT use plain text dashes. Copy this template character-for-character:** +**Copy this EXACTLY, replacing only the {placeholders}:** ``` --- ✅ All agents reported back! -├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments -├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts (via Bird/xAI) -├─ 🌐 Web: {n} pages │ {domains} -└─ 🗣️ Top voices: @{handle1} ({n}K likes), @{handle2} │ r/{sub1}, r/{sub2} -``` - -**BAD (DO NOT DO THIS):** -``` ----All agents reported back! -- Reddit: 10 threads | 54 upvotes -- X: 3 posts | 21 likes -``` - -**GOOD (DO THIS):** -``` +├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments +├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts (via Bird/xAI) +├─ 🌐 Web: {N} pages │ {domain1}, {domain2}, {domain3} +└─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2} --- -✅ All agents reported back! -├─ 🟠 Reddit: 10 threads │ 54 upvotes │ 144 comments -├─ 🔵 X: 3 posts │ 21 likes │ 10 reposts (via Bird) -├─ 🌐 Web: 20 pages │ digitalocean.com, dev.to, medium.com -└─ 🗣️ Top voices: @yhemi0pe (10 likes, 10rt), @0x1BMW │ r/openclaw, r/AI_Agents ``` +If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)" +NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji. + **LAST - Invitation:** ``` --- diff --git a/docs/plans/2026-02-06-fix-last30days-v2-formatting-reddit-citations-plan.md b/docs/plans/2026-02-06-fix-last30days-v2-formatting-reddit-citations-plan.md new file mode 100644 index 0000000..4bfac03 --- /dev/null +++ b/docs/plans/2026-02-06-fix-last30days-v2-formatting-reddit-citations-plan.md @@ -0,0 +1,167 @@ +--- +title: "fix: last30days v2 formatting, Reddit results, and citation verbosity" +type: fix +date: 2026-02-06 +--- + +# fix: last30days v2 Formatting, Reddit Results, and Citation Verbosity + +## Overview + +Four bugs found during v2 testing across 4 queries (kanye west, howie.ai, nano banana pro prompting, open claw). The skill IS executing (the agent:Explore removal worked) but output quality has regressed from v1. + +## Problem Statement + +| # | Bug | Severity | Where | +|---|-----|----------|-------| +| 1 | Stats emoji tree format ignored 3/4 times - agent renders plain text dashes instead | High | `SKILL.md` | +| 2 | Reddit returns 0 results for popular topics (kanye west, howie.ai) | High | `scripts/lib/openai_reddit.py` | +| 3 | Citations too verbose - every sentence has `(per @x, @y, @z; r/sub)` making summary unreadable | Medium | `SKILL.md` | +| 4 | Kanye summary is wall of text - no bold headers or paragraph breaks like nano banana pro got | Medium | `SKILL.md` | + +## Proposed Fixes + +### Fix 1: Stats Emoji Format Enforcement + +**Root cause:** The agent ignores the emoji tree template even with BAD/GOOD examples. The template uses box-drawing characters (├─ └─) that the agent treats as decorative, not mandatory. + +**Approach:** Instead of relying on the agent to copy box-drawing characters, provide the template as a **literal fill-in-the-blank** with placeholders that are impossible to misinterpret. + +**File:** `SKILL.md` (stats section, currently around line 190) + +**Change:** Replace the current template + BAD/GOOD examples with a single, strict fill-in format: + +``` +Copy this EXACTLY, replacing only the {placeholders}: + +--- +✅ All agents reported back! +├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments +├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts (via Bird/xAI) +├─ 🌐 Web: {N} pages │ {domain1}, {domain2}, {domain3} +└─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2} +--- + +If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)" +NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji. +``` + +Remove the separate BAD/GOOD section (it adds length without helping). + +### Fix 2: Reddit Returning 0 Results + +**Root cause (from code analysis):** + +1. `openai_reddit.py:53-93` - The `REDDIT_SEARCH_PROMPT` instructs the OpenAI model to strip noise words before searching. For "kanye west" this isn't the issue (no noise words), but for "howie.ai" it might strip "ai". + +2. `openai_reddit.py:160-166` - The search is restricted to `allowed_domains: ["reddit.com"]` which depends on OpenAI's web_search indexing of Reddit. + +3. `last30days.py:474-490` - Post-retrieval filtering: `normalize.filter_by_date_range()` + `score.score_reddit_items()` + `dedupe.dedupe_reddit()` can discard all results if date confidence is low. + +4. `score.py:151-157` - Items with no engagement metrics get `-10` penalty, low date confidence gets `-10`. Combined that's `-20` which may push score below threshold. + +**Approach (multi-layered):** + +**A. Add subreddit-targeted search fallback** in `openai_reddit.py`: +- When the first search returns < 3 results, add a second search prompt that explicitly queries: `"r/{topic} site:reddit.com"` and `"{topic} subreddit site:reddit.com"` +- This catches cases where OpenAI's web_search doesn't find the obvious subreddit + +**B. Soften post-retrieval scoring** in `score.py`: +- Change the no-engagement penalty from `-10` to `-3` (missing metrics ≠ irrelevant) +- Change low date confidence penalty from `-10` to `-5` + +**C. Add minimum result guarantee** in `last30days.py`: +- If scoring filters out ALL results, keep the top 3 by raw relevance regardless of score +- Log a warning: "All Reddit results scored below threshold, keeping top 3 by relevance" + +**Files to change:** +- `scripts/lib/openai_reddit.py` - Add subreddit fallback search (lines ~160-180) +- `scripts/lib/score.py` - Soften penalties (lines ~151-157) +- `scripts/last30days.py` - Add minimum result guarantee (lines ~474-490) + +### Fix 3: Citations Too Verbose + +**Root cause:** The SKILL.md instruction says "Every insight MUST cite at least one source" with a GOOD example showing `(per @XXX, 15 likes; r/kanye thread with 200 upvotes)` - this is too much detail per citation and the agent over-applies it. + +**Approach:** Dial back to "cite 1-2 sources per KEY PATTERN, not per sentence. Use short format." + +**File:** `SKILL.md` (citation section, currently around line 158) + +**Change the citation rule to:** + +``` +CITATION RULE: Cite sources sparingly to prove research is real. +- In the "What I learned" intro: cite 1-2 top sources total, not every sentence +- In KEY PATTERNS: cite 1 source per pattern, short format: "per @handle" or "per r/sub" +- Do NOT include engagement metrics in citations (likes, upvotes) - save those for stats box +- Do NOT chain multiple citations: "per @x, @y, @z" is too much. Pick the strongest one. + +BAD: "His album is set for March 20 (per @cocoabutterbf; Rolling Stone; HotNewHipHop; Complex)." +GOOD: "His album BULLY is set for March 20 via Gamma, per Rolling Stone." +``` + +### Fix 4: Summary Formatting (Wall of Text vs Structured) + +**Root cause:** The SKILL.md template for PROMPTING/NEWS/GENERAL shows: +``` +What I learned: +[2-4 sentences synthesizing...] +``` + +This gives the agent permission to write a dense paragraph. The nano banana pro test got good formatting because PROMPTING queries naturally produce structured patterns. NEWS queries (kanye) produce narratives that become walls of text. + +**Approach:** Add explicit structure to the NEWS/GENERAL format with bold topic headers. + +**File:** `SKILL.md` (summary display section, around line 158) + +**Change the PROMPTING/NEWS/GENERAL template to:** + +``` +What I learned: + +**{Topic 1}** — [1-2 sentences about this storyline, per source] + +**{Topic 2}** — [1-2 sentences, per source] + +**{Topic 3}** — [1-2 sentences, per source] + +KEY PATTERNS from the research: +1. [Pattern] — per @handle +2. [Pattern] — per r/sub +3. [Pattern] — per source +``` + +The bold topic headers force structure. Each topic gets its own paragraph with a line break. No more wall-of-text narratives. + +## Acceptance Criteria + +- [ ] **Fix 1:** Stats box uses emoji tree format ├─ 🟠 🔵 🌐 └─ 🗣️ in 4/4 test queries +- [ ] **Fix 2:** "kanye west" returns >0 Reddit threads (r/kanye exists and is active) +- [ ] **Fix 3:** Summary citations are 1 per insight, short format, no engagement metrics inline +- [ ] **Fix 4:** NEWS/GENERAL summaries use bold topic headers with paragraph breaks, not wall of text + +## Test Plan + +Re-run the same 4 queries after fixes: +1. `/last30days kanye west` — NEWS: should get Reddit results, structured summary, emoji stats +2. `/last30days howie.ai` — GENERAL: should get Reddit if available, citations not verbose +3. `/last30days nano banana pro prompting` — PROMPTING: should maintain current good quality, reduce citation density +4. `/last30days open claw` — GENERAL: should cite @handles in summary, emoji stats + +## Files to Modify + +| File | Fix | Change | +|------|-----|--------| +| `SKILL.md` | 1, 3, 4 | Stats template, citation rules, summary structure | +| `scripts/lib/openai_reddit.py` | 2 | Add subreddit fallback search | +| `scripts/lib/score.py` | 2 | Soften scoring penalties | +| `scripts/last30days.py` | 2 | Add minimum result guarantee | + +## References + +- Current SKILL.md: `~/.claude/skills/last30days/SKILL.md` +- Private repo: `/Users/mvanhorn/last30days-skill-private/` +- Old working SKILL.md: `~/.claude/skills/last30days.backup-v1/SKILL.md` +- Reddit search module: `scripts/lib/openai_reddit.py:53-93` (prompt), `:160-166` (API call) +- Scoring module: `scripts/lib/score.py:151-157` (penalties) +- Main pipeline: `scripts/last30days.py:474-490` (filtering) diff --git a/scripts/last30days.py b/scripts/last30days.py index 9561b1f..edf8d0d 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -114,6 +114,25 @@ def _search_reddit( except Exception: pass + # Subreddit-targeted fallback if still < 3 results + if len(reddit_items) < 3 and not mock and not reddit_error: + sub_query = openai_reddit._build_subreddit_query(topic) + try: + sub_raw = openai_reddit.search_reddit( + config["OPENAI_API_KEY"], + selected_models["openai"], + sub_query, + from_date, to_date, + depth=depth, + ) + sub_items = openai_reddit.parse_reddit_response(sub_raw) + existing_urls = {item.get("url") for item in reddit_items} + for item in sub_items: + if item.get("url") not in existing_urls: + reddit_items.append(item) + except Exception: + pass + return reddit_items, raw_openai, reddit_error @@ -488,6 +507,13 @@ def main(): deduped_reddit = dedupe.dedupe_reddit(sorted_reddit) deduped_x = dedupe.dedupe_x(sorted_x) + # Minimum result guarantee: if all Reddit results were filtered out but + # we had raw results, keep top 3 by relevance regardless of score + if not deduped_reddit and normalized_reddit: + print("[REDDIT WARNING] All results scored below threshold, keeping top 3 by relevance", file=sys.stderr) + by_relevance = sorted(normalized_reddit, key=lambda item: item.relevance, reverse=True) + deduped_reddit = by_relevance[:3] + progress.end_processing() # Create report diff --git a/scripts/lib/openai_reddit.py b/scripts/lib/openai_reddit.py index a2af19c..68da853 100644 --- a/scripts/lib/openai_reddit.py +++ b/scripts/lib/openai_reddit.py @@ -103,6 +103,18 @@ def _extract_core_subject(topic: str) -> str: return ' '.join(result[:3]) or topic # Keep max 3 words +def _build_subreddit_query(topic: str) -> str: + """Build a subreddit-targeted search query for fallback. + + When standard search returns few results, try searching for the + subreddit itself: 'r/kanye', 'r/howie', etc. + """ + core = _extract_core_subject(topic) + # Remove dots and special chars for subreddit name guess + sub_name = core.replace('.', '').replace(' ', '').lower() + return f"r/{sub_name} site:reddit.com" + + def search_reddit( api_key: str, model: str, diff --git a/scripts/lib/score.py b/scripts/lib/score.py index 0f9eb69..2f06833 100644 --- a/scripts/lib/score.py +++ b/scripts/lib/score.py @@ -21,7 +21,7 @@ WEBSEARCH_NO_DATE_PENALTY = 20 # Heavy penalty for no date signals (low confide # Default engagement score for unknown DEFAULT_ENGAGEMENT = 35 -UNKNOWN_ENGAGEMENT_PENALTY = 10 +UNKNOWN_ENGAGEMENT_PENALTY = 3 def log1p_safe(x: Optional[int]) -> float: @@ -152,9 +152,9 @@ def score_reddit_items(items: List[schema.RedditItem]) -> List[schema.RedditItem # Apply penalty for low date confidence if item.date_confidence == "low": - overall -= 10 - elif item.date_confidence == "med": overall -= 5 + elif item.date_confidence == "med": + overall -= 2 item.score = max(0, min(100, int(overall))) @@ -212,9 +212,9 @@ def score_x_items(items: List[schema.XItem]) -> List[schema.XItem]: # Apply penalty for low date confidence if item.date_confidence == "low": - overall -= 10 - elif item.date_confidence == "med": overall -= 5 + elif item.date_confidence == "med": + overall -= 2 item.score = max(0, min(100, int(overall)))