Compare commits
320 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 61d46b54ee | |||
| e7b7e61237 | |||
| a547a0a948 | |||
| 8ea048b988 | |||
| 1b23a3e900 | |||
| 35f12cb9ea | |||
| a1afbce84c | |||
| 15781bfcc0 | |||
| d9f606ff75 | |||
| 4a30923892 | |||
| 9fb19eae63 | |||
| d1cc29d338 | |||
| ded52062e6 | |||
| 164d7ae6ed | |||
| f1ce7533e6 | |||
| 0b939bf703 | |||
| 9f95efb215 | |||
| ff54c07a3b | |||
| e276c30477 | |||
| 2f277dfc66 | |||
| 6c2c55733c | |||
| 997708ad48 | |||
| c913e1cf89 | |||
| 54db014c7c | |||
| 80a1a47eef | |||
| c845f483d6 | |||
| dc934ddb6a | |||
| 80392061d4 | |||
| c04bd67922 | |||
| b1773be8f3 | |||
| 5b87cca886 | |||
| bbf892aecc | |||
| 2acbf8a869 | |||
| e6b89f2644 | |||
| 2c2755b49c | |||
| 18b5658674 | |||
| 145adc9f56 | |||
| b100caf2df | |||
| dc0cb9850b | |||
| ceec99b24c | |||
| d1823a2d05 | |||
| 17caa0526d | |||
| f03cb866aa | |||
| 72495c1c14 | |||
| 1f7e85a03f | |||
| 949bcf8942 | |||
| 00d01933e0 | |||
| 5f054380c5 | |||
| ff21243517 | |||
| 4e91f4e754 | |||
| 952a876536 | |||
| 1f23e3f980 | |||
| 5269806a75 | |||
| adac4c377a | |||
| 3107325443 | |||
| 1da9c601c3 | |||
| 4388fed46a | |||
| a7d6ef051a | |||
| b7df5ecd2d | |||
| a0d61b0dc6 | |||
| 5f218aaac5 | |||
| a709d66e2a | |||
| 4d9f29d2ed | |||
| 52fb0e50cb | |||
| f635f78e4a | |||
| a070a584a4 | |||
| 8e18d0142c | |||
| e8105df4fd | |||
| 361e9d6c13 | |||
| 58845df312 | |||
| d14814a9b0 | |||
| e9911ae2ae | |||
| 7cee41509f | |||
| 371f62a403 | |||
| bad1d312ef | |||
| 0103324701 | |||
| f09c6850bc | |||
| 3499c246b8 | |||
| 53b8e33d13 | |||
| 73b4bd6ac6 | |||
| a2850e3d19 | |||
| 9c1e253dcc | |||
| f4a3cc104b | |||
| a220632186 | |||
| 082efe03e3 | |||
| 242e38ef56 | |||
| c12dd3adbf | |||
| c5b03adffc | |||
| 38a1c27e2e | |||
| ed80797564 | |||
| 68c3420f9f | |||
| 12167ee19e | |||
| 21b8e5c6d3 | |||
| 1157ea8afe | |||
| 9f3be8bbda | |||
| beb54e9e9d | |||
| 8d8ca68781 | |||
| 0949b870e0 | |||
| 4b07ba02a6 | |||
| 039fc89874 | |||
| 2b506e90f5 | |||
| deb9f33437 | |||
| cb88bd2eed | |||
| 23fc6c7061 | |||
| 9dd3f21476 | |||
| e395c1d57f | |||
| 33502d2a07 | |||
| bdc71cfd07 | |||
| 65be6196c1 | |||
| 7dc530b4c9 | |||
| b159f8b1ff | |||
| cff005b038 | |||
| 460565c107 | |||
| e6493033b0 | |||
| ca00cacf83 | |||
| b982ed5b30 | |||
| a9d13d695a | |||
| 877706da4d | |||
| 1a6d8d07d0 | |||
| 3bc12cdc57 | |||
| ad59e60269 | |||
| 9d037786f2 | |||
| 8b67378964 | |||
| 2b015b64ab | |||
| d3972a6523 | |||
| 56cabf33c6 | |||
| 13dcea781d | |||
| 01812ec185 | |||
| 86b2b9dd69 | |||
| 6e7c0ba7aa | |||
| 71e0492840 | |||
| 8b2cf41f13 | |||
| 3d57db9644 | |||
| 99b167d03a | |||
| 65fcf6be65 | |||
| 9ef9d38b90 | |||
| 86e1d77ad7 | |||
| ac692e85bc | |||
| 5196bf68a7 | |||
| 6bbb4300f5 | |||
| 62584631bb | |||
| 43c8d6c29c | |||
| ccd2a4065d | |||
| bb7c956db5 | |||
| 9be0780c46 | |||
| 341da37218 | |||
| 4b3458776b | |||
| 6436290cc1 | |||
| 5316d92fc6 | |||
| 1f350d2b30 | |||
| 474cc4ad7a | |||
| 80892d31c6 | |||
| 5c4383d661 | |||
| 101c4724f4 | |||
| c09ca59747 | |||
| dad97f1b05 | |||
| eef3547c37 | |||
| 252c8222f1 | |||
| 6a4071a9fd | |||
| 5896c9582b | |||
| 718fe9547b | |||
| 6b3de9170e | |||
| ca4ffb9633 | |||
| 79a5c3ea94 | |||
| a41bf5d8e7 | |||
| 319ec796bd | |||
| 86df1e7d4b | |||
| 45f596ca0c | |||
| 4fab23a357 | |||
| 9405aa3fb4 | |||
| 2020156591 | |||
| b6d97a571d | |||
| 0e5faa7a82 | |||
| 9de2398106 | |||
| fd6ec55f07 | |||
| 65399eb4eb | |||
| b7d1a38ff6 | |||
| 89e6ee29fd | |||
| dedf615114 | |||
| f926d6507a | |||
| bcc4694fa9 | |||
| f09fe202e8 | |||
| 57ec92c299 | |||
| 681d05d7ee | |||
| 9c203d3595 | |||
| 77f67c1bd9 | |||
| 565deb443e | |||
| 0a9ff16dfc | |||
| 61904b31e3 | |||
| bb79d54b2b | |||
| 2eb9cd6fba | |||
| 775596ce21 | |||
| 4d6224f79a | |||
| 499074b564 | |||
| 6a5a0013c0 | |||
| 0c6358ab98 | |||
| 7ffa508bb4 | |||
| c2e18dbbeb | |||
| 52a22f5cb1 | |||
| 3830274e11 | |||
| 569745c0ed | |||
| 058c4e1899 | |||
| c711e443fe | |||
| 8c1dce95e8 | |||
| 3a0f3d8b19 | |||
| 8eda5fad5c | |||
| 946af84f9a | |||
| 0e46c7cb33 | |||
| b489663450 | |||
| 3aaf31b08d | |||
| dd9a3f1482 | |||
| cbee987f65 | |||
| d8d2b97716 | |||
| 036bcd2ae3 | |||
| 6c402f66b7 | |||
| 046795c4ae | |||
| 1002f1f020 | |||
| c5be117701 | |||
| 38caae3288 | |||
| 96948cc7c0 | |||
| dc88c215be | |||
| fa42a5d031 | |||
| d667586597 | |||
| ce8e289692 | |||
| 588cff3e00 | |||
| 859f6c5829 | |||
| 4fde52459d | |||
| ef7c0f05dd | |||
| e568ef8af9 | |||
| 3e9e2f632b | |||
| 9ca84e495e | |||
| b38703e53d | |||
| b6fd5ff406 | |||
| adb5a672d9 | |||
| ecf90c0281 | |||
| 9a1059ee9d | |||
| 4b7087e136 | |||
| 82a006280e | |||
| a7398c50cc | |||
| b1a0e2bcfc | |||
| 75e4b8e2cd | |||
| 25e27bdade | |||
| 3e3615e2c8 | |||
| 9a58fe6481 | |||
| 4756c20ec0 | |||
| 8a9f734d14 | |||
| 0979f506db | |||
| 627947fc2c | |||
| 2f16ff1ee8 | |||
| 8f7fb5a7fe | |||
| 4503da7920 | |||
| 28dff6e7b2 | |||
| 32992834ee | |||
| 7dd8379c61 | |||
| 408ac148ec | |||
| c9559252cd | |||
| f70370a6f4 | |||
| fad26d41fd | |||
| ef1f380cda | |||
| e690d61a12 | |||
| 6d5acb9121 | |||
| 28d223e43c | |||
| 471badd329 | |||
| 5b1636f94f | |||
| 18f6273f7c | |||
| 9950d01ab4 | |||
| cc774d5e69 | |||
| 8cbbe87c3e | |||
| 49d993b162 | |||
| 6afc094bb1 | |||
| f6a1769e35 | |||
| 4d35b53eab | |||
| 2247800003 | |||
| 7048fe7b83 | |||
| 30b973f62e | |||
| 09b09946c0 | |||
| d4328b5598 | |||
| 788514ce8e | |||
| db75f9e341 | |||
| 740dcc5789 | |||
| e03046bd49 | |||
| 1d18bee1a2 | |||
| fb00856bff | |||
| d7b354b2cf | |||
| 7c5763d048 | |||
| 61729b9ae7 | |||
| b990aed40e | |||
| d4ac57f041 | |||
| 1db0b6054a | |||
| 5e5d586f7d | |||
| 94b6b6eb7b | |||
| 6ae4b16791 | |||
| 82efa6100b | |||
| a52ed30109 | |||
| 78678e3919 | |||
| 04bfb5381d | |||
| d7bff81757 | |||
| 1ed990a081 | |||
| 27b4865d60 | |||
| 9ee1291033 | |||
| e48d84b1d0 | |||
| 309c8e37c6 | |||
| 2ff9b6f6c1 | |||
| 4c82309c36 | |||
| 9d9e7e89d9 | |||
| 994a4ab2ca | |||
| 52503f8e68 | |||
| 44a662ef94 | |||
| 7009039ac4 | |||
| 41c8742fcf | |||
| 71903b596f | |||
| bbaaf28d2e | |||
| 4f584a4e96 | |||
| bed0557b65 | |||
| 0591f55f0e | |||
| f60a4359a0 | |||
| 7a9f447231 | |||
| 38a7ea253e | |||
| 427a4e453d | |||
| 451ebb3e22 |
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Research"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,17 +1,24 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"name": "last30days-skill",
|
||||
"owner": {
|
||||
"name": "mvanhorn",
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Research any topic from the last 30 days across Reddit, X, YouTube, and the web",
|
||||
"version": "2.1.0"
|
||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": "."
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.2.3",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"source": "./",
|
||||
"category": "productivity",
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic from the last 30 days across Reddit, X, YouTube, and the web",
|
||||
"version": "2.1.0",
|
||||
"version": "3.2.3",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "mvanhorn"
|
||||
"name": "Matt Van Horn",
|
||||
"email": "mvanhorn@gmail.com",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": ["research", "reddit", "twitter", "x", "youtube", "trends", "prompts"],
|
||||
"skills": ["./"]
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Exclude binary assets and dev/test artifacts from ClawHub bundle
|
||||
assets/
|
||||
docs/
|
||||
fixtures/
|
||||
tests/
|
||||
plans/
|
||||
agents/
|
||||
variants/
|
||||
release-notes.md
|
||||
SPEC.md
|
||||
TASKS.md
|
||||
SKILL-original.md
|
||||
*.jsonl
|
||||
*.mp3
|
||||
*.jpeg
|
||||
*.jpg
|
||||
*.png
|
||||
*.gif
|
||||
@@ -0,0 +1,46 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by skills/last30days/scripts/build-skill.sh to produce a
|
||||
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
|
||||
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
# (mirrors anthropics/skills/skills/skill-creator/scripts/package_skill.py)
|
||||
__pycache__/ export-ignore
|
||||
node_modules/ export-ignore
|
||||
*.pyc export-ignore
|
||||
.DS_Store export-ignore
|
||||
evals/ export-ignore
|
||||
|
||||
# Dev, docs, test, and media - not needed at skill runtime
|
||||
tests/ export-ignore
|
||||
docs/ export-ignore
|
||||
fixtures/ export-ignore
|
||||
assets/ export-ignore
|
||||
|
||||
# NOTE: skills/ and .claude-plugin/ are NOT export-ignored here because
|
||||
# Claude Code's /plugin install fetches this same git archive tarball.
|
||||
# Removing those from the archive (as v3.0.1 did) silently breaks installs.
|
||||
# claude.ai-bundle-specific exclusions live in scripts/build-skill.sh.
|
||||
|
||||
# Historical + repo-only manifests
|
||||
SKILL-original.md export-ignore
|
||||
SPEC.md export-ignore
|
||||
TASKS.md export-ignore
|
||||
test-run.log export-ignore
|
||||
CONTRIBUTORS.md export-ignore
|
||||
HERMES_SETUP.md export-ignore
|
||||
release-notes.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
uv.lock export-ignore
|
||||
|
||||
# Platform adapters are kept in git archives because Claude Code and Codex
|
||||
# plugin installs use the same repository archive as their source payload.
|
||||
.hermes-plugin/ export-ignore
|
||||
|
||||
# CI workflows - repo-only, not needed at skill runtime
|
||||
.github/ export-ignore
|
||||
|
||||
# Build config itself
|
||||
.clawhubignore export-ignore
|
||||
.gitignore export-ignore
|
||||
.gitattributes export-ignore
|
||||
@@ -0,0 +1,53 @@
|
||||
name: Bug Report
|
||||
description: Report a bug or unexpected behavior
|
||||
labels: [bug]
|
||||
body:
|
||||
- type: textarea
|
||||
id: summary
|
||||
attributes:
|
||||
label: Summary
|
||||
description: What happened?
|
||||
placeholder: Describe the bug in 1-2 sentences.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: How can we reproduce this?
|
||||
placeholder: |
|
||||
1. Run `python3 scripts/last30days.py "topic" --emit compact`
|
||||
2. ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected Behavior
|
||||
description: What should have happened?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: traceback
|
||||
attributes:
|
||||
label: Error / Traceback
|
||||
description: Paste the full traceback or error output.
|
||||
render: text
|
||||
- type: dropdown
|
||||
id: install
|
||||
attributes:
|
||||
label: Install Method
|
||||
options:
|
||||
- Claude Code plugin
|
||||
- Gemini CLI extension
|
||||
- Codex plugin
|
||||
- Hermes skill
|
||||
- Manual (git clone)
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: OS
|
||||
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Feature Request
|
||||
description: Suggest a new feature or improvement
|
||||
labels: [enhancement]
|
||||
body:
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem
|
||||
description: What problem does this solve?
|
||||
placeholder: When I try to ..., I can't ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: How should this work?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
description: Other approaches you thought of (optional).
|
||||
@@ -0,0 +1,19 @@
|
||||
## Summary
|
||||
|
||||
<!-- What does this PR do? 1-3 sentences. -->
|
||||
|
||||
## Changes
|
||||
|
||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
||||
|
||||
-
|
||||
|
||||
## Testing
|
||||
|
||||
<!-- How did you verify this works? -->
|
||||
|
||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
||||
|
||||
## Related Issues
|
||||
|
||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
||||
@@ -0,0 +1,125 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
# Build the existing .skill artifact (Claude Code / Codex / Cursor install
|
||||
# surface). Unchanged from prior versions; just isolated into its own job
|
||||
# so the .mcpb matrix can run in parallel.
|
||||
build-skill:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Upload skill artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: last30days-skill
|
||||
path: dist/last30days.skill
|
||||
|
||||
# Cross-compile the Go MCP server for each Claude Desktop platform and
|
||||
# package each as a .mcpb. printing-press bundle handles the manifest +
|
||||
# zip layout; we only supply the pre-built binary via --skip-build.
|
||||
build-mcpb:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- goos: darwin
|
||||
goarch: arm64
|
||||
platform: darwin/arm64
|
||||
- goos: darwin
|
||||
goarch: amd64
|
||||
platform: darwin/amd64
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
platform: linux/amd64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: stable
|
||||
|
||||
- name: Install printing-press
|
||||
# Pin to a known-good PP release so the bundle command's behavior
|
||||
# is deterministic across our tags. Bump deliberately when adopting
|
||||
# a newer PP version. GOSUMDB=off skips the sumdb 404 some
|
||||
# private-namespaced go install calls hit even when the repo is
|
||||
# public; harmless here because the module path is fully qualified.
|
||||
env:
|
||||
GOPRIVATE: github.com/mvanhorn/*
|
||||
GOSUMDB: "off"
|
||||
run: go install github.com/mvanhorn/cli-printing-press/v4/cmd/printing-press@v4.8.0
|
||||
|
||||
- name: Sync engine into vendored/
|
||||
run: bash mcp/scripts/sync-engine.sh
|
||||
|
||||
- name: Build MCP binary
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: "0"
|
||||
run: |
|
||||
mkdir -p mcp/build
|
||||
go -C mcp build \
|
||||
-ldflags "-X main.Version=${{ github.ref_name }}" \
|
||||
-o build/last30days-pp-mcp \
|
||||
./cmd/last30days-pp-mcp
|
||||
|
||||
- name: Bundle .mcpb
|
||||
# printing-press bundle reads manifest.json from the cli dir and
|
||||
# rewrites the binary into bin/<entry_point> inside the zip. The
|
||||
# --platform tag drives the output filename suffix; the binary
|
||||
# itself is whatever we just cross-compiled.
|
||||
run: |
|
||||
printing-press bundle mcp \
|
||||
--skip-build \
|
||||
--binary mcp/build/last30days-pp-mcp \
|
||||
--platform ${{ matrix.platform }} \
|
||||
--output mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
|
||||
- name: Upload .mcpb artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
|
||||
# Gather every platform artifact and attach to one GitHub release.
|
||||
# release-notes generation reads commits since the prior tag.
|
||||
release:
|
||||
needs: [build-skill, build-mcpb]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
|
||||
- name: Create GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
dist/last30days.skill
|
||||
dist/last30days-pp-mcp-*.mcpb
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
prerelease: false
|
||||
@@ -0,0 +1,26 @@
|
||||
name: Validate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
plugin-contract:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run plugin contract tests
|
||||
run: uv run pytest tests/test_plugin_contract.py tests/test_version_consistency.py
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
# Private benchmark / evaluation artifacts — never push to upstream
|
||||
docs/comparison-results/
|
||||
scripts/evaluate-synthesis.py
|
||||
scripts/generate-synthesis-inputs.py
|
||||
fixtures/polymarket_sample.json
|
||||
docs/v2.1-tweets.md
|
||||
docs/30-day-anniversary-thread.md
|
||||
docs/30-day-anniversary-tweets.md
|
||||
variants/open/references/research.md
|
||||
|
||||
# OS / tool files
|
||||
.DS_Store
|
||||
.claude/
|
||||
.entire/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
mise.toml
|
||||
.memsearch/
|
||||
.venv/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Root vendor/ is accidental - real vendored client lives at scripts/lib/vendor/bird-search/
|
||||
/vendor/
|
||||
|
||||
# build artifact from scripts/build-skill.sh
|
||||
/dist/
|
||||
|
||||
# Go MCP bundle build outputs - source of truth for vendored/ stays under
|
||||
# skills/last30days/scripts/; build/ holds cross-compiled binaries + .mcpb files.
|
||||
# vendored/ lives inside the engine package because //go:embed cannot reach
|
||||
# outside its own package directory; the .gitkeep anchor stays tracked so
|
||||
# the embed pattern always finds a match even before sync-engine runs.
|
||||
/mcp/internal/engine/vendored/*
|
||||
!/mcp/internal/engine/vendored/.gitkeep
|
||||
/mcp/build/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
+378
-1
@@ -5,6 +5,380 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Changed
|
||||
|
||||
- Rename "Digg AI 1000" to just "Digg" in user-facing output (footer line, source label, inline-quote suffix, why_relevant, container attribution). Internal references to the upstream Digg AI 1000 product remain in code comments and docstrings.
|
||||
- Bump `POSTS_PER_CLUSTER` from 3 to 5 and the render-side display limit from 2 to 3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous 3/2 caps routinely truncated cluster context (e.g. dropped a Jason Calacanis quote tweet on a `cli-printing-press` run).
|
||||
- Rewrite SKILL.md path resolution. STEP 0 narrows from a global canonical-path enforcement to a Claude-Code-marketplaces-only stale-clone guard. Step 1 SKILL_ROOT resolver walks a single precedence list (Claude plugin cache, then `~/.codex/skills/`, `~/.agents/skills/`, repo checkout, `./.skills/last30days` for `npx skills add`, CWD, Gemini). Adds SKILL.md frontmatter fallback to `render.py::_skill_version` so the badge no longer prints `v?` on installs that don't include `.claude-plugin/plugin.json`.
|
||||
|
||||
- Switch SKILL.md's `--plan` and `--competitors-plan` invocation templates from inline single-quoted JSON to heredoc-written tmpfiles. Apostrophes in resolved context strings ("McDonald's", "people's choice", "developer's") previously closed the outer single-quote and broke shell parsing before the engine started — observed in a Codex run during PR #400 testing. The engine's `parse_plan()` / `parse_competitors_plan()` already supported file paths (via `os.path.isfile()` probe); only the template prose changed. Fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403).
|
||||
|
||||
### Removed
|
||||
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1. Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`.** The maintainer dev-deploy script is gone. Every job it did has a better replacement: `npx skills add . -g -y` symlinks the working tree into every detected harness's skill dir (better than sync.sh's copy model — edits propagate live), `hermes skills install mvanhorn/last30days-skill --force` handles Hermes, `clawhub install last30days-official` handles OpenClaw, and the Claude marketplace cache target was a "test against the official install path" hack we shouldn't have been recommending in the first place. The `test_sync_cache_path_uses_skill_version` test was dropped along with it. CLAUDE.md, HERMES_SETUP.md, the PR template, and a render.py docstring were updated to drop references; CHANGELOG and historical docs (release notes, plan files) keep their existing mentions as accurate history.
|
||||
|
||||
## [3.2.0] - 2026-05-09
|
||||
|
||||
### Added
|
||||
|
||||
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
|
||||
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
|
||||
|
||||
## [3.1.1] - 2026-04-24
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
|
||||
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
|
||||
|
||||
## [3.1.0] - 2026-04-22
|
||||
|
||||
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
|
||||
|
||||
### Added
|
||||
|
||||
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
|
||||
|
||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||
|
||||
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
|
||||
|
||||
## [3.0.14] - 2026-04-22
|
||||
|
||||
### Changed
|
||||
|
||||
- **Comparison-mode title attribution.** The synthesis title for vs-mode and `--competitors` outputs changes from `What the Community Says (Last 30 Days)` to `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range. Three SKILL.md occurrences updated; pure documentation change.
|
||||
|
||||
## [3.0.13] - 2026-04-22
|
||||
|
||||
### Changed
|
||||
|
||||
- **vs mode runs N full passes in parallel, one per entity.** Architectural revert of the 3-pass → 1-pass latency optimization from an earlier version. `/last30days "OpenAI vs Anthropic vs xAI"` now runs three full `pipeline.run()` calls in parallel via the same fanout `--competitors` uses, producing three `*-raw.md` save files plus a merged comparison output. Each entity gets its own Step 0.55-grade targeting, own primary X handle weight, own subreddit scoping — apples-to-apples depth instead of the one-pool merged retrieval the single-pass path produced. Parallel execution keeps wall clock ≈ single pass.
|
||||
- **`--competitors` is now a SKILL.md-level shortcut for vs-mode with auto-discovery.** The hosting reasoning model (Claude Code, Codex, Hermes, Gemini, any agent with WebSearch) performs discovery and Step 0.55 per entity via its own WebSearch tool, then invokes the engine with a vs-topic and `--competitors-plan` JSON. The engine flag remains for headless/cron use with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys (engine-internal `auto_resolve` stays as fallback).
|
||||
- **LAW 7-style stderr for `--competitors` with no backend** now leads with the hosting-model path (WebSearch + Step 0.55 + `--competitors-plan`) instead of `BRAVE_API_KEY`. API-key framing moved to a secondary "headless" section.
|
||||
|
||||
### Added
|
||||
|
||||
- **`--competitors-plan` JSON flag** for per-entity Step 0.55 targeting. Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`. Accepts inline JSON or a file path (matches `--plan`). When present for an entity, skips engine-internal `auto_resolve` and uses the provided values; missing fields fall back to `auto_resolve` (if backend) or planner defaults. Case-insensitive entity matching. The `subrun_kwargs_for` helper is the single source of truth for per-entity kwargs — no closure-default fallthrough from main scope.
|
||||
- **Per-entity save files** when `--save-dir` is set on a vs-mode or `--competitors` run. Each entity's sub-run produces its own `{slug}-raw.md` with a single-row Resolved Entities block — matches historical vs-mode behavior (N passes → N save files).
|
||||
- **`--polymarket-keywords "kw1,kw2"`** to filter Polymarket matches for ambiguous single-token topics (e.g., "Warriors" → `nba,gsw,golden-state` kills Glasgow Warriors rugby and Honor of Kings Rogue Warriors noise).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **BRAVE/SERPER footer nudge suppressed** when `--plan` or `--competitors-plan` is present. The nudge told Claude Code users to set an API key when they already have WebSearch via the hosting model. Nudge still fires for true headless runs (no `--plan`, no backend) where the advice is correct.
|
||||
- **Override-leak regression testing.** 3.0.12 already fixed the main-topic `--subreddits` / `--x-handle` / `--github-*` from leaking into peer sub-runs via explicit per-entity kwargs scrubbing. This release adds a 4-test regression suite (`test_competitor_subrun_isolation.py`) locking in the invariant.
|
||||
|
||||
## [3.0.12] - 2026-04-22
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Per-entity Step 0.55 resolution for competitor sub-runs.** In 3.0.11, only the main topic got X handle / subreddit / GitHub resolution; competitor sub-runs ran with planner defaults and produced visibly thinner evidence (Reddit 403 fallbacks, single-word queries). Each competitor sub-run now calls `resolve.auto_resolve()` inside `fanout.run_competitor_fanout` when a web backend is available, mirroring the main topic's pre-flight resolution. Per-entity X handle, subreddit list, GitHub user/repos, and news context are threaded into each sub-run's `pipeline.run()` call. Deep-copied config per sub-run prevents `_auto_resolve_context` cross-leak. Surfaces in a new `## Resolved Entities` output block so the resolution coverage is visible without reading stderr.
|
||||
- **LAW 7 false-positive on internal fan-out sub-runs.** Each competitor sub-run was emitting the `[Planner] No --plan passed... YOU ARE the planner` stderr warning. LAW 7 targets the hosting-reasoning-model path, not engine-internal fan-out. New `internal_subrun=True` keyword on `planner.plan_query` and `pipeline.run` suppresses the warning for sub-runs only; the default path is unchanged.
|
||||
- **Marketplace-stale SKILL.md trap.** Added a STEP 0 canonical-path self-check at the top of SKILL.md. Two of three 2026-04-22 test runs loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (Claude-Code-managed git clone pinned to origin/main, lagging the versioned cache), then ran `--help` against the same stale path, did not see `--competitors`, and fell back to a manual comparison plan. The STEP 0 block forces any reader to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` and re-read from the versioned cache if not.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Default `--competitors` count is now 2 (3-way total: original + 2 peers).** Previously 3. `--competitors=N` still customizes (range 1..6). Matches the feature description's canonical example (`Kanye vs Drake vs Kendrick`).
|
||||
|
||||
### Added
|
||||
|
||||
- **`## Resolved Entities` block** in `render_comparison_multi` output. Shows per-entity X handle, subreddits, GitHub user/repos, and truncated context for every entity in the comparison. Block is omitted entirely when no entity has a resolved payload (mock mode, no backend).
|
||||
|
||||
## [3.0.11] - 2026-04-22
|
||||
|
||||
### Added
|
||||
|
||||
- **`--competitors` flag for auto-discovered comparison fan-out.** Pass `--competitors` on a single-entity topic and the engine discovers 2-6 peer entities via web search, then runs the full pipeline on each in parallel and emits one N-way comparison. `last30days Kanye West --competitors` resolves Drake, Kendrick Lamar, and one more peer. `last30days OpenAI --competitors` resolves Anthropic, xAI, Google Gemini. `--competitors=N` controls count, `--competitors-list="A,B,C"` skips discovery and uses the explicit list. Discovery mirrors the `auto_resolve` pattern (Brave / Exa / Serper / Parallel) with deterministic text extraction - no internal LLM call. Sub-runs inherit the main `--quick`/`--deep`/`--days`, run in a `ThreadPoolExecutor`, and degrade gracefully when at least 2 entities survive. Output reuses the existing 9-axis `## Head-to-Head` scaffold.
|
||||
|
||||
## [3.0.10] - 2026-04-21
|
||||
|
||||
### Added
|
||||
|
||||
- **Commenter handles on evidence lines.** Top-comment rendering now includes the commenter's handle - `u/author` for Reddit, `@handle` for TikTok/YouTube/Instagram/Bluesky/X/Threads. The enrichment adapters already captured `author`; the render layer just was not using it. Evidence lines change from `- Comment (6822 upvotes): Finally, John Apple` to `- u/Cyrisaurus (6822 upvotes): Finally, John Apple`. Person-level citations make synthesis-side inline markdown links per LAW 8 much more natural. Both the compact and full render paths are covered.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
||||
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
|
||||
|
||||
### Behavior fallback
|
||||
|
||||
- When an author is empty, `[deleted]`, or `[removed]`, the render falls back to the legacy `Comment (...)` shape - no `u/` or `@` prefix with an empty handle is ever emitted.
|
||||
|
||||
## [3.0.9] - 2026-04-18 - The Self-Debug Release
|
||||
|
||||
### Highlights
|
||||
|
||||
v3.0.9 adds the engine-side Class 1 keyword-trap refuse-gate ("birthday gift for 40 year old" now gets a clarifying question, not 5 minutes of junk), promotes TikTok and YouTube top comments to the same first-class rendering Reddit's got, lands Hermes AI Agent as a first-class deploy target, and moves the SKILL.md formatting contract from line 1094 to the top of the file.
|
||||
|
||||
"The Self-Debug Release" refers to how the fixes in 3.0.6-3.0.9 were written: 5 separate Opus 4.7 instances each debugged their own failed outputs. Three converged on "SKILL.md is too big and the LAWs are too deep." Two converged on "the engine should refuse demographic-shopping queries." I shipped exactly what they said. Validation: 5/5 canonical compliance.
|
||||
|
||||
### Added
|
||||
|
||||
- **Engine Class 1 keyword-trap refuse-gate** (`scripts/lib/preflight.py`, new). Pattern-matches demographic-shopping queries at main() front-door. Exit code 2 with structured REFUSE message. Escape hatch: `LAST30DAYS_SKIP_PREFLIGHT=1`. 29 tests in `tests/test_preflight.py`.
|
||||
- **TikTok + YouTube top comments** rendered with same `💬 Top comment` prominence as Reddit's. Shipped in [#260](https://github.com/mvanhorn/last30days-skill/pull/260); enrichment fixed in [#265](https://github.com/mvanhorn/last30days-skill/pull/265).
|
||||
- **Hermes AI Agent as a deploy target** - thanks @stephenmcconnachie ([#228](https://github.com/mvanhorn/last30days-skill/pull/228)). `scripts/sync.sh` detects `~/.hermes/skills/research` and deploys automatically.
|
||||
- **Multi-key SCRAPECREATORS_API_KEY rotation** - thanks @zaydiscold ([#268](https://github.com/mvanhorn/last30days-skill/pull/268)). Set `SCRAPECREATORS_API_KEY_1`, `_2`, etc. Engine rotates on rate-limit.
|
||||
- **Offline quality evaluation fixture** - thanks @j-sperling ([#233](https://github.com/mvanhorn/last30days-skill/pull/233)). `eval_topics.json` lets contributors run quality regressions without burning live API credits.
|
||||
- **END-OF-CANONICAL-OUTPUT boundary** in `render_compact()`. Engine now emits an explicit pass-through instruction so re-synthesis requires actively ignoring a visible boundary.
|
||||
- **LAW 1 verbatim-pattern override.** LAW 1 now quotes the exact WebSearch tool-result reminder ("CRITICAL REQUIREMENT: MUST include Sources: section") and declares it OVERRIDDEN inside last30days output.
|
||||
|
||||
### Changed
|
||||
|
||||
- **SKILL.md restructure.** VOICE CONTRACT LAWs and BADGE MANDATORY block moved from line 1094 to lines 75-150. Grounded in 3 separate Opus 4.7 self-debugs.
|
||||
- **Engine emits the badge as stdout.** `🌐 last30days v3.0.9 · synced YYYY-MM-DD` is the first line of every compact emit. Pass-through is now the default-correct behavior.
|
||||
- **Reddit client HTTP consolidation** - thanks @iliaal ([#207](https://github.com/mvanhorn/last30days-skill/pull/207)). Migrated to `http.get(params=...)` helper.
|
||||
- **ScrapeCreators header consolidation** - thanks @iliaal ([#209](https://github.com/mvanhorn/last30days-skill/pull/209)). `_sc_headers` refactored into `http.scrapecreators_headers`.
|
||||
- **Simpler Hermes sync.** `scripts/sync.sh` Hermes branch now always uses main SKILL.md (previously had a `.hermes-plugin/SKILL.md` fallback that created a wrong-file-capture hazard).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Peter Steinberger trailing Sources leak.** 2026-04-18 validation failure where the model appended a TechCrunch / TED / Fortune / Wikipedia Sources list after the invitation. Now structurally prevented at three layers: engine emits the canonical body, LAW 1 quotes the exact WebSearch reminder, closing boundary names the anti-pattern.
|
||||
- **Wrong-file SKILL.md capture.** Deleted `.agents/skills/last30days/SKILL.md` (1382 lines, April 13 snapshot) and `.hermes-plugin/SKILL.md` (269 lines). One SKILL.md per plugin now, at the plugin root.
|
||||
- **GitHub date parsing garbage** - thanks @iliaal ([#208](https://github.com/mvanhorn/last30days-skill/pull/208)). `_parse_date` now rejects invalid input cleanly.
|
||||
- **Windows Bird X stability** - thanks @Chelebii ([#227](https://github.com/mvanhorn/last30days-skill/pull/227)).
|
||||
- **Linux `check_perms` false-warn** - thanks @george231224 ([#216](https://github.com/mvanhorn/last30days-skill/pull/216)). Uses GNU stat first.
|
||||
- **UTF-8 saved output** - thanks @Gujiassh ([#225](https://github.com/mvanhorn/last30days-skill/pull/225)).
|
||||
- **Version metadata alignment** - thanks @Gujiassh ([#217](https://github.com/mvanhorn/last30days-skill/pull/217)) and @shalomma ([#229](https://github.com/mvanhorn/last30days-skill/pull/229)).
|
||||
- **`--days` alias backcompat** - thanks @BryanTegomoh ([#230](https://github.com/mvanhorn/last30days-skill/pull/230)).
|
||||
- **`INCLUDE_SOURCES` env default** - thanks @hnshah ([#223](https://github.com/mvanhorn/last30days-skill/pull/223)).
|
||||
- **Bird X all-None engagement** - thanks @j-sperling ([#234](https://github.com/mvanhorn/last30days-skill/pull/234)).
|
||||
|
||||
### Contributors
|
||||
|
||||
@j-sperling, @stephenmcconnachie, @zaydiscold, @iliaal, @Chelebii, @Gujiassh, @hnshah, @george231224, @shalomma, @BryanTegomoh for PRs since v3.0.0. @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, @Jah-yee, @dannyshmueli, @Cody-Coyote for issues and PRs that shaped the v3 roadmap.
|
||||
|
||||
### Recovery
|
||||
|
||||
```
|
||||
/plugin update last30days
|
||||
/reload-plugins
|
||||
```
|
||||
|
||||
Verify: `cat ~/.claude/plugins/cache/last30days-skill/last30days/*/.claude-plugin/plugin.json | grep version` returns `"version": "3.0.9"`.
|
||||
|
||||
Smoke test: `/last30days birthday gift for 40 year old` should ask a clarifying question before running.
|
||||
|
||||
## [3.0.5] - 2026-04-15
|
||||
|
||||
### Added
|
||||
|
||||
- **`/last30days` slash command for plugin users.** New `commands/last30days.md` registers a Claude Code slash command. Users type `/last30days <topic>` and Claude Code's autocomplete prefix-matches it to the canonical `/last30days:last30days` form (the same way `/ce:plan` resolves to `/compound-engineering:ce-plan`). The command delegates to the existing `last30days` skill body — no skill behavior changes.
|
||||
|
||||
### Removed
|
||||
|
||||
- **`skills/last30days-nux/`** — byte-identical duplicate of root `SKILL.md` that created confusing `/last30days:last30days-nux` autocomplete entries via Claude Code's plugin namespacing. The root `SKILL.md` remains the canonical skill source.
|
||||
|
||||
### Recovery
|
||||
|
||||
```
|
||||
/plugin update last30days
|
||||
/reload-plugins
|
||||
```
|
||||
|
||||
Then type `/last30days <topic>` to invoke the skill via slash command. Natural-language invocation ("search the last 30 days for X") continues to work unchanged.
|
||||
|
||||
## [3.0.4] - 2026-04-15
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Cleared `/doctor` path-escape error on Claude Code v2.1.109+.** `.claude-plugin/plugin.json` previously declared `"skills": ["./"]`. That value shipped unchanged from v2.1.0 through v3.0.3 and worked on older Claude Code, but current versions reject `./` with `Path escapes plugin directory: ./ (skills)`. The `"skills"` key is now omitted entirely, matching the pattern used by every other plugin in the Claude Code marketplace ecosystem. Claude Code auto-discovers `skills/*/SKILL.md` when the key is absent.
|
||||
|
||||
### Recovery
|
||||
|
||||
If `/doctor` reports a path-escape error for last30days, run `/plugin update last30days` then `/reload-plugins`. If errors persist, uninstall and reinstall the plugin.
|
||||
|
||||
## [3.0.3] - 2026-04-15
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Restored `skills/` and `.claude-plugin/` to the plugin install tarball.** v3.0.1 added `.gitattributes` rules that excluded both directories from `git archive` output to shrink the claude.ai `.skill` bundle. Claude Code's `/plugin install` fetches the same archive, so users installing v3.0.1 or v3.0.2 received a tarball with no plugin manifest and no skill files. `git archive v3.0.0` contained 8 files under those paths; `v3.0.1` and `v3.0.2` contained 0. This release reverts those `.gitattributes` lines.
|
||||
- **Reverted `plugin.json` `"skills"` field to `["./"]`.** v3.0.2 changed this to `["skills"]` based on a misdiagnosis — the manifest change had no effect because the manifest wasn't in the tarball at all. The historical `["./"]` value shipped in every release from v2.1.0 through v3.0.0 without issues and is restored here.
|
||||
|
||||
### Recovery
|
||||
|
||||
Users on v3.0.1 or v3.0.2: run `/plugin update last30days` then `/reload-plugins`. If autoUpdate is enabled, the next session start will pull v3.0.3 automatically. Users on cached v3.0.0 or earlier installs were unaffected.
|
||||
|
||||
### Notes
|
||||
|
||||
- The claude.ai `.skill` bundle built by `scripts/build-skill.sh` still works — the archive grew from 89 to 97 files, well under the 200-file cap.
|
||||
- claude.ai-specific exclusions (avoiding duplicate `SKILL.md` files in the bundle) should move into `scripts/build-skill.sh` rather than `.gitattributes` in a future release, since `.gitattributes` cannot distinguish between the two distribution channels.
|
||||
|
||||
## [3.0.2] - 2026-04-15
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`/last30days` slash command now registers on Claude Code v2.1.105+.** `.claude-plugin/plugin.json` declared `"skills": ["./"]`, which newer Claude Code rejects with `Path escapes plugin directory: ./ (skills)`. The skill silently failed to register, so `/last30days <query>` returned "Unknown command" even though `/plugin list` showed the plugin as installed. Fix: `"skills": ["skills"]` so the loader scans the real skill subdirectory.
|
||||
- **Version drift between manifests.** `.claude-plugin/marketplace.json` was pinned to `3.0.0` while `.claude-plugin/plugin.json` advertised `3.0.1`. The `/plugin` resolver used the marketplace version and could install stale cached metadata alongside the correct build. Both manifests now agree on `3.0.2`.
|
||||
|
||||
### Recovery
|
||||
|
||||
If `/last30days` stopped working for you, run `/plugin update last30days` then `/reload-plugins`. If `/doctor` still reports errors, uninstall and reinstall the plugin from the marketplace.
|
||||
|
||||
## [3.0.1] - 2026-04-14
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Skill upload packaging** - `scripts/build-skill.sh` produces a claude.ai-upload-ready `.skill` file that fits under the 200-file cap. Previously, zipping the repo hit 406 files and the "Upload skill" UI rejected it outright.
|
||||
- **SKILL.md description length** - trimmed from 228 to 167 chars (Anthropic caps descriptions at 200).
|
||||
|
||||
### Removed
|
||||
|
||||
- Unused root `vendor/` directory (215 files from an accidental commit in PR #48 - the real vendored X client lives at `scripts/lib/vendor/bird-search/`).
|
||||
- Legacy top-level `plans/` directory (superseded by `docs/plans/`; both plans described work that was already shipped in v3).
|
||||
|
||||
### Added
|
||||
|
||||
- `.gitattributes` with `export-ignore` entries so `git archive` drops tests, docs, fixtures, assets, historical manifests, and internal skill subdirs. Mirrors Anthropic's canonical `package_skill.py` exclusions.
|
||||
- `scripts/build-skill.sh` - one-command path to produce `dist/last30days.skill` with a single top-level `last30days/` folder, defensive `=200` file check, and dirty-tree refusal.
|
||||
- `README.md` section documenting the claude.ai skill upload workflow.
|
||||
|
||||
## [3.0.0] - 2026-04-11
|
||||
|
||||
### Highlights
|
||||
|
||||
Intelligent search, fun judge, cross-source cluster merging, single-pass comparisons, and OpenClaw as a first-class citizen. The v3 engine doesn't just search for your topic -- it figures out *where* to search before the search begins. Engine architecture by @j-sperling.
|
||||
|
||||
### Added
|
||||
|
||||
- **Intelligent pre-research** -- Resolves X handles, subreddits, TikTok hashtags, and YouTube channels via a new Python brain before any API calls fire. Bidirectional: person to company, product to founder.
|
||||
- **Fun judge / Best Takes** -- Second parallel LLM judge scores humor, cleverness, and virality. Surfaces the best reactions in a dedicated output section.
|
||||
- **Cross-source cluster merging** -- Entity-based overlap detection merges the same story across Reddit, X, YouTube into one cluster instead of three separate items.
|
||||
- **Single-pass comparisons** -- "X vs Y" runs one pass with entity-aware subqueries instead of three serial passes. 3 minutes instead of 12+.
|
||||
- **GitHub as a source** -- Stars, reactions, and comments from repos and issues.
|
||||
- **OpenClaw first-class citizen** -- Auto-resolve for engine-side pre-research. Device auth for frictionless ScrapeCreators signup.
|
||||
- **Per-author cap** -- Max 3 items per author prevents single-voice dominance.
|
||||
- **Entity disambiguation** -- Synthesis trusts resolved handles over keyword matches.
|
||||
- **Perplexity Sonar Pro as additive source** -- AI-synthesized research with citations via OpenRouter. Opt-in via `INCLUDE_SOURCES=perplexity`. Returns structured narratives that complement social data.
|
||||
- **Perplexity Deep Research** -- `--deep-research` flag for exhaustive 50+ citation reports (~$0.90/query). Premium opt-in for serious investigation.
|
||||
- **OpenRouter as reasoning provider** -- One OPENROUTER_API_KEY powers planning, reranking, and Perplexity search. Auto-detected after Gemini/OpenAI/xAI.
|
||||
- **Parallel AI grounding backend** -- `--web-backend parallel` or auto-detected via PARALLEL_API_KEY.
|
||||
- **Grounding in planner** -- Grounding source properly registered in SOURCE_CAPABILITIES instead of force-injected.
|
||||
|
||||
### Changed
|
||||
|
||||
- YouTube transcript candidate pool widened 3x past music videos to reach talk/review content with captions
|
||||
- Reddit comment enrichment sorted by total engagement (upvotes + comments), not just upvotes
|
||||
- Polymarket display shows % odds only; dollar volumes removed
|
||||
- 852 tests passing
|
||||
|
||||
### Fixed
|
||||
|
||||
- Marketplace validation: duplicate `name: last30days` collision in `skills/last30days/SKILL.md` caused strict validators to reject the plugin. Resolved by renaming the internal v3 architecture spec to `last30days-v3-spec` with `user-invocable: false`. Fixed in #214 (reported by @Cody-Coyote in #204).
|
||||
- Stale README link to the deleted `skills/last30days-v3/` path from the v3 directory rename. Fixed in #214.
|
||||
- OpenAI Codex CLI discoverability: added `.agents/skills/last30days/SKILL.md` as a real file (Codex's loader skips symlinked files) plus `.codex-plugin/plugin.json` as the namespace marker. The skill now registers as `last30days:last30days` when Codex runs in a checkout of the repo. Fixed in #219 (inspired by @Jah-yee in #153 and @dannyshmueli on X).
|
||||
|
||||
### Contributors
|
||||
|
||||
- @j-sperling -- v3 engine architecture, Python pre-research brain
|
||||
- @hnshah -- Watchlist features
|
||||
- @Cody-Coyote -- Marketplace validation bug report (#204)
|
||||
- @Jah-yee -- Codex CLI integration inspiration (#153)
|
||||
|
||||
## [2.9.4] - 2026-03-06
|
||||
|
||||
### Changed
|
||||
|
||||
- Move save into Python script via `--save-dir` flag - raw research data saved during the existing script Bash call, zero extra tool calls after invitation
|
||||
- Remove entire "Save Research to Documents" section from SKILL.md (~45 lines removed)
|
||||
- No more `📎` footer, no Bash heredoc, no `(No output)`, no multi-minute cogitation after research
|
||||
|
||||
## [2.9.3] - 2026-03-06
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Critical:** Switch save from `run_in_background` to foreground Bash - background callbacks caused model to re-engage, hallucinate fake user messages, and generate unsolicited multi-paragraph responses
|
||||
- Save uses foreground `cat >` heredoc (executes sub-second, no callback, no delayed notification)
|
||||
|
||||
## [2.9.2] - 2026-03-06
|
||||
|
||||
### Fixed
|
||||
|
||||
- Save research silently using background Bash heredoc instead of Write tool (eliminates "Wrote N lines..." clutter)
|
||||
- Suppress follow-up text after background save completes (no more "Research briefing saved..." noise)
|
||||
- Add `📎` footer line for save path instead of verbose confirmation
|
||||
|
||||
## [2.9.1] - 2026-03-05
|
||||
|
||||
### Highlights
|
||||
|
||||
Auto-save research briefings to the default memory directory as topic-named .md files. Every run now builds a personal research library automatically - no more manual copy-paste.
|
||||
|
||||
### Added
|
||||
|
||||
- Auto-save complete research briefings (synthesis, stats, follow-up suggestions) to the default memory directory after every run
|
||||
- Kebab-case filename generation from topic (e.g., "Claude Code skills" -> `claude-code-skills.md`)
|
||||
- Duplicate topic handling: appends date suffix instead of overwriting (e.g., `claude-code-skills-2026-03-05.md`)
|
||||
- Agent mode (`--agent`) also saves research files
|
||||
- Brief confirmation after save with the saved file path
|
||||
|
||||
### Credits
|
||||
|
||||
- [@devin_explores](https://x.com/devin_explores) -- Inspired this feature by sharing their workflow of saving every last30days run into organized .md files ([PR #51](https://github.com/mvanhorn/last30days-skill/pull/51))
|
||||
|
||||
## [2.9.0] - 2026-03-05
|
||||
|
||||
### Highlights
|
||||
|
||||
ScrapeCreators Reddit as the default backend (one `SCRAPECREATORS_API_KEY` covers Reddit + TikTok + Instagram), smart subreddit discovery with relevance-weighted scoring, and top comments elevated with 10% scoring weight and prominent display.
|
||||
|
||||
### Added
|
||||
|
||||
- ScrapeCreators Reddit backend (`scripts/lib/reddit.py`) — keyword search, subreddit discovery, comment enrichment, all via `api.scrapecreators.com`
|
||||
- Smart subreddit discovery with relevance-weighted scoring: frequency × recency × topic-word match, replacing pure frequency count
|
||||
- `UTILITY_SUBS` blocklist to filter noise subreddits (r/tipofmytongue, r/whatisthisthing, etc.) from discovery results
|
||||
- Top comment scoring: 10% weight in engagement formula via `log1p(top_comment_score)`
|
||||
- Top comment rendering: `💬 Top comment` lines with upvote counts in compact and full report output
|
||||
- Comment excerpt length increased from 300 → 400 chars; `comment_insights` limit raised from 7 → 10
|
||||
|
||||
### Changed
|
||||
|
||||
- `primaryEnv` switched from `OPENAI_API_KEY` to `SCRAPECREATORS_API_KEY` — one key now powers Reddit, TikTok, and Instagram
|
||||
- Reddit engagement scoring formula: `0.55/0.40/0.05` (score/comments/ratio) → `0.50/0.35/0.05/0.10` (score/comments/ratio/top-comment)
|
||||
- SKILL.md synthesis instructions updated to emphasize quoting top comments
|
||||
|
||||
### Fixed
|
||||
|
||||
- Utility subreddit noise in discovery (e.g., r/tipofmytongue appearing for unrelated topics)
|
||||
- Reddit search no longer requires `OPENAI_API_KEY` — ScrapeCreators API handles search directly
|
||||
|
||||
## [2.8.0] - 2026-03-04
|
||||
|
||||
### Highlights
|
||||
|
||||
Instagram Reels as the 8th signal source, TikTok migrated from Apify to ScrapeCreators API, and SKILL.md quality improvements. One API key (`SCRAPECREATORS_API_KEY`) now covers both TikTok and Instagram.
|
||||
|
||||
### Added
|
||||
|
||||
- Instagram Reels as 8th research source via ScrapeCreators API — keyword search, engagement metrics (views, likes, comments), spoken-word transcript extraction (`scripts/lib/instagram.py`)
|
||||
- `InstagramItem` dataclass, normalization, scoring (45% relevance / 25% recency / 30% engagement), deduplication, cross-source linking, and rendering
|
||||
- Instagram in SKILL.md: stats template (`📸 Instagram:`), citation priority, item format description, output footer
|
||||
- URL-to-name extraction examples in SKILL.md for cleaner web source display
|
||||
- `--search=instagram` flag support
|
||||
|
||||
### Changed
|
||||
|
||||
- TikTok backend migrated from Apify to ScrapeCreators API (`api.scrapecreators.com`)
|
||||
- `APIFY_API_TOKEN` replaced by `SCRAPECREATORS_API_KEY` in config
|
||||
- SKILL.md version bumped to v2.8
|
||||
- WebSearch citation instruction strengthened to prevent trailing Sources: blocks
|
||||
- Security section updated: Apify → ScrapeCreators references
|
||||
|
||||
### Fixed
|
||||
|
||||
- Web stats line showing full URLs instead of plain domain names
|
||||
- Trailing "Sources:" block appearing after skill invitation (WebSearch tool mandate conflict)
|
||||
- Instagram/TikTok not running in web-only mode when `--search=instagram` used without Reddit/X
|
||||
- `$ARGUMENTS` quoting in SKILL.md for correct flag forwarding
|
||||
|
||||
## [2.1.0] - 2026-02-15
|
||||
|
||||
### Highlights
|
||||
@@ -51,7 +425,6 @@ Three headline features: watchlists for always-on bots, YouTube transcripts as a
|
||||
|
||||
### Credits
|
||||
|
||||
- @steipete -- Bird CLI (vendored X search) and yt-dlp/summarize inspiration for YouTube transcripts
|
||||
- @galligan -- Marketplace plugin inspiration
|
||||
- @hutchins -- Pushed for YouTube feature
|
||||
|
||||
@@ -59,5 +432,9 @@ Three headline features: watchlists for always-on bots, YouTube transcripts as a
|
||||
|
||||
Initial public release. Reddit + X search via OpenAI Responses API and xAI API.
|
||||
|
||||
[3.0.9]: https://github.com/mvanhorn/last30days-skill/compare/v3.0.5...v3.0.9
|
||||
[2.9.1]: https://github.com/mvanhorn/last30days-skill/compare/v2.9.0...v2.9.1
|
||||
[2.9.0]: https://github.com/mvanhorn/last30days-skill/compare/v2.8.0...v2.9.0
|
||||
[2.8.0]: https://github.com/mvanhorn/last30days-skill/compare/v2.6.0...v2.8.0
|
||||
[2.1.0]: https://github.com/mvanhorn/last30days-skill/compare/v1.0.0...v2.1.0
|
||||
[1.0.0]: https://github.com/mvanhorn/last30days-skill/releases/tag/v1.0.0
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# last30days Skill
|
||||
|
||||
Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||
Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
||||
```
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
@@ -0,0 +1,59 @@
|
||||
# Contributors
|
||||
|
||||
last30days is built by [@mvanhorn](https://github.com/mvanhorn) with help from the community.
|
||||
|
||||
## v3 Inspiration
|
||||
|
||||
These contributors submitted PRs and issues that directly inspired v3 features. The v3 engine was a ground-up rewrite, so their original code wasn't merged, but their ideas shaped what shipped.
|
||||
|
||||
Want to claim your entry? Submit a PR replacing the placeholder line below your name with your bio, website, or anything you'd like.
|
||||
|
||||
---
|
||||
|
||||
### @uppinote20
|
||||
[PR #143](https://github.com/mvanhorn/last30days-skill/pull/143) - Rich Reddit comments, top 3 per post
|
||||
v3 ships top comments with upvote counts on every thread.
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @zerone0x
|
||||
[Issue #134](https://github.com/mvanhorn/last30days-skill/issues/134) + [PR #136](https://github.com/mvanhorn/last30days-skill/pull/136) - GitHub as a first-class data source
|
||||
v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos with live star counts.
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @thinkun
|
||||
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
||||
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @thomasmktong
|
||||
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
||||
v3 Reddit is 100% pure Python with zero external dependencies.
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @fanispoulinakisai-boop
|
||||
[Issue #100](https://github.com/mvanhorn/last30days-skill/issues/100) - Reddit timeout report
|
||||
Drove the timeout resilience work that made v3 Reddit bulletproof.
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @pejmanjohn
|
||||
[Issue #78](https://github.com/mvanhorn/last30days-skill/issues/78) - ScrapeCreators silent failures
|
||||
v3 surfaces all API errors with clear diagnostics instead of silently returning empty results.
|
||||
> Repping the mighty MI; home of the most cracked agentic engineers. https://github.com/pejmanjohn
|
||||
|
||||
### @zl190
|
||||
[PR #115](https://github.com/mvanhorn/last30days-skill/pull/115) - HN trending merge
|
||||
v3 merges trending and keyword HN results with deduplication for better coverage.
|
||||
> Healthcare AI engineer. [Blog](https://zl190.github.io/blog)
|
||||
|
||||
### @hnshah
|
||||
[PR #84](https://github.com/mvanhorn/last30days-skill/pull/84), [#85](https://github.com/mvanhorn/last30days-skill/pull/85), [#86](https://github.com/mvanhorn/last30days-skill/pull/86) - Watchlist delivery, 90-day scanning window, HN/Polymarket storage
|
||||
v3 has durable watchlist with multi-source storage and extended time windows.
|
||||
> Hiten Shah. Founder. Builds in public. https://github.com/hnshah
|
||||
|
||||
---
|
||||
|
||||
## Past Contributors
|
||||
|
||||
- [@JosephOIbrahim](https://github.com/JosephOIbrahim) - Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17))
|
||||
- [@levineam](https://github.com/levineam) - Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16))
|
||||
- [@jonthebeef](https://github.com/jonthebeef) - Early testing and feedback
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
# Hermes Setup Guide for last30days
|
||||
|
||||
This guide covers installing last30days on Hermes AI Agent.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Hermes installed** - See https://github.com/mercurial-tf/hermes
|
||||
2. **Python 3.12+** - `brew install python@3.12` or similar
|
||||
3. **yt-dlp** (optional, for YouTube) - `brew install yt-dlp`
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
```
|
||||
|
||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
||||
|
||||
### Developer / live-edit alternative
|
||||
|
||||
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
mkdir -p ~/.hermes/skills/research
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
In Hermes, invoke with:
|
||||
|
||||
```
|
||||
last30days "your research topic"
|
||||
```
|
||||
|
||||
Or with options:
|
||||
```
|
||||
last30days "best mechanical keyboards 2025" --search=reddit,youtube
|
||||
last30days "AI news" --days=7 --deep
|
||||
```
|
||||
|
||||
## First Run Setup
|
||||
|
||||
On first run, the skill will guide you through setup:
|
||||
|
||||
1. **Auto setup** (~30 seconds)
|
||||
- Scans browser cookies for X/Twitter
|
||||
- Checks/installs yt-dlp for YouTube
|
||||
- Configures free sources (Reddit, HN, Polymarket)
|
||||
|
||||
2. **Optional: ScrapeCreators**
|
||||
- Adds TikTok, Instagram, Reddit backup
|
||||
- 10,000 free API calls
|
||||
- Sign up at scrapecreators.com
|
||||
|
||||
3. **Optional: API Keys**
|
||||
- XAI_API_KEY for X/Twitter (alternative to browser cookies)
|
||||
- BRAVE_API_KEY for web search
|
||||
|
||||
## Available Sources
|
||||
|
||||
### Free (No API Key)
|
||||
- **Reddit** - Public discussions and comments
|
||||
- **Hacker News** - Tech discussions via Algolia
|
||||
- **Polymarket** - Prediction markets
|
||||
- **YouTube** - Search and transcripts (requires yt-dlp)
|
||||
|
||||
### Requires API Key
|
||||
- **X/Twitter** - xAI API key or browser cookies
|
||||
- **TikTok** - ScrapeCreators API
|
||||
- **Instagram** - ScrapeCreators API
|
||||
- **Web Search** - Brave Search API
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Python not found
|
||||
```bash
|
||||
# Find Python 3.12+
|
||||
which python3.12 python3.13 python3.14
|
||||
|
||||
# If not installed
|
||||
brew install python@3.12
|
||||
```
|
||||
|
||||
### yt-dlp not found
|
||||
```bash
|
||||
brew install yt-dlp
|
||||
# or
|
||||
pip install yt-dlp
|
||||
```
|
||||
|
||||
### Check what's configured
|
||||
```bash
|
||||
cd ~/.hermes/skills/research/last30days
|
||||
python3.12 scripts/last30days.py --diagnose
|
||||
```
|
||||
|
||||
## Updating
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
```
|
||||
|
||||
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step.
|
||||
|
||||
## Support
|
||||
|
||||
- Original repo: https://github.com/mvanhorn/last30days-skill
|
||||
- Hermes: https://github.com/mercurial-tf/hermes
|
||||
- Issues: Please report in the original repo
|
||||
+1
-2
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Peter Steinberger
|
||||
Copyright (c) 2026 Matt Van Horn
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## Overview
|
||||
|
||||
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet.
|
||||
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. OpenAI auth can come from `OPENAI_API_KEY` or Codex login credentials.
|
||||
|
||||
The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability.
|
||||
|
||||
@@ -10,7 +10,7 @@ The skill operates in three modes depending on available API keys: **reddit-only
|
||||
|
||||
The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`:
|
||||
|
||||
- **env.py**: Load and validate API keys from `~/.config/last30days/.env`
|
||||
- **env.py**: Load API keys from `~/.config/last30days/.env` and Codex auth from `~/.codex/auth.json`
|
||||
- **dates.py**: Date range calculation and confidence scoring
|
||||
- **cache.py**: 24-hour TTL caching keyed by topic + date range
|
||||
- **http.py**: stdlib-only HTTP client with retry logic
|
||||
@@ -18,6 +18,8 @@ The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalizat
|
||||
- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit
|
||||
- **xai_x.py**: xAI Responses API + x_search for X
|
||||
- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics
|
||||
- **hackernews.py**: Hacker News search via Algolia API (free, no auth)
|
||||
- **polymarket.py**: Polymarket prediction market search via Gamma API (free, no auth)
|
||||
- **normalize.py**: Convert raw API responses to canonical schema
|
||||
- **score.py**: Compute popularity-aware scores (relevance + recency + engagement)
|
||||
- **dedupe.py**: Near-duplicate detection via text similarity
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
description: Research what people actually say about any topic in the last 30 days across Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.
|
||||
argument-hint: <topic> — e.g. "nvidia earnings reaction" or "best noise cancelling headphones"
|
||||
allowed-tools: [Bash, Read, Write, AskUserQuestion, WebSearch]
|
||||
---
|
||||
|
||||
Invoke the `last30days` skill with the user's arguments: $ARGUMENTS
|
||||
|
||||
Use the skill's canonical pipeline (plan → retrieve → normalize → fuse → rerank → cluster → render). If the user provided no arguments, ask them for a topic before proceeding.
|
||||
@@ -0,0 +1,195 @@
|
||||
# How Reddit & X Search Work in last30days
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
User: /last30days "kanye west"
|
||||
↓
|
||||
┌─────┴─────┐
|
||||
↓ ↓ (concurrent via ThreadPoolExecutor)
|
||||
[REDDIT] [X/TWITTER]
|
||||
↓ ↓
|
||||
OpenAI Bundled Bird or
|
||||
API xAI API
|
||||
↓ ↓
|
||||
Parse Parse
|
||||
↓ ↓
|
||||
Enrich ───┘
|
||||
(fetch ↓
|
||||
actual [MERGE]
|
||||
upvotes) ↓
|
||||
↓ [NORMALIZE → FILTER → SCORE → DEDUPE]
|
||||
└───────────↓
|
||||
[OUTPUT to SKILL.md agent]
|
||||
```
|
||||
|
||||
Both searches run **in parallel** using Python's `ThreadPoolExecutor(max_workers=2)`.
|
||||
|
||||
---
|
||||
|
||||
## Reddit Search
|
||||
|
||||
### How it works
|
||||
|
||||
Reddit search uses the **OpenAI Responses API** with the `web_search` tool, domain-filtered to `reddit.com` only.
|
||||
|
||||
**API Call:**
|
||||
```
|
||||
POST https://api.openai.com/v1/responses
|
||||
Authorization: Bearer {OPENAI_API_KEY}
|
||||
```
|
||||
|
||||
**Payload:**
|
||||
```json
|
||||
{
|
||||
"model": "gpt-5.2",
|
||||
"tools": [{
|
||||
"type": "web_search",
|
||||
"filters": { "allowed_domains": ["reddit.com"] }
|
||||
}],
|
||||
"input": "Search Reddit for threads about {topic}..."
|
||||
}
|
||||
```
|
||||
|
||||
The prompt asks the model to:
|
||||
1. Extract core subject (strip noise words like "best", "tips", "top")
|
||||
2. Search 3 patterns: `"{topic} site:reddit.com"`, `"reddit {topic}"`, `"{topic} reddit"`
|
||||
3. Return JSON with `title`, `url`, `subreddit`, `date`, `relevance`
|
||||
4. URLs must contain `/r/` AND `/comments/` (real threads only)
|
||||
|
||||
**Model fallback chain:** `gpt-5.2 → gpt-5.1 → gpt-5 → gpt-4.1 → gpt-4o → gpt-4o-mini`
|
||||
Triggers on HTTP 400/403 with access error keywords.
|
||||
|
||||
### Enrichment (the secret sauce)
|
||||
|
||||
After search, each thread gets **enriched** by hitting Reddit's free JSON API:
|
||||
|
||||
```
|
||||
GET https://reddit.com/r/{sub}/comments/{id}/{slug}/.json
|
||||
```
|
||||
|
||||
No API key needed. This returns the actual thread data:
|
||||
|
||||
| Data Point | Source |
|
||||
|---|---|
|
||||
| Upvotes (score) | Reddit JSON API |
|
||||
| Comment count | Reddit JSON API |
|
||||
| Upvote ratio | Reddit JSON API |
|
||||
| Top 10 comments (text + score) | Reddit JSON API |
|
||||
| 7 key comment insights | Extracted via heuristics |
|
||||
| Actual post date | `created_utc` timestamp |
|
||||
|
||||
**This is why Reddit results have real engagement metrics** — the enrichment step fetches actual upvote/comment data, not AI estimates.
|
||||
|
||||
### Depth settings
|
||||
|
||||
| Depth | Threads requested | Timeout |
|
||||
|---|---|---|
|
||||
| `--quick` | 15-25 | 90s |
|
||||
| default | 30-50 | 120s |
|
||||
| `--deep` | 70-100 | 180s |
|
||||
|
||||
---
|
||||
|
||||
## X/Twitter Search
|
||||
|
||||
X search has **two backends** — the skill auto-detects which to use.
|
||||
|
||||
### Priority: Bundled Bird (env auth) → xAI API (paid)
|
||||
|
||||
```python
|
||||
if node_available and AUTH_TOKEN and CT0:
|
||||
use bundled Bird # Free, popup-free, env-authenticated
|
||||
elif XAI_API_KEY:
|
||||
use xAI API # Paid, uses grok-4-1-fast
|
||||
else:
|
||||
skip X entirely # No X results
|
||||
```
|
||||
|
||||
### Backend 1: xAI API
|
||||
|
||||
**API Call:**
|
||||
```
|
||||
POST https://api.x.ai/v1/responses
|
||||
Authorization: Bearer {XAI_API_KEY}
|
||||
```
|
||||
|
||||
**Payload:**
|
||||
```json
|
||||
{
|
||||
"model": "grok-4-1-fast",
|
||||
"tools": [{ "type": "x_search" }],
|
||||
"input": "Search X for posts about {topic} from {from_date} to {to_date}..."
|
||||
}
|
||||
```
|
||||
|
||||
The prompt asks grok to return JSON with:
|
||||
- `text`, `url`, `author_handle`, `date`
|
||||
- `engagement`: `{ likes, reposts, replies, quotes }`
|
||||
- `why_relevant`, `relevance` score
|
||||
|
||||
**Engagement data comes from grok's x_search tool** - it has direct access to X's data.
|
||||
|
||||
### Backend 2: Bundled Bird client (free alternative)
|
||||
|
||||
The repo vendors a search-only subset of Bird's Twitter GraphQL client and shells out to it with Node.js. No global `bird` install is required. The Python wrapper passes `AUTH_TOKEN` and `CT0` via env, which keeps normal local runs headless and avoids browser-cookie prompts.
|
||||
|
||||
**Bundled Bird returns raw X API data** - likes, reposts, replies are real engagement metrics from X's API, not estimates.
|
||||
|
||||
| Metric | Bundled Bird | xAI API |
|
||||
|---|---|---|
|
||||
| Post text | Real | Real |
|
||||
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
||||
| Replies/quotes | Real | Real |
|
||||
| Author handle | Real | Real |
|
||||
| Relevance score | Default 0.7 (re-ranked by score.py) | AI-assessed 0.0-1.0 |
|
||||
|
||||
### Depth settings
|
||||
|
||||
| Depth | xAI posts | Bundled Bird results | xAI timeout | Bird timeout |
|
||||
|---|---|---|---|---|
|
||||
| `--quick` | 8-12 | 12 | 90s | 30s |
|
||||
| default | 20-30 | 30 | 120s | 45s |
|
||||
| `--deep` | 40-60 | 60 | 180s | 60s |
|
||||
|
||||
---
|
||||
|
||||
## Post-Processing (both sources)
|
||||
|
||||
After both searches complete:
|
||||
|
||||
1. **Normalize** — consistent formatting, timezone handling
|
||||
2. **Date filter** — hard filter to requested date range
|
||||
3. **Score** — relevance scoring (engagement-weighted)
|
||||
4. **Sort** — highest scores first
|
||||
5. **Deduplicate** — remove duplicate URLs
|
||||
6. **Fallback** — if all items filtered out, keep top 3 by relevance
|
||||
|
||||
---
|
||||
|
||||
## Error Handling
|
||||
|
||||
| Layer | Strategy |
|
||||
|---|---|
|
||||
| HTTP requests | 3 retries with exponential backoff (1s → 2s → 3s) |
|
||||
| Model access errors | Automatic fallback to next model in chain |
|
||||
| Reddit enrichment | Per-item try/catch; keeps unenriched item on failure |
|
||||
| X source detection | Silent fallback from Bird → xAI → skip |
|
||||
| Overall pipeline | Errors stored as `reddit_error`/`x_error`, shown to user |
|
||||
|
||||
---
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `scripts/last30days.py` | Main orchestrator, concurrent execution |
|
||||
| `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API |
|
||||
| `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `scripts/lib/models.py` | Auto-select best available model |
|
||||
| `scripts/lib/env.py` | API key loading, source detection |
|
||||
| `scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `scripts/lib/score.py` | Relevance scoring |
|
||||
| `scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
@@ -1,929 +0,0 @@
|
||||
# Bird CLI Integration Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Add Bird CLI as a free, zero-config alternative to xAI for X/Twitter searches with interactive installation.
|
||||
|
||||
**Architecture:** New `bird_x.py` module handles Bird detection, installation prompts, and search. Modified `env.py` determines X source priority (Bird → xAI → WebSearch). Main script prompts for Bird install if not found.
|
||||
|
||||
**Tech Stack:** Python 3, subprocess for Bird CLI calls, existing lib modules for normalization/scoring.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Create bird_x.py - Detection Functions
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/lib/bird_x.py`
|
||||
|
||||
**Step 1: Create the module with detection functions**
|
||||
|
||||
```python
|
||||
"""Bird CLI client for X (Twitter) search."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
"""Log to stderr."""
|
||||
sys.stderr.write(f"[Bird] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def is_bird_installed() -> bool:
|
||||
"""Check if Bird CLI is installed."""
|
||||
return shutil.which("bird") is not None
|
||||
|
||||
|
||||
def is_bird_authenticated() -> Optional[str]:
|
||||
"""Check if Bird is authenticated by running 'bird whoami'.
|
||||
|
||||
Returns:
|
||||
Username if authenticated, None otherwise.
|
||||
"""
|
||||
if not is_bird_installed():
|
||||
return None
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["bird", "whoami"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
# Output is typically the username
|
||||
return result.stdout.strip().split('\n')[0]
|
||||
return None
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, Exception):
|
||||
return None
|
||||
|
||||
|
||||
def check_npm_available() -> bool:
|
||||
"""Check if npm is available for installation."""
|
||||
return shutil.which("npm") is not None
|
||||
```
|
||||
|
||||
**Step 2: Verify the module loads**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib import bird_x; print('OK')"`
|
||||
Expected: `OK`
|
||||
|
||||
**Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/lib/bird_x.py
|
||||
git commit -m "feat(bird): add detection functions for Bird CLI"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Add Bird Installation Functions
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/bird_x.py`
|
||||
|
||||
**Step 1: Add installation function**
|
||||
|
||||
Add after `check_npm_available()`:
|
||||
|
||||
```python
|
||||
def install_bird() -> Tuple[bool, str]:
|
||||
"""Install Bird CLI via npm.
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message).
|
||||
"""
|
||||
if not check_npm_available():
|
||||
return False, "npm not found. Install Node.js first, or install Bird manually: https://github.com/steipete/bird"
|
||||
|
||||
try:
|
||||
_log("Installing Bird CLI...")
|
||||
result = subprocess.run(
|
||||
["npm", "install", "-g", "@steipete/bird"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return True, "Bird CLI installed successfully!"
|
||||
else:
|
||||
error = result.stderr.strip() or result.stdout.strip() or "Unknown error"
|
||||
return False, f"Installation failed: {error}"
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "Installation timed out"
|
||||
except Exception as e:
|
||||
return False, f"Installation error: {e}"
|
||||
|
||||
|
||||
def get_bird_status() -> Dict[str, Any]:
|
||||
"""Get comprehensive Bird status.
|
||||
|
||||
Returns:
|
||||
Dict with keys: installed, authenticated, username, can_install
|
||||
"""
|
||||
installed = is_bird_installed()
|
||||
username = is_bird_authenticated() if installed else None
|
||||
|
||||
return {
|
||||
"installed": installed,
|
||||
"authenticated": username is not None,
|
||||
"username": username,
|
||||
"can_install": check_npm_available(),
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Verify functions work**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib import bird_x; print(bird_x.get_bird_status())"`
|
||||
Expected: Dict with installed/authenticated status
|
||||
|
||||
**Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/lib/bird_x.py
|
||||
git commit -m "feat(bird): add installation and status functions"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Add Bird Search Function
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/bird_x.py`
|
||||
|
||||
**Step 1: Add depth config and search function**
|
||||
|
||||
Add after imports at top:
|
||||
|
||||
```python
|
||||
# Depth configurations: number of results to request
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 12,
|
||||
"default": 30,
|
||||
"deep": 60,
|
||||
}
|
||||
```
|
||||
|
||||
Add after `get_bird_status()`:
|
||||
|
||||
```python
|
||||
def search_x(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search X using Bird CLI.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
depth: Research depth - "quick", "default", or "deep"
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
|
||||
# Build command
|
||||
cmd = [
|
||||
"bird", "search",
|
||||
topic,
|
||||
"--since", from_date,
|
||||
"-n", str(count),
|
||||
"--json",
|
||||
]
|
||||
|
||||
# Adjust timeout based on depth
|
||||
timeout = 30 if depth == "quick" else 45 if depth == "default" else 60
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
# Parse JSON output
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
return json.loads(output)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"error": "Search timed out", "items": []}
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "items": []}
|
||||
```
|
||||
|
||||
**Step 2: Verify search function signature**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib import bird_x; import inspect; print(inspect.signature(bird_x.search_x))"`
|
||||
Expected: `(topic: str, from_date: str, to_date: str, depth: str = 'default') -> Dict[str, Any]`
|
||||
|
||||
**Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/lib/bird_x.py
|
||||
git commit -m "feat(bird): add search_x function"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Add Bird Response Parser
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/bird_x.py`
|
||||
|
||||
**Step 1: Add parse function**
|
||||
|
||||
Add at end of file:
|
||||
|
||||
```python
|
||||
def parse_bird_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Parse Bird response to match xai_x output format.
|
||||
|
||||
Args:
|
||||
response: Raw Bird JSON response
|
||||
|
||||
Returns:
|
||||
List of normalized item dicts matching xai_x.parse_x_response() format.
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Check for errors
|
||||
if "error" in response and response["error"]:
|
||||
_log(f"Bird error: {response['error']}")
|
||||
return items
|
||||
|
||||
# Bird returns a list of tweets directly or under a key
|
||||
raw_items = response if isinstance(response, list) else response.get("items", response.get("tweets", []))
|
||||
|
||||
if not isinstance(raw_items, list):
|
||||
return items
|
||||
|
||||
for i, tweet in enumerate(raw_items):
|
||||
if not isinstance(tweet, dict):
|
||||
continue
|
||||
|
||||
# Extract URL - Bird uses permanent_url or we construct from id
|
||||
url = tweet.get("permanent_url") or tweet.get("url", "")
|
||||
if not url and tweet.get("id"):
|
||||
screen_name = tweet.get("user", {}).get("screen_name", "")
|
||||
if screen_name:
|
||||
url = f"https://x.com/{screen_name}/status/{tweet['id']}"
|
||||
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Parse date from created_at (e.g., "Wed Jan 15 14:30:00 +0000 2026")
|
||||
date = None
|
||||
created_at = tweet.get("created_at", "")
|
||||
if created_at:
|
||||
try:
|
||||
# Try ISO format first
|
||||
if "T" in created_at:
|
||||
dt = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||
else:
|
||||
# Twitter format: "Wed Jan 15 14:30:00 +0000 2026"
|
||||
dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
|
||||
date = dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Extract user info
|
||||
user = tweet.get("user", {})
|
||||
author_handle = user.get("screen_name", "") or tweet.get("author_handle", "")
|
||||
|
||||
# Build engagement dict
|
||||
engagement = {
|
||||
"likes": tweet.get("like_count") or tweet.get("favorite_count"),
|
||||
"reposts": tweet.get("retweet_count"),
|
||||
"replies": tweet.get("reply_count"),
|
||||
"quotes": tweet.get("quote_count"),
|
||||
}
|
||||
# Convert to int where possible
|
||||
for key in engagement:
|
||||
if engagement[key] is not None:
|
||||
try:
|
||||
engagement[key] = int(engagement[key])
|
||||
except (ValueError, TypeError):
|
||||
engagement[key] = None
|
||||
|
||||
# Build normalized item
|
||||
item = {
|
||||
"id": f"X{i+1}",
|
||||
"text": str(tweet.get("text", tweet.get("full_text", ""))).strip()[:500],
|
||||
"url": url,
|
||||
"author_handle": author_handle.lstrip("@"),
|
||||
"date": date,
|
||||
"engagement": engagement if any(v is not None for v in engagement.values()) else None,
|
||||
"why_relevant": "", # Bird doesn't provide relevance explanations
|
||||
"relevance": 0.7, # Default relevance, let score.py re-rank
|
||||
}
|
||||
|
||||
items.append(item)
|
||||
|
||||
return items
|
||||
```
|
||||
|
||||
**Step 2: Verify parser handles empty input**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib import bird_x; print(bird_x.parse_bird_response({}))"`
|
||||
Expected: `[]`
|
||||
|
||||
**Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/lib/bird_x.py
|
||||
git commit -m "feat(bird): add response parser matching xai_x format"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Add UI Functions for Bird Prompts
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/ui.py`
|
||||
|
||||
**Step 1: Add Bird-related messages and prompts**
|
||||
|
||||
Add after `PROMO_SINGLE_KEY_PLAIN` dict (around line 128):
|
||||
|
||||
```python
|
||||
# Bird CLI prompts
|
||||
BIRD_INSTALL_PROMPT = f"""
|
||||
{Colors.CYAN}{Colors.BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━{Colors.RESET}
|
||||
{Colors.CYAN}🐦 FREE X/TWITTER SEARCH AVAILABLE{Colors.RESET}
|
||||
|
||||
Bird CLI provides free X search using your browser session (no API key needed).
|
||||
|
||||
"""
|
||||
|
||||
BIRD_INSTALL_PROMPT_PLAIN = """
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
🐦 FREE X/TWITTER SEARCH AVAILABLE
|
||||
|
||||
Bird CLI provides free X search using your browser session (no API key needed).
|
||||
|
||||
"""
|
||||
|
||||
BIRD_AUTH_HELP = f"""
|
||||
{Colors.YELLOW}Bird authentication failed.{Colors.RESET}
|
||||
|
||||
To fix this:
|
||||
1. Log into X (twitter.com) in Safari, Chrome, or Firefox
|
||||
2. Run: {Colors.BOLD}bird check{Colors.RESET} to verify credentials
|
||||
3. Try again
|
||||
|
||||
For manual setup, see: https://github.com/steipete/bird#authentication
|
||||
"""
|
||||
|
||||
BIRD_AUTH_HELP_PLAIN = """
|
||||
Bird authentication failed.
|
||||
|
||||
To fix this:
|
||||
1. Log into X (twitter.com) in Safari, Chrome, or Firefox
|
||||
2. Run: bird check to verify credentials
|
||||
3. Try again
|
||||
|
||||
For manual setup, see: https://github.com/steipete/bird#authentication
|
||||
"""
|
||||
```
|
||||
|
||||
**Step 2: Add prompt functions to ProgressDisplay class**
|
||||
|
||||
Add these methods to the `ProgressDisplay` class (after `show_promo` method, around line 310):
|
||||
|
||||
```python
|
||||
def prompt_bird_install(self) -> bool:
|
||||
"""Prompt user to install Bird CLI.
|
||||
|
||||
Returns:
|
||||
True if user wants to install, False otherwise.
|
||||
"""
|
||||
if IS_TTY:
|
||||
sys.stderr.write(BIRD_INSTALL_PROMPT)
|
||||
else:
|
||||
sys.stderr.write(BIRD_INSTALL_PROMPT_PLAIN)
|
||||
sys.stderr.flush()
|
||||
|
||||
try:
|
||||
response = input("Install Bird CLI now? (y/n): ").strip().lower()
|
||||
return response in ('y', 'yes')
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
return False
|
||||
|
||||
def show_bird_install_success(self, username: str):
|
||||
"""Show Bird installation success message."""
|
||||
msg = f"{Colors.GREEN}✓ Bird installed and authenticated as @{username}{Colors.RESET}\n" if IS_TTY else f"✓ Bird installed and authenticated as @{username}\n"
|
||||
sys.stderr.write(msg)
|
||||
sys.stderr.flush()
|
||||
|
||||
def show_bird_install_failed(self, error: str):
|
||||
"""Show Bird installation failure message."""
|
||||
msg = f"{Colors.RED}✗ Bird installation failed: {error}{Colors.RESET}\n" if IS_TTY else f"✗ Bird installation failed: {error}\n"
|
||||
sys.stderr.write(msg)
|
||||
sys.stderr.flush()
|
||||
|
||||
def show_bird_auth_help(self):
|
||||
"""Show Bird authentication help."""
|
||||
if IS_TTY:
|
||||
sys.stderr.write(BIRD_AUTH_HELP)
|
||||
else:
|
||||
sys.stderr.write(BIRD_AUTH_HELP_PLAIN)
|
||||
sys.stderr.flush()
|
||||
```
|
||||
|
||||
**Step 3: Verify new methods exist**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib.ui import ProgressDisplay; p = ProgressDisplay('test', show_banner=False); print(hasattr(p, 'prompt_bird_install'))"`
|
||||
Expected: `True`
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/lib/ui.py
|
||||
git commit -m "feat(ui): add Bird CLI install prompts and auth help"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Update env.py with X Source Detection
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/env.py`
|
||||
|
||||
**Step 1: Add get_x_source function**
|
||||
|
||||
Add at end of file:
|
||||
|
||||
```python
|
||||
def get_x_source(config: Dict[str, Any]) -> Optional[str]:
|
||||
"""Determine the best available X/Twitter source.
|
||||
|
||||
Priority: Bird (free) → xAI (paid API)
|
||||
|
||||
Args:
|
||||
config: Configuration dict from get_config()
|
||||
|
||||
Returns:
|
||||
'bird' if Bird is installed and authenticated,
|
||||
'xai' if XAI_API_KEY is configured,
|
||||
None if no X source available.
|
||||
"""
|
||||
# Import here to avoid circular dependency
|
||||
from . import bird_x
|
||||
|
||||
# Check Bird first (free option)
|
||||
if bird_x.is_bird_installed():
|
||||
username = bird_x.is_bird_authenticated()
|
||||
if username:
|
||||
return 'bird'
|
||||
|
||||
# Fall back to xAI if key exists
|
||||
if config.get('XAI_API_KEY'):
|
||||
return 'xai'
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Get detailed X source status for UI decisions.
|
||||
|
||||
Returns:
|
||||
Dict with keys: source, bird_installed, bird_authenticated,
|
||||
bird_username, xai_available, can_install_bird
|
||||
"""
|
||||
from . import bird_x
|
||||
|
||||
bird_status = bird_x.get_bird_status()
|
||||
xai_available = bool(config.get('XAI_API_KEY'))
|
||||
|
||||
# Determine active source
|
||||
if bird_status["authenticated"]:
|
||||
source = 'bird'
|
||||
elif xai_available:
|
||||
source = 'xai'
|
||||
else:
|
||||
source = None
|
||||
|
||||
return {
|
||||
"source": source,
|
||||
"bird_installed": bird_status["installed"],
|
||||
"bird_authenticated": bird_status["authenticated"],
|
||||
"bird_username": bird_status["username"],
|
||||
"xai_available": xai_available,
|
||||
"can_install_bird": bird_status["can_install"],
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Verify function works**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib import env; print(env.get_x_source_status(env.get_config()))"`
|
||||
Expected: Dict with source status
|
||||
|
||||
**Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/lib/env.py
|
||||
git commit -m "feat(env): add X source detection with Bird priority"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Update __init__.py to Export bird_x
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/__init__.py`
|
||||
|
||||
**Step 1: Add bird_x to imports**
|
||||
|
||||
Replace file contents with:
|
||||
|
||||
```python
|
||||
# last30days library modules
|
||||
from . import bird_x
|
||||
```
|
||||
|
||||
**Step 2: Verify import works**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib import bird_x; print('OK')"`
|
||||
Expected: `OK`
|
||||
|
||||
**Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/lib/__init__.py
|
||||
git commit -m "feat(lib): export bird_x module"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Integrate Bird into Main Script - Part 1 (Setup Phase)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py`
|
||||
|
||||
**Step 1: Add bird_x import**
|
||||
|
||||
Add `bird_x` to the imports from lib (around line 36):
|
||||
|
||||
```python
|
||||
from lib import (
|
||||
bird_x,
|
||||
dates,
|
||||
dedupe,
|
||||
env,
|
||||
http,
|
||||
models,
|
||||
normalize,
|
||||
openai_reddit,
|
||||
reddit_enrich,
|
||||
render,
|
||||
schema,
|
||||
score,
|
||||
ui,
|
||||
websearch,
|
||||
xai_x,
|
||||
)
|
||||
```
|
||||
|
||||
**Step 2: Add Bird setup function**
|
||||
|
||||
Add after the imports, before `load_fixture`:
|
||||
|
||||
```python
|
||||
def setup_bird_if_needed(progress: ui.ProgressDisplay) -> Optional[str]:
|
||||
"""Check Bird status and offer installation if needed.
|
||||
|
||||
Returns:
|
||||
'bird' if Bird is ready to use,
|
||||
'declined' if user declined install,
|
||||
None if Bird not available and couldn't be installed.
|
||||
"""
|
||||
status = bird_x.get_bird_status()
|
||||
|
||||
# Already working
|
||||
if status["authenticated"]:
|
||||
return 'bird'
|
||||
|
||||
# Installed but not authenticated
|
||||
if status["installed"]:
|
||||
progress.show_bird_auth_help()
|
||||
return None
|
||||
|
||||
# Not installed - offer to install if npm available
|
||||
if status["can_install"]:
|
||||
if progress.prompt_bird_install():
|
||||
success, message = bird_x.install_bird()
|
||||
if success:
|
||||
# Check if auth works now
|
||||
username = bird_x.is_bird_authenticated()
|
||||
if username:
|
||||
progress.show_bird_install_success(username)
|
||||
return 'bird'
|
||||
else:
|
||||
progress.show_bird_auth_help()
|
||||
return None
|
||||
else:
|
||||
progress.show_bird_install_failed(message)
|
||||
return None
|
||||
else:
|
||||
return 'declined'
|
||||
|
||||
return None
|
||||
```
|
||||
|
||||
**Step 3: Verify script still loads**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "import scripts.last30days; print('OK')"`
|
||||
Expected: `OK`
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/last30days.py
|
||||
git commit -m "feat(main): add Bird setup function"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Integrate Bird into Main Script - Part 2 (Search Dispatch)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py`
|
||||
|
||||
**Step 1: Modify _search_x function to support Bird**
|
||||
|
||||
Replace the `_search_x` function (around line 119-159) with:
|
||||
|
||||
```python
|
||||
def _search_x(
|
||||
topic: str,
|
||||
config: dict,
|
||||
selected_models: dict,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str,
|
||||
mock: bool,
|
||||
x_source: str = "xai",
|
||||
) -> tuple:
|
||||
"""Search X via Bird CLI or xAI (runs in thread).
|
||||
|
||||
Args:
|
||||
x_source: 'bird' or 'xai' - which backend to use
|
||||
|
||||
Returns:
|
||||
Tuple of (x_items, raw_response, error)
|
||||
"""
|
||||
raw_response = None
|
||||
x_error = None
|
||||
|
||||
if mock:
|
||||
raw_response = load_fixture("xai_sample.json")
|
||||
x_items = xai_x.parse_x_response(raw_response or {})
|
||||
return x_items, raw_response, x_error
|
||||
|
||||
# Use Bird if specified
|
||||
if x_source == "bird":
|
||||
try:
|
||||
raw_response = bird_x.search_x(
|
||||
topic,
|
||||
from_date,
|
||||
to_date,
|
||||
depth=depth,
|
||||
)
|
||||
except Exception as e:
|
||||
raw_response = {"error": str(e)}
|
||||
x_error = f"{type(e).__name__}: {e}"
|
||||
|
||||
x_items = bird_x.parse_bird_response(raw_response or {})
|
||||
|
||||
# Check for error in response
|
||||
if raw_response and raw_response.get("error") and not x_error:
|
||||
x_error = raw_response["error"]
|
||||
|
||||
return x_items, raw_response, x_error
|
||||
|
||||
# Use xAI (original behavior)
|
||||
try:
|
||||
raw_response = xai_x.search_x(
|
||||
config["XAI_API_KEY"],
|
||||
selected_models["xai"],
|
||||
topic,
|
||||
from_date,
|
||||
to_date,
|
||||
depth=depth,
|
||||
)
|
||||
except http.HTTPError as e:
|
||||
raw_response = {"error": str(e)}
|
||||
x_error = f"API error: {e}"
|
||||
except Exception as e:
|
||||
raw_response = {"error": str(e)}
|
||||
x_error = f"{type(e).__name__}: {e}"
|
||||
|
||||
x_items = xai_x.parse_x_response(raw_response or {})
|
||||
|
||||
return x_items, raw_response, x_error
|
||||
```
|
||||
|
||||
**Step 2: Update run_research to accept x_source parameter**
|
||||
|
||||
Find the `run_research` function signature (around line 161) and add `x_source` parameter:
|
||||
|
||||
```python
|
||||
def run_research(
|
||||
topic: str,
|
||||
sources: str,
|
||||
config: dict,
|
||||
selected_models: dict,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
mock: bool = False,
|
||||
progress: ui.ProgressDisplay = None,
|
||||
x_source: str = "xai",
|
||||
) -> tuple:
|
||||
```
|
||||
|
||||
Then update the `_search_x` call inside (around line 218-222) to pass `x_source`:
|
||||
|
||||
```python
|
||||
x_future = executor.submit(
|
||||
_search_x, topic, config, selected_models,
|
||||
from_date, to_date, depth, mock, x_source
|
||||
)
|
||||
```
|
||||
|
||||
**Step 3: Verify script syntax is valid**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -m py_compile scripts/last30days.py && echo "OK"`
|
||||
Expected: `OK`
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/last30days.py
|
||||
git commit -m "feat(main): dispatch X search to Bird or xAI"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Integrate Bird into Main Script - Part 3 (Main Function)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py`
|
||||
|
||||
**Step 1: Update main() to check Bird before research**
|
||||
|
||||
In the `main()` function, after loading config and before checking available sources (around line 345-355), add Bird setup:
|
||||
|
||||
Find this section:
|
||||
```python
|
||||
# Load config
|
||||
config = env.get_config()
|
||||
|
||||
# Check available sources
|
||||
available = env.get_available_sources(config)
|
||||
```
|
||||
|
||||
Replace with:
|
||||
```python
|
||||
# Load config
|
||||
config = env.get_config()
|
||||
|
||||
# Initialize progress display early for Bird prompts
|
||||
progress = ui.ProgressDisplay(args.topic, show_banner=True)
|
||||
|
||||
# Check Bird availability and offer install if needed
|
||||
x_source_status = env.get_x_source_status(config)
|
||||
x_source = x_source_status["source"]
|
||||
|
||||
# If no X source and Bird can be installed, offer it
|
||||
if x_source is None and x_source_status["can_install_bird"]:
|
||||
bird_result = setup_bird_if_needed(progress)
|
||||
if bird_result == 'bird':
|
||||
x_source = 'bird'
|
||||
# Refresh status
|
||||
x_source_status = env.get_x_source_status(config)
|
||||
|
||||
# Check available sources (now accounting for Bird)
|
||||
available = env.get_available_sources(config)
|
||||
|
||||
# Override available if Bird is ready
|
||||
if x_source == 'bird':
|
||||
if available == 'reddit':
|
||||
available = 'both' # Now have both Reddit + X (via Bird)
|
||||
elif available == 'web':
|
||||
available = 'x' # Now have X via Bird
|
||||
```
|
||||
|
||||
**Step 2: Remove duplicate progress initialization**
|
||||
|
||||
Find and remove the later `progress = ui.ProgressDisplay(...)` line (around line 371) since we now create it earlier.
|
||||
|
||||
**Step 3: Pass x_source to run_research**
|
||||
|
||||
Find the `run_research` call (around line 413) and add `x_source` parameter:
|
||||
|
||||
```python
|
||||
reddit_items, x_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error = run_research(
|
||||
args.topic,
|
||||
sources,
|
||||
config,
|
||||
selected_models,
|
||||
from_date,
|
||||
to_date,
|
||||
depth,
|
||||
args.mock,
|
||||
progress,
|
||||
x_source=x_source or "xai",
|
||||
)
|
||||
```
|
||||
|
||||
**Step 4: Verify script runs with --help**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 scripts/last30days.py --help`
|
||||
Expected: Help text displays without errors
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/last30days.py
|
||||
git commit -m "feat(main): integrate Bird setup into main flow"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Test End-to-End with Mock Mode
|
||||
|
||||
**Files:**
|
||||
- None (testing only)
|
||||
|
||||
**Step 1: Test mock mode still works**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 scripts/last30days.py "Claude Code" --mock --emit=compact 2>&1 | head -20`
|
||||
Expected: Output showing research results without errors
|
||||
|
||||
**Step 2: Test Bird detection (informational)**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && python3 -c "from scripts.lib import env; import json; print(json.dumps(env.get_x_source_status(env.get_config()), indent=2))"`
|
||||
Expected: JSON showing current Bird/xAI status
|
||||
|
||||
**Step 3: Commit any fixes if needed, then final commit**
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "feat(bird): complete Bird CLI integration
|
||||
|
||||
- Add bird_x.py module for Bird CLI detection, install, and search
|
||||
- Add UI prompts for interactive Bird installation
|
||||
- Update env.py with X source priority (Bird > xAI)
|
||||
- Integrate Bird into main research flow
|
||||
- Bird uses browser cookies (free, no API key needed)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 12: Push to Private Repo
|
||||
|
||||
**Files:**
|
||||
- None (git only)
|
||||
|
||||
**Step 1: Push all changes**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && git push origin main`
|
||||
Expected: Changes pushed to private repo
|
||||
|
||||
**Step 2: Verify commit history**
|
||||
|
||||
Run: `cd /Users/mvanhorn/last30days-skill-private && git log --oneline -10`
|
||||
Expected: Shows Bird integration commits
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
After completing all tasks, the skill will:
|
||||
|
||||
1. Check if Bird CLI is installed on startup
|
||||
2. If not installed but npm available, prompt user to install
|
||||
3. If installed, verify authentication via `bird whoami`
|
||||
4. If authenticated, use Bird for all X searches (free)
|
||||
5. If not, fall back to xAI (if key exists) or WebSearch
|
||||
6. Output format identical regardless of backend used
|
||||
@@ -1,102 +0,0 @@
|
||||
# Bird CLI Integration Design
|
||||
|
||||
**Date:** 2026-02-03
|
||||
**Status:** Approved
|
||||
|
||||
## Overview
|
||||
|
||||
Add Bird CLI as an alternative X/Twitter search source for the last30days skill. Bird uses browser cookie authentication (free, no API key) and provides direct access to X's GraphQL API.
|
||||
|
||||
## Goals
|
||||
|
||||
- Provide free X search without requiring xAI API key
|
||||
- Seamless fallback: Bird → xAI → WebSearch
|
||||
- Interactive onboarding for users without Bird installed
|
||||
- Output parity with existing xAI implementation
|
||||
|
||||
## Detection & Priority Flow
|
||||
|
||||
```
|
||||
On startup:
|
||||
1. Check: Is Bird installed? (`which bird`)
|
||||
├─ No → Offer to install: "Bird CLI not found. Install for free X search? (y/n)"
|
||||
│ ├─ Yes → Run `npm install -g @steipete/bird`
|
||||
│ └─ No → Continue to step 2
|
||||
│
|
||||
└─ Yes → Check: Is Bird authenticated? (`bird whoami`)
|
||||
├─ Success → Use Bird for X searches
|
||||
└─ Fail → Show: "Bird auth failed. Run `bird check` to diagnose."
|
||||
Continue to step 2
|
||||
|
||||
2. Fall back to xAI if XAI_API_KEY exists
|
||||
3. Fall back to WebSearch if nothing else available
|
||||
```
|
||||
|
||||
**Priority order:** Bird → xAI → WebSearch
|
||||
|
||||
## New Module: `scripts/lib/bird_x.py`
|
||||
|
||||
### Functions
|
||||
|
||||
- `is_bird_installed()` → checks `which bird`, returns bool
|
||||
- `is_bird_authenticated()` → runs `bird whoami`, returns username or None
|
||||
- `install_bird()` → runs `npm install -g @steipete/bird`, returns success bool
|
||||
- `search_x(topic, from_date, to_date, depth)` → runs `bird search` with JSON output
|
||||
- `parse_bird_response(json)` → converts to same format as `xai_x.parse_x_response()`
|
||||
|
||||
### Search Command
|
||||
|
||||
```bash
|
||||
bird search "Claude Code skills" --since 2026-01-04 -n 30 --json
|
||||
```
|
||||
|
||||
- `--since` filters to last 30 days
|
||||
- `-n 30` controls result count (maps to depth: quick=12, default=30, deep=60)
|
||||
- `--json` gives machine-readable output
|
||||
|
||||
### Output Mapping
|
||||
|
||||
| Bird field | Our field |
|
||||
|------------|-----------|
|
||||
| `text` | `text` |
|
||||
| `permanent_url` | `url` |
|
||||
| `user.screen_name` | `author_handle` |
|
||||
| `created_at` | `date` (parse to YYYY-MM-DD) |
|
||||
| `like_count` | `engagement.likes` |
|
||||
| `retweet_count` | `engagement.reposts` |
|
||||
| `reply_count` | `engagement.replies` |
|
||||
| `quote_count` | `engagement.quotes` |
|
||||
|
||||
Relevance: Default to 0.7, let `score.py` re-rank based on engagement.
|
||||
|
||||
## Modified Files
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `env.py` | Add `get_x_source()` → returns `'bird'`, `'xai'`, or `None` |
|
||||
| `last30days.py` | Check Bird availability with interactive install prompt before research |
|
||||
| `last30days.py` | In `_search_x()`, dispatch to `bird_x` or `xai_x` based on source |
|
||||
| `ui.py` | Add `prompt_bird_install()` and `show_bird_auth_help()` |
|
||||
|
||||
## Unchanged Files
|
||||
|
||||
- `normalize.py` - Bird output matches xAI format after parsing
|
||||
- `score.py` - Same scoring logic applies
|
||||
- `dedupe.py` - Same deduplication logic
|
||||
- `render.py` - X results labeled as "X" regardless of backend
|
||||
|
||||
## Error Handling
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|----------|
|
||||
| Bird installed but no browser cookies | Show `bird check` guidance, fall back to xAI |
|
||||
| Bird search returns 0 results | Retry with simplified query (same as xAI logic) |
|
||||
| Bird search times out | Fall back to xAI if available, else WebSearch |
|
||||
| npm not installed (can't install Bird) | Skip Bird, continue with xAI/WebSearch |
|
||||
| User declines Bird install | Remember for session, don't ask again |
|
||||
|
||||
**Timeout:** 30 seconds for Bird commands
|
||||
|
||||
## Output Labels
|
||||
|
||||
Results labeled as "X" regardless of whether Bird or xAI was used. Users care about the data, not the backend.
|
||||
@@ -1,391 +0,0 @@
|
||||
---
|
||||
title: "feat: Release last30days v2 with Bird CLI to GitHub"
|
||||
type: feat
|
||||
date: 2026-02-06
|
||||
---
|
||||
|
||||
# Release last30days v2 (Bird CLI) to GitHub
|
||||
|
||||
## Overview
|
||||
|
||||
Replace the current public `last30days-skill` on GitHub with the new Bird CLI-enhanced version from `last30days-skill-private`. The new version adds free X/Twitter search via Bird CLI while maintaining backward compatibility with xAI API keys.
|
||||
|
||||
**Goal:** Ship with confidence. No rollbacks.
|
||||
|
||||
## Current State
|
||||
|
||||
| | Old (Public) | New (Private) |
|
||||
|---|---|---|
|
||||
| **Repo** | `mvanhorn/last30days-skill` | `mvanhorn/last30days-skill-private` |
|
||||
| **Local path** | `~/.claude/skills/last30days/` | `~/.claude/skills/last30daystest/` (symlink) |
|
||||
| **Remote** | `origin` → public repo | `origin` → private, `upstream` → public |
|
||||
| **Key addition** | -- | Bird CLI (`@steipete/bird`) for free X search |
|
||||
| **X source chain** | xAI API only | Bird (free) → xAI (paid) → WebSearch |
|
||||
| **Uses** | 136 | 18 |
|
||||
| **Latest commit** | `cc892d7` | `4230fa2` |
|
||||
|
||||
**Why both show as `/last30days`:** Both `SKILL.md` files declare `name: last30days` in frontmatter. Claude Code discovers both from `~/.claude/skills/` and lists them separately.
|
||||
|
||||
---
|
||||
|
||||
## Phase 0: Clean Swap (Day 1)
|
||||
|
||||
Remove the old skill so only the new one is active. This eliminates ambiguity during testing.
|
||||
|
||||
### Steps
|
||||
|
||||
1. **Back up the old skill** (safety net):
|
||||
```bash
|
||||
mv ~/.claude/skills/last30days ~/.claude/skills/last30days.backup-v1
|
||||
```
|
||||
|
||||
2. **Promote the new skill to primary**:
|
||||
```bash
|
||||
# Remove the test symlink
|
||||
rm ~/.claude/skills/last30daystest
|
||||
|
||||
# Create new symlink with the primary name
|
||||
ln -s /Users/mvanhorn/last30days-skill-private ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
3. **Verify only one `/last30days` appears**:
|
||||
- Open a new Claude Code session
|
||||
- Type `/last` and confirm only ONE `/last30days` shows in autocomplete
|
||||
- Confirm description mentions Bird CLI
|
||||
|
||||
4. **Rollback procedure** (if something goes wrong):
|
||||
```bash
|
||||
rm ~/.claude/skills/last30days
|
||||
mv ~/.claude/skills/last30days.backup-v1 ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Only one `/last30days` appears in Claude Code autocomplete
|
||||
- [ ] Old skill preserved at `~/.claude/skills/last30days.backup-v1`
|
||||
- [ ] New skill responds to `/last30days` invocation
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Claude's Test Plan (Automated)
|
||||
|
||||
These are tests Claude can run autonomously to validate the new skill before the user touches it.
|
||||
|
||||
### 1.1 Script-Level Smoke Tests
|
||||
|
||||
Run the Python scripts directly to verify core functionality without invoking the full skill.
|
||||
|
||||
#### Bird CLI Detection
|
||||
```bash
|
||||
# Test: Bird is installed and authenticated
|
||||
python3 -c "
|
||||
import sys; sys.path.insert(0, '/Users/mvanhorn/last30days-skill-private/scripts/lib')
|
||||
import bird_x
|
||||
print('installed:', bird_x.is_bird_installed())
|
||||
print('authenticated:', bird_x.is_bird_authenticated())
|
||||
print('status:', bird_x.get_bird_status())
|
||||
"
|
||||
```
|
||||
- [ ] `is_bird_installed()` returns True (or False with clear message)
|
||||
- [ ] `is_bird_authenticated()` returns True if logged into X in browser
|
||||
- [ ] `get_bird_status()` returns a dict with `installed`, `authenticated`, `available` keys
|
||||
|
||||
#### Environment & Source Detection
|
||||
```bash
|
||||
python3 -c "
|
||||
import sys; sys.path.insert(0, '/Users/mvanhorn/last30days-skill-private/scripts/lib')
|
||||
import env
|
||||
config = env.load_config()
|
||||
print('x_source:', env.get_x_source(config))
|
||||
print('has_openai:', bool(config.get('OPENAI_API_KEY')))
|
||||
"
|
||||
```
|
||||
- [ ] `get_x_source()` returns `'bird'` if Bird available, `'xai'` if API key set, `None` otherwise
|
||||
- [ ] Config loads from `~/.config/last30days/.env`
|
||||
|
||||
#### Bird Search (Direct)
|
||||
```bash
|
||||
python3 -c "
|
||||
import sys, json; sys.path.insert(0, '/Users/mvanhorn/last30days-skill-private/scripts/lib')
|
||||
import bird_x
|
||||
result = bird_x.search_x('Claude Code tips', '2026-01-07', '2026-02-06', 'quick')
|
||||
print(json.dumps(result, indent=2, default=str)[:2000])
|
||||
"
|
||||
```
|
||||
- [ ] Returns search results (list of dicts with `url`, `text`, `author_handle`)
|
||||
- [ ] No Python tracebacks
|
||||
- [ ] Results are from the expected date range
|
||||
|
||||
#### Full Research Pipeline (Compact Output)
|
||||
```bash
|
||||
cd /Users/mvanhorn/last30days-skill-private
|
||||
python3 scripts/last30days.py "Claude Code tips" --emit=compact --quick 2>&1 | head -100
|
||||
```
|
||||
- [ ] Completes without error
|
||||
- [ ] Output includes X results (via Bird or xAI)
|
||||
- [ ] Output includes Reddit results (via OpenAI) if key configured
|
||||
- [ ] Stats summary shows source counts
|
||||
|
||||
### 1.2 Source Fallback Tests
|
||||
|
||||
Verify graceful degradation when sources are unavailable.
|
||||
|
||||
#### Bird unavailable, xAI available
|
||||
```bash
|
||||
# Temporarily hide Bird
|
||||
PATH_BACKUP="$PATH"
|
||||
export PATH=$(echo "$PATH" | tr ':' '\n' | grep -v "$(dirname $(which bird 2>/dev/null))" | tr '\n' ':')
|
||||
|
||||
python3 -c "
|
||||
import sys; sys.path.insert(0, '/Users/mvanhorn/last30days-skill-private/scripts/lib')
|
||||
import env
|
||||
config = env.load_config()
|
||||
print('x_source (no bird):', env.get_x_source(config))
|
||||
"
|
||||
|
||||
export PATH="$PATH_BACKUP"
|
||||
```
|
||||
- [ ] Falls back to `'xai'` when Bird not in PATH
|
||||
- [ ] No crash or unhandled exception
|
||||
|
||||
#### No X source at all
|
||||
```bash
|
||||
python3 -c "
|
||||
import sys; sys.path.insert(0, '/Users/mvanhorn/last30days-skill-private/scripts/lib')
|
||||
import env
|
||||
config = {} # empty config, no keys
|
||||
print('x_source (nothing):', env.get_x_source(config))
|
||||
"
|
||||
```
|
||||
- [ ] Returns `None`
|
||||
- [ ] No crash
|
||||
|
||||
### 1.3 Response Parsing Tests
|
||||
|
||||
Validate that Bird responses are correctly normalized to the canonical schema.
|
||||
|
||||
```bash
|
||||
python3 -c "
|
||||
import sys; sys.path.insert(0, '/Users/mvanhorn/last30days-skill-private/scripts/lib')
|
||||
import bird_x
|
||||
|
||||
# Test with sample Bird response format
|
||||
sample = {
|
||||
'tweets': [{
|
||||
'permanentUrl': 'https://x.com/user/status/123',
|
||||
'text': 'Test tweet about Claude Code',
|
||||
'username': 'testuser',
|
||||
'likeCount': 42,
|
||||
'retweetCount': 10,
|
||||
'replyCount': 5,
|
||||
'timeParsed': '2026-02-01T12:00:00.000Z'
|
||||
}]
|
||||
}
|
||||
parsed = bird_x.parse_bird_response(sample)
|
||||
print('Parsed count:', len(parsed))
|
||||
print('First item keys:', sorted(parsed[0].keys()) if parsed else 'EMPTY')
|
||||
print('URL:', parsed[0].get('url'))
|
||||
print('Author:', parsed[0].get('author_handle'))
|
||||
"
|
||||
```
|
||||
- [ ] Parses correctly with expected keys
|
||||
- [ ] Handles both camelCase and snake_case fields
|
||||
- [ ] URL, text, author, engagement metrics all present
|
||||
|
||||
### 1.4 SKILL.md Validation
|
||||
|
||||
```bash
|
||||
# Verify YAML frontmatter parses correctly
|
||||
python3 -c "
|
||||
import yaml
|
||||
with open('/Users/mvanhorn/last30days-skill-private/SKILL.md') as f:
|
||||
content = f.read()
|
||||
# Extract YAML between --- markers
|
||||
parts = content.split('---', 2)
|
||||
meta = yaml.safe_load(parts[1])
|
||||
print('name:', meta.get('name'))
|
||||
print('context:', meta.get('context'))
|
||||
print('agent:', meta.get('agent'))
|
||||
print('allowed-tools:', meta.get('allowed-tools'))
|
||||
"
|
||||
```
|
||||
- [ ] `name` is `last30days` (not `last30daystest`)
|
||||
- [ ] `context` is `fork`
|
||||
- [ ] `agent` is `Explore`
|
||||
- [ ] `allowed-tools` includes `Bash`, `WebSearch`
|
||||
|
||||
### 1.5 Diff Audit (Old vs New)
|
||||
|
||||
```bash
|
||||
# Verify the only meaningful addition is bird_x.py
|
||||
diff -rq ~/.claude/skills/last30days.backup-v1/scripts/lib/ \
|
||||
/Users/mvanhorn/last30days-skill-private/scripts/lib/ 2>/dev/null
|
||||
```
|
||||
- [ ] Only new file is `bird_x.py`
|
||||
- [ ] Modified files: `env.py` (source detection), `__init__.py` (exports)
|
||||
- [ ] No unexpected deletions or renames
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: User's Test Plan (Manual)
|
||||
|
||||
These require human judgment - evaluating quality, UX, and real-world behavior.
|
||||
|
||||
### 2.1 Basic Invocation (5 min)
|
||||
|
||||
Open a fresh Claude Code session after Phase 0 is complete.
|
||||
|
||||
| # | Test | Command | Pass Criteria |
|
||||
|---|------|---------|---------------|
|
||||
| 1 | Simple topic | `/last30days AI music generation` | Returns results, shows source stats |
|
||||
| 2 | Topic + tool | `/last30days Suno prompts for music production` | Returns results + generates a prompt |
|
||||
| 3 | Quick mode | `/last30days --quick TypeScript tips` | Faster, fewer results, still valid |
|
||||
| 4 | Empty input | `/last30days` | Prompts for topic (doesn't crash) |
|
||||
|
||||
### 2.2 Bird CLI Verification (5 min)
|
||||
|
||||
| # | Test | What to Check |
|
||||
|---|------|---------------|
|
||||
| 1 | Source indicator | Output shows Bird as X source (not xAI) |
|
||||
| 2 | X results quality | X/Twitter results are real, recent, have engagement metrics |
|
||||
| 3 | Bird promo | If Bird NOT installed, shows non-blocking info banner |
|
||||
| 4 | Mixed sources | Both Reddit (OpenAI) and X (Bird) results appear |
|
||||
|
||||
### 2.3 Fallback Behavior (5 min)
|
||||
|
||||
| # | Test | Setup | Expected |
|
||||
|---|------|-------|----------|
|
||||
| 1 | No Bird | `npm uninstall -g @steipete/bird` temporarily | Falls back to xAI or WebSearch |
|
||||
| 2 | No API keys | Rename `~/.config/last30days/.env` temporarily | WebSearch-only mode works |
|
||||
| 3 | Restore | Reinstall bird + restore .env | Full mode returns |
|
||||
|
||||
### 2.4 Output Quality (10 min)
|
||||
|
||||
Run 3 real research queries you care about. For each, evaluate:
|
||||
|
||||
- [ ] Results are actually from the last 30 days (not stale)
|
||||
- [ ] Engagement metrics (likes, upvotes) are present and reasonable
|
||||
- [ ] No duplicate results
|
||||
- [ ] Sources are properly cited with URLs
|
||||
- [ ] Synthesis is grounded in actual results (not hallucinated)
|
||||
- [ ] Generated prompts (if requested) are usable
|
||||
|
||||
### 2.5 Comparison Test (10 min)
|
||||
|
||||
Before removing the backup, run the SAME query on both versions:
|
||||
|
||||
```bash
|
||||
# New version (active)
|
||||
/last30days [your topic]
|
||||
|
||||
# Old version (temporarily restore)
|
||||
rm ~/.claude/skills/last30days
|
||||
mv ~/.claude/skills/last30days.backup-v1 ~/.claude/skills/last30days
|
||||
# New Claude Code session
|
||||
/last30days [same topic]
|
||||
# Then swap back
|
||||
```
|
||||
|
||||
- [ ] New version produces equal or better results
|
||||
- [ ] No features regressed
|
||||
- [ ] Bird results add value over xAI-only
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Release Plan (30-Day Timeline)
|
||||
|
||||
**Target release date:** March 1, 2026 (conservative buffer before March 8 deadline)
|
||||
|
||||
### Week 1: Feb 6-12 - Clean & Test
|
||||
|
||||
| Day | Task | Owner |
|
||||
|-----|------|-------|
|
||||
| Feb 6 | Phase 0: Clean swap (remove old, activate new) | User |
|
||||
| Feb 6 | Phase 1: Claude runs automated tests | Claude |
|
||||
| Feb 7-8 | Phase 2: User runs manual tests (2.1-2.4) | User |
|
||||
| Feb 9 | Phase 2.5: Comparison test | User |
|
||||
| Feb 10-12 | Fix any issues found during testing | Claude + User |
|
||||
|
||||
### Week 2: Feb 13-19 - Harden
|
||||
|
||||
| Day | Task | Owner |
|
||||
|-----|------|-------|
|
||||
| Feb 13 | Run edge cases: unicode topics, very long topics, special chars | Claude |
|
||||
| Feb 14 | Test with Bird logged out (auth expiry scenario) | User |
|
||||
| Feb 15 | Review all error messages for clarity | Claude |
|
||||
| Feb 16-17 | Update README.md with Bird CLI setup instructions | Claude |
|
||||
| Feb 18-19 | Buffer for fixes | Claude + User |
|
||||
|
||||
### Week 3: Feb 20-26 - Pre-Release
|
||||
|
||||
| Day | Task | Owner |
|
||||
|-----|------|-------|
|
||||
| Feb 20 | Final diff audit: private repo vs public repo | Claude |
|
||||
| Feb 21 | Strip any private/test artifacts (test symlinks, debug prints) | Claude |
|
||||
| Feb 22 | Update SKILL.md description if needed | Claude |
|
||||
| Feb 23 | Dry-run: push to a branch on public repo (not main) | User |
|
||||
| Feb 24 | Test installation from the branch (fresh `~/.claude/skills/`) | User |
|
||||
| Feb 25-26 | Buffer for fixes | Claude + User |
|
||||
|
||||
### Week 4: Feb 27 - Mar 1 - Ship
|
||||
|
||||
| Day | Task | Owner |
|
||||
|-----|------|-------|
|
||||
| Feb 27 | Merge branch to main on public repo | User |
|
||||
| Feb 28 | Create GitHub release with changelog | Claude + User |
|
||||
| Mar 1 | Delete backup: `rm -rf ~/.claude/skills/last30days.backup-v1` | User |
|
||||
| Mar 1 | Archive private repo (optional) | User |
|
||||
|
||||
### Release Checklist (Final Gate)
|
||||
|
||||
Before merging to `main` on the public repo:
|
||||
|
||||
- [ ] All Phase 1 automated tests pass
|
||||
- [ ] All Phase 2 manual tests pass
|
||||
- [ ] Comparison test shows new >= old quality
|
||||
- [ ] SKILL.md frontmatter is correct (`name: last30days`, not `last30daystest`)
|
||||
- [ ] README.md documents Bird CLI setup
|
||||
- [ ] No debug/test artifacts in codebase
|
||||
- [ ] No hardcoded paths (e.g., `/Users/mvanhorn/...`)
|
||||
- [ ] `.env` files are gitignored
|
||||
- [ ] Git history is clean (no "test" or "WIP" commits on main)
|
||||
- [ ] Bird CLI failure doesn't break the skill (graceful fallback verified)
|
||||
|
||||
### Rollback Plan (Emergency)
|
||||
|
||||
If something goes wrong after release:
|
||||
|
||||
```bash
|
||||
# Option 1: Revert to backup (if still exists)
|
||||
rm ~/.claude/skills/last30days
|
||||
mv ~/.claude/skills/last30days.backup-v1 ~/.claude/skills/last30days
|
||||
|
||||
# Option 2: Git revert on public repo
|
||||
cd ~/.claude/skills/last30days
|
||||
git log --oneline -5 # find the last good commit
|
||||
git revert HEAD # revert the merge commit
|
||||
git push origin main
|
||||
|
||||
# Option 3: Pin to old version
|
||||
cd ~/.claude/skills/last30days
|
||||
git checkout cc892d7 # last known good commit from old version
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Risk Analysis
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|-----------|--------|------------|
|
||||
| Bird CLI breaks after X API changes | Medium | Low | Fallback to xAI/WebSearch still works |
|
||||
| Bird auth expires silently | Medium | Low | `is_bird_authenticated()` check + user message |
|
||||
| Old xAI workflows regress | Low | High | Comparison test in Phase 2.5 |
|
||||
| Hardcoded paths in codebase | Low | Medium | Grep for `/Users/mvanhorn` before release |
|
||||
| SKILL.md name still says `last30daystest` | Low | High | Already fixed in commit `4e972d0` |
|
||||
|
||||
## References
|
||||
|
||||
- Private repo: `https://github.com/mvanhorn/last30days-skill-private.git`
|
||||
- Public repo: `https://github.com/mvanhorn/last30days-skill.git`
|
||||
- Bird CLI: `https://github.com/steipete/bird`
|
||||
- Bird implementation plan: `docs/plans/2026-02-03-bird-cli-implementation.md`
|
||||
- Bird integration design: `docs/plans/2026-02-03-bird-cli-integration-design.md`
|
||||
@@ -1,91 +0,0 @@
|
||||
---
|
||||
title: "feat: Add visible query parsing display before research starts"
|
||||
type: feat
|
||||
date: 2026-02-06
|
||||
---
|
||||
|
||||
# feat: Add Visible Query Parsing Display
|
||||
|
||||
## Overview
|
||||
|
||||
The last30days skill parses user intent (TOPIC, QUERY_TYPE, TARGET_TOOL) internally but never shows the user what it understood. The agent jumps straight from the user's `/last30days kanye west` into running tools with a generic "I'll start the research script and web searches in parallel."
|
||||
|
||||
Users expect to see a reformulation of their query — confirming what the agent understood before it starts searching. This builds trust and lets users course-correct before waiting for results.
|
||||
|
||||
## Problem Statement
|
||||
|
||||
Current behavior:
|
||||
```
|
||||
User: /last30days kanye west
|
||||
|
||||
Agent: I'll start the research script and web searches in parallel.
|
||||
[immediately runs bash + WebSearch]
|
||||
```
|
||||
|
||||
Expected behavior:
|
||||
```
|
||||
User: /last30days kanye west
|
||||
|
||||
Agent: 🔍 **kanye west** · News
|
||||
Searching Reddit, X, and the web for the latest on kanye west...
|
||||
|
||||
[then runs bash + WebSearch]
|
||||
```
|
||||
|
||||
The "Parse User Intent" section in SKILL.md tells the agent to store variables internally but never instructs it to **display** them.
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
Add an explicit "Display your parsing" instruction between the "Parse User Intent" section and "Research Execution" section in SKILL.md. One new block of text — no code changes, no script changes.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] Agent displays parsed TOPIC and QUERY_TYPE before running any tools
|
||||
- [ ] Display is concise (1-2 lines, not a verbose block)
|
||||
- [ ] Agent still runs script + WebSearch in parallel after displaying
|
||||
- [ ] No changes to Python scripts — SKILL.md only
|
||||
|
||||
## Implementation
|
||||
|
||||
### SKILL.md Change
|
||||
|
||||
**File:** `/Users/mvanhorn/last30days-skill-private/SKILL.md`
|
||||
|
||||
After the "Store these variables" block (line ~38) and before "Research Execution" (line ~42), add:
|
||||
|
||||
```markdown
|
||||
**DISPLAY your parsing to the user.** Before running any tools, output a single line:
|
||||
|
||||
🔍 **{TOPIC}** · {QUERY_TYPE}
|
||||
Searching Reddit, X, and the web for {natural language description of what you'll look for}...
|
||||
|
||||
Example outputs:
|
||||
- 🔍 **kanye west** · News — Searching Reddit, X, and the web for the latest kanye west news and discussions...
|
||||
- 🔍 **best MCP servers** · Recommendations — Searching Reddit, X, and the web for the most recommended MCP servers...
|
||||
- 🔍 **nano banana pro prompting** · Prompting — Searching Reddit, X, and the web for nano banana pro prompting techniques and tips...
|
||||
- 🔍 **open claw** · General — Searching Reddit, X, and the web for what people are saying about open claw...
|
||||
|
||||
If TARGET_TOOL is known, mention it: "...for nano banana pro prompting techniques to use in ChatGPT..."
|
||||
|
||||
This text MUST appear before you call any tools. It confirms to the user that you understood their request.
|
||||
```
|
||||
|
||||
### Sync
|
||||
|
||||
After editing SKILL.md:
|
||||
```bash
|
||||
cp /Users/mvanhorn/last30days-skill-private/SKILL.md ~/.claude/skills/last30days/SKILL.md
|
||||
```
|
||||
|
||||
## Test Plan
|
||||
|
||||
Run in a NEW Claude Code session:
|
||||
1. `/last30days kanye west` — should display: 🔍 **kanye west** · News
|
||||
2. `/last30days best MCP servers` — should display: 🔍 **best MCP servers** · Recommendations
|
||||
3. `/last30days nano banana pro prompting for ChatGPT` — should display with tool mention
|
||||
|
||||
## Files to Modify
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `SKILL.md` | Add display instruction between Parse User Intent and Research Execution |
|
||||
@@ -1,167 +0,0 @@
|
||||
---
|
||||
title: "fix: last30days v2 formatting, Reddit results, and citation verbosity"
|
||||
type: fix
|
||||
date: 2026-02-06
|
||||
---
|
||||
|
||||
# fix: last30days v2 Formatting, Reddit Results, and Citation Verbosity
|
||||
|
||||
## Overview
|
||||
|
||||
Four bugs found during v2 testing across 4 queries (kanye west, howie.ai, nano banana pro prompting, open claw). The skill IS executing (the agent:Explore removal worked) but output quality has regressed from v1.
|
||||
|
||||
## Problem Statement
|
||||
|
||||
| # | Bug | Severity | Where |
|
||||
|---|-----|----------|-------|
|
||||
| 1 | Stats emoji tree format ignored 3/4 times - agent renders plain text dashes instead | High | `SKILL.md` |
|
||||
| 2 | Reddit returns 0 results for popular topics (kanye west, howie.ai) | High | `scripts/lib/openai_reddit.py` |
|
||||
| 3 | Citations too verbose - every sentence has `(per @x, @y, @z; r/sub)` making summary unreadable | Medium | `SKILL.md` |
|
||||
| 4 | Kanye summary is wall of text - no bold headers or paragraph breaks like nano banana pro got | Medium | `SKILL.md` |
|
||||
|
||||
## Proposed Fixes
|
||||
|
||||
### Fix 1: Stats Emoji Format Enforcement
|
||||
|
||||
**Root cause:** The agent ignores the emoji tree template even with BAD/GOOD examples. The template uses box-drawing characters (├─ └─) that the agent treats as decorative, not mandatory.
|
||||
|
||||
**Approach:** Instead of relying on the agent to copy box-drawing characters, provide the template as a **literal fill-in-the-blank** with placeholders that are impossible to misinterpret.
|
||||
|
||||
**File:** `SKILL.md` (stats section, currently around line 190)
|
||||
|
||||
**Change:** Replace the current template + BAD/GOOD examples with a single, strict fill-in format:
|
||||
|
||||
```
|
||||
Copy this EXACTLY, replacing only the {placeholders}:
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments
|
||||
├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts (via Bird/xAI)
|
||||
├─ 🌐 Web: {N} pages │ {domain1}, {domain2}, {domain3}
|
||||
└─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2}
|
||||
---
|
||||
|
||||
If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)"
|
||||
NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji.
|
||||
```
|
||||
|
||||
Remove the separate BAD/GOOD section (it adds length without helping).
|
||||
|
||||
### Fix 2: Reddit Returning 0 Results
|
||||
|
||||
**Root cause (from code analysis):**
|
||||
|
||||
1. `openai_reddit.py:53-93` - The `REDDIT_SEARCH_PROMPT` instructs the OpenAI model to strip noise words before searching. For "kanye west" this isn't the issue (no noise words), but for "howie.ai" it might strip "ai".
|
||||
|
||||
2. `openai_reddit.py:160-166` - The search is restricted to `allowed_domains: ["reddit.com"]` which depends on OpenAI's web_search indexing of Reddit.
|
||||
|
||||
3. `last30days.py:474-490` - Post-retrieval filtering: `normalize.filter_by_date_range()` + `score.score_reddit_items()` + `dedupe.dedupe_reddit()` can discard all results if date confidence is low.
|
||||
|
||||
4. `score.py:151-157` - Items with no engagement metrics get `-10` penalty, low date confidence gets `-10`. Combined that's `-20` which may push score below threshold.
|
||||
|
||||
**Approach (multi-layered):**
|
||||
|
||||
**A. Add subreddit-targeted search fallback** in `openai_reddit.py`:
|
||||
- When the first search returns < 3 results, add a second search prompt that explicitly queries: `"r/{topic} site:reddit.com"` and `"{topic} subreddit site:reddit.com"`
|
||||
- This catches cases where OpenAI's web_search doesn't find the obvious subreddit
|
||||
|
||||
**B. Soften post-retrieval scoring** in `score.py`:
|
||||
- Change the no-engagement penalty from `-10` to `-3` (missing metrics ≠ irrelevant)
|
||||
- Change low date confidence penalty from `-10` to `-5`
|
||||
|
||||
**C. Add minimum result guarantee** in `last30days.py`:
|
||||
- If scoring filters out ALL results, keep the top 3 by raw relevance regardless of score
|
||||
- Log a warning: "All Reddit results scored below threshold, keeping top 3 by relevance"
|
||||
|
||||
**Files to change:**
|
||||
- `scripts/lib/openai_reddit.py` - Add subreddit fallback search (lines ~160-180)
|
||||
- `scripts/lib/score.py` - Soften penalties (lines ~151-157)
|
||||
- `scripts/last30days.py` - Add minimum result guarantee (lines ~474-490)
|
||||
|
||||
### Fix 3: Citations Too Verbose
|
||||
|
||||
**Root cause:** The SKILL.md instruction says "Every insight MUST cite at least one source" with a GOOD example showing `(per @XXX, 15 likes; r/kanye thread with 200 upvotes)` - this is too much detail per citation and the agent over-applies it.
|
||||
|
||||
**Approach:** Dial back to "cite 1-2 sources per KEY PATTERN, not per sentence. Use short format."
|
||||
|
||||
**File:** `SKILL.md` (citation section, currently around line 158)
|
||||
|
||||
**Change the citation rule to:**
|
||||
|
||||
```
|
||||
CITATION RULE: Cite sources sparingly to prove research is real.
|
||||
- In the "What I learned" intro: cite 1-2 top sources total, not every sentence
|
||||
- In KEY PATTERNS: cite 1 source per pattern, short format: "per @handle" or "per r/sub"
|
||||
- Do NOT include engagement metrics in citations (likes, upvotes) - save those for stats box
|
||||
- Do NOT chain multiple citations: "per @x, @y, @z" is too much. Pick the strongest one.
|
||||
|
||||
BAD: "His album is set for March 20 (per @cocoabutterbf; Rolling Stone; HotNewHipHop; Complex)."
|
||||
GOOD: "His album BULLY is set for March 20 via Gamma, per Rolling Stone."
|
||||
```
|
||||
|
||||
### Fix 4: Summary Formatting (Wall of Text vs Structured)
|
||||
|
||||
**Root cause:** The SKILL.md template for PROMPTING/NEWS/GENERAL shows:
|
||||
```
|
||||
What I learned:
|
||||
[2-4 sentences synthesizing...]
|
||||
```
|
||||
|
||||
This gives the agent permission to write a dense paragraph. The nano banana pro test got good formatting because PROMPTING queries naturally produce structured patterns. NEWS queries (kanye) produce narratives that become walls of text.
|
||||
|
||||
**Approach:** Add explicit structure to the NEWS/GENERAL format with bold topic headers.
|
||||
|
||||
**File:** `SKILL.md` (summary display section, around line 158)
|
||||
|
||||
**Change the PROMPTING/NEWS/GENERAL template to:**
|
||||
|
||||
```
|
||||
What I learned:
|
||||
|
||||
**{Topic 1}** — [1-2 sentences about this storyline, per source]
|
||||
|
||||
**{Topic 2}** — [1-2 sentences, per source]
|
||||
|
||||
**{Topic 3}** — [1-2 sentences, per source]
|
||||
|
||||
KEY PATTERNS from the research:
|
||||
1. [Pattern] — per @handle
|
||||
2. [Pattern] — per r/sub
|
||||
3. [Pattern] — per source
|
||||
```
|
||||
|
||||
The bold topic headers force structure. Each topic gets its own paragraph with a line break. No more wall-of-text narratives.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] **Fix 1:** Stats box uses emoji tree format ├─ 🟠 🔵 🌐 └─ 🗣️ in 4/4 test queries
|
||||
- [ ] **Fix 2:** "kanye west" returns >0 Reddit threads (r/kanye exists and is active)
|
||||
- [ ] **Fix 3:** Summary citations are 1 per insight, short format, no engagement metrics inline
|
||||
- [ ] **Fix 4:** NEWS/GENERAL summaries use bold topic headers with paragraph breaks, not wall of text
|
||||
|
||||
## Test Plan
|
||||
|
||||
Re-run the same 4 queries after fixes:
|
||||
1. `/last30days kanye west` — NEWS: should get Reddit results, structured summary, emoji stats
|
||||
2. `/last30days howie.ai` — GENERAL: should get Reddit if available, citations not verbose
|
||||
3. `/last30days nano banana pro prompting` — PROMPTING: should maintain current good quality, reduce citation density
|
||||
4. `/last30days open claw` — GENERAL: should cite @handles in summary, emoji stats
|
||||
|
||||
## Files to Modify
|
||||
|
||||
| File | Fix | Change |
|
||||
|------|-----|--------|
|
||||
| `SKILL.md` | 1, 3, 4 | Stats template, citation rules, summary structure |
|
||||
| `scripts/lib/openai_reddit.py` | 2 | Add subreddit fallback search |
|
||||
| `scripts/lib/score.py` | 2 | Soften scoring penalties |
|
||||
| `scripts/last30days.py` | 2 | Add minimum result guarantee |
|
||||
|
||||
## References
|
||||
|
||||
- Current SKILL.md: `~/.claude/skills/last30days/SKILL.md`
|
||||
- Private repo: `/Users/mvanhorn/last30days-skill-private/`
|
||||
- Old working SKILL.md: `~/.claude/skills/last30days.backup-v1/SKILL.md`
|
||||
- Reddit search module: `scripts/lib/openai_reddit.py:53-93` (prompt), `:160-166` (API call)
|
||||
- Scoring module: `scripts/lib/score.py:151-157` (penalties)
|
||||
- Main pipeline: `scripts/last30days.py:474-490` (filtering)
|
||||
@@ -1,177 +0,0 @@
|
||||
---
|
||||
title: "fix: Skill execution broken - fork mode subagent ignores bash and text instructions"
|
||||
type: fix
|
||||
date: 2026-02-06
|
||||
---
|
||||
|
||||
# fix: Skill Execution Broken in Fork Mode
|
||||
|
||||
## Overview
|
||||
|
||||
The last30days v2 skill stopped running its Python script and stopped showing acknowledgment text. The agent jumps straight to WebSearch, ignoring all instructions to run bash first or output text. Five attempted fixes all failed.
|
||||
|
||||
## Root Cause (Confirmed via Research)
|
||||
|
||||
**The old v1 skill worked by accident.** GitHub Issue #17283 documented that `context: fork` and `agent: Explore` were **silently ignored** in older Claude Code versions. The skill ran **inline** in the main conversation — not in a forked subagent. That's why:
|
||||
- The user saw acknowledgment text (output inline to conversation)
|
||||
- The bash script ran (main model followed instructions inline)
|
||||
- Progress was visible (tool calls shown normally)
|
||||
|
||||
**Claude Code 2.1+ fixed the bug** and now properly honors `context: fork`. The skill now truly runs in an isolated subagent where:
|
||||
- The model decides tool ordering independently
|
||||
- Text output instructions are deprioritized vs tool calls
|
||||
- "RUN THIS FIRST" instructions are **suggestions**, not commands
|
||||
- There is **no mechanism** to force tool ordering in a forked subagent
|
||||
|
||||
**This is why every SKILL.md rewrite failed** — the problem isn't the instructions, it's `context: fork` itself.
|
||||
|
||||
## Evidence
|
||||
|
||||
| Attempt | What we tried | Result |
|
||||
|---------|--------------|--------|
|
||||
| 1 | "YOUR FIRST ACTION: Run this command. EXECUTE." | Agent ran script sometimes, never showed ack text |
|
||||
| 2 | "YOUR FIRST OUTPUT — before ANY tool calls" + progress block | Agent ignored text, jumped to WebSearch |
|
||||
| 3 | Moved progress block to very first section | Agent ignored it entirely |
|
||||
| 4 | "DO NOT skip this. DO NOT jump to tool calls first." | Agent still jumped to WebSearch |
|
||||
| 5 | Embedded echo in bash block + "Do NOT start with WebSearch" | Agent still jumped to WebSearch, never ran bash |
|
||||
|
||||
## Proposed Fix
|
||||
|
||||
### Option A: Remove `context: fork` (Recommended)
|
||||
|
||||
**Remove `context: fork` from frontmatter.** The skill runs inline in the main conversation, exactly like the old v1 skill accidentally did.
|
||||
|
||||
**Why this works:**
|
||||
- Inline execution follows instructions sequentially
|
||||
- Text output appears directly to the user
|
||||
- Bash commands run when instructed
|
||||
- This is how the "working" v1 skill actually operated
|
||||
|
||||
**File:** `SKILL.md` frontmatter
|
||||
|
||||
**Change from:**
|
||||
```yaml
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web...
|
||||
argument-hint: '"[topic] for [tool]" or "[topic]"'
|
||||
context: fork
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
```
|
||||
|
||||
**Change to:**
|
||||
```yaml
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web...
|
||||
argument-hint: '"[topic] for [tool]" or "[topic]"'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
```
|
||||
|
||||
That's it. Remove the one line.
|
||||
|
||||
**Then restore the old v1 instruction flow:**
|
||||
1. "Parse User Intent" section FIRST (generates acknowledgment text)
|
||||
2. "Research Execution" with bash command
|
||||
3. "Do WebSearch" while script runs
|
||||
4. Synthesize and present
|
||||
|
||||
### Option B: Keep `context: fork` + Use `!`command`` Preprocessing
|
||||
|
||||
Use shell preprocessing syntax (`!`command``) to run the script **before** the model even sees the prompt:
|
||||
|
||||
```markdown
|
||||
## Research data (auto-fetched)
|
||||
!`python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1`
|
||||
```
|
||||
|
||||
**Risk:** Not confirmed that `$ARGUMENTS` works in `!`command`` context. More complex. The user still won't see progress text during preprocessing.
|
||||
|
||||
### Recommendation: Option A
|
||||
|
||||
Remove `context: fork`. It's one line. The old skill worked inline. The v2 skill should too. Option B is a backup if inline mode causes context window issues.
|
||||
|
||||
## Implementation
|
||||
|
||||
### Step 1: Remove `context: fork` from frontmatter
|
||||
|
||||
Single line removal in `SKILL.md`.
|
||||
|
||||
### Step 2: Restore v1-style instruction flow
|
||||
|
||||
The SKILL.md opening should match the public v1 pattern:
|
||||
|
||||
```markdown
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
[... topic/tool/query type parsing ...]
|
||||
|
||||
Store these variables:
|
||||
- TOPIC = ...
|
||||
- TARGET_TOOL = ...
|
||||
- QUERY_TYPE = ...
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
**Step 2: Do WebSearch** (while script runs)
|
||||
[... websearch queries based on QUERY_TYPE ...]
|
||||
|
||||
**Step 3: Wait for script to complete**
|
||||
[... synthesis instructions ...]
|
||||
```
|
||||
|
||||
The key structural elements from v1 that need to return:
|
||||
1. Descriptive intro paragraph
|
||||
2. "Parse User Intent" BEFORE any tool calls
|
||||
3. Script execution as a clearly labeled step
|
||||
4. WebSearch as step 2 (not step 1)
|
||||
|
||||
### Step 3: Keep all v2 improvements
|
||||
|
||||
The v2-specific improvements (citation rules, stats template, Reddit fallback, scoring changes) stay. Only the frontmatter and instruction flow change.
|
||||
|
||||
### Step 4: Sync and test
|
||||
|
||||
Copy to `~/.claude/skills/last30days/SKILL.md`, test in new session.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] `context: fork` removed from SKILL.md frontmatter
|
||||
- [ ] Agent outputs acknowledgment text before running tools
|
||||
- [ ] Python script actually executes (Reddit + X results appear)
|
||||
- [ ] WebSearch supplements, doesn't replace, script results
|
||||
- [ ] Stats emoji tree format renders correctly
|
||||
- [ ] Citations are sparse (1 per insight, not 3-5)
|
||||
|
||||
## Test Plan
|
||||
|
||||
Run in a NEW Claude Code session:
|
||||
1. `/last30days kanye west` — should see ack text, script runs, Reddit results
|
||||
2. `/last30days open claw` — should see ack text, X results via Bird
|
||||
|
||||
## Files to Modify
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `SKILL.md` | Remove `context: fork`, restore v1 instruction flow |
|
||||
|
||||
## References
|
||||
|
||||
- GitHub Issue #17283: `context: fork` was silently ignored (the bug that made v1 work)
|
||||
- Claude Code Skills docs: `!`command`` preprocessing syntax
|
||||
- Claude Code Subagents docs: `agent: Explore` uses Haiku, read-only tools
|
||||
- Public v1 SKILL.md: `github.com/mvanhorn/last30days-skill`
|
||||
@@ -1,385 +0,0 @@
|
||||
---
|
||||
title: "test: Compare v1 (public) vs v2 (private) last30days output quality"
|
||||
type: test
|
||||
date: 2026-02-06
|
||||
---
|
||||
|
||||
# test: V1 vs V2 Comparison Test Plan
|
||||
|
||||
## Overview
|
||||
|
||||
Run the same queries through both the public v1 and private v2 of last30days, compare output quality across 7 dimensions, and determine if v2 is ready to ship as the new public version.
|
||||
|
||||
**This plan also includes a full feature audit** identifying everything v1 has that v2 is missing — some of those gaps need fixing before shipping.
|
||||
|
||||
---
|
||||
|
||||
## How to Run the Comparison
|
||||
|
||||
### Setup
|
||||
|
||||
**V1 (public upstream):** Check out upstream SKILL.md temporarily:
|
||||
```bash
|
||||
# Save current v2
|
||||
cp ~/.claude/skills/last30days/SKILL.md ~/.claude/skills/last30days/SKILL.md.v2
|
||||
|
||||
# Install v1 from upstream
|
||||
cd /Users/mvanhorn/last30days-skill-private
|
||||
git show upstream/main:SKILL.md > ~/.claude/skills/last30days/SKILL.md
|
||||
```
|
||||
|
||||
Run test queries in a NEW Claude Code session (one session per query to avoid context bleed). Save output.
|
||||
|
||||
**V2 (private current):** Restore v2:
|
||||
```bash
|
||||
cp ~/.claude/skills/last30days/SKILL.md.v2 ~/.claude/skills/last30days/SKILL.md
|
||||
```
|
||||
|
||||
Run same queries in NEW sessions. Save output.
|
||||
|
||||
---
|
||||
|
||||
## ALL Test Queries
|
||||
|
||||
### From README Examples (13 documented use cases)
|
||||
|
||||
Every single example from the README, in order:
|
||||
|
||||
| # | Query | Type | README Section |
|
||||
|---|-------|------|---------------|
|
||||
| 1 | `prompting techniques for chatgpt for legal questions` | PROMPTING + TOOL | Example: Legal Prompting |
|
||||
| 2 | `best clawdbot use cases` | RECOMMENDATIONS | Example: ClawdBot Use Cases |
|
||||
| 3 | `how to best setup clawdbot` | HOW-TO | Example: ClawdBot Setup |
|
||||
| 4 | `prompting tips for nano banana pro for ios designs` | PROMPTING + TOOL | Example: iOS App Mockup |
|
||||
| 5 | `top claude code skills` | RECOMMENDATIONS | Example: Top Claude Code Skills |
|
||||
| 6 | `using ChatGPT to make images of dogs` | GENERAL | Example: Dog as Human |
|
||||
| 7 | `research best practices for beautiful remotion animation videos in claude code` | PROMPTING | Example: Remotion Launch Video |
|
||||
| 8 | `photorealistic people in nano banana pro` | PROMPTING | Example: Photorealistic Portraits |
|
||||
| 9 | `What are the best rap songs lately` | RECOMMENDATIONS | Example: Best Rap Songs |
|
||||
| 10 | `what are people saying about DeepSeek R1` | NEWS | Example: DeepSeek R1 |
|
||||
| 11 | `best practices for cursor rules files for Cursor` | PROMPTING | Example: Cursor Rules |
|
||||
| 12 | `prompt advice for using suno to make killer songs in simple mode` | PROMPTING | Example: Suno AI Music |
|
||||
| 13 | `how do I use Codex with Claude Code on same app to make it better` | HOW-TO | Example: Codex + Claude Code |
|
||||
|
||||
### From Plan Documents (4 additional battle-tested queries)
|
||||
|
||||
| # | Query | Type | Source |
|
||||
|---|-------|------|--------|
|
||||
| 14 | `kanye west` | NEWS | fix-v2-formatting plan, most-tested query |
|
||||
| 15 | `howie.ai` | GENERAL | fix-v2-formatting plan, edge case (domain as topic) |
|
||||
| 16 | `open claw` | GENERAL | fix-v2-formatting plan, X-heavy sources |
|
||||
| 17 | `nano banana pro prompting` | PROMPTING | fix-v2-formatting plan |
|
||||
|
||||
### Follow-up Vision Tests (pick 4 from above, ask a follow-up)
|
||||
|
||||
These test the prompt-generation phase specifically:
|
||||
|
||||
| Base Query | Follow-up Vision |
|
||||
|------------|-----------------|
|
||||
| #4 (nano banana pro ios) | "make a mock-up of an app for moms who swim" |
|
||||
| #6 (ChatGPT dog images) | "what would my dog look like as a human prompt" |
|
||||
| #12 (suno music) | "Rap song about self aware AI that loves Claude Code" |
|
||||
| #13 (codex + claude code) | "how do I build a review loop workflow" |
|
||||
|
||||
---
|
||||
|
||||
## FEATURE AUDIT: V1 vs V2
|
||||
|
||||
### Section-by-section comparison
|
||||
|
||||
I diffed the full v1 (upstream/main) SKILL.md against the current v2. Here's everything.
|
||||
|
||||
#### KEPT (in both versions) ✅
|
||||
|
||||
| Feature | V1 Location | V2 Location | Notes |
|
||||
|---------|------------|------------|-------|
|
||||
| Parse User Intent section | Lines 23-48 | Lines 12-38 | Same logic |
|
||||
| QUERY_TYPE detection (4 types) | Lines 29-36 | Lines 18-22 | Same types |
|
||||
| "Don't ask about tool before research" | Lines 49-51 | Lines 31-33 | Same rule |
|
||||
| Store variables block | Lines 53-56 | Lines 35-38 | Same |
|
||||
| Research script execution | Lines 81-86 | Lines 59-62 | Same command |
|
||||
| WebSearch by QUERY_TYPE | Lines 99-127 | Lines 77-98 | Same queries |
|
||||
| "Use user's exact terminology" | Lines 129-133 | Lines 100-101 | V2 shorter but same intent |
|
||||
| Judge Agent synthesis | Lines 143-151 | Lines 113-124 | Same logic |
|
||||
| Internalize research (ground in actual content) | Lines 159-165 | Lines 128-135 | V2 shorter |
|
||||
| RECOMMENDATIONS: extract specific names | Lines 167-177 | Lines 137-145 | Same, v2 removes BAD/GOOD example |
|
||||
| Prompt format matching | Lines 193-196 | Lines 149-153 | Same |
|
||||
| Summary + Stats + Invitation flow | Lines 200-250 | Lines 157-236 | Same structure, different details |
|
||||
| Wait for user's vision | Lines 254-258 | Lines 240-242 | Same |
|
||||
| Write ONE perfect prompt | Lines 262-275 | Lines 246-266 | Same structure |
|
||||
| Context memory | Lines 298-316 | Lines 278-288 | V2 shorter |
|
||||
| Output summary footer | Lines 320-340 | Lines 292-302 | Different format |
|
||||
| Depth options (quick/default/deep) | Lines 135-139 | Lines 106-109 | Same |
|
||||
|
||||
#### ADDED in V2 (improvements) ✨
|
||||
|
||||
| Feature | What it does | V2 Location |
|
||||
|---------|-------------|------------|
|
||||
| **Query parsing display** | Shows `🔍 **{TOPIC}** · {QUERY_TYPE}` before tools | Lines 40-53 |
|
||||
| **Sparse citation rules** | BAD/GOOD examples, "1 per pattern, short format" | Lines 186-193 |
|
||||
| **Bold topic headers** | `**{Topic 1}** — [1-2 sentences, per source]` format | Lines 195-208 |
|
||||
| **Strict stats template** | "NEVER use plain text dashes", fill-in-blank | Lines 217-230 |
|
||||
| **RECOMMENDATIONS source attribution** | Each item MUST have Sources: line with @handles | Lines 178-182 |
|
||||
| **Reddit 0 results handling** | Explicit instruction for 0-thread line | Line 229 |
|
||||
| **Bird CLI in stats** | "(via Bird/xAI)" notation | Line 223 |
|
||||
|
||||
#### ❌ MISSING FROM V2 — Features V1 Has That V2 Dropped
|
||||
|
||||
These are the regressions. Some are intentional simplifications, others are real gaps.
|
||||
|
||||
**1. Use Cases Block (intro section)**
|
||||
- **V1 has:** 4 use case examples right after the intro: Prompting, Recommendations, News, General — with concrete examples
|
||||
- **V2 has:** Nothing. Just the intro paragraph.
|
||||
- **Impact:** LOW. The query type detection handles this. But it was nice onboarding.
|
||||
- **Verdict:** Skip — not needed for execution quality.
|
||||
|
||||
**2. Setup Check Section (API key guidance)**
|
||||
- **V1 has:** Full section explaining 3 modes (Full/Partial/Web-Only), first-time setup bash script, "API keys are OPTIONAL" messaging
|
||||
- **V2 has:** Nothing. Script auto-detects.
|
||||
- **Impact:** LOW for experienced users. HIGH for first-time users who don't have keys.
|
||||
- **Verdict:** Skip for now — script handles auto-detection. Consider adding back for public release.
|
||||
|
||||
**3. Anti-Pattern Examples (synthesis quality guard)**
|
||||
- **V1 has:** Explicit anti-pattern block: "If user asks about 'clawdbot skills' and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as 'Claude Code skills' just because both involve 'skills'." Plus BAD/GOOD synthesis examples for RECOMMENDATIONS.
|
||||
- **V2 has:** Only "Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge" — no concrete examples.
|
||||
- **Impact:** MEDIUM-HIGH. Without concrete anti-patterns, the agent may conflate similar-sounding things.
|
||||
- **Verdict:** ⚠️ ADD BACK. At minimum, restore the BAD/GOOD RECOMMENDATIONS example and the "don't conflate" warning.
|
||||
|
||||
**4. Self-Check Instruction (pre-display validation)**
|
||||
- **V1 has:** "SELF-CHECK before displaying: Re-read your 'What I learned' section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it."
|
||||
- **V2 has:** Nothing.
|
||||
- **Impact:** MEDIUM. The self-check forces the model to validate its own output.
|
||||
- **Verdict:** ⚠️ ADD BACK. One line costs nothing and catches hallucination.
|
||||
|
||||
**5. Quality Checklist for Prompts ⭐**
|
||||
- **V1 has:** Explicit checklist before delivering a prompt:
|
||||
```
|
||||
### Quality Checklist:
|
||||
- [ ] FORMAT MATCHES RESEARCH - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
```
|
||||
- **V2 has:** Only "If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT." — one line instead of 5 checks.
|
||||
- **Impact:** HIGH. This is likely what the user noticed as missing — v1 prompts felt more polished because the agent ran a checklist before delivering.
|
||||
- **Verdict:** ⚠️ ADD BACK. This is the "that's a great prompt" quality feel.
|
||||
|
||||
**6. Prompt Format Anti-Pattern**
|
||||
- **V1 has:** "ANTI-PATTERN: Research says 'use JSON prompts with device specs' but you write plain prose. This defeats the entire purpose of the research."
|
||||
- **V2 has:** Only the positive instruction (use the format research recommends).
|
||||
- **Impact:** MEDIUM. Negative examples ("don't do this") are powerful for LLMs.
|
||||
- **Verdict:** ⚠️ ADD BACK. One line.
|
||||
|
||||
**7. "IF USER ASKS FOR MORE OPTIONS" Section**
|
||||
- **V1 has:** "Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested."
|
||||
- **V2 has:** Nothing about handling multi-prompt requests.
|
||||
- **Impact:** LOW-MEDIUM. Without it, agent might dump multiple prompts unprompted.
|
||||
- **Verdict:** ⚠️ ADD BACK. Two lines.
|
||||
|
||||
**8. Web-Only Mode Stats Template + Promo**
|
||||
- **V1 has:** Separate stats template for web-only mode with "💡 Want engagement metrics? Add API keys..." promo
|
||||
- **V2 has:** Only the full-mode template. If running web-only, agent has no guidance.
|
||||
- **Impact:** MEDIUM for users without API keys.
|
||||
- **Verdict:** Consider adding back for public release. Lower priority for now.
|
||||
|
||||
**9. TARGET_TOOL Question Template**
|
||||
- **V1 has:** Explicit AskUserQuestion block with 4 options: [Most relevant tool], Nano Banana Pro, ChatGPT/Claude, Other
|
||||
- **V2 has:** "run research first, then ask AFTER showing results" — but no actual question template.
|
||||
- **Impact:** LOW-MEDIUM. Agent will still ask, just less structured.
|
||||
- **Verdict:** Skip — not critical.
|
||||
|
||||
**10. Context Memory: "Don't re-search" Instructions**
|
||||
- **V1 has:** Explicit "DO NOT run new WebSearches — you already have the research. Answer from what you learned. Cite the Reddit threads, X posts, and web sources."
|
||||
- **V2 has:** Only "Only do new research if the user explicitly asks about a DIFFERENT topic."
|
||||
- **Impact:** MEDIUM. Without the explicit ban, agent may re-search on follow-ups, wasting time.
|
||||
- **Verdict:** ⚠️ ADD BACK. Three lines.
|
||||
|
||||
**11. Output Summary Footer (emoji + engagement counts)**
|
||||
- **V1 has:** `📚 Expert in: {TOPIC} for {TARGET_TOOL}` and `📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages`
|
||||
- **V2 has:** `Expert in: {TOPIC} for {TARGET_TOOL}` and `Based on: {n} Reddit threads + {n} X posts + {n} web pages` — no emoji, no engagement counts.
|
||||
- **Impact:** LOW but noticeable. The emoji + counts make the footer feel more substantial.
|
||||
- **Verdict:** ⚠️ ADD BACK. Trivial fix.
|
||||
|
||||
---
|
||||
|
||||
## Priority Fix List (Before Shipping V2 as Public)
|
||||
|
||||
Based on the audit, these should be restored in V2 before it replaces V1:
|
||||
|
||||
### Must Fix (affects output quality)
|
||||
|
||||
| # | Missing Feature | Why | Effort |
|
||||
|---|----------------|-----|--------|
|
||||
| 1 | **Quality Checklist for prompts** | The "that's a great prompt" feel. V1's 5-point checklist made prompts more polished. | Add 8 lines to SKILL.md |
|
||||
| 2 | **Anti-pattern examples** | BAD/GOOD synthesis examples prevent agent from conflating research. | Add 5 lines |
|
||||
| 3 | **Self-check instruction** | One-line pre-display validation catches hallucination. | Add 2 lines |
|
||||
| 4 | **Context Memory: don't re-search** | Prevents wasting time re-searching on follow-ups. | Add 3 lines |
|
||||
|
||||
### Should Fix (polish)
|
||||
|
||||
| # | Missing Feature | Why | Effort |
|
||||
|---|----------------|-----|--------|
|
||||
| 5 | **Prompt format anti-pattern** | Negative example reinforces "match the format". | Add 2 lines |
|
||||
| 6 | **"IF USER ASKS FOR MORE OPTIONS"** | Prevents prompt dumping. | Add 2 lines |
|
||||
| 7 | **Output footer emoji + engagement counts** | More polished footer. | Edit 3 lines |
|
||||
|
||||
### Skip for Now (nice-to-have for public release)
|
||||
|
||||
| # | Missing Feature | Why Skip |
|
||||
|---|----------------|----------|
|
||||
| 8 | Use cases block (intro) | Doesn't affect execution |
|
||||
| 9 | Setup Check section | Script auto-detects; add back for public README |
|
||||
| 10 | Web-only mode stats + promo | Lower priority, most users have keys |
|
||||
| 11 | TARGET_TOOL question template | Agent handles this naturally |
|
||||
|
||||
---
|
||||
|
||||
## Scoring Dimensions (1-5 scale, 7 dimensions)
|
||||
|
||||
### 1. Query Parsing Display
|
||||
Does the agent show what it understood before starting research?
|
||||
|
||||
| Score | Criteria |
|
||||
|-------|----------|
|
||||
| 1 | No acknowledgment, jumps straight to tools |
|
||||
| 2 | Generic "I'll research this" with no specifics |
|
||||
| 3 | Mentions the topic but not query type |
|
||||
| 4 | Shows topic + query type clearly |
|
||||
| 5 | Shows topic + query type + reformulated search terms |
|
||||
|
||||
### 2. Source Coverage
|
||||
Did it actually use Reddit, X, AND web — or skip sources?
|
||||
|
||||
| Score | Criteria |
|
||||
|-------|----------|
|
||||
| 1 | WebSearch only, script didn't run |
|
||||
| 2 | Script ran but returned 0 from one major source |
|
||||
| 3 | 2 of 3 sources returned results |
|
||||
| 4 | All 3 sources returned results |
|
||||
| 5 | All 3 sources + good volume (10+ Reddit, 10+ X, 5+ web) |
|
||||
|
||||
### 3. Citation Quality
|
||||
Are citations sparse and useful, or verbose and noisy?
|
||||
|
||||
| Score | Criteria |
|
||||
|-------|----------|
|
||||
| 1 | Every sentence has 3+ citations chained |
|
||||
| 2 | Most sentences have multiple citations |
|
||||
| 3 | 1-2 citations per insight, some over-citing |
|
||||
| 4 | 1 citation per pattern, short format |
|
||||
| 5 | Sparse citations that prove research is real without cluttering |
|
||||
|
||||
### 4. Summary Structure
|
||||
Is the "What I learned" section scannable or a wall of text?
|
||||
|
||||
| Score | Criteria |
|
||||
|-------|----------|
|
||||
| 1 | Single paragraph wall of text |
|
||||
| 2 | Multiple paragraphs but no structure |
|
||||
| 3 | Some bold text but inconsistent |
|
||||
| 4 | Bold topic headers with 1-2 sentence explanations |
|
||||
| 5 | Clean topic headers + KEY PATTERNS list, easy to scan |
|
||||
|
||||
### 5. Stats Box Format
|
||||
Does the emoji stats tree render correctly?
|
||||
|
||||
| Score | Criteria |
|
||||
|-------|----------|
|
||||
| 1 | No stats shown |
|
||||
| 2 | Stats shown but plain text dashes, no emoji |
|
||||
| 3 | Partial emoji format, some lines wrong |
|
||||
| 4 | Correct ├─ └─ │ format with emoji, minor issues |
|
||||
| 5 | Perfect emoji tree with accurate counts and top voices |
|
||||
|
||||
### 6. Research Grounding
|
||||
Does the synthesis reflect the ACTUAL research, or generic pre-training knowledge?
|
||||
|
||||
| Score | Criteria |
|
||||
|-------|----------|
|
||||
| 1 | Entirely generic knowledge, no research content |
|
||||
| 2 | Mentions some research but mostly generic |
|
||||
| 3 | Mix of research and generic, some conflation |
|
||||
| 4 | Clearly grounded in research, minor generic leakage |
|
||||
| 5 | Every insight traceable to a specific source from the research |
|
||||
|
||||
### 7. Prompt Quality (follow-up tests only)
|
||||
When user shares vision, is the generated prompt good?
|
||||
|
||||
| Score | Criteria |
|
||||
|-------|----------|
|
||||
| 1 | Generic prompt that ignores research |
|
||||
| 2 | Mentions research topics but generic structure |
|
||||
| 3 | Uses some research insights, decent prompt |
|
||||
| 4 | Tailored to research, correct format for target tool |
|
||||
| 5 | Uses research-recommended format, specific techniques, ready to paste, "that's a great prompt" feel |
|
||||
|
||||
---
|
||||
|
||||
## Comparison Scorecard Template
|
||||
|
||||
```
|
||||
Query: [query text]
|
||||
Version: V1 / V2
|
||||
Date: YYYY-MM-DD
|
||||
|
||||
| Dimension | Score (1-5) | Notes |
|
||||
|---------------------|-------------|-------|
|
||||
| Query Parsing | | |
|
||||
| Source Coverage | | |
|
||||
| Citation Quality | | |
|
||||
| Summary Structure | | |
|
||||
| Stats Box Format | | |
|
||||
| Research Grounding | | |
|
||||
| Prompt Quality | | (follow-up tests only) |
|
||||
| **TOTAL** | **/35** | |
|
||||
|
||||
Script output:
|
||||
- Reddit: ___ threads / ___ upvotes / ___ comments
|
||||
- X: ___ posts / ___ likes / ___ reposts
|
||||
- Web: ___ pages
|
||||
|
||||
Observations:
|
||||
[Free text notes]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Execution Plan
|
||||
|
||||
### Phase 1: Fix the gaps first
|
||||
Apply the 7 "Must Fix" + "Should Fix" items from the audit to V2 SKILL.md. This takes ~20 minutes since it's all small text additions.
|
||||
|
||||
### Phase 2: Smoke test (4 queries)
|
||||
Run queries #14 (kanye west), #2 (best clawdbot use cases), #8 (photorealistic nano banana pro), #10 (DeepSeek R1) on V2 only. Verify the fixes work.
|
||||
|
||||
### Phase 3: Full comparison (all 17 queries)
|
||||
Run all 17 queries on both V1 and V2. Fill scorecards.
|
||||
|
||||
### Phase 4: Follow-up vision tests (4 queries)
|
||||
Run the 4 follow-up vision tests. Compare prompt quality — this is where the quality checklist fix matters most.
|
||||
|
||||
### Phase 5: Analysis
|
||||
- Sum scores per version across all queries
|
||||
- Identify any dimension where v1 consistently beats v2
|
||||
- Decision: ship v2, or fix more gaps first
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] Feature audit complete (this document)
|
||||
- [x] Must-fix gaps restored in V2 SKILL.md
|
||||
- [ ] All 17 queries run on V2
|
||||
- [ ] At least 4 queries run on V1 for comparison
|
||||
- [ ] 4 follow-up vision tests completed
|
||||
- [ ] Scorecards filled for each
|
||||
- [ ] Total score comparison documented
|
||||
- [ ] Any V1 > V2 regressions identified with fix plan
|
||||
- [ ] Go/no-go decision on shipping v2 as public
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `docs/plans/2026-02-06-test-v1-vs-v2-comparison-plan.md` | This plan |
|
||||
| `SKILL.md` | Apply Must Fix + Should Fix items |
|
||||
| `docs/test-results/v1-vs-v2-comparison.md` | Results (to be created) |
|
||||
@@ -1,208 +0,0 @@
|
||||
---
|
||||
title: "feat: Bundle Bird X search client to eliminate npm dependency"
|
||||
type: feat
|
||||
date: 2026-02-07
|
||||
---
|
||||
|
||||
# feat: Bundle Bird X search client to eliminate npm dependency
|
||||
|
||||
## Overview
|
||||
|
||||
Replace the `subprocess.run(["bird", "search", ...])` dependency in `bird_x.py` with a vendored Node.js module that calls Twitter's GraphQL search API directly. This eliminates the need for users to `npm install -g @steipete/bird` and protects against the package being removed from npm.
|
||||
|
||||
Bird is MIT-licensed. We have the full compiled package archived at `vendor/steipete-bird-0.8.0.tgz` and forked to `github.com/mvanhorn/bird-cli-archive`.
|
||||
|
||||
## Problem Statement
|
||||
|
||||
@steipete deleted Bird's GitHub repo on 2026-02-07. The npm package still works today, but if he unpublishes it from npm:
|
||||
- New users can't `npm install -g @steipete/bird`
|
||||
- The `bird` binary disappears from PATH on fresh installs
|
||||
- `bird_x.py` returns 0 X results for everyone without an xAI API key
|
||||
- /last30days V2's headline feature ("free X search") stops working for new users
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
**Vendor Bird's search-only subset as a Node.js module inside /last30days, called from Python via `subprocess.run(["node", ...])`.**
|
||||
|
||||
This is the minimal-change approach:
|
||||
- Keep Python as the orchestrator (bird_x.py stays mostly the same)
|
||||
- Replace `subprocess.run(["bird", "search", ...])` with `subprocess.run(["node", "vendor/bird-search.mjs", ...])`
|
||||
- Extract only the search-related code from Bird (not posting, bookmarks, lists, etc.)
|
||||
- Cookie auth stays the same (environment variables or browser extraction)
|
||||
|
||||
### Why not rewrite in pure Python?
|
||||
|
||||
Bird's search client uses Twitter's internal GraphQL API with:
|
||||
- Rotating QueryIDs (hardcoded + runtime refresh from x.com)
|
||||
- Specific request header construction (bearer token, csrf, client UUIDs)
|
||||
- Cursor-based pagination with Twitter-specific response parsing
|
||||
- The `@steipete/sweet-cookie` dependency for browser cookie extraction
|
||||
|
||||
Porting all of this to Python is ~1000 lines of fragile reverse-engineering. Vendoring the working JS code is faster, safer, and easier to maintain since the archive includes source maps for debugging.
|
||||
|
||||
## Technical Approach
|
||||
|
||||
### What we need from Bird
|
||||
|
||||
Only 8 files from `dist/lib/` (out of 30+):
|
||||
|
||||
1. `twitter-client-base.js` - HTTP client, auth headers, rate limiting
|
||||
2. `twitter-client-search.js` - Search mixin (the core feature)
|
||||
3. `twitter-client-utils.js` - Tweet parsing, cursor extraction
|
||||
4. `twitter-client-constants.js` - API endpoints, QueryIDs
|
||||
5. `twitter-client-types.js` - TypeScript type stubs
|
||||
6. `cookies.js` - Cookie resolution (env vars, browser extraction)
|
||||
7. `runtime-query-ids.js` - QueryID refresh from x.com
|
||||
8. `paginate-cursor.js` - Cursor pagination helper
|
||||
|
||||
Plus:
|
||||
- `features.json` - GraphQL feature flags
|
||||
- `query-ids.json` - Hardcoded QueryID fallbacks
|
||||
|
||||
### What we DON'T need
|
||||
|
||||
Posting, bookmarks, lists, timelines, engagement, follow, media, news, user lookup, user tweets - all the non-search mixins. This cuts the vendored code roughly in half.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
scripts/
|
||||
lib/
|
||||
bird_x.py # MODIFIED - calls node instead of bird binary
|
||||
vendor/
|
||||
bird-search/
|
||||
bird-search.mjs # NEW - thin CLI wrapper, ~40 lines
|
||||
lib/ # VENDORED - subset of Bird's dist/lib/
|
||||
twitter-client-base.js
|
||||
twitter-client-search.js
|
||||
twitter-client-utils.js
|
||||
twitter-client-constants.js
|
||||
twitter-client-types.js
|
||||
cookies.js
|
||||
runtime-query-ids.js
|
||||
paginate-cursor.js
|
||||
features.json
|
||||
query-ids.json
|
||||
node_modules/ # VENDORED - sweet-cookie only
|
||||
@steipete/
|
||||
sweet-cookie/
|
||||
package.json # Minimal, points to bird-search.mjs
|
||||
LICENSE # Bird's MIT license (required by MIT terms)
|
||||
```
|
||||
|
||||
### Implementation
|
||||
|
||||
#### 1. Create `bird-search.mjs` wrapper (~40 lines)
|
||||
|
||||
A minimal Node.js script that:
|
||||
- Accepts: `node bird-search.mjs <query> --count <n> --json`
|
||||
- Creates a TwitterClient with search mixin only
|
||||
- Resolves cookies (env vars first, then browser extraction)
|
||||
- Calls `client.search(query, count)`
|
||||
- Outputs JSON to stdout
|
||||
- Exits with code 0 on success, 1 on error
|
||||
|
||||
This replaces the full `bird` CLI binary. Same interface, fraction of the code.
|
||||
|
||||
#### 2. Modify `bird_x.py` - change subprocess target
|
||||
|
||||
```python
|
||||
# BEFORE (current)
|
||||
cmd = ["bird", "search", query, "-n", str(count), "--json"]
|
||||
|
||||
# AFTER (vendored)
|
||||
bird_search = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
|
||||
cmd = ["node", bird_search, query, "--count", str(count), "--json"]
|
||||
```
|
||||
|
||||
Same subprocess pattern. Same JSON output format. Minimal diff.
|
||||
|
||||
#### 3. Update auth check functions
|
||||
|
||||
```python
|
||||
# BEFORE
|
||||
def is_bird_installed() -> bool:
|
||||
return shutil.which("bird") is not None
|
||||
|
||||
# AFTER
|
||||
def is_bird_installed() -> bool:
|
||||
bird_search = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
|
||||
return bird_search.exists() and shutil.which("node") is not None
|
||||
```
|
||||
|
||||
`is_bird_authenticated()` changes from `bird whoami` to a quick Node.js cookie check or environment variable check.
|
||||
|
||||
`install_bird()` becomes a no-op (already vendored) or removes itself entirely.
|
||||
|
||||
#### 4. Vendor sweet-cookie
|
||||
|
||||
`@steipete/sweet-cookie` is the only runtime dependency. It handles browser cookie extraction on macOS/Linux. Options:
|
||||
|
||||
**Option A (recommended):** Vendor sweet-cookie into `vendor/bird-search/node_modules/`. It's small (one file). This makes the skill fully self-contained with zero npm installs.
|
||||
|
||||
**Option B:** Fall back to environment variables only (no browser cookie extraction). Users would need to manually set `AUTH_TOKEN` and `CT0` env vars. Simpler but worse UX.
|
||||
|
||||
Recommend Option A - vendor it.
|
||||
|
||||
#### 5. Update user-facing docs
|
||||
|
||||
- `README.md` - Remove "Install Bird CLI" section, replace with "Requires Node.js 22+"
|
||||
- `SKILL.md` - Remove Bird CLI installation instructions
|
||||
- Keep the fallback chain: vendored Bird search -> xAI API key -> web-only
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] `bird_x.py` calls vendored Node.js module instead of `bird` binary
|
||||
- [ ] `search_x()` returns identical JSON format (no downstream changes needed)
|
||||
- [ ] `search_handles()` works with vendored module
|
||||
- [ ] Cookie auth works via environment variables (`AUTH_TOKEN`, `CT0`)
|
||||
- [ ] Cookie auth works via browser extraction (sweet-cookie)
|
||||
- [ ] `is_bird_installed()` checks for vendored module + Node.js
|
||||
- [ ] `install_bird()` removed or returns success immediately
|
||||
- [ ] No `npm install -g @steipete/bird` required anywhere
|
||||
- [ ] Bird's MIT LICENSE included in vendor directory
|
||||
- [ ] README updated to remove Bird CLI install steps
|
||||
- [ ] SKILL.md updated to remove Bird CLI references
|
||||
- [ ] Works on macOS (primary) and Linux
|
||||
- [ ] Fallback to xAI API key still works if vendored search fails
|
||||
|
||||
## Files Changed
|
||||
|
||||
| File | Action | Description |
|
||||
|------|--------|-------------|
|
||||
| `scripts/lib/bird_x.py` | MODIFY | Replace `["bird", ...]` subprocess calls with `["node", "vendor/bird-search/bird-search.mjs", ...]` |
|
||||
| `scripts/lib/vendor/bird-search/bird-search.mjs` | CREATE | Thin Node.js wrapper that imports Bird's search client and outputs JSON |
|
||||
| `scripts/lib/vendor/bird-search/lib/*.js` | VENDOR | 8 files from Bird's dist/lib/ (search subset only) |
|
||||
| `scripts/lib/vendor/bird-search/lib/features.json` | VENDOR | GraphQL feature flags |
|
||||
| `scripts/lib/vendor/bird-search/lib/query-ids.json` | VENDOR | Hardcoded QueryID fallbacks |
|
||||
| `scripts/lib/vendor/bird-search/node_modules/` | VENDOR | sweet-cookie package |
|
||||
| `scripts/lib/vendor/bird-search/package.json` | CREATE | Minimal package.json for module resolution |
|
||||
| `scripts/lib/vendor/bird-search/LICENSE` | COPY | Bird's MIT license |
|
||||
| `README.md` | MODIFY | Remove Bird CLI install section, add Node.js 22+ requirement |
|
||||
| `SKILL.md` | MODIFY | Remove Bird CLI references |
|
||||
|
||||
## Dependencies & Risks
|
||||
|
||||
**Node.js 22+ required** - Users who had Bird CLI already have Node.js. This is not a new dependency, just a version requirement. Claude Code environments typically have Node.js.
|
||||
|
||||
**Twitter API changes** - The GraphQL QueryIDs may rotate. Bird includes a runtime refresh mechanism (`runtime-query-ids.js`) that fetches new IDs from x.com. This is vendored and will continue working.
|
||||
|
||||
**sweet-cookie platform support** - Browser cookie extraction only works on macOS (Safari, Chrome, Firefox) and Linux (Chrome, Firefox). Windows users need manual env vars. This matches Bird CLI's existing behavior.
|
||||
|
||||
**Legal** - Bird is MIT licensed. MIT requires including the license notice in copies. We include `LICENSE` in the vendor directory. Using Twitter's internal API is the same legal gray area Bird always operated in - user accepted this when they used Bird.
|
||||
|
||||
## What This Does NOT Change
|
||||
|
||||
- Python remains the orchestrator - bird_x.py still does query construction, retry logic, response parsing
|
||||
- The fallback chain stays: vendored search -> xAI API -> web-only
|
||||
- Cookie auth mechanism is identical (env vars or browser extraction)
|
||||
- JSON output format is identical - no changes needed in score.py or format.py
|
||||
- xai_x.py is completely untouched
|
||||
|
||||
## References
|
||||
|
||||
- Bird CLI archive: `github.com/mvanhorn/bird-cli-archive`
|
||||
- Local vendor tarball: `vendor/steipete-bird-0.8.0.tgz`
|
||||
- Bird search implementation: `bird-cli-archive/dist/lib/twitter-client-search.js`
|
||||
- Current bird_x.py: `scripts/lib/bird_x.py`
|
||||
- Fallback chain: `scripts/lib/env.py:get_x_source()`
|
||||
@@ -1,263 +0,0 @@
|
||||
---
|
||||
title: "feat: Smart Supplemental Search — Entity-Aware Secondary Passes for Reddit & X"
|
||||
type: feat
|
||||
date: 2026-02-07
|
||||
---
|
||||
|
||||
# feat: Smart Supplemental Search — Entity-Aware Secondary Passes for Reddit & X
|
||||
|
||||
## Overview
|
||||
|
||||
Add an intelligent "discover → drill down" second pass to both Reddit and X searches. After the initial broad search, extract entities (handles, subreddits, hashtags) from results and run targeted secondary searches to surface content the broad pass missed. This supplements — does not replace — the existing search pipeline.
|
||||
|
||||
## Problem Statement / Motivation
|
||||
|
||||
The current search pipeline does a single broad pass per source (with Reddit having 2 fallbacks for low-result scenarios). This works well for general topics, but misses content that lives in:
|
||||
|
||||
- **Niche subreddits** that don't rank for generic queries (e.g., searching "Nano Banana Pro" finds r/generativeAI but misses r/nanobanana, r/localLLaMA)
|
||||
- **Key accounts on X** that are the authorities on a topic but whose individual posts don't rank for broad keyword search (e.g., @steipete for Open Claw, @karpathy for AI training)
|
||||
- **Conversation threads** where the most valuable discussion happens in replies, not the original tweet
|
||||
|
||||
The product works great today. This is about squeezing 20-30% more high-quality results from sources we already have access to.
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
### Architecture: Two-Phase Search
|
||||
|
||||
```
|
||||
CURRENT (Phase 1 — unchanged):
|
||||
Broad topic search → Reddit results + X results
|
||||
↓
|
||||
NEW (Phase 2 — supplemental):
|
||||
Extract entities from Phase 1 results
|
||||
↓ ↓
|
||||
[SUBREDDITS] [@HANDLES + #HASHTAGS]
|
||||
↓ ↓
|
||||
Targeted Reddit Targeted X searches
|
||||
searches per sub per handle/hashtag
|
||||
↓ ↓
|
||||
Merge + dedupe with Phase 1 results
|
||||
```
|
||||
|
||||
Phase 2 only runs if Phase 1 returned results (entities need to come from somewhere). Phase 2 results are merged and deduped against Phase 1 — the existing `dedupe.py` handles this.
|
||||
|
||||
### Feature 1: Entity Extraction Module (NEW FILE)
|
||||
|
||||
**File: `scripts/lib/entity_extract.py`**
|
||||
|
||||
A lightweight module that parses Phase 1 results and extracts:
|
||||
|
||||
**From X results:**
|
||||
- `@handles` — from `author_handle` field + any @mentions in post text
|
||||
- `#hashtags` — from post text
|
||||
- Rank by frequency: handles that appear 2+ times are "key voices"
|
||||
|
||||
**From Reddit results:**
|
||||
- `subreddit` names — from the `subreddit` field on each result
|
||||
- Cross-referenced subreddits — from enriched comment text mentioning "r/othersub"
|
||||
- Rank by frequency: subreddits with 2+ threads are "core communities"
|
||||
|
||||
**Output:**
|
||||
```python
|
||||
{
|
||||
"x_handles": ["steipete", "openclaw", "karpathy"], # ranked by frequency
|
||||
"x_hashtags": ["#openclaw", "#aitools"],
|
||||
"reddit_subreddits": ["generativeAI", "localLLaMA", "nanobanana"],
|
||||
"reddit_cross_refs": ["singularity", "MachineLearning"], # mentioned in comments
|
||||
}
|
||||
```
|
||||
|
||||
**Rules:**
|
||||
- No hardcoded entities — everything discovered dynamically from Phase 1
|
||||
- Cap at top 5 handles, top 3 hashtags, top 5 subreddits
|
||||
- Skip generic handles (@elonmusk, @OpenAI) that appear everywhere — maintain a small exclusion list of "too common" handles (< 20 entries)
|
||||
- Skip the original topic's "obvious" subreddit if it was already searched
|
||||
|
||||
### Feature 2: Supplemental X Search (Bird)
|
||||
|
||||
**File: modify `scripts/lib/bird_x.py`**
|
||||
|
||||
Add a `search_handles()` function:
|
||||
|
||||
```python
|
||||
def search_handles(handles: list[str], topic: str, from_date: str, count_per: int = 5) -> list:
|
||||
"""Search top handles for topic-related content."""
|
||||
results = []
|
||||
for handle in handles[:5]:
|
||||
# Uses Bird's support for X search operators
|
||||
query = f"from:{handle} {topic} since:{from_date}"
|
||||
cmd = ["bird", "search", query, "-n", str(count_per), "--json"]
|
||||
# ... parse results, add to list
|
||||
return results
|
||||
```
|
||||
|
||||
**Why Bird, not xAI:** Bird is free (uses your X login). Running 5 secondary searches via xAI would cost ~$0.025 per run, which adds up. Bird costs nothing.
|
||||
|
||||
**xAI alternative for users without Bird:** If Bird is not available but xAI is, use `allowed_x_handles` parameter:
|
||||
|
||||
```python
|
||||
# xAI supports filtering to specific handles (max 10)
|
||||
tools = [{
|
||||
"type": "x_search",
|
||||
"x_handles": {"allowed_x_handles": top_handles[:10]}
|
||||
}]
|
||||
```
|
||||
|
||||
### Feature 3: Supplemental Reddit Search
|
||||
|
||||
**File: modify `scripts/lib/openai_reddit.py`**
|
||||
|
||||
Add a `search_subreddits()` function:
|
||||
|
||||
```python
|
||||
def search_subreddits(subreddits: list[str], topic: str, ...) -> list:
|
||||
"""Search discovered subreddits for topic-related content."""
|
||||
# Build multi-subreddit query for the OpenAI web_search prompt
|
||||
sub_query = " OR ".join(f"r/{sub}" for sub in subreddits[:5])
|
||||
prompt = f"Search Reddit for threads about {topic} in these communities: {sub_query}"
|
||||
# ... single OpenAI API call, same pattern as existing search
|
||||
```
|
||||
|
||||
**Alternative approach — Reddit JSON API (free, no API key):**
|
||||
|
||||
```python
|
||||
def search_subreddit_json(subreddit: str, topic: str) -> list:
|
||||
"""Search a specific subreddit via Reddit's free JSON endpoint."""
|
||||
url = f"https://www.reddit.com/r/{subreddit}/search/.json"
|
||||
params = {"q": topic, "restrict_sr": "on", "sort": "new", "limit": 10}
|
||||
# ... parse JSON response
|
||||
```
|
||||
|
||||
This is free, requires no API key, and gives us structured data. The `.json` endpoint trick is well-documented and widely used.
|
||||
|
||||
### Feature 4: Orchestration Changes
|
||||
|
||||
**File: modify `scripts/last30days.py`**
|
||||
|
||||
After Phase 1 completes and enrichment is done, run Phase 2:
|
||||
|
||||
```python
|
||||
# Phase 1 (existing — unchanged)
|
||||
reddit_items, x_items = run_parallel_search(...)
|
||||
|
||||
# Phase 2 (new — supplemental)
|
||||
if reddit_items or x_items:
|
||||
entities = entity_extract.extract(reddit_items, x_items)
|
||||
|
||||
supplemental_reddit = []
|
||||
supplemental_x = []
|
||||
|
||||
# Run supplemental searches in parallel
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
if entities["reddit_subreddits"]:
|
||||
reddit_future = executor.submit(
|
||||
openai_reddit.search_subreddits,
|
||||
entities["reddit_subreddits"], topic, ...
|
||||
)
|
||||
if entities["x_handles"] and bird_available:
|
||||
x_future = executor.submit(
|
||||
bird_x.search_handles,
|
||||
entities["x_handles"], topic, from_date, ...
|
||||
)
|
||||
|
||||
# Merge with Phase 1
|
||||
all_reddit = reddit_items + supplemental_reddit
|
||||
all_x = x_items + supplemental_x
|
||||
|
||||
# Dedupe handles the rest
|
||||
```
|
||||
|
||||
**Depth-dependent behavior:**
|
||||
| Depth | Phase 2 behavior |
|
||||
|---|---|
|
||||
| `--quick` | Skip Phase 2 entirely (speed matters) |
|
||||
| default | Run Phase 2 with caps: 3 handles, 3 subreddits, 3 results each |
|
||||
| `--deep` | Run Phase 2 with caps: 5 handles, 5 subreddits, 5 results each |
|
||||
|
||||
### Feature 5: Thread Expansion for High-Engagement Posts (stretch goal)
|
||||
|
||||
**File: modify `scripts/lib/bird_x.py`**
|
||||
|
||||
For X posts with very high engagement (top 1-2 by likes), expand the conversation thread:
|
||||
|
||||
```python
|
||||
def expand_thread(tweet_id: str) -> list:
|
||||
"""Fetch full thread for a high-engagement tweet."""
|
||||
cmd = ["bird", "thread", tweet_id, "--json"]
|
||||
# ... parse thread, extract key replies
|
||||
```
|
||||
|
||||
This surfaces the discussion around viral posts — often more valuable than the original tweet. Only trigger for posts with 100+ likes to avoid noise.
|
||||
|
||||
## Technical Considerations
|
||||
|
||||
### Performance
|
||||
- Phase 2 adds 2-5 seconds for Bird (5 subprocess calls) and 3-8 seconds for Reddit subreddit search (1 API call)
|
||||
- On `--quick` mode, Phase 2 is skipped entirely — zero performance impact
|
||||
- Phase 2 runs AFTER Phase 1, not in parallel with it (needs Phase 1 results for entity extraction)
|
||||
|
||||
### Cost
|
||||
- Reddit subreddit search: 1 additional OpenAI API call (~$0.005) OR free via `.json` endpoint
|
||||
- X handle search via Bird: Free (uses your X login)
|
||||
- X handle search via xAI (fallback): 1 additional API call (~$0.005)
|
||||
- Thread expansion: Free via Bird
|
||||
|
||||
### No New Dependencies
|
||||
- Entity extraction is string parsing — no NLP libraries needed
|
||||
- Reddit `.json` endpoint uses existing `http.py` transport
|
||||
- Bird CLI calls use existing subprocess pattern from `bird_x.py`
|
||||
|
||||
### Backward Compatibility
|
||||
- Phase 2 is purely additive — all existing behavior unchanged
|
||||
- If Phase 2 finds nothing, output is identical to current
|
||||
- Deduplication handles any overlap between Phase 1 and Phase 2
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] Entity extraction module correctly parses handles, hashtags, and subreddits from search results
|
||||
- [x] Supplemental X searches via Bird find additional content from key handles
|
||||
- [x] Supplemental Reddit searches find content in discovered subreddits
|
||||
- [x] Phase 2 results are properly merged and deduped with Phase 1
|
||||
- [x] `--quick` mode skips Phase 2 entirely
|
||||
- [x] `--deep` mode searches more handles/subreddits with higher per-query limits
|
||||
- [x] No performance regression on `--quick` mode
|
||||
- [ ] Default mode adds < 10 seconds of latency
|
||||
- [x] Works with Bird-only, xAI-only, and both-available configurations
|
||||
- [x] Output format unchanged (Phase 2 results look identical to Phase 1 results)
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. `scripts/lib/entity_extract.py` — Entity extraction from results (new file)
|
||||
2. `scripts/lib/bird_x.py` — Add `search_handles()` function
|
||||
3. `scripts/lib/openai_reddit.py` — Add `search_subreddits()` function
|
||||
4. `scripts/last30days.py` — Orchestration: Phase 2 after Phase 1
|
||||
5. Test with real queries: "Open Claw", "Nano Banana Pro", "kanye west"
|
||||
6. (Stretch) Thread expansion for high-engagement posts
|
||||
|
||||
## Research Sources
|
||||
|
||||
### Reddit Search Techniques
|
||||
- [reddit-research-mcp](https://github.com/king-of-the-grackles/reddit-research-mcp) — MCP server with semantic subreddit discovery via 20K+ pre-indexed communities
|
||||
- [anvaka/sayit](https://github.com/anvaka/sayit) — Subreddit similarity graph via collaborative filtering (Jaccard similarity on user overlap)
|
||||
- [YARS](https://github.com/datavorous/yars) — No-API-key Reddit scraper using `.json` endpoint trick
|
||||
- Reddit's free JSON search endpoint: `reddit.com/r/{sub}/search/.json?q=QUERY&restrict_sr=on` — no auth needed
|
||||
- Reddit search operators: `subreddit:`, `title:`, `selftext:`, `author:`, `flair:` (Lucene-style)
|
||||
|
||||
### X/Twitter Search Techniques
|
||||
- [igorbrigadir/twitter-advanced-search](https://github.com/igorbrigadir/twitter-advanced-search) — Canonical reference of all X search operators
|
||||
- Bird CLI supports all X operators: `from:`, `to:`, `conversation_id:`, `min_retweets:`, `#hashtag`, `list:`
|
||||
- xAI x_search `allowed_x_handles` parameter — filter to max 10 specific handles
|
||||
- xAI x_search semantic search — finds conceptually related content without exact keyword matches
|
||||
- [Bellingcat OSINT Toolkit](https://bellingcat.gitbook.io/toolkit) — Multi-pass handle discovery methodology
|
||||
|
||||
### Key Insight
|
||||
The biggest gap in the current implementation is that **neither X nor Reddit search does entity extraction from initial results to inform follow-up queries.** Every tool/project researched that achieves better-than-basic results does some form of "discover entities → search entities" two-pass strategy.
|
||||
|
||||
## What We're NOT Doing
|
||||
|
||||
- **Not adding new API dependencies** — everything uses existing OpenAI, xAI, or Bird infrastructure
|
||||
- **Not adding NLP/ML libraries** — entity extraction is simple string parsing
|
||||
- **Not changing the output format** — Phase 2 results merge seamlessly
|
||||
- **Not hardcoding any entities** — all discovery is dynamic from search results
|
||||
- **Not slowing down `--quick` mode** — Phase 2 is skipped entirely
|
||||
- **Not replacing the current search** — Phase 2 supplements Phase 1
|
||||
@@ -1,147 +0,0 @@
|
||||
---
|
||||
title: "fix: X search query too restrictive, returns 0 results on popular topics"
|
||||
type: fix
|
||||
date: 2026-02-07
|
||||
---
|
||||
|
||||
# fix: X search query too restrictive, returns 0 results on popular topics
|
||||
|
||||
## Problem
|
||||
|
||||
`/last30days vibe motion best prompt techniques` returned **0 X posts** despite Vibe Motion being actively discussed on X (screenshots show posts from @Godid242, @KamilStanuch, @ColdStartTheory, @higgsfield_ai).
|
||||
|
||||
Root cause: `_extract_core_subject()` in `bird_x.py` produces overly specific queries. Bird/X search uses **literal keyword AND matching** — ALL words must appear in a tweet. The function kept 4 keywords (`vibe motion prompt techniques`) when only 2 (`vibe motion`) were needed.
|
||||
|
||||
## Three Bugs Found
|
||||
|
||||
### Bug 1: Multi-word noise phrases never match
|
||||
|
||||
```python
|
||||
# Current code (bird_x.py:24-38)
|
||||
noise = ['best', ..., 'what are', 'what is', 'how to', 'tips for', ...]
|
||||
words = topic.lower().split() # splits into individual words
|
||||
result = [w for w in words if w not in noise] # compares "what" against "what are" → no match!
|
||||
```
|
||||
|
||||
`"what are people saying about DeepSeek R1"` → keeps `"what are people saying"` → **LOSES THE ENTIRE TOPIC**.
|
||||
|
||||
The multi-word entries (`"what are"`, `"how to"`, `"tips for"`, `"use cases"`) are dead code. They never match because `.split()` creates individual words but the noise list has multi-word strings.
|
||||
|
||||
### Bug 2: Missing meta/research words
|
||||
|
||||
The noise list has `"prompting"` but not `"prompt"`, `"prompts"`, `"techniques"`, `"tips"`, `"tricks"`, `"methods"`, etc.
|
||||
|
||||
- `"vibe motion best prompt techniques"` → `"vibe motion prompt techniques"` (4 words, should be 2)
|
||||
- `"nano banana pro prompts for gemini"` → `"nano banana pro prompts"` (4 words, should be 3)
|
||||
|
||||
### Bug 3: No retry on 0 results
|
||||
|
||||
Reddit has multi-stage retry: full query → simplified core → subreddit fallback. X search runs once and accepts whatever comes back, even 0 results.
|
||||
|
||||
## Proposed Fix
|
||||
|
||||
All changes in `scripts/lib/bird_x.py`.
|
||||
|
||||
### Step 1: Fix `_extract_core_subject()` — strip phrases first, then words
|
||||
|
||||
```python
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for X search."""
|
||||
text = topic.lower()
|
||||
|
||||
# Phase 1: Strip multi-word prefixes/suffixes (order matters - longest first)
|
||||
prefixes = ['what are the best', 'what is the best', 'what are', 'what is',
|
||||
'how to', 'how do i', 'tips for', 'best practices for']
|
||||
for p in prefixes:
|
||||
if text.startswith(p):
|
||||
text = text[len(p):].strip()
|
||||
break
|
||||
|
||||
suffixes = ['best practices', 'use cases', 'prompt techniques',
|
||||
'prompting techniques']
|
||||
for s in suffixes:
|
||||
if text.endswith(s):
|
||||
text = text[:-len(s)].strip()
|
||||
break
|
||||
|
||||
# Phase 2: Split and filter individual noise words
|
||||
noise = {'best', 'top', 'practices', 'features', 'killer', 'guide',
|
||||
'tutorial', 'recommendations', 'advice', 'prompting', 'prompt',
|
||||
'prompts', 'techniques', 'tips', 'tricks', 'methods',
|
||||
'strategies', 'review', 'reviews', 'uses', 'usecases',
|
||||
'examples', 'using', 'for', 'with', 'the', 'of', 'in', 'on',
|
||||
'about', 'latest', 'new', 'news', 'update', 'updates',
|
||||
'good', 'great', 'awesome', 'and', 'or', 'a', 'an', 'is',
|
||||
'are', 'was', 'were', 'people', 'saying', 'think', 'said'}
|
||||
words = text.split()
|
||||
result = [w for w in words if w not in noise]
|
||||
|
||||
return ' '.join(result[:3]) or topic # Max 3 words (was 4)
|
||||
```
|
||||
|
||||
**Expected results after fix:**
|
||||
|
||||
| Input | Before | After |
|
||||
|-------|--------|-------|
|
||||
| `vibe motion best prompt techniques` | `vibe motion prompt techniques` | `vibe motion` |
|
||||
| `what are people saying about DeepSeek R1` | `what are people saying` | `deepseek r1` |
|
||||
| `nano banana pro prompts for gemini` | `nano banana pro prompts` | `nano banana pro` |
|
||||
| `open claw best uses` | `open claw uses` | `open claw` |
|
||||
| `best claude code skills` | `claude code skills` | `claude code skills` |
|
||||
| `kanye west` | `kanye west` | `kanye west` |
|
||||
|
||||
### Step 2: Add retry with simplified query on 0 results
|
||||
|
||||
In `search_x()`, after the initial search, if 0 items returned, retry with just the first 2 words of the core subject:
|
||||
|
||||
```python
|
||||
def search_x(topic, from_date, to_date, depth="default"):
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
query = f"{core_topic} since:{from_date}"
|
||||
|
||||
# ... existing Bird search code ...
|
||||
|
||||
items = parse_bird_response(response)
|
||||
|
||||
# Retry with fewer keywords if 0 results
|
||||
if not items and len(core_topic.split()) > 2:
|
||||
shorter = ' '.join(core_topic.split()[:2])
|
||||
_log(f"0 results for '{core_topic}', retrying with '{shorter}'")
|
||||
query = f"{shorter} since:{from_date}"
|
||||
# ... retry Bird search ...
|
||||
items = parse_bird_response(retry_response)
|
||||
|
||||
return response # or merged response
|
||||
```
|
||||
|
||||
### Step 3 (optional): Cross-pollinate Reddit entities into X Phase 2
|
||||
|
||||
When X Phase 1 returns 0 results but Reddit found threads, extract brand/product names from Reddit thread titles and use them as X search fallback queries. This is lower priority — Steps 1-2 should fix most cases.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] `vibe motion best prompt techniques` returns >0 X posts (12 posts found)
|
||||
- [x] `what are people saying about DeepSeek R1` produces query containing "deepseek r1" not "what are people saying"
|
||||
- [x] No regressions on working queries (`kanye west`, `claude code skills`, `open claw`)
|
||||
- [x] Retry fires when initial query returns 0, logged to stderr
|
||||
- [x] `openai_reddit.py`'s `_extract_core_subject()` NOT changed (Reddit uses semantic search, not literal matching — the current function works fine there)
|
||||
|
||||
## Files to Change
|
||||
|
||||
- `scripts/lib/bird_x.py` — `_extract_core_subject()` rewrite + retry logic in `search_x()`
|
||||
- `scripts/lib/bird_x.py` — `search_handles()` benefits automatically (calls `_extract_core_subject()`)
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
# Mock mode (quick syntax check)
|
||||
python3 scripts/last30days.py "vibe motion best prompt techniques" --mock --emit=compact 2>&1
|
||||
|
||||
# Live queries to verify X results
|
||||
python3 scripts/last30days.py "vibe motion best prompt techniques" --quick --emit=compact 2>&1 | grep -E "X:|posts"
|
||||
python3 scripts/last30days.py "what are people saying about DeepSeek R1" --quick --emit=compact 2>&1 | grep -E "X:|posts"
|
||||
|
||||
# Regression check
|
||||
python3 scripts/last30days.py "kanye west" --quick --emit=compact 2>&1 | grep -E "X:|posts"
|
||||
```
|
||||
@@ -1,243 +0,0 @@
|
||||
---
|
||||
title: "feat: Add Codex CLI compatibility"
|
||||
type: feat
|
||||
date: 2026-02-14
|
||||
---
|
||||
|
||||
# feat: Add Codex CLI Compatibility
|
||||
|
||||
## Overview
|
||||
|
||||
Make /last30days work as a Codex CLI skill alongside Claude Code. Both platforms use `SKILL.md` with YAML frontmatter — the gap is small but the details matter. Inspired by PR #24 (el-analista) and PR #5 (jblwilliams) on the public repo, applied to the v2.1 codebase.
|
||||
|
||||
## Research Findings
|
||||
|
||||
### How Codex Skills Work (from [official docs](https://developers.openai.com/codex/skills))
|
||||
|
||||
**Format:** Identical to Claude Code — `SKILL.md` with YAML frontmatter + Markdown body.
|
||||
|
||||
**Required frontmatter:** Only `name` and `description`. The official skill-creator guidance says "Do not include any other fields in YAML frontmatter." This is stricter than Claude Code which allows `version`, `allowed-tools`, `argument-hint`, etc.
|
||||
|
||||
**Discovery:** Codex uses "progressive disclosure" — it reads ONLY the `description` field to decide whether to invoke a skill. The body loads only after triggering. This means the description must be comprehensive about when to use/not use the skill.
|
||||
|
||||
**Invocation:** Users invoke with `$skill-name` or `/skills` menu. Codex can also implicitly match based on the description (configurable via `agents/openai.yaml`).
|
||||
|
||||
**Installation paths** (scanned in order):
|
||||
| Scope | Path |
|
||||
|-------|------|
|
||||
| Folder | `$CWD/.agents/skills/` |
|
||||
| Repo | `$REPO_ROOT/.agents/skills/` |
|
||||
| User | `$HOME/.agents/skills/` |
|
||||
| Admin | `/etc/codex/skills/` |
|
||||
| System | Bundled |
|
||||
|
||||
Note: Some docs also mention `~/.codex/skills/` as an alias for `$HOME/.agents/skills/`. Both should be checked.
|
||||
|
||||
**`agents/openai.yaml`** (optional sidecar):
|
||||
```yaml
|
||||
interface:
|
||||
display_name: "User-facing name"
|
||||
short_description: "Brief description"
|
||||
default_prompt: "Surrounding prompt template"
|
||||
brand_color: "#hex"
|
||||
policy:
|
||||
allow_implicit_invocation: true
|
||||
dependencies:
|
||||
tools:
|
||||
- type: "mcp"
|
||||
value: "toolName"
|
||||
```
|
||||
|
||||
**Size guidance:** Keep SKILL.md under 500 lines. Use `references/` directory for detailed docs that load on demand.
|
||||
|
||||
**Scripts:** Put executable code in `scripts/`. These can run without being loaded into context — good for our Python research engine.
|
||||
|
||||
### What Real Codex Skills Look Like (from [openai/skills catalog](https://github.com/openai/skills))
|
||||
|
||||
**openai-docs skill** — Uses MCP tools (`mcp__openaiDeveloperDocs__search_openai_docs`). Has a workflow section, fallback instructions if MCP isn't set up, and quality rules. Clean and focused.
|
||||
|
||||
**pdf skill** — Runs scripts (`pdftoppm`, `reportlab`), has file conventions (`tmp/pdfs/`, `output/pdf/`), specifies dependencies. Good example of a skill that shells out to tools like we do.
|
||||
|
||||
**skill-creator** — The meta-skill. Emphasizes "the context window is a public good" and treating the LLM as "already very smart — only add information it genuinely lacks." Has 6 creation steps, validation scripts, and naming conventions.
|
||||
|
||||
### Key Insight: Frontmatter Compatibility Problem
|
||||
|
||||
Claude Code SKILL.md uses:
|
||||
```yaml
|
||||
name: last30days
|
||||
version: "2.1"
|
||||
description: Research a topic...
|
||||
argument-hint: 'nano banana pro prompts...'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
```
|
||||
|
||||
Codex wants only `name` and `description`. The question: does Codex error on unknown frontmatter fields, or ignore them?
|
||||
|
||||
**Safe answer:** Codex uses standard YAML parsing and likely ignores unknown keys. But the official guidance says "Do not include any other fields" — meaning it's untested territory and could break in future Codex updates.
|
||||
|
||||
**Our approach:** Keep one SKILL.md with Claude-specific fields. If Codex chokes, we add a thin wrapper. This is pragmatic — maintaining two SKILL.md files defeats the purpose of cross-platform compatibility.
|
||||
|
||||
### PR #24 Analysis (el-analista)
|
||||
|
||||
Good ideas to incorporate:
|
||||
- Portable script path resolution (repo → Claude → Codex → agents)
|
||||
- `agents/openai.yaml` for Codex discovery
|
||||
- Platform-neutral output text ("assistant" instead of "Claude")
|
||||
- Sandbox-friendly cache/output dir fallbacks with env var overrides
|
||||
- Last-chance retry for Bird search (better query noise stripping)
|
||||
|
||||
Not applicable to v2.1:
|
||||
- Based on v2.0 codebase — doesn't have YouTube, vendored Bird, or pipeline changes
|
||||
- We'll cherry-pick the ideas, not the code
|
||||
|
||||
### PR #5 Analysis (jblwilliams)
|
||||
|
||||
Not needed:
|
||||
- Codex JWT auth — our OpenAI API calls work natively in Codex already
|
||||
- SSE response handling — we don't stream responses
|
||||
- The 403 enrichment issues they hit are specific to Codex-hosted auth, not our use case
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
Five changes, all additive — zero impact on existing Claude Code behavior:
|
||||
|
||||
### 1. Add `agents/openai.yaml` for Codex discovery
|
||||
|
||||
```yaml
|
||||
interface:
|
||||
display_name: "Last 30 Days"
|
||||
short_description: "Research any topic across Reddit, X, YouTube, and the web from the last 30 days. Returns synthesized expert answers and copy-paste prompts."
|
||||
default_prompt: "Research this topic from the last 30 days across Reddit, X, YouTube, and web. Synthesize what people are actually saying, upvoting, and sharing right now."
|
||||
brand_color: "#FF6B35"
|
||||
|
||||
policy:
|
||||
allow_implicit_invocation: true
|
||||
```
|
||||
|
||||
### 2. Make SKILL.md script path portable
|
||||
|
||||
Replace the hardcoded Claude path with a lookup that checks multiple install locations:
|
||||
|
||||
```bash
|
||||
# Find the skill root
|
||||
for dir in \
|
||||
"." \
|
||||
"${CLAUDE_PLUGIN_ROOT:-}" \
|
||||
"$HOME/.claude/skills/last30days" \
|
||||
"$HOME/.agents/skills/last30days" \
|
||||
"$HOME/.codex/skills/last30days"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
|
||||
if [ -z "${SKILL_ROOT:-}" ]; then
|
||||
echo "ERROR: Could not find scripts/last30days.py" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 "${SKILL_ROOT}/scripts/last30days.py" "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
### 3. Platform-neutral Python output text
|
||||
|
||||
Replace "Claude" with "assistant" in LLM-facing output strings only. Human-facing docs (README, etc.) stay as-is.
|
||||
|
||||
Files:
|
||||
- `scripts/last30days.py` — web search marker text (~3 lines)
|
||||
- `scripts/lib/render.py` — docstrings + web-only banner (~4 lines)
|
||||
- `scripts/lib/http.py` — User-Agent string (~1 line)
|
||||
|
||||
### 4. Sandbox-friendly cache/output dirs
|
||||
|
||||
Codex runs sandboxed. Add env var overrides + tempdir fallback (from PR #24):
|
||||
|
||||
**`scripts/lib/cache.py`:**
|
||||
- Check `LAST30DAYS_CACHE_DIR` env var
|
||||
- Catch `PermissionError`, fall back to `tempfile.gettempdir()/last30days/cache`
|
||||
|
||||
**`scripts/lib/render.py`:**
|
||||
- Check `LAST30DAYS_OUTPUT_DIR` env var
|
||||
- Catch `PermissionError`, fall back to `tempfile.gettempdir()/last30days/out`
|
||||
|
||||
### 5. README + installation docs
|
||||
|
||||
Add a "Codex Compatibility" section to README:
|
||||
|
||||
```markdown
|
||||
## Codex Compatibility
|
||||
|
||||
This skill works in both Claude Code and OpenAI Codex CLI.
|
||||
|
||||
**Claude Code:** `git clone` into `~/.claude/skills/last30days`
|
||||
**Codex CLI:** `git clone` into `~/.agents/skills/last30days`
|
||||
|
||||
Both use the same SKILL.md, same Python engine, same scripts.
|
||||
The `agents/openai.yaml` provides Codex-specific discovery metadata.
|
||||
```
|
||||
|
||||
## What We're NOT Doing
|
||||
|
||||
- **Separate SKILL.md for Codex** — One file, both platforms. Claude-specific frontmatter fields (`allowed-tools`, `version`, `argument-hint`) are likely ignored by Codex's YAML parser. If this breaks, we'll address it then.
|
||||
- **Codex JWT auth (PR #5)** — Our OpenAI Responses API calls work natively in Codex. No special handling needed.
|
||||
- **SSE streaming (PR #5)** — Not our use case.
|
||||
- **Codex-specific tool names in SKILL.md** — Both LLMs understand "do a web search" and "run this bash command." The instructions work cross-platform as-is.
|
||||
- **Publishing to openai/skills catalog** — Out of scope for now. Users install via git clone.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] `agents/openai.yaml` exists with proper `interface` and `policy` sections
|
||||
- [x] SKILL.md uses portable path resolution (repo checkout, `~/.claude/skills/`, `~/.agents/skills/`, `~/.codex/skills/`)
|
||||
- [x] Python scripts use "assistant" instead of "Claude" in LLM-facing output (~8 string replacements)
|
||||
- [x] Cache dir falls back gracefully in sandboxed environments (`LAST30DAYS_CACHE_DIR` env var + `PermissionError` catch)
|
||||
- [x] Output dir falls back gracefully in sandboxed environments (`LAST30DAYS_OUTPUT_DIR` env var + `PermissionError` catch)
|
||||
- [x] Existing Claude Code behavior is unchanged (zero regressions)
|
||||
- [x] README documents Codex installation path (`~/.agents/skills/last30days`)
|
||||
- [x] `python3 scripts/last30days.py "test topic" --mock --emit=compact` still works
|
||||
|
||||
## Files to Create/Modify
|
||||
|
||||
### New Files
|
||||
- `agents/openai.yaml` — Codex discovery metadata (~10 lines)
|
||||
|
||||
### Modified Files
|
||||
- `SKILL.md` — Portable script path resolution (~15 lines changed)
|
||||
- `README.md` — Add "Codex Compatibility" section (~15 lines)
|
||||
- `scripts/last30days.py` — "Claude" → "assistant" in output strings (~3 lines)
|
||||
- `scripts/lib/render.py` — "Claude" → "assistant" + output dir fallback (~15 lines)
|
||||
- `scripts/lib/cache.py` — Cache dir env override + fallback (~12 lines)
|
||||
- `scripts/lib/http.py` — User-Agent string (~1 line)
|
||||
|
||||
### Total scope: ~70 lines changed across 7 files. Small, additive, low risk.
|
||||
|
||||
## Dependencies & Risks
|
||||
|
||||
| Risk | Likelihood | Mitigation |
|
||||
|------|-----------|------------|
|
||||
| Codex rejects unknown YAML frontmatter (`allowed-tools`, etc.) | Low-Medium | Standard YAML parsers ignore unknown keys. If it breaks, strip Claude-specific fields and use `agents/openai.yaml` for metadata. |
|
||||
| Codex sandbox blocks Node.js (vendored Bird) | Medium | Bird failure already falls back to xAI API. If no xAI key, X search skipped gracefully. |
|
||||
| yt-dlp not in Codex sandbox PATH | Medium | YouTube already degrades gracefully — "yt-dlp not installed, skipping YouTube." |
|
||||
| Codex sandbox blocks `~/.cache/` writes | Medium | Env var override + tempdir fallback handles this. (Proven approach from PR #24) |
|
||||
| Codex changes skill discovery paths | Low | We check 5 paths. Easy to add more. |
|
||||
| Codex description matching triggers on wrong queries | Low | Write description with clear "use when" / "do not use when" boundaries per official guidance. |
|
||||
|
||||
## References
|
||||
|
||||
### Community PRs
|
||||
- [PR #24](https://github.com/mvanhorn/last30days-skill/pull/24) (el-analista) — Codex compatibility, portable paths, platform-neutral text
|
||||
- [PR #5](https://github.com/mvanhorn/last30days-skill/pull/5) (jblwilliams) — Codex auth support
|
||||
|
||||
### Official Codex Docs
|
||||
- [Agent Skills](https://developers.openai.com/codex/skills) — SKILL.md format, discovery, installation paths
|
||||
- [AGENTS.md Guide](https://developers.openai.com/codex/guides/agents-md/) — Custom instructions, hierarchical loading
|
||||
- [Codex CLI Features](https://developers.openai.com/codex/cli/features/) — Overview of CLI capabilities
|
||||
- [Configuration Reference](https://developers.openai.com/codex/config-reference/) — config.toml, skill enable/disable
|
||||
|
||||
### Examples
|
||||
- [openai/skills catalog](https://github.com/openai/skills) — Official curated skills
|
||||
- [skill-creator](https://github.com/openai/skills/blob/main/skills/.system/skill-creator/SKILL.md) — Meta-skill for creating skills, best practices
|
||||
- [pdf skill](https://github.com/openai/skills/blob/main/skills/.curated/pdf/SKILL.md) — Example of skill that runs external scripts
|
||||
- [openai-docs skill](https://github.com/openai/skills/blob/main/skills/.curated/openai-docs/SKILL.md) — Example of MCP-backed skill
|
||||
|
||||
### Community Analysis
|
||||
- [Skills in OpenAI Codex](https://blog.fsck.com/2025/12/19/codex-skills/) — Jesse Vincent's deep dive on skill internals
|
||||
- [Simon Willison on skills adoption](https://simonw.substack.com/p/openai-are-quietly-adopting-skills) — Cross-platform skill format analysis
|
||||
- [SkillsMP marketplace](https://skillsmp.com/) — Community marketplace supporting both Claude Code and Codex skills
|
||||
@@ -1,224 +0,0 @@
|
||||
---
|
||||
title: "feat: Merge OpenClaw variant into main repo"
|
||||
type: feat
|
||||
date: 2026-02-14
|
||||
---
|
||||
|
||||
# feat: Merge OpenClaw Variant into Main Repo
|
||||
|
||||
## Overview
|
||||
|
||||
Consolidate the `last30days-openclaw` project into `last30days-skill-private` so there's one unified Python engine powering both the main skill (Claude Code / Codex) and an "open" variant with watchlist, briefing, history, and built-in web search. The open variant also gets YouTube and Bird CLI — features the main project already has but openclaw was built before they existed.
|
||||
|
||||
## Problem Statement / Motivation
|
||||
|
||||
Right now there are two separate repos with diverging codebases:
|
||||
|
||||
- **`last30days-skill-private`** (main, Feb 14) — YouTube, vendored Bird, better scoring/normalization, Codex compat. But no built-in web search APIs and no persistence layer.
|
||||
- **`last30days-openclaw`** (Feb 10) — SQLite store, watchlist, briefings, 3 web search backends (Parallel AI, Brave, OpenRouter). But frozen without YouTube or latest engine improvements.
|
||||
|
||||
They share ~80% of the same `scripts/lib/` files but are drifting apart. Maintaining two codebases is unsustainable.
|
||||
|
||||
**Goal:** One repo, one Python engine, two SKILL.md variants. Install once, works everywhere.
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
Use `last30days-skill-private` as the base (it's 4 days newer with better code) and port the OpenClaw-exclusive features in:
|
||||
|
||||
### What gets ported from OpenClaw
|
||||
|
||||
| File | What it does | Destination |
|
||||
|------|-------------|-------------|
|
||||
| `scripts/store.py` | SQLite research accumulator (WAL, FTS5, dedup) | `scripts/store.py` |
|
||||
| `scripts/watchlist.py` | Topic watchlist CLI (add/remove/list/run) | `scripts/watchlist.py` |
|
||||
| `scripts/briefing.py` | Morning briefing generator (daily/weekly) | `scripts/briefing.py` |
|
||||
| `scripts/lib/brave_search.py` | Brave Search API (free tier, 2K/mo) | `scripts/lib/brave_search.py` |
|
||||
| `scripts/lib/parallel_search.py` | Parallel AI search (LLM-optimized) | `scripts/lib/parallel_search.py` |
|
||||
| `scripts/lib/openrouter_search.py` | OpenRouter/Sonar Pro search | `scripts/lib/openrouter_search.py` |
|
||||
| `references/research.md` | One-shot research instructions | `variants/open/references/research.md` |
|
||||
| `references/watchlist.md` | Watchlist mode instructions | `variants/open/references/watchlist.md` |
|
||||
| `references/briefing.md` | Briefing mode instructions | `variants/open/references/briefing.md` |
|
||||
| `references/history.md` | History query instructions | `variants/open/references/history.md` |
|
||||
|
||||
### What gets upgraded in the ported code
|
||||
|
||||
- **`store.py`**: No changes needed — it's self-contained SQLite, works as-is
|
||||
- **`watchlist.py`**: Remove OpenClaw cron-specific code, make cron setup generic (launchd on macOS, systemd on Linux, or manual cron)
|
||||
- **`briefing.py`**: No changes needed
|
||||
- **`scripts/lib/env.py`**: Merge OpenClaw's web search key support (`PARALLEL_API_KEY`, `BRAVE_API_KEY`, `OPENROUTER_API_KEY`) and `has_web_search_keys()` / `get_web_search_source()` functions into main's env.py. Drop the OpenClaw config loader (`~/.openclaw/openclaw.json`) — just use env vars and `~/.config/last30days/.env`
|
||||
- **`scripts/last30days.py`**: Add OpenClaw's `_search_web()` function so the script can do web search natively when API keys are available (instead of always delegating to the assistant)
|
||||
|
||||
### What gets DROPPED from OpenClaw
|
||||
|
||||
| File | Why |
|
||||
|------|-----|
|
||||
| `scripts/cron_setup.py` | Too OpenClaw-platform-specific. Replace with generic scheduling docs. |
|
||||
| OpenClaw config loader in `env.py` | `~/.openclaw/openclaw.json` path is platform-specific. Use env vars instead. |
|
||||
| `.clawhubignore` | OpenClaw marketplace artifact, not needed in unified repo |
|
||||
|
||||
### New file: Open variant SKILL.md
|
||||
|
||||
Create `variants/open/SKILL.md` — the multi-mode skill with command routing:
|
||||
|
||||
```
|
||||
variants/open/
|
||||
├── SKILL.md # Router: watch, briefing, history, or one-shot
|
||||
├── references/
|
||||
│ ├── research.md # One-shot research instructions
|
||||
│ ├── watchlist.md # Watchlist management instructions
|
||||
│ ├── briefing.md # Briefing mode instructions
|
||||
│ └── history.md # History query instructions
|
||||
└── context.md # Agent memory (user preferences, source quality)
|
||||
```
|
||||
|
||||
The open variant's SKILL.md points to `{baseDir}/scripts/last30days.py` (same engine) but adds the router and reference file system. It also adds the `--store` flag for persistence.
|
||||
|
||||
### How YouTube and Bird CLI get added to the open variant
|
||||
|
||||
They're already in `scripts/lib/youtube_yt.py` and `scripts/lib/vendor/bird/`. The open variant's SKILL.md just needs to mention YouTube in its description and the research.md reference file gets the YouTube stats line in the output format. No code changes needed — the Python engine already supports all four sources.
|
||||
|
||||
## Technical Considerations
|
||||
|
||||
### File Structure After Merge
|
||||
|
||||
```
|
||||
last30days-skill-private/
|
||||
├── SKILL.md # Main skill (Claude Code / Codex)
|
||||
├── agents/openai.yaml # Codex discovery (existing)
|
||||
├── variants/
|
||||
│ └── open/
|
||||
│ ├── SKILL.md # Open variant with routing
|
||||
│ ├── references/
|
||||
│ │ ├── research.md
|
||||
│ │ ├── watchlist.md
|
||||
│ │ ├── briefing.md
|
||||
│ │ └── history.md
|
||||
│ └── context.md
|
||||
├── scripts/
|
||||
│ ├── last30days.py # Unified engine (+ native web search)
|
||||
│ ├── store.py # SQLite accumulator (from openclaw)
|
||||
│ ├── watchlist.py # Watchlist CLI (from openclaw, genericized)
|
||||
│ ├── briefing.py # Briefing generator (from openclaw)
|
||||
│ └── lib/
|
||||
│ ├── ... (existing files)
|
||||
│ ├── brave_search.py # NEW from openclaw
|
||||
│ ├── parallel_search.py # NEW from openclaw
|
||||
│ ├── openrouter_search.py # NEW from openclaw
|
||||
│ ├── youtube_yt.py # Existing
|
||||
│ └── vendor/bird/ # Existing
|
||||
└── README.md # Updated with open variant docs
|
||||
```
|
||||
|
||||
### Installation for open variant users
|
||||
|
||||
```bash
|
||||
# Claude Code (main skill — unchanged)
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||
|
||||
# Open variant (with watchlist, briefings, history)
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||
# Then in Claude Code settings, point skill to variants/open/SKILL.md
|
||||
# OR symlink:
|
||||
ln -sf ~/.claude/skills/last30days/variants/open/SKILL.md ~/.claude/skills/last30days-open/SKILL.md
|
||||
```
|
||||
|
||||
### env.py merge strategy
|
||||
|
||||
Main's `env.py` is the base. Add from OpenClaw:
|
||||
- Three new key names: `PARALLEL_API_KEY`, `BRAVE_API_KEY`, `OPENROUTER_API_KEY`
|
||||
- `has_web_search_keys()` function
|
||||
- `get_web_search_source()` function — returns `'parallel'`, `'brave'`, or `'openrouter'`
|
||||
- `get_available_sources()` update to include web-search-capable modes
|
||||
|
||||
### last30days.py merge strategy
|
||||
|
||||
Main's `last30days.py` is the base. Add from OpenClaw:
|
||||
- `_search_web()` function that calls the appropriate web search backend
|
||||
- `--store` CLI flag to persist findings to SQLite
|
||||
- `--diagnose` CLI flag for source availability diagnostics
|
||||
- Web results integration into the existing report pipeline (normalize → score → dedupe → render)
|
||||
|
||||
Keep main's:
|
||||
- YouTube integration
|
||||
- Phase 2 supplemental search
|
||||
- 3-tier Reddit fallback
|
||||
- Better error handling
|
||||
- Minimum result guarantee
|
||||
|
||||
### Portable path resolution (already done)
|
||||
|
||||
The main SKILL.md already has portable path resolution (from Codex compat work):
|
||||
```bash
|
||||
for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "$HOME/.claude/skills/last30days" ...
|
||||
```
|
||||
|
||||
The open variant's SKILL.md uses `{baseDir}` which resolves to the skill root. Both approaches work — we just need to make sure the open variant's references use `{baseDir}` consistently.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] `scripts/store.py` ported and working (SQLite creates on first use)
|
||||
- [x] `scripts/watchlist.py` ported with generic scheduling (no OpenClaw cron dependency)
|
||||
- [x] `scripts/briefing.py` ported and generates daily/weekly briefings
|
||||
- [x] `scripts/lib/brave_search.py` ported and functional
|
||||
- [x] `scripts/lib/parallel_search.py` ported and functional
|
||||
- [x] `scripts/lib/openrouter_search.py` ported and functional
|
||||
- [x] `scripts/lib/env.py` updated with web search key support
|
||||
- [x] `scripts/last30days.py` has native `_search_web()` + `--store` + `--diagnose`
|
||||
- [x] `variants/open/SKILL.md` exists with command routing (watch, briefing, history, research)
|
||||
- [x] `variants/open/references/*.md` — all 4 reference files ported
|
||||
- [x] Open variant mentions YouTube in description and research output format
|
||||
- [x] Open variant uses same portable path resolution as main
|
||||
- [x] Main SKILL.md behavior is unchanged (zero regressions)
|
||||
- [x] `python3 scripts/last30days.py "test topic" --mock --emit=compact` still works
|
||||
- [x] `python3 scripts/last30days.py "test topic" --diagnose` shows source availability
|
||||
- [x] README documents open variant installation and usage
|
||||
|
||||
## Dependencies & Risks
|
||||
|
||||
| Risk | Likelihood | Mitigation |
|
||||
|------|-----------|------------|
|
||||
| OpenClaw's store.py has import dependencies we don't have | Low | store.py uses only stdlib (sqlite3, json, datetime). Self-contained. |
|
||||
| Web search backends need API keys to test | Medium | Each has a `--mock` or dry-run path. Test with real keys if available, mock otherwise. |
|
||||
| watchlist.py depends on OpenClaw cron API | High | Known — strip cron_setup.py dependency, replace with generic docs for launchd/systemd/crontab. |
|
||||
| Open variant SKILL.md is too long (>500 lines) | Medium | Use reference file pattern (already planned). Router SKILL.md stays under 100 lines. |
|
||||
| env.py merge introduces regressions | Low | Main's env.py is well-tested. Additive changes only — new keys, new functions. |
|
||||
| Two SKILL.md files = maintenance burden | Low | They serve different purposes. Main is simple one-shot. Open adds routing. Core engine is shared. |
|
||||
|
||||
## Files to Create/Modify
|
||||
|
||||
### New Files
|
||||
- `variants/open/SKILL.md` — Open variant router (~100 lines)
|
||||
- `variants/open/references/research.md` — One-shot research instructions (from openclaw, updated with YouTube)
|
||||
- `variants/open/references/watchlist.md` — Watchlist management instructions (from openclaw)
|
||||
- `variants/open/references/briefing.md` — Briefing mode instructions (from openclaw)
|
||||
- `variants/open/references/history.md` — History query instructions (from openclaw)
|
||||
- `variants/open/context.md` — Agent memory template
|
||||
- `scripts/store.py` — SQLite accumulator (from openclaw, as-is)
|
||||
- `scripts/watchlist.py` — Watchlist CLI (from openclaw, genericized)
|
||||
- `scripts/briefing.py` — Briefing generator (from openclaw, as-is)
|
||||
- `scripts/lib/brave_search.py` — Brave Search API (from openclaw)
|
||||
- `scripts/lib/parallel_search.py` — Parallel AI search (from openclaw)
|
||||
- `scripts/lib/openrouter_search.py` — OpenRouter/Sonar Pro search (from openclaw)
|
||||
|
||||
### Modified Files
|
||||
- `scripts/lib/env.py` — Add web search key support (~30 lines added)
|
||||
- `scripts/last30days.py` — Add `_search_web()`, `--store`, `--diagnose` (~80 lines added)
|
||||
- `README.md` — Add open variant section (~20 lines)
|
||||
|
||||
### Total scope: ~12 new files (mostly copied), ~130 lines of new code in existing files.
|
||||
|
||||
## References
|
||||
|
||||
### Internal
|
||||
- OpenClaw plan: `/Users/mvanhorn/last30days-openclaw/docs/plans/2026-02-10-feat-openclaw-last30days-skill-plan.md` (989 lines, comprehensive spec)
|
||||
- Codex compat plan: `docs/plans/2026-02-14-feat-codex-skill-compatibility-plan.md` (portable paths, platform-neutral text)
|
||||
- OpenClaw source: `/Users/mvanhorn/last30days-openclaw/`
|
||||
|
||||
### Key files to port
|
||||
- `store.py`: `/Users/mvanhorn/last30days-openclaw/scripts/store.py` (20KB, SQLite with FTS5)
|
||||
- `watchlist.py`: `/Users/mvanhorn/last30days-openclaw/scripts/watchlist.py` (10KB)
|
||||
- `briefing.py`: `/Users/mvanhorn/last30days-openclaw/scripts/briefing.py` (8KB)
|
||||
- `brave_search.py`: `/Users/mvanhorn/last30days-openclaw/scripts/lib/brave_search.py` (6KB)
|
||||
- `parallel_search.py`: `/Users/mvanhorn/last30days-openclaw/scripts/lib/parallel_search.py` (4KB)
|
||||
- `openrouter_search.py`: `/Users/mvanhorn/last30days-openclaw/scripts/lib/openrouter_search.py` (7KB)
|
||||
- `env.py` (openclaw version): `/Users/mvanhorn/last30days-openclaw/scripts/lib/env.py` (9KB — has web search key functions)
|
||||
@@ -1,315 +0,0 @@
|
||||
---
|
||||
title: "feat: Add YouTube transcript search as 4th source"
|
||||
type: feat
|
||||
date: 2026-02-14
|
||||
---
|
||||
|
||||
# feat: Add YouTube Transcript Search
|
||||
|
||||
## Overview
|
||||
|
||||
Add YouTube as a 4th research source alongside Reddit, X, and Web. Search for recent videos on the user's topic, fetch transcripts from the top results, and feed the transcript text into the synthesis — giving the Judge Agent access to what people are *saying* in video form, not just what they're posting on social media.
|
||||
|
||||
**Why this matters:** For many topics (tutorials, product reviews, drama breakdowns), the best content lives on YouTube, not Reddit or X. A 20-minute video review contains 10x the signal of a tweet. The skill currently misses all of it.
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
Use **yt-dlp** (already installed via Homebrew) for both YouTube search and transcript extraction. No new API keys, no new dependencies. Follows the same "zero friction" philosophy as vendored Bird search.
|
||||
|
||||
### Two-step process per research run:
|
||||
|
||||
1. **Search**: `yt-dlp "ytsearch{N}:{topic}" --dateafter {30d_ago} --flat-playlist --print` → top videos by view count
|
||||
2. **Transcripts**: For top 5 videos, extract auto-generated subtitles via `yt-dlp --write-auto-subs --skip-download`, clean VTT to plaintext in Python
|
||||
|
||||
### Why NOT use `summarize` CLI:
|
||||
|
||||
- Adds 146MB brew dependency (arm64-only binary)
|
||||
- Calls OpenAI API per video ($0.01-0.03 each) — adds cost on top of existing API usage
|
||||
- yt-dlp already extracts raw transcripts for free (covers ~95% of videos with auto-captions)
|
||||
- Raw transcripts are better for synthesis anyway — the LLM doing synthesis (Claude) should interpret the content itself, not get a pre-summarized version
|
||||
|
||||
`summarize` is a great standalone tool, but for integration into a research pipeline where an LLM already synthesizes everything, raw transcripts are the right input.
|
||||
|
||||
## Technical Approach
|
||||
|
||||
### Architecture
|
||||
|
||||
New file: `scripts/lib/youtube_yt.py` (mirrors `bird_x.py` pattern)
|
||||
|
||||
```
|
||||
yt-dlp search → metadata (title, views, channel, date)
|
||||
↓
|
||||
sort by views, take top N
|
||||
↓
|
||||
yt-dlp subtitle extraction → raw VTT files
|
||||
↓
|
||||
VTT cleanup → plaintext transcripts
|
||||
↓
|
||||
truncate to ~500 words per video
|
||||
↓
|
||||
normalize → YouTubeItem objects
|
||||
↓
|
||||
score, dedupe, render (same pipeline as Reddit/X)
|
||||
```
|
||||
|
||||
### Implementation Phases
|
||||
|
||||
#### Phase 1: Search + Metadata (the fast part)
|
||||
|
||||
**New file: `scripts/lib/youtube_yt.py`**
|
||||
|
||||
Core search function:
|
||||
```python
|
||||
def search_youtube(topic: str, from_date: str, to_date: str, depth: str = "default") -> Dict[str, Any]:
|
||||
"""Search YouTube via yt-dlp. No API key needed.
|
||||
|
||||
Returns:
|
||||
Dict with 'items' list of video metadata dicts.
|
||||
"""
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
date_filter = from_date.replace("-", "") # YYYYMMDD format
|
||||
|
||||
# yt-dlp search with metadata extraction
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
f"ytsearch{count}:{topic}",
|
||||
"--dateafter", date_filter,
|
||||
"--flat-playlist",
|
||||
"--print", "%(view_count)s\t%(id)s\t%(title)s\t%(channel)s\t%(upload_date)s\t%(like_count)s\t%(comment_count)s",
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
|
||||
# Parse tab-separated output, sort by views, return top N
|
||||
...
|
||||
```
|
||||
|
||||
Depth config (matches existing pattern):
|
||||
```python
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 10, # search 10, transcript top 3
|
||||
"default": 20, # search 20, transcript top 5
|
||||
"deep": 40, # search 40, transcript top 8
|
||||
}
|
||||
|
||||
TRANSCRIPT_LIMITS = {
|
||||
"quick": 3,
|
||||
"default": 5,
|
||||
"deep": 8,
|
||||
}
|
||||
```
|
||||
|
||||
**Key detail**: `yt-dlp --flat-playlist` returns exit code 0 with empty stdout when `--dateafter` filters out everything. Check for empty output, not error codes.
|
||||
|
||||
#### Phase 2: Transcript Extraction (the slow part)
|
||||
|
||||
For top N videos (by view count), fetch transcripts:
|
||||
|
||||
```python
|
||||
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
||||
"""Fetch auto-generated transcript for a YouTube video.
|
||||
|
||||
Returns:
|
||||
Plaintext transcript string, or None if no captions available.
|
||||
"""
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"--write-auto-subs",
|
||||
"--sub-lang", "en",
|
||||
"--sub-format", "vtt",
|
||||
"--skip-download",
|
||||
"-o", f"{temp_dir}/%(id)s",
|
||||
f"https://www.youtube.com/watch?v={video_id}",
|
||||
]
|
||||
subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
|
||||
vtt_path = Path(temp_dir) / f"{video_id}.en.vtt"
|
||||
if not vtt_path.exists():
|
||||
return None
|
||||
|
||||
return _clean_vtt(vtt_path.read_text())
|
||||
```
|
||||
|
||||
VTT cleanup (~10 lines of Python):
|
||||
```python
|
||||
def _clean_vtt(vtt_text: str) -> str:
|
||||
"""Convert VTT subtitle format to clean plaintext."""
|
||||
text = re.sub(r'^WEBVTT.*?\n\n', '', vtt_text, flags=re.DOTALL)
|
||||
text = re.sub(r'\d{2}:\d{2}:\d{2}\.\d{3} --> \d{2}:\d{2}:\d{2}\.\d{3}.*\n', '', text)
|
||||
text = re.sub(r'<[^>]+>', '', text)
|
||||
lines = text.strip().split('\n')
|
||||
seen = set()
|
||||
unique = []
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped and stripped not in seen:
|
||||
seen.add(stripped)
|
||||
unique.append(stripped)
|
||||
return re.sub(r'\s+', ' ', ' '.join(unique)).strip()
|
||||
```
|
||||
|
||||
**Parallelization**: Run transcript fetches in parallel using ThreadPoolExecutor (same pattern as Phase 2 supplemental searches for Reddit/X):
|
||||
|
||||
```python
|
||||
def fetch_transcripts_parallel(video_ids: List[str], max_workers: int = 5) -> Dict[str, Optional[str]]:
|
||||
"""Fetch transcripts for multiple videos in parallel."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {
|
||||
executor.submit(fetch_transcript, vid, temp_dir): vid
|
||||
for vid in video_ids
|
||||
}
|
||||
results = {}
|
||||
for future in as_completed(futures):
|
||||
vid = futures[future]
|
||||
results[vid] = future.result()
|
||||
return results
|
||||
```
|
||||
|
||||
#### Phase 3: Integration into Pipeline
|
||||
|
||||
**Update `scripts/lib/schema.py`** — add YouTubeItem:
|
||||
```python
|
||||
@dataclass
|
||||
class YouTubeItem:
|
||||
id: str # video_id
|
||||
title: str
|
||||
url: str
|
||||
channel_name: str
|
||||
date: Optional[str]
|
||||
date_confidence: str # always "high" for YouTube
|
||||
engagement: Engagement # views, likes, comments
|
||||
transcript_snippet: str # first ~500 words of transcript
|
||||
relevance: float
|
||||
why_relevant: str
|
||||
subs: Optional[SubScores] = None
|
||||
score: int = 0
|
||||
```
|
||||
|
||||
Update `Report` to add:
|
||||
```python
|
||||
youtube: List[YouTubeItem] = field(default_factory=list)
|
||||
youtube_error: Optional[str] = None
|
||||
```
|
||||
|
||||
**Update `scripts/lib/score.py`** — YouTube-specific engagement weights:
|
||||
```python
|
||||
def compute_youtube_engagement_raw(views, likes, comments):
|
||||
"""YouTube engagement: views dominate, likes secondary, comments tertiary."""
|
||||
return (
|
||||
0.50 * math.log1p(views or 0) +
|
||||
0.35 * math.log1p(likes or 0) +
|
||||
0.15 * math.log1p(comments or 0)
|
||||
)
|
||||
```
|
||||
|
||||
**Update `scripts/last30days.py`** — add YouTube to ThreadPoolExecutor:
|
||||
```python
|
||||
with ThreadPoolExecutor(max_workers=3) as executor: # was 2
|
||||
if run_reddit:
|
||||
reddit_future = executor.submit(_search_reddit, ...)
|
||||
if run_x:
|
||||
x_future = executor.submit(_search_x, ...)
|
||||
if run_youtube:
|
||||
youtube_future = executor.submit(_search_youtube, ...)
|
||||
```
|
||||
|
||||
**Update `scripts/lib/render.py`** — YouTube section in compact output:
|
||||
```
|
||||
### YouTube Videos
|
||||
|
||||
**{id}** (score:{score}) {channel_name} ({date}) [{views} views, {likes} likes]
|
||||
{title}
|
||||
https://www.youtube.com/watch?v={id}
|
||||
{transcript_snippet[:200]}...
|
||||
*{why_relevant}*
|
||||
```
|
||||
|
||||
**Update `scripts/lib/env.py`** — YouTube availability detection:
|
||||
```python
|
||||
def is_ytdlp_available() -> bool:
|
||||
return shutil.which("yt-dlp") is not None
|
||||
```
|
||||
|
||||
No API key needed. YouTube search is available whenever yt-dlp is in PATH.
|
||||
|
||||
#### Phase 4: SKILL.md Updates
|
||||
|
||||
Stats box adds YouTube line:
|
||||
```
|
||||
├─ 🎥 YouTube: {N} videos │ {N} views │ {N} transcripts
|
||||
```
|
||||
|
||||
Citation priority updated:
|
||||
```
|
||||
1. @handles from X
|
||||
2. YouTube creators — "per [Channel Name] on YouTube"
|
||||
3. r/subreddits from Reddit
|
||||
4. Web sources
|
||||
```
|
||||
|
||||
Synthesis instructions updated to weight YouTube transcripts highly — a 20-minute video transcript with 500K views is a stronger signal than a tweet with 50 likes.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [x] `yt-dlp` search returns videos matching topic within date range
|
||||
- [x] Transcripts extracted for top N videos (auto-generated captions)
|
||||
- [x] Videos without captions gracefully skipped (no error)
|
||||
- [x] YouTube results appear in compact output with engagement metrics
|
||||
- [x] YouTube items scored and ranked alongside Reddit/X items
|
||||
- [x] YouTube auto-activates when yt-dlp is available (no --sources flag needed)
|
||||
- [x] SKILL.md stats box includes YouTube line
|
||||
- [x] Transcript snippets (first ~500 words) included in output for LLM synthesis
|
||||
- [ ] Total YouTube search + transcript extraction completes within 30 seconds
|
||||
- [x] Works when yt-dlp is not installed (graceful degradation, no crash)
|
||||
- [ ] Mock mode works for testing without network
|
||||
|
||||
## Dependencies & Risks
|
||||
|
||||
**Dependencies:**
|
||||
- `yt-dlp` (Homebrew) — already installed, widely available via brew/pip/standalone
|
||||
- No API keys needed
|
||||
- No new Python packages (just subprocess + regex)
|
||||
|
||||
**Risks:**
|
||||
| Risk | Likelihood | Mitigation |
|
||||
|------|-----------|------------|
|
||||
| yt-dlp search is slow (>10s) | Medium | Set 30s timeout, run in parallel with Reddit/X |
|
||||
| YouTube blocks yt-dlp | Low | yt-dlp is actively maintained with anti-bot updates. Degrade gracefully. |
|
||||
| Videos lack auto-captions | Medium (~5%) | Skip those videos, note in output. Transcript is enrichment, not required. |
|
||||
| Transcript extraction adds latency | High | Only fetch top 3-5, run in parallel, use tempdir |
|
||||
| yt-dlp not installed for some users | Medium | Auto-detect, skip YouTube with info message, don't error |
|
||||
| Linux `--dateafter` date format differs | Low | Use Python to format date, not shell `date -v` |
|
||||
|
||||
## Files to Create/Modify
|
||||
|
||||
### New Files
|
||||
- `scripts/lib/youtube_yt.py` — search, transcript extraction, parsing
|
||||
- `tests/test_youtube_yt.py` — unit tests
|
||||
- `fixtures/youtube_sample.json` — mock data for tests
|
||||
|
||||
### Modified Files
|
||||
- `scripts/lib/schema.py` — add YouTubeItem, update Report
|
||||
- `scripts/lib/normalize.py` — add normalize_youtube_items()
|
||||
- `scripts/lib/score.py` — add YouTube engagement scoring
|
||||
- `scripts/lib/dedupe.py` — add YouTube dedup (title + channel Jaccard)
|
||||
- `scripts/lib/render.py` — add YouTube section to compact + full report
|
||||
- `scripts/lib/env.py` — add yt-dlp availability check, update source detection
|
||||
- `scripts/last30days.py` — add _search_youtube(), update run_research(), update arg parser
|
||||
- `SKILL.md` — update stats box, citation rules, synthesis instructions
|
||||
- `README.md` — document YouTube source, yt-dlp requirement
|
||||
|
||||
## Alternative Approaches Considered
|
||||
|
||||
**1. YouTube Data API v3** — Rejected. Requires API key + Google Cloud project. Adds friction, counter to "zero config" philosophy. 10K quota/day limit. yt-dlp has no limits.
|
||||
|
||||
**2. steipete/summarize for transcripts** — Rejected for MVP. Adds 146MB dependency, requires brew tap, calls OpenAI API per video (adds cost). Raw transcripts via yt-dlp are better input for our synthesis LLM anyway. Could revisit as optional enhancement for captionless videos.
|
||||
|
||||
**3. youtube-transcript-api Python package** — Considered. Lightweight, Python-native transcript fetcher. But adds a pip dependency to a project that currently has zero Python deps. yt-dlp is already a brew dependency we can auto-detect.
|
||||
|
||||
**4. Skip transcripts, just use metadata** — Rejected. Titles + view counts alone don't give the synthesis LLM enough to work with. Transcripts are what make YouTube a *research* source vs just a link list.
|
||||
|
||||
## Cost Impact
|
||||
|
||||
**Zero additional API cost.** yt-dlp scrapes YouTube directly. No API keys, no token usage. The only cost is the existing OpenAI/xAI calls for Reddit/X search, which are unchanged.
|
||||
|
||||
**Time impact:** Adds ~10-20 seconds to research (search + parallel transcript extraction), running in parallel with Reddit/X so effective wall-clock increase is minimal.
|
||||
@@ -0,0 +1,303 @@
|
||||
---
|
||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
||||
type: feat
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
---
|
||||
|
||||
# feat: --competitors flag for auto-discovered comparison fan-out
|
||||
|
||||
## Overview
|
||||
|
||||
Add a `--competitors` flag to the last30days engine that auto-discovers 2-4 peer entities for the topic, runs the full retrieval pipeline on each in parallel, and renders a multi-entity comparison. Invoking `last30days Kanye West --competitors` should resolve to "Kanye vs Drake vs Kendrick Lamar" and emit a comparison report covering all three. Invoking `last30days OpenAI --competitors` should resolve to "OpenAI vs Anthropic vs xAI vs Gemini" and emit a four-way comparison.
|
||||
|
||||
Discovery mirrors the existing `resolve.auto_resolve()` pattern used for X handles and subreddits at pipeline start — web search (Brave / Exa / Serper) plus deterministic extraction. Not an internal LLM call.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
Users who want a comparison today must type "OpenAI vs Anthropic vs xAI" themselves. The `planner._comparison_entities()` path already handles explicit multi-entity topics and `render._render_comparison_scaffold()` already emits a 9-axis comparison table. What is missing is the discovery half — a user who types a single entity with `--competitors` should get the comparison for free.
|
||||
|
||||
This is also the natural next step after the Step 0.55 category-peer subreddit work (PR #305, merged 2026-04-22). That feature widens the subreddit set within a single topic; this feature widens the entity set into peer entities.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. New `--competitors` boolean flag that triggers competitor discovery and multi-entity fan-out.
|
||||
- R2. New `--competitors-list="A,B,C"` to explicitly skip discovery (mirrors `--plan`, `--subreddits`, `--x-handle` overrides).
|
||||
- R3. New `--competitors=N` short form to set competitor count inline (N in 1..6).
|
||||
- R4. Default count is 3 competitors (original + 3 = 4-way comparison).
|
||||
- R5. Competitor retrieval depth inherits the main run's depth (`--quick` / `--deep`); all entities run in parallel so wall clock stays close to a single run.
|
||||
- R6. Discovery mirrors `resolve.auto_resolve()`: web search for peers, deterministic text extraction. No internal LLM dependency.
|
||||
- R7. If no web search backend is configured and no `--competitors-list` was passed, engine emits a LAW 7-style stderr telling the host agent to pass `--competitors-list` and exits non-zero.
|
||||
- R8. Output rendering is a single comparison report covering all entities, reusing the existing 9-axis scaffold from `render._render_comparison_scaffold()` where applicable.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- Synthesis prompt changes beyond wiring N reports into the existing comparison scaffold are out of scope.
|
||||
- `--competitors` does not replace the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`; both paths coexist.
|
||||
- No caching layer for discovery results in v1.
|
||||
- No UI/SKILL.md rewrite of the entire comparison section; only the new flag is documented.
|
||||
- No new web search backend.
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Caching of competitor lookups: separate follow-up once hit rate justifies it.
|
||||
- Disambiguation UX for topics with multiple common entities ("Amazon" the company vs the river): separate brainstorm.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py:168-249` — `build_parser()` argparse definitions. Existing depth flags (`--quick`, `--deep`) and override flags (`--plan`, `--subreddits`, `--x-handle`, `--auto-resolve`) set the convention to mirror.
|
||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the reference pattern: web search fan-out via `ThreadPoolExecutor`, per-query extraction functions, graceful empty-dict return when no backend is available.
|
||||
- `scripts/lib/resolve.py:98-140` — `_extract_x_handle()` and sibling extractors show the deterministic text-mining style competitor extraction should mirror.
|
||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature is the fan-out target. One call per entity, each returning a `schema.Report`.
|
||||
- `scripts/lib/planner.py:430-564` — Existing comparison-intent handling and `_comparison_entities()` entity extraction. The new flag feeds the same mental model but populates entities from discovery instead of from the topic string.
|
||||
- `scripts/lib/render.py:333-392` — `_render_comparison_scaffold()` already emits a 9-axis markdown comparison table. The new multi-report renderer should reuse this helper by assembling a synthetic "A vs B vs C" topic header for it.
|
||||
- `scripts/lib/grounding.py` + `scripts/lib/providers.py` — Web search backend resolution (Brave / Exa / Serper). Reused as-is.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- No existing `docs/solutions/` entries for competitor discovery or multi-entity fan-out.
|
||||
- Recent plan `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md` established the precedent of deterministic peer expansion; this plan extends that idea from subreddits to entities.
|
||||
|
||||
### External References
|
||||
|
||||
- None gathered — local patterns are strong. `resolve.auto_resolve()` is a direct template.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Discovery mirrors auto_resolve, not plan_query.** Web search + regex extraction, not an LLM call. Matches the user's explicit direction ("use the python brain the same way it searches for X handles"). Cheaper, no provider credential requirement, deterministic.
|
||||
- **Orchestration lives in `last30days.py` main, not inside `pipeline.run()`.** The fan-out is a top-level concern — one pipeline run per entity, each independent. Keeps `pipeline.run()` single-entity and unchanged except for sharing a `ThreadPoolExecutor` factory.
|
||||
- **Sub-runs inherit main depth and run in parallel.** Wall clock ≈ single run; token cost scales linearly with N. User-controlled via the existing `--quick`/`--deep` flags.
|
||||
- **New module `scripts/lib/competitors.py` instead of adding to `resolve.py`.** Keeps resolve focused on single-entity entity-bundle discovery (handles/subreddits/github); competitors.py owns peer-entity discovery. Similar shape, different responsibility.
|
||||
- **Multi-report render is additive in `render.py`.** New `render_comparison_multi(reports: list[Report]) -> str` composes a synthetic "A vs B vs C" topic and delegates to the existing scaffold + synthesis path where possible. No rewrite of the single-entity render path.
|
||||
- **Default count = 3 competitors (4-way comparison).** Hard cap at 6.
|
||||
- **LAW 7-style stderr when no backend and no list.** Matches how `planner.plan_query()` already tells the hosting agent to pass `--plan`.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **Discovery mechanism:** Web search via `grounding.web_search()`, not an internal LLM. User confirmed the auto_resolve pattern is the target.
|
||||
- **Default competitor count:** 3 (original + 3 = 4-way).
|
||||
- **Sub-run depth:** Inherit main depth, parallel execution.
|
||||
- **Flag naming:** `--competitors` (standard argparse double-dash). `--competitors=N` for inline count. `--competitors-list="A,B,C"` to skip discovery.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact extraction heuristics for competitor names across Brave / Exa / Serper result shapes. The SERP text varies (listicles, comparison pages, "vs" pages); the initial implementation will start with listicle parsing plus a "X vs Y" pattern match, and harden against real results in the test phase.
|
||||
- Handling of topic ambiguity ("Amazon", "Apple"). Initial behavior: trust whatever web search returns for the topic verbatim; disambiguation is a separate concern.
|
||||
- Merge strategy when two entities return overlapping URLs (e.g., an "OpenAI vs Anthropic" article shows up in both runs). Likely dedupe at the clustering step, but defer the exact policy until we see how often it happens.
|
||||
- Whether to expose competitor discovery artifacts (the raw web search results) as a debug emit. Follow the existing `--debug` conventions.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: CLI flag parsing and validation**
|
||||
|
||||
**Goal:** Add `--competitors`, `--competitors=N`, and `--competitors-list` to the argparse surface, validate values, and thread them into the main orchestration.
|
||||
|
||||
**Requirements:** R1, R2, R3, R4
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py`
|
||||
- Test: `tests/test_cli_competitors.py`
|
||||
|
||||
**Approach:**
|
||||
- Add three mutually cooperative flags near line 205 in `build_parser()`:
|
||||
- `--competitors` with `nargs="?"` and `const=3` so bare `--competitors` defaults to 3, `--competitors=4` is honored, and `--competitors=0` is rejected
|
||||
- `--competitors-list` free-text CSV
|
||||
- Normalize in `main()`: if `--competitors-list` is present, skip discovery and use the list. If `--competitors` is set and no list, trigger discovery with count = the flag value. Clamp count to 1..6 with a stderr warning at boundary.
|
||||
- Thread the resulting entity list into the orchestrator added in Unit 3.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `--plan` argument at `scripts/last30days.py:187` — same skip-discovery-when-explicit shape.
|
||||
- `--subreddits` / `--x-handle` at `scripts/last30days.py:180,189` — same override semantics.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: bare `--competitors` parses to count=3, empty list.
|
||||
- Happy path: `--competitors=4` parses to count=4.
|
||||
- Happy path: `--competitors-list="A,B,C"` parses to count=3, list=["A","B","C"], and is preferred over any discovery signal.
|
||||
- Edge case: `--competitors=0` and `--competitors=-1` are rejected with a clear error.
|
||||
- Edge case: `--competitors=99` clamps to 6 with a stderr warning.
|
||||
- Edge case: `--competitors` combined with `--competitors-list` uses the list and logs that discovery was skipped.
|
||||
- Edge case: `--competitors-list` value with whitespace ("A, B , C") normalizes correctly.
|
||||
|
||||
**Verification:**
|
||||
- Running the binary with each flag variation produces the expected post-parse state without calling out to the network.
|
||||
|
||||
- [ ] **Unit 2: `scripts/lib/competitors.py` discovery module**
|
||||
|
||||
**Goal:** Discover peer entities for a topic using web search + deterministic extraction, mirroring `resolve.auto_resolve()`.
|
||||
|
||||
**Requirements:** R6, R7
|
||||
|
||||
**Dependencies:** None (pure module; wired by Unit 3)
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/lib/competitors.py`
|
||||
- Test: `tests/test_competitors.py`
|
||||
|
||||
**Approach:**
|
||||
- Public entry point `discover_competitors(topic: str, count: int, config: dict) -> list[str]`.
|
||||
- Early return `[]` when `_has_backend(config)` is false (reuse the helper from `resolve.py`; factor if needed).
|
||||
- Fan out 2-3 web searches in a `ThreadPoolExecutor`:
|
||||
- `"{topic} competitors"`
|
||||
- `"{topic} alternatives"`
|
||||
- `"{topic} vs"` (captures "X vs Y" articles)
|
||||
- Feed results into a deterministic `_extract_peer_entities(results, topic)` that:
|
||||
- Mines titles and snippets for capitalized noun phrases other than the topic itself
|
||||
- Scores by frequency across results
|
||||
- Filters stopwords and the topic's own tokens
|
||||
- Returns top `count` unique entities ordered by score
|
||||
- Emit a single-line stderr log mirroring the `resolve._log` format.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/lib/resolve.py:179-258` for the function shape, executor usage, and empty-result fallback.
|
||||
- `scripts/lib/resolve.py:98-140` for extractor style (small, deterministic, no external state).
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: canned SERP fixtures for "OpenAI" return ["Anthropic", "xAI", "Google"] or close peers in the top 3.
|
||||
- Happy path: canned SERP fixtures for "Kanye West" return rap peers (Drake, Kendrick) in the top 3.
|
||||
- Edge case: empty SERP results return `[]` without raising.
|
||||
- Edge case: extractor filters out the topic itself (case- and punctuation-insensitive).
|
||||
- Edge case: near-duplicate entities ("OpenAI" vs "Open AI") dedupe to one slot.
|
||||
- Error path: web search backend raises — the failure is logged and the function returns `[]`.
|
||||
- Edge case: count=1 returns a single-element list; count=6 returns up to six entities.
|
||||
|
||||
**Verification:**
|
||||
- Unit tests pass with fixtures committed under `tests/fixtures/competitors-*.json`.
|
||||
- Manual run against a live backend for one topic confirms sensible output (recorded as a notes file, not a test assertion).
|
||||
|
||||
- [ ] **Unit 3: Parallel fan-out orchestrator**
|
||||
|
||||
**Goal:** Run `pipeline.run()` once per entity (topic + discovered competitors) in parallel, collect `schema.Report` per entity, and hand them to the comparison renderer.
|
||||
|
||||
**Requirements:** R5, R7
|
||||
|
||||
**Dependencies:** Unit 1, Unit 2
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py`
|
||||
- Possibly create: `scripts/lib/fanout.py` if the orchestrator grows past ~60 lines
|
||||
- Test: `tests/test_competitor_fanout.py`
|
||||
|
||||
**Approach:**
|
||||
- After arg parsing and before the existing `pipeline.run()` call, branch on `args.competitors`:
|
||||
- If a list was provided or discovery returned entities, build `entities = [topic, *competitors]`.
|
||||
- Spawn one `pipeline.run()` per entity via `ThreadPoolExecutor(max_workers=len(entities))`, passing the same `config`, `depth`, and all sub-run-relevant args (mock, plan, etc.). Respect `--plan` — if a plan is passed it applies to the main topic only; competitors use the internal planner fallback for v1.
|
||||
- Collect `{entity: Report}` mapping. A per-entity failure logs a stderr warning and drops that entity from the comparison; the run continues as long as 2 entities succeed.
|
||||
- If fewer than 2 entities survive, exit with a clear error.
|
||||
- LAW 7-style stderr:
|
||||
- If `args.competitors` is set, no list was passed, no web search backend is configured, emit a LAW 7 stderr message pointing to the `--competitors-list` override and exit non-zero. Reuse the tone from `planner.plan_query()` fallback (`scripts/lib/planner.py:125-135`).
|
||||
|
||||
**Execution note:** Start with a failing integration test that exercises the full main → orchestrator → mocked pipeline.run path; the orchestrator is where bugs hide.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/lib/resolve.py:225-239` for ThreadPoolExecutor + as_completed + per-future error handling.
|
||||
- `scripts/lib/pipeline.py:310+` for how ThreadPoolExecutor is already used inside a single run (same idiom, outer layer).
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: main + 2 competitors, all three `pipeline.run()` calls succeed (mocked), orchestrator returns 3 Reports.
|
||||
- Happy path: discovery returns the competitor list; orchestrator fans out accordingly.
|
||||
- Edge case: one of three competitor pipelines raises — the run continues with the surviving 2 and emits a warning.
|
||||
- Edge case: all competitors fail but the main topic succeeds — orchestrator exits non-zero with a clear error rather than silently degrading to a single-entity render.
|
||||
- Edge case: `--competitors` set, no backend, no list — orchestrator emits the LAW 7 stderr and exits non-zero before any pipeline call.
|
||||
- Integration: wall-clock time for 3 mocked pipelines in parallel is close to the slowest single run, not the sum (timing assertion with generous margin).
|
||||
|
||||
**Verification:**
|
||||
- End-to-end test with mocked `pipeline.run()` and mocked competitors discovery produces 3 Reports and hands them to a stubbed renderer.
|
||||
|
||||
- [ ] **Unit 4: Multi-report comparison renderer**
|
||||
|
||||
**Goal:** Compose N `schema.Report`s into a single comparison-mode output, reusing the existing 9-axis scaffold.
|
||||
|
||||
**Requirements:** R8
|
||||
|
||||
**Dependencies:** Unit 3
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/render.py`
|
||||
- Test: `tests/test_render_comparison_multi.py`
|
||||
|
||||
**Approach:**
|
||||
- Add `render_comparison_multi(reports: list[schema.Report], *, emit: str) -> str`.
|
||||
- Build a synthetic comparison topic: `f"{entity_a} vs {entity_b} vs {entity_c}"`.
|
||||
- Reuse `_render_comparison_scaffold()` for the table skeleton. Each entity column is populated from its own Report's top clusters and citations.
|
||||
- For the narrative synthesis block, concatenate per-entity highlights, clearly labeled by entity, under a shared "Comparison" header.
|
||||
- Preserve existing emit modes (`compact`, `md`, `json`, `context`). In `json` emit, return a `{"entities": [...], "reports": [...]}` shape; single-Report consumers remain unaffected because the single-report render path is untouched.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/lib/render.py:333-392` (`_parse_comparison_entities`, `_render_comparison_scaffold`) — the scaffold is the contract.
|
||||
- `scripts/lib/render.py` single-report rendering — for per-entity narrative blocks.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: 3 Reports with distinct clusters render into a 3-column table and a "Comparison" section that mentions each entity at least once.
|
||||
- Happy path: 2 Reports render as a 2-column table without breaking the scaffold.
|
||||
- Edge case: a Report with an empty cluster list renders as "(no significant discussion this month)" in its column rather than crashing.
|
||||
- Edge case: Reports with overlapping URLs (same article cited by two entities) dedupe citations at the footer but keep both column entries.
|
||||
- Emit variants: `--emit=compact`, `--emit=md`, `--emit=json`, `--emit=context` each produce valid output with all entities represented.
|
||||
- Integration: end-to-end snapshot test using fixture Reports, checked against a stored expected output (with a clear update path when the scaffold intentionally evolves).
|
||||
|
||||
**Verification:**
|
||||
- Snapshot tests pass. Manual review of one real 3-way comparison confirms readability.
|
||||
|
||||
- [ ] **Unit 5: Docs, SKILL.md mention, and sync**
|
||||
|
||||
**Goal:** Document the new flag so the hosting agent and human users both know it exists, and run the sync script.
|
||||
|
||||
**Requirements:** R1-R8 (surfaces them to users)
|
||||
|
||||
**Dependencies:** Units 1-4
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md`
|
||||
- Modify: `README.md` (brief flag reference)
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
|
||||
**Approach:**
|
||||
- Add a compact "Competitor mode" subsection under the existing comparison docs in `SKILL.md`. Document the flag, the default count, the override flag, and the LAW 7 fallback stderr.
|
||||
- Keep `README.md` addition to a single example line.
|
||||
- CHANGELOG entry mirrors the voice of recent entries (imperative, outcome-first).
|
||||
- Sync via `scripts/sync.sh` per CLAUDE.md rules so `~/.claude/`, `~/.agents/`, `~/.codex/` pick up the new SKILL.md.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — documentation and sync only. Verification is by inspection and by running `sync.sh` and confirming target directories updated.
|
||||
|
||||
**Verification:**
|
||||
- `sync.sh` completes without errors.
|
||||
- `SKILL.md` rendered preview mentions `--competitors` in the comparison section.
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** `last30days.py main()` now orchestrates multiple `pipeline.run()` calls instead of one. No other callers of `pipeline.run()` are affected (it remains single-entity).
|
||||
- **Error propagation:** Per-entity failures degrade gracefully as long as ≥2 entities survive; fewer survivors exits non-zero. Discovery failure with `--competitors` and no list is fatal.
|
||||
- **State lifecycle risks:** Each sub-run uses its own `pipeline.run()` state; no shared mutable config. The `config` dict is read-only in `pipeline.run()` today — verify before committing to shared-reference passing, else deep-copy per sub-run.
|
||||
- **API surface parity:** `--competitors` coexists with the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`. Both produce comparable output formats; the only difference is where the entity list came from.
|
||||
- **Integration coverage:** The fan-out orchestrator crosses CLI → discovery → N pipelines → render; integration tests in Unit 3 and Unit 4 must exercise the full path end to end, not just unit-level.
|
||||
- **Unchanged invariants:** `pipeline.run()` signature and single-entity semantics are unchanged. The single-entity render path in `render.py` is unchanged. No changes to `planner.plan_query()`. No changes to existing flags.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Competitor discovery returns garbage entities for niche topics. | `--competitors-list` override lets the user (or hosting agent) correct it. Unit tests with edge-case fixtures. Log discovery output to stderr under `--debug`. |
|
||||
| Token cost scales linearly with N sub-runs. | Default count capped at 3, hard max 6, inherit `--quick` to let users throttle. Wall clock stays parallel. Emit a cost hint to stderr when N ≥ 4. |
|
||||
| Merge conflicts against the single-entity render path during refactoring. | Keep the multi-report renderer strictly additive; do not modify the single-Report code path. |
|
||||
| Config dict mutation inside sub-runs could leak state between entities. | Verify read-only usage before sharing references. If any sub-component mutates, deep-copy per sub-run before spawning threads. |
|
||||
| A SERP extractor that works on Brave fixtures breaks on Exa/Serper result shapes. | Test fixtures for all three backends. Extractor operates on a normalized shape from `grounding.web_search()` (already the case), not raw provider output. |
|
||||
| Hosting agent (Claude Code, Codex) unaware of the new flag when it could usefully pass `--competitors-list`. | SKILL.md updated in Unit 5 documents the flag in the same style as `--plan` and `--auto-resolve`. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first: per `CLAUDE.md`, experimental changes go to `mvanhorn/last30days-skill-private` on the `/last30days-beta` command. Land this on the private repo first, shake out on real topics for a day or two, then cherry-pick to public.
|
||||
- After land-merge: run `scripts/sync.sh` to deploy SKILL.md + scripts to `~/.claude/`, `~/.agents/`, `~/.codex/`.
|
||||
- Release notes entry in CHANGELOG.md follows the v3.0.9 voice — outcome-first, one paragraph.
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Related code: `scripts/lib/resolve.py:179` (`auto_resolve`), `scripts/lib/pipeline.py:162` (`pipeline.run`), `scripts/lib/planner.py:80` (`plan_query` LAW 7 fallback), `scripts/lib/render.py:333` (comparison scaffold)
|
||||
- Related PRs: #305 (Step 0.55 category-peer subreddit expansion — the precedent for deterministic peer expansion, merged 2026-04-22)
|
||||
- Related plan: `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md`
|
||||
@@ -0,0 +1,349 @@
|
||||
---
|
||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
||||
type: fix
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
origin: docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md
|
||||
---
|
||||
|
||||
# fix: per-entity resolution, default-2, and stale-path guard for --competitors
|
||||
|
||||
## Overview
|
||||
|
||||
Three test runs of v3.0.11 `--competitors` surfaced four real bugs plus one product tweak. This plan fixes all of them in a single follow-up:
|
||||
|
||||
1. Competitor sub-runs get no Step 0.55 resolution (no X handle, no subreddits, no GitHub repo). Drake / Kendrick / Travis ran with deterministic-fallback single-word queries while Kanye had the full targeting package. User called it "lazy" and was right.
|
||||
2. Two of three test windows (Linear, Coinbase) never invoked the new flag at all. They loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (a Claude-Code-managed git clone pinned to origin/main, which predates PR #308) instead of `plugins/cache/last30days-skill/last30days/3.0.11/`, so `--help` showed no `--competitors` flag and the model fell back to the manual comparison path.
|
||||
3. Each competitor sub-run emits a scary `[Planner] No --plan passed... deterministic fallback` stderr line because LAW 7 targets the hosting-model path, not internal fan-out sub-runs.
|
||||
4. Default competitor count is 3 (→ 4-way comparison). User wants default 2 (→ 3-way: original + 2 peers). Flag keeps `--competitors=N` to customize.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
The 3 test runs (Kanye, Linear, Coinbase) showed a pattern:
|
||||
|
||||
| Window | Loaded SKILL.md from | Invoked --competitors? | Per-entity resolution? | Outcome |
|
||||
|--------|----------------------|-----------------------|------------------------|---------|
|
||||
| Kanye | cache/3.0.11/ (correct) | Yes | Only for main topic (Kanye) | Drake/Kendrick/Travis thin; Reddit 403 fallbacks |
|
||||
| Linear | marketplaces/ (stale) | No — fell back to manual comparison | No | Thin run with noisy subreddits |
|
||||
| Coinbase | marketplaces/ (stale) | No — fell back to manual comparison | Main only; keyword-search poisoned pool | Top subs: r/survivor, r/Airpodsmax (noise) |
|
||||
|
||||
Root causes:
|
||||
- **Per-entity resolution gap:** `scripts/lib/fanout.py` calls `pipeline.run()` with topic + depth + web_backend + lookback_days only. It does not call `resolve.auto_resolve()` per entity, so sub-runs have no X handle, subreddit, or GitHub targeting. The original plan (`2026-04-22-002`) acknowledged this as a deliberate v1 simplification ("competitor sub-runs use planner defaults"). In practice this produces visibly asymmetric output and triggers downstream retrieval issues (403 fallbacks, keyword-search noise).
|
||||
- **Stale-path loading:** Claude Code's skill loader alphabetizes `find` results with `marketplaces/` before `cache/`, and the model reads the first plausible SKILL.md it sees. SKILL.md line 823's `SKILL_ROOT` resolver is the correct path but only fires in engine-invocation blocks, not in the skill-load step.
|
||||
- **LAW 7 in sub-runs:** LAW 7 exists because the *hosting reasoning model* is supposed to pass `--plan`. For competitor sub-runs, there is no hosting-model planning — it's an engine-internal fan-out. The warning is a false positive there.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. Default `--competitors` count is 2 peers (3-way comparison: original + 2).
|
||||
- R2. Each competitor sub-run performs Step 0.55 resolution (X handle, subreddits, GitHub user/repos, news context) before its pipeline runs — not just the main topic.
|
||||
- R3. Sub-runs do not emit the LAW 7 `No --plan passed` warning; they are internal fan-out, not hosting-model calls.
|
||||
- R4. The rendered comparison output includes a visible "Resolved entities" block showing per-entity handles/subs/github for debug transparency (answers "did it resolve everyone?" without the user having to read stderr).
|
||||
- R5. SKILL.md has a canonical-path self-check at the top: if the reader loaded it from anywhere other than `plugins/cache/last30days-skill/last30days/{VERSION}/`, re-read from the versioned path before proceeding.
|
||||
- R6. Version bumps to 3.0.12; CHANGELOG entry; `scripts/sync.sh` deploys.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No new discovery strategy. The web-search + regex extraction in `scripts/lib/competitors.py` stays as-is.
|
||||
- No new CLI flags beyond the behavior changes above. Specifically: no per-entity override flags like `--competitor-handles`. The hosting-model escape hatch remains `--competitors-list`.
|
||||
- No changes to the explicit `A vs B` comparison path (topic-string parsing in `planner._comparison_entities`).
|
||||
- No marketplace-clone auto-restore fix — that's Claude Code harness behavior. This plan only guards against the symptom on the skill side.
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Caching of per-entity resolution results: separate follow-up once hit rate justifies it.
|
||||
- Fan-out rate-limiting tuning (currently `max_workers=len(entities)+1`, capped at 6): defer until we see real-world quota exhaustion.
|
||||
- Pre-flight cost hint when N ≥ 4 (noted in `2026-04-22-002` risks): defer.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py:205-219` — `--competitors` / `--competitors-list` argparse definition (const=3 today; changing to 2).
|
||||
- `scripts/last30days.py:220-290` — `resolve_competitors_args()` validator; update `COMPETITORS_DEFAULT`.
|
||||
- `scripts/last30days.py:438-520` — main() fan-out orchestration; currently passes only topic/depth to each `_competitor_runner`.
|
||||
- `scripts/lib/fanout.py:40-95` — `run_competitor_fanout()` signature. The `competitor_runner` callable is where per-entity resolution needs to happen.
|
||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the exact per-entity resolver to reuse. Already does X handle + subreddits + GitHub user/repos + news context in parallel via ThreadPoolExecutor.
|
||||
- `scripts/lib/planner.py:80-135` — `plan_query()` emits the LAW 7 stderr. A `quiet: bool` keyword or `internal_subrun: bool` flag will suppress it.
|
||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature. Needs a new keyword to propagate quiet-mode down to the planner.
|
||||
- `scripts/lib/render.py:render_comparison_multi` — where the "Resolved entities" block is inserted.
|
||||
- `SKILL.md` line 823 — canonical `SKILL_ROOT` resolver already exists but fires in engine bash, not at skill-load time.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md` acknowledged the per-entity-resolution gap as a v1 tradeoff. This plan closes that gap.
|
||||
- Kanye run stderr: `[Planner] No --plan passed... deterministic fallback` × 3 (once per competitor sub-run). That's the LAW 7 noise R3 targets.
|
||||
- Linear / Coinbase runs loaded `plugins/marketplaces/last30days-skill/CLAUDE.md` as the first hit. That's the stale-path issue R5 targets.
|
||||
|
||||
### External References
|
||||
|
||||
- None. All patterns are in-repo.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Per-entity resolve happens inside fanout, not in SKILL.md.** The user-facing promise of `--competitors` is "one flag, engine does the work." Pushing resolution onto the hosting model creates another path-of-least-resistance trap (model skips it, output looks lazy). Auto-resolve inside each sub-run when a web backend is available makes the feature self-contained.
|
||||
- **Stale-path guard is a SKILL.md self-check, not a code change.** We cannot stop Claude Code from auto-restoring the marketplace clone. But we can put a 3-line banner at the top of SKILL.md that forces any path-mismatched read to re-read from the versioned cache. Both the marketplace copy (once main catches up) and the cache copy carry the guard.
|
||||
- **LAW 7 suppression is opt-in via `internal_subrun=True` keyword.** Do not remove the warning from the default path — it's load-bearing for the hosting-model contract. Add an explicit bypass for engine-internal fan-out only.
|
||||
- **Default 2, hard max 6 unchanged.** "Original + 2" matches the Kanye/Drake/Kendrick mental model from the feature description. Still allow `--competitors=N` from 1 to 6.
|
||||
- **Resolved block is inside the EVIDENCE envelope, not above it.** Keeps the rendered output structure stable for the synthesis contract (LAW 1–8). The block is context, not output.
|
||||
- **Skip auto-resolve when `--mock` or no web backend.** Mirrors the existing `resolve.auto_resolve()` fast-fail and keeps the mock test path deterministic.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **Where does per-entity resolve live?** Inside `fanout.run_competitor_fanout`, not in `main()`. Each sub-run calls `auto_resolve()` just before `pipeline.run()`.
|
||||
- **Should the hosting model still be able to override?** Yes — `--competitors-list` remains the escape hatch. When an explicit list is passed, the engine still does auto-resolve per entity; the user's list just skips discovery.
|
||||
- **Should sub-runs run auto-resolve in parallel with each other?** Yes. The existing `ThreadPoolExecutor` in fanout already parallelizes sub-runs; auto-resolve happens inside each sub-run's thread, so resolve calls for different entities run concurrently.
|
||||
- **Default count:** 2 peers (3-way). Confirmed.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Whether to expose a `--no-auto-resolve-competitors` flag for power users who want the fast, shallow behavior. Probably not needed v2; ship auto-resolve always-on and revisit if someone complains about cost.
|
||||
- Whether to surface the per-entity resolution context back into the main topic's planner (cross-entity context sharing). Stays deferred.
|
||||
- Whether the Resolved block should be collapsible or always inline. Start inline; revisit based on output length feedback.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: Default `--competitors` to 2 peers**
|
||||
|
||||
**Goal:** Change the bare `--competitors` default from 3 to 2 per user feedback. `--competitors=N` still overrides; range 1..6 unchanged.
|
||||
|
||||
**Requirements:** R1
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (`COMPETITORS_DEFAULT`, `--competitors` const, stderr messages if any reference 3)
|
||||
- Modify: `SKILL.md` Competitor mode section ("discovered 2-6" wording, bare-flag default line)
|
||||
- Modify: `README.md` auto-discovered example line (if it references count)
|
||||
- Test: `tests/test_cli_competitors.py`
|
||||
|
||||
**Approach:**
|
||||
- Change `COMPETITORS_DEFAULT = 3` → `2` in `scripts/last30days.py`.
|
||||
- Change argparse `--competitors` `const=3` → `const=2`.
|
||||
- Update any SKILL.md / README copy referencing "3 peers" to "2 peers" (default) or "2-6 peers" (range).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing default constants in `scripts/last30days.py` argparse block.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: bare `--competitors` yields count=2, enabled=True, empty explicit_list.
|
||||
- Edge case: `--competitors=3` still works (explicit override).
|
||||
- Edge case: existing `test_bare_flag_defaults_to_three` test is updated to `test_bare_flag_defaults_to_two` and asserts count=2.
|
||||
- Edge case: `--competitors=5` with a `--competitors-list` of length 2 still logs the mismatch warning and uses the list.
|
||||
|
||||
**Verification:**
|
||||
- `pytest tests/test_cli_competitors.py -v` passes with the updated default.
|
||||
|
||||
- [ ] **Unit 2: Per-entity Step 0.55 resolution inside fanout**
|
||||
|
||||
**Goal:** Each competitor sub-run auto-resolves its own X handle, subreddits, GitHub user/repos, and news context via `resolve.auto_resolve()` before its `pipeline.run()` call — just like the main topic.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** None (but Unit 3 should land together so sub-runs don't emit LAW 7 stderr while the resolution context is being passed)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/fanout.py`
|
||||
- Modify: `scripts/last30days.py` (`_competitor_runner` closure builds the resolved args)
|
||||
- Test: `tests/test_competitor_fanout.py`
|
||||
- Test: `tests/test_competitors_resolve_integration.py` (new; covers the auto-resolve path)
|
||||
|
||||
**Approach:**
|
||||
- `_competitor_runner(entity)` in main() does:
|
||||
1. Call `resolve.auto_resolve(entity, config)` when `not args.mock` and a web backend is configured (reuse `_has_backend`).
|
||||
2. Extract resolved x_handle, subreddits, github_user, github_repos, context.
|
||||
3. Pass them to `pipeline.run()` for that sub-run.
|
||||
4. Inject resolved context into a per-entity config copy (so `_auto_resolve_context` does not leak across sub-runs — deep-copy the config or use a local dict).
|
||||
5. Store the resolved block on the Report's `artifacts` so the renderer can surface it (Unit 4).
|
||||
- When `args.mock` is True or no backend is available, skip auto-resolve (fall through to planner defaults, matching the existing `auto_resolve()` early-return contract).
|
||||
- Update `fanout.run_competitor_fanout` docstring to note that auto-resolve happens inside the caller-provided runner.
|
||||
|
||||
**Execution note:** Start with a failing integration test that exercises two-entity fanout + auto-resolve via a mocked `resolve.auto_resolve` and asserts that `pipeline.run` receives the resolved x_handle/subreddits for each entity.
|
||||
|
||||
**Patterns to follow:**
|
||||
- `scripts/last30days.py` main topic branch (`if args.auto_resolve and not external_plan`) already calls `resolve.auto_resolve` and propagates results — mirror the shape for competitors.
|
||||
- Config isolation: `scripts/lib/pipeline.py:162-220` reads config as-is; use `dict(config)` to avoid cross-sub-run mutation of `_auto_resolve_context`.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: 3 entities, mocked `auto_resolve` returns distinct handles per entity; `pipeline.run` receives `x_handle=@drake` for Drake, `x_handle=@kendricklamar` for Kendrick, etc.
|
||||
- Happy path: the main topic still uses the user-supplied `--x-handle` / `--subreddits` overrides (not overwritten by auto-resolve for the main). Competitors use their own auto-resolved values.
|
||||
- Edge case: `--mock` skips auto-resolve entirely for all sub-runs (no `resolve.auto_resolve` calls).
|
||||
- Edge case: `resolve.auto_resolve` returns empty dicts for one entity (low-signal topic) — the sub-run still executes with planner defaults; doesn't crash.
|
||||
- Edge case: no web backend configured — auto-resolve returns empty for every entity, sub-runs fall through to planner defaults, no stack trace.
|
||||
- Error path: `resolve.auto_resolve` raises — the sub-run logs a warning and continues with planner defaults (does not fail the whole comparison).
|
||||
- Integration: config `_auto_resolve_context` from entity A does not leak into entity B's `pipeline.run`. Assert each sub-run gets its own context string.
|
||||
|
||||
**Verification:**
|
||||
- New integration test passes.
|
||||
- End-to-end smoke (mock mode + explicit list): each sub-run's stderr shows `[AutoResolve]` lines per entity with distinct values.
|
||||
|
||||
- [ ] **Unit 3: Suppress LAW 7 warning for engine-internal sub-runs**
|
||||
|
||||
**Goal:** The `[Planner] No --plan passed... deterministic fallback` warning does not fire during competitor sub-runs. LAW 7 is load-bearing for hosting-model contracts and must stay on the default path; this is an opt-in bypass for internal fan-out only.
|
||||
|
||||
**Requirements:** R3
|
||||
|
||||
**Dependencies:** Unit 2 (so the sub-run call site is already being modified)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/planner.py` (`plan_query` signature + conditional stderr)
|
||||
- Modify: `scripts/lib/pipeline.py` (`run` signature + propagation)
|
||||
- Modify: `scripts/last30days.py` or `scripts/lib/fanout.py` (pass `internal_subrun=True` for competitor runners)
|
||||
- Test: `tests/test_planner_v3.py` (or new `tests/test_planner_quiet_mode.py`)
|
||||
- Test: `tests/test_competitor_fanout.py` (assert sub-runs don't emit LAW 7 stderr)
|
||||
|
||||
**Approach:**
|
||||
- Add a keyword `internal_subrun: bool = False` to `planner.plan_query`. When True, skip the two `print(..., file=sys.stderr)` blocks that emit the LAW 7 banner and the `[Planner] No --plan passed` capability message.
|
||||
- Add the same keyword to `pipeline.run()`; pass through to `plan_query`.
|
||||
- In main()/fanout, set `internal_subrun=True` for every competitor sub-run's pipeline.run call. The main topic's pipeline.run keeps the default (LAW 7 stays on for the hosting-model path).
|
||||
- Also suppress the LAW 7-triggered degraded-run warning block in the render layer for sub-reports when the envelope is going to be merged into a comparison output (or accept that the block is per-entity and surfaces once per entity).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing keyword-only parameters on `pipeline.run` (`mock`, `x_handle`, etc.).
|
||||
- `planner.plan_query` signature is already keyword-only.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `plan_query(..., internal_subrun=True, provider=None, model=None)` returns the deterministic fallback plan WITHOUT writing the LAW 7 stderr block.
|
||||
- Happy path: `plan_query(...)` with default `internal_subrun=False` still writes the LAW 7 warning (unchanged behavior).
|
||||
- Integration: end-to-end competitor fanout; assert captured stderr contains zero occurrences of `No --plan passed` and zero of `YOU ARE the planner`.
|
||||
- Integration: main topic is not part of competitor mode; if the user invokes bare `/last30days OpenAI` without `--plan`, LAW 7 stderr fires exactly once (regression test).
|
||||
|
||||
**Verification:**
|
||||
- Running the Kanye-style smoke test shows zero `[Planner] No --plan passed` lines for Drake / Kendrick / Travis sub-runs.
|
||||
|
||||
- [ ] **Unit 4: "Resolved entities" block in comparison output**
|
||||
|
||||
**Goal:** The rendered comparison output includes a visible block listing per-entity handles, subreddits, GitHub user, and resolved context. Answers "did it resolve everyone?" at a glance without reading stderr.
|
||||
|
||||
**Requirements:** R4
|
||||
|
||||
**Dependencies:** Unit 2 (needs resolved data on report artifacts)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/render.py` (`render_comparison_multi` and `render_comparison_multi_context`)
|
||||
- Test: `tests/test_render_comparison_multi.py`
|
||||
|
||||
**Approach:**
|
||||
- When each entity's `Report.artifacts` contains a `resolved` dict (populated by Unit 2), `render_comparison_multi` emits a `## Resolved Entities` block early in the EVIDENCE envelope:
|
||||
```
|
||||
## Resolved Entities
|
||||
- **Kanye West**: X @kanyewest | Subs r/Kanye, r/hiphopheads | GitHub: — | Context: BULLY released, UK ban…
|
||||
- **Drake**: X @Drake | Subs r/DrakeTheType, r/hiphopheads | GitHub: — | Context: ICEMAN rollout…
|
||||
- **Kendrick Lamar**: X @kendricklamar | Subs r/KendrickLamar | GitHub: — | Context: Grammy wins, dormant…
|
||||
```
|
||||
- Missing fields render as `—` not empty.
|
||||
- When no entity has a `resolved` payload (mock mode, no web backend), omit the block entirely rather than emit an empty section.
|
||||
- Context strings are truncated at 120 chars to keep the block scannable.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing `render_comparison_multi` envelope structure (lines ~395-480 in render.py).
|
||||
- Existing per-entity evidence block format (`## {label}`) for consistency.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: 3 entities each with a `resolved` artifact → block lists all 3 with their fields.
|
||||
- Happy path: 2 entities, one with full resolution, one with partial (x_handle only) → missing fields render as `—`.
|
||||
- Edge case: no entity has a resolved artifact → block is omitted entirely.
|
||||
- Edge case: context string > 120 chars → truncated with ellipsis.
|
||||
- Integration: rendered output passes through the same EVIDENCE envelope comments and synthesis contract (LAW 1–8 unchanged).
|
||||
|
||||
**Verification:**
|
||||
- Snapshot tests confirm the block appears in the right spot with the right formatting.
|
||||
- End-to-end smoke shows a realistic 3-entity Resolved block in the rendered output.
|
||||
|
||||
- [ ] **Unit 5: SKILL.md canonical-path self-check**
|
||||
|
||||
**Goal:** A top-of-file SKILL.md directive forces any reader (Claude Code, Codex, Hermes, Gemini) to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` before proceeding. If loaded from `marketplaces/` or any other path, re-read from the pinned versioned cache.
|
||||
|
||||
**Requirements:** R5
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md` (prepend a STEP 0 block before the existing STEP 0 / LAW list)
|
||||
|
||||
**Approach:**
|
||||
- Add a numbered first step at the top (before or bundled with existing "STEP 0: ToolSearch preload"):
|
||||
```
|
||||
## STEP 0: Canonical Path Self-Check (must run first)
|
||||
|
||||
Before reading anything else below, verify you loaded this SKILL.md from
|
||||
the versioned cache, not the marketplace clone:
|
||||
|
||||
CANONICAL=$HOME/.claude/plugins/cache/last30days-skill/last30days/
|
||||
CANONICAL_LATEST=$(ls -d "$CANONICAL"*/ 2>/dev/null | sort -V | tail -1)
|
||||
|
||||
If the SKILL.md you just read is not under $CANONICAL_LATEST, STOP. Re-read
|
||||
$CANONICAL_LATEST/SKILL.md and restart from here. Marketplace clones
|
||||
(`plugins/marketplaces/last30days-skill/`) are pinned to origin/main and
|
||||
can be stale; the versioned cache is the ground truth.
|
||||
```
|
||||
- Reinforce in the existing LAW 7 block that `--help` output must be read from the same pinned `SKILL_ROOT` to avoid flag-list skew.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing STEP 0 ToolSearch preload (top of SKILL.md) for tone / imperative voice.
|
||||
- Existing `SKILL_ROOT` resolver snippet (line ~823).
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — SKILL.md is documentation; no unit test, verified by follow-up user invocation.
|
||||
|
||||
**Verification:**
|
||||
- In a fresh Claude Code window, `/last30days Test --competitors` loads SKILL.md, the model executes the STEP 0 self-check, and (if it had loaded from marketplaces/) switches to the cache path before running `--help` or the engine. Observable via the model's announced reasoning / task list.
|
||||
|
||||
- [ ] **Unit 6: Version bump, CHANGELOG, sync**
|
||||
|
||||
**Goal:** Ship 3.0.12 and deploy to all local targets.
|
||||
|
||||
**Requirements:** R6
|
||||
|
||||
**Dependencies:** Units 1-5
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json` (version 3.0.11 → 3.0.12)
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG entry under `## [3.0.12]` dated 2026-04-22 covering the four fixes (Fixed: per-entity resolution; Fixed: LAW 7 sub-run noise; Changed: default count 3→2; Added: Resolved entities block; Added: canonical-path self-check in SKILL.md).
|
||||
- `sync.sh` deploys to `~/.claude/plugins/cache/last30days-skill-private/...`, `~/.agents/`, `~/.codex/`, Hermes.
|
||||
- Manual hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.12/` so the public `/last30days` slash command picks up the new version before PR merge (matches the 3.0.11 testing pattern).
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging only. Verification is by inspection.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` returns `3.0.12`.
|
||||
- `sync.sh` exits 0 with "Import check: OK" for each target.
|
||||
- Hot-copied 3.0.12 directory contains the new files and `/last30days` picks up the new version (highest-version resolver).
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** Fanout sub-runs now call `resolve.auto_resolve` per entity. Each sub-run is independent; no shared mutable state with other sub-runs or with the main topic.
|
||||
- **Error propagation:** `auto_resolve` failures inside a sub-run log a warning and degrade to planner defaults; do not propagate up to abort the comparison. Same contract as today for the main topic.
|
||||
- **State lifecycle risks:** Config dict is mutated by `auto_resolve` (via `config["_auto_resolve_context"]`). Must deep-copy per sub-run or scope context to a local mapping — otherwise two sub-runs' context strings race.
|
||||
- **API surface parity:** `pipeline.run` gains a keyword (`internal_subrun`); callers that don't pass it get the existing behavior. `planner.plan_query` gains the same. Backward compatible.
|
||||
- **Integration coverage:** New integration test for the fanout + auto-resolve + render chain. Existing snapshot tests update to include the Resolved block.
|
||||
- **Unchanged invariants:** Single-entity `/last30days` invocations (no `--competitors`) behave identically. Explicit `A vs B` comparison topics behave identically. LAW 7 still fires on the default hosting-model path. `render_compact` path is untouched.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Auto-resolving per competitor triples the WebSearch call volume (4 queries × 3 competitors = 12 extra web searches). | Fast-fail when no backend; user can pass `--competitors-list` to skip discovery but still get auto-resolve. Cost note in CHANGELOG. |
|
||||
| Config mutation across sub-runs via `_auto_resolve_context`. | Unit 2 deep-copies config per sub-run before each `auto_resolve` + `pipeline.run` call. Integration test asserts no cross-entity leak. |
|
||||
| LAW 7 suppression leaks onto the hosting-model path via a wrong default. | Default `internal_subrun=False`. Only fanout's competitor sub-runs set True. Unit test asserts bare-topic invocation still emits LAW 7. |
|
||||
| SKILL.md STEP 0 banner gets ignored by the model (same failure mode as line 823 today). | Put it in the guaranteed-read top band (before LAW 1, above all other content), imperative voice, concrete `STOP` verb. Still not bulletproof but strictly better than current. |
|
||||
| Default count change breaks assumptions in downstream tools or existing user muscle memory. | Changelog calls it out as Changed; `--competitors=3` still works for users who want the old default. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first: merge behind `/last30days-beta` via the private repo before cherry-picking to public. Follows the same process as 3.0.11.
|
||||
- Version 3.0.12 is a fix release; no marketing post required.
|
||||
- After merge, add a line to the PR description pointing at this plan.
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Origin plan: `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
||||
- Related PR: #308 (v3.0.11 shipping --competitors)
|
||||
- Test windows that surfaced the bugs: Kanye, Linear, Coinbase (2026-04-22 session)
|
||||
- Related code: `scripts/lib/fanout.py`, `scripts/lib/resolve.py` (`auto_resolve`), `scripts/lib/planner.py` (`plan_query`), `scripts/lib/render.py` (`render_comparison_multi`)
|
||||
+394
@@ -0,0 +1,394 @@
|
||||
---
|
||||
title: "fix: --competitors runs a full last30days per entity with hosting-model pre-resolve"
|
||||
type: fix
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
origin: docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md
|
||||
---
|
||||
|
||||
# fix: --competitors runs a full last30days per entity with hosting-model pre-resolve
|
||||
|
||||
## Overview
|
||||
|
||||
User intent confirmed 2026-04-22: `--competitors` should run a full single-entity `last30days` pipeline for the main topic AND for each discovered peer — three independent full-depth passes, each with its own Step 0.55 resolution, own X handle primary weight, own subreddit targeting, own GitHub repo scoping. Then merge them into the comparison output.
|
||||
|
||||
3.0.12 already built the N-parallel-pipelines orchestration (`scripts/lib/fanout.py`). What it got wrong: it tried to do per-entity Step 0.55 engine-side via `resolve.auto_resolve()`, which requires a web search backend key (BRAVE/EXA/SERPER/PARALLEL/OPENROUTER). Matt runs from Claude Code, which has its own WebSearch tool. The engine has none of those keys, so per-entity auto_resolve silently no-ops and all peer sub-runs fall through to deterministic single-word planner queries.
|
||||
|
||||
Four 2026-04-22 test runs (Warriors, Seattle, Arizona Wildcats, Kanye West) confirmed this via engine receipts:
|
||||
|
||||
- Compact Resolved Entities block shows peers as `X - | Subs - | GitHub - | Context: -`.
|
||||
- Sub-run planner lines show `source=deterministic, subqueries=1` — the "I gave up and keyword-searched" shape.
|
||||
- Engine footer keeps nudging `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY`, which is wrong advice for a Claude Code user who already has WebSearch.
|
||||
- Kanye run leaked main topic's `--subreddits` into Drake's and Kendrick's sub-runs (regression bug).
|
||||
|
||||
The fix is to flip the resolution responsibility: the hosting model (Claude Code, Codex, Hermes, Gemini) does Step 0.55 via its own WebSearch tool for every entity, then passes the resolved targeting to the engine via a new `--competitors-plan` JSON flag. Engine fan-out remains — each peer still runs a full `pipeline.run()`. The difference is the peers now arrive with full targeting, equivalent to the main topic, so retrieval is apples-to-apples.
|
||||
|
||||
Why not just reuse vs-mode? vs-mode is a SINGLE `pipeline.run()` with a comparison-optimized plan. It pre-resolves Step 0.55 per entity but merges everything into one retrieval pool with lower-weight `--x-related` for peers, merged subreddits, and cross-entity keyword noise. That is not "three full passes." The user explicitly wants three full passes.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
3.0.12's architecture was correct; its data dependency was wrong.
|
||||
|
||||
| Capability | 3.0.12 path | Target path (this plan) |
|
||||
|---|---|---|
|
||||
| Fan out to N parallel pipelines | Yes (`fanout.run_competitor_fanout`) | Same — keep |
|
||||
| Per-entity Step 0.55 resolution | Engine-internal `resolve.auto_resolve()` — needs BRAVE/EXA/SERPER/PARALLEL key | Hosting model does it via its own WebSearch, passes to engine |
|
||||
| Per-entity targeting threaded into `pipeline.run()` | Main topic only via outer flags; peers via auto_resolve (failing) or nothing | Main topic via outer flags; peers via `--competitors-plan` JSON |
|
||||
| Footer nudge | Unconditional BRAVE/SERPER | Suppressed when `--plan` or `--competitors-plan` present |
|
||||
| Resolved Entities block in raw save file | Stdout only | Also in `--save-dir` raw file |
|
||||
| Override-leak from main into peers | Present (Kanye receipt) | Fixed via explicit per-entity kwargs scrub |
|
||||
| Polymarket noise on ambiguous topics | Present (Warriors, Arizona receipts) | `--polymarket-keywords` + auto-skip for single-token-ambiguous |
|
||||
|
||||
The key architectural change is who owns per-entity resolution. The engine stops trying to do it itself; the hosting model does it upstream (it already has WebSearch) and passes results in.
|
||||
|
||||
This is the same pattern `--plan` already uses for the main topic: hosting model generates the plan via its own reasoning, passes it in, engine accepts. We apply the pattern to peers.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. New `--competitors-plan` JSON flag accepting per-entity targeting: `x_handle`, `x_related`, `subreddits`, `github_user`, `github_repos`, `context`. Implies `--competitors`. Per-entity values thread into that entity's `pipeline.run()`. Bypasses engine-internal `auto_resolve` for covered entities.
|
||||
- R2. SKILL.md "Competitor mode" rewritten to make the hosting-model path canonical: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers) via WebSearch, (c) assemble `--competitors-plan` JSON, (d) invoke engine. Engine-internal auto_resolve remains as headless fallback.
|
||||
- R3. The LAW 7-style stderr emitted when `--competitors` has no list, no plan, no backend is reframed: leads with "hosting reasoning model, use your WebSearch to run Step 0.55 per entity and pass `--competitors-plan`." Does not lead with BRAVE_API_KEY.
|
||||
- R4. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` OR `--competitors-plan` was passed. Signal: hosting model is driving and already has WebSearch.
|
||||
- R5. Override-leak fix: competitor sub-runs do not inherit main topic's `--subreddits`, `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`. Sub-runs use only their own per-entity targeting (from `--competitors-plan` if provided, else engine-internal auto_resolve if backend, else planner defaults).
|
||||
- R6. The `## Resolved Entities` block is also appended to the saved raw file when `--save-dir` is in use. Each entity's effective targeting (whatever was actually passed to its `pipeline.run()`) is visible on audit.
|
||||
- R6b. When `--save-dir` is in use with a comparison run, each entity's sub-run ALSO saves its own standalone raw file — same format as a single-entity run. `/last30days Kanye West --competitors` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md` (one per entity) plus the merged comparison file. Matches the historical vs-mode behavior when it ran as N passes.
|
||||
- R7. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
||||
- R8. Default `--competitors` count remains 2 (3-way: main + 2 peers). Unchanged from 3.0.12.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No changes to `scripts/lib/fanout.py` architecture. N parallel pipelines stays. Only the data each sub-run receives changes.
|
||||
- No changes to the vs-mode (topic contains "vs" / "versus") behavior. That path is independent.
|
||||
- No new emit modes. Comparison output format unchanged.
|
||||
- No deprecation of `--competitors-list`. Stays as the minimum escape hatch for hosting models that skip per-entity Step 0.55 (names-only).
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Cache layer for hosting-model competitor resolution: separate plan once cost evidence exists.
|
||||
- Cross-source disambiguation beyond Polymarket: separate plan.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py` — `--competitors` / `--competitors-list` argparse block, `resolve_competitors_args` validator, `_main_runner` closure, `_competitor_runner` closure, the `[Competitors] --competitors requires...` stderr block. Primary file for this plan.
|
||||
- `scripts/lib/fanout.py` — `run_competitor_fanout` orchestrator. Signature unchanged; `_competitor_runner` closure now builds kwargs from `--competitors-plan`.
|
||||
- `scripts/lib/pipeline.py` — `pipeline.run()` signature; no changes required (all per-entity flags already exist as kwargs).
|
||||
- `scripts/lib/planner.py` — existing `--plan` parsing and validation, pattern to mirror for `--competitors-plan`.
|
||||
- `scripts/lib/render.py` `_render_resolved_entities_block` (added in 3.0.12) — already reads `report.artifacts["resolved"]`; no change needed.
|
||||
- `scripts/last30days.py` `save_output` / `render.render_full` — the save path. Needs to include the Resolved Entities block for comparison runs.
|
||||
- `scripts/lib/quality_nudge.py` — where the BRAVE/SERPER footer nudge is emitted. Needs a context-aware suppression check.
|
||||
- `scripts/lib/polymarket.py` — source adapter. Entry point for `--polymarket-keywords` filter and single-token-ambiguous auto-skip.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- 3.0.11 plan (`2026-04-22-002`): built the initial fanout, deferred per-entity resolve as "v1 simplification."
|
||||
- 3.0.12 plan (`2026-04-22-003`): tried to close the gap via engine-internal `auto_resolve`. Works only with backend keys. Fails silently without.
|
||||
- 2026-04-22 test session receipts: confirmed all four fixes in this plan are real, reproducible bugs.
|
||||
- User's architectural steer 2026-04-22: "runs a full last30days on all 3 topics" — this plan encodes that explicitly as N full `pipeline.run()` calls with pre-resolved targeting per entity.
|
||||
|
||||
### External References
|
||||
|
||||
- None. All patterns in-repo.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **`--competitors-plan` is a single JSON flag, not a fan of separate flags.** Mirrors `--plan`. Stable schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Accept inline JSON or a file path (matches `--plan`).
|
||||
- **Hosting-model-driven resolution is the documented default.** Engine-internal `auto_resolve` is the headless / cron fallback. SKILL.md routes hosting models to the JSON-flag path; engine keeps auto_resolve alive for BRAVE/EXA/SERPER users running CI.
|
||||
- **Override-leak fix is call-site scrubbing, not a signature change.** `_competitor_runner` builds an explicit kwargs dict per entity from `_subrun_kwargs(entity, plan_entry)`. No closure-default fallthrough from main scope. The 3.0.12 `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag.
|
||||
- **Footer nudge becomes context-aware.** Suppressed when `--plan` or `--competitors-plan` present. Not suppressed for bare `--competitors-list` or bare invocations. Headless cron without keys still sees the nudge.
|
||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known list of single-token-ambiguous names (states, common nouns). Stderr notes the skip so it is observable and overridable.
|
||||
- **Per-entity sub-runs get the full `pipeline.run()` pass.** Same depth, same sources, same API cost per entity as a single-topic run. This is the explicit user intent — three full passes, not one merged pass.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **JSON or multi-flag?** JSON. Matches `--plan`.
|
||||
- **Default count?** 2 peers (3-way comparison). Unchanged from 3.0.12.
|
||||
- **Does engine-internal auto_resolve stay alive?** Yes, for entities not covered by `--competitors-plan` when a backend is configured. Headless/cron users with keys keep the current 3.0.12 behavior.
|
||||
- **vs-mode or fanout?** Fanout. User's explicit ask: three full passes, not one merged pass. vs-mode merges into one pipeline with lower peer weighting, which is not what the user wants.
|
||||
- **Does the save file need per-entity clusters?** Start with the Resolved block appended. Per-entity cluster sections can follow in a separate task; they are nice-to-have, not blocking.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact trace of override-leak source. Candidates: closure capture of `subreddits` in `_competitor_runner`, shared `_auto_resolve_context` leak, Reddit adapter inheriting global config. Test-first; trace at implementation time.
|
||||
- Heuristic for "single-token-ambiguous topic" auto-skip. Start with a short hard-coded list (US state names, US city names, common nouns like "Warriors", "Suns", "Jets"); revisit after dogfood.
|
||||
- Whether per-entity coverage warnings fire when `--competitors-plan` under-resolves an entity (e.g., only `x_handle`, no subreddits). Start with stderr logging; revisit UX.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: `--competitors-plan` JSON flag + per-entity kwargs threading**
|
||||
|
||||
**Goal:** New CLI flag accepting per-entity targeting JSON. Each covered entity's `pipeline.run()` receives its own `x_handle` / `x_related` / `subreddits` / `github_user` / `github_repos` / `context`. Skips engine-internal `auto_resolve` for covered entities.
|
||||
|
||||
**Requirements:** R1, R5 (primary leak fix site)
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner`)
|
||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; verify)
|
||||
- Test: `tests/test_cli_competitors.py` (extend)
|
||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON OR a file path (mirror `--plan`).
|
||||
- Validation: parse JSON; must be a dict; each value must be a dict; unknown fields log warnings; malformed input exits 2.
|
||||
- Schema per entity: optional fields `x_handle` (str), `x_related` (list), `subreddits` (list), `github_user` (str), `github_repos` (list), `context` (str).
|
||||
- Case-insensitive matching against `--competitors-list` / discovered entities.
|
||||
- Build `_subrun_kwargs(entity, plan_entry)` helper. Returns a complete, explicit kwargs dict for `pipeline.run()` with no closure-default fallthrough from main scope. This helper is the single source of truth for per-entity call args. It also fixes the override-leak (R5) by scrubbing all per-entity flags to None unless the plan (or auto_resolve) sets them.
|
||||
- `_competitor_runner(entity)`:
|
||||
1. Look up `plan_entry` from `--competitors-plan` (if any).
|
||||
2. If plan covers entity fully, build kwargs from it; skip `auto_resolve`.
|
||||
3. If plan partially covers or is absent, fall back to `auto_resolve` (3.0.12 behavior) when a backend is configured. Plan values win over auto_resolve values on conflict.
|
||||
4. If neither plan nor backend, fall through to `pipeline.run()` with per-entity kwargs all None — engine uses planner defaults for that entity only (no leak).
|
||||
- Deep-copy config per sub-run (already done in 3.0.12); merge per-entity `context` into `entity_config["_auto_resolve_context"]` only.
|
||||
|
||||
**Execution note:** Test-first for the override-leak regression (pass `--subreddits=A,B` on main + a peer, assert peer's `pipeline.run(subreddits=...)` is None or peer-specific).
|
||||
|
||||
**Patterns to follow:**
|
||||
- `--plan` parsing at `scripts/last30days.py` (inline JSON or file path).
|
||||
- 3.0.12's `_competitor_runner` closure for scope; extract the kwargs-build into `_subrun_kwargs` helper.
|
||||
- `entity_config = dict(config)` deep-copy pattern from 3.0.12.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors-plan '{"Drake": {"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's `pipeline.run` receives `x_handle="Drake"` and `subreddits=["Drizzy"]`; no `auto_resolve` call for Drake.
|
||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered entities skip auto_resolve; third falls back to auto_resolve.
|
||||
- Happy path: plan file path accepted like `--plan` file path.
|
||||
- Happy path: case-insensitive entity match (`Drake` in plan, `drake` in list).
|
||||
- Edge case: unknown fields in plan entry → logged, ignored, run continues.
|
||||
- Edge case: plan entry for entity not in list → ignored with warning.
|
||||
- Error path: malformed JSON → exit 2.
|
||||
- Error path: top-level JSON is list not dict → exit 2.
|
||||
- Regression (leak fix): main `--subreddits=A,B` + `--competitors-list "Drake"` + no plan → Drake's `pipeline.run` receives `subreddits=None` (no leak).
|
||||
- Regression (leak fix): same for `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`.
|
||||
- Regression (leak fix): main `--x-handle=kanyewest` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanyewest"`.
|
||||
- Integration: full main + 2 peers run via `--competitors-plan`; assert each sub-run's effective kwargs match expected per-entity values.
|
||||
|
||||
**Verification:**
|
||||
- All new and regression tests pass.
|
||||
- Smoke run (mock mode + `--competitors-plan`): stderr shows `[Competitors] Drake: x=@Drake subs=Drizzy` line per entity; no `[AutoResolve]` calls for plan-covered entities; no leak of main topic's flags.
|
||||
|
||||
- [ ] **Unit 2: Reframe LAW 7-style stderr for hosting-model context**
|
||||
|
||||
**Goal:** When `--competitors` has no `--competitors-list`, no `--competitors-plan`, and no backend, stderr tells the hosting reasoning model to use its WebSearch tool for Step 0.55 per entity and pass `--competitors-plan`. Stops leading with BRAVE_API_KEY.
|
||||
|
||||
**Requirements:** R3
|
||||
|
||||
**Dependencies:** Unit 1 (flag must exist)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (the existing `[Competitors] --competitors requires...` block)
|
||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Rewrite stderr in this order:
|
||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent runtime with a WebSearch tool), YOU should: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers), (c) assemble a `--competitors-plan` JSON, (d) re-invoke. Skip this step and quality degrades — peer entities will run with planner defaults."
|
||||
2. "If you are running headless (cron, CI, no hosting model), set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
||||
3. "Minimum escape hatch: `--competitors-list "A,B,C"` skips discovery but does not pre-resolve peers. Use only for quick tests."
|
||||
- Exits non-zero as today.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing LAW 7 stderr in `planner.plan_query` for tone.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: stderr leads with "If you are the hosting reasoning model" and names `--competitors-plan` before any backend key.
|
||||
- Happy path: stderr explicitly names `--competitors-plan` as the preferred override.
|
||||
- Happy path: stderr does NOT say "requires either a configured web search backend OR an explicit --competitors-list" (the current 3.0.12 wording).
|
||||
|
||||
**Verification:**
|
||||
- Test asserts ordering and required phrases.
|
||||
|
||||
- [ ] **Unit 3: Suppress BRAVE/SERPER footer nudge when hosting-model-driven**
|
||||
|
||||
**Goal:** The `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY` footer is suppressed when `--plan` or `--competitors-plan` was passed (signal: hosting model is driving and already has WebSearch).
|
||||
|
||||
**Requirements:** R4
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever nudge is emitted; verify during implementation)
|
||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Locate the nudge emission point.
|
||||
- Add a suppression check: if `--plan` OR `--competitors-plan` was passed, skip the nudge. Otherwise, current behavior.
|
||||
- Don't suppress the nudge for bare `--competitors-list` alone — that path isn't necessarily hosting-model-driven.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--plan` passed, no backend → nudge does NOT fire.
|
||||
- Happy path: `--competitors-plan` passed, no backend → nudge does NOT fire.
|
||||
- Happy path: `--competitors-list` only, no backend → nudge fires (current behavior).
|
||||
- Happy path: no `--competitors`, no `--plan`, no backend → nudge fires (current behavior unchanged).
|
||||
|
||||
**Verification:**
|
||||
- All four scenarios produce expected nudge presence/absence.
|
||||
|
||||
- [ ] **Unit 4: Per-entity save files + Resolved block in each**
|
||||
|
||||
**Goal:** When `--save-dir` is in use with a comparison run, each entity's sub-run saves its own standalone raw file (same format as a single-entity run), and each file includes the `## Resolved Entities` block so audits can see what targeting that entity received. Matches the historical vs-mode behavior when it was N passes.
|
||||
|
||||
**Requirements:** R6, R6b
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (`save_output`, the save loop after fanout completes)
|
||||
- Possibly modify: `scripts/lib/render.py` (`render_full` branch to include Resolved block when artifact is present)
|
||||
- Test: `tests/test_save_raw_competitor_files.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]`. For each `(entity, entity_report)` tuple, call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)` — same path a single-entity run takes.
|
||||
- Each saved file uses its entity's slug as the filename (`drake-raw.md`, `kendrick-lamar-raw.md`). Main topic keeps the existing `kanye-west-raw.md` filename.
|
||||
- Each file includes its own `## Resolved Entities` block (single-entity variant: one row for that entity only). This makes each sub-run's file self-describing — you can see what targeting was used without opening the comparison file.
|
||||
- The merged comparison output (stdout) still includes the 3-row Resolved Entities block.
|
||||
- Optional: also save a comparison summary file (e.g., `kanye-west-comparison-raw.md`) holding the merged multi-entity render. Start with per-entity files only; comparison summary is a follow-up if stdout-plus-individual-files is insufficient.
|
||||
- Single-entity runs unchanged (no additional files, no block change).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing `save_output` invocation for single-entity runs (line 501 of current `scripts/last30days.py`).
|
||||
- Existing slug generation (`slugify(topic)`) for filename consistency.
|
||||
- `_render_resolved_entities_block` from 3.0.12 for the single-entity variant.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors-list "Drake,Kendrick Lamar"` + `--save-dir=/tmp/x` → `/tmp/x/kanye-west-raw.md`, `/tmp/x/drake-raw.md`, `/tmp/x/kendrick-lamar-raw.md` all exist.
|
||||
- Happy path: each peer file's first sections include that entity's Resolved Entities block with its own row only.
|
||||
- Happy path: single-entity run with `--save-dir` → one file, unchanged from today's behavior.
|
||||
- Edge case: entity slug collides with existing file → overwrite (matches single-entity behavior).
|
||||
- Edge case: `--save-suffix=v3` → all 3 files get the suffix (`kanye-west-raw-v3.md`, `drake-raw-v3.md`, `kendrick-lamar-raw-v3.md`).
|
||||
- Edge case: comparison run with one peer whose sub-run failed → that entity's file is NOT saved; others are.
|
||||
- Integration: stderr after save shows three `[last30days] Saved output to <path>` lines, one per entity.
|
||||
|
||||
**Verification:**
|
||||
- After `/last30days Kanye West --competitors-list "Drake,Kendrick Lamar" --save-dir=/tmp/x`: `ls /tmp/x/*-raw.md` shows 3 files. Each contains its entity's Resolved block.
|
||||
|
||||
- [ ] **Unit 5: SKILL.md "Competitor mode" rewrite — hosting-model Step 0.55 canonical**
|
||||
|
||||
**Goal:** SKILL.md documents the hosting-model-driven path as canonical: discover N peers via WebSearch, run Step 0.55 per entity, assemble `--competitors-plan`, invoke engine. Engine-internal `auto_resolve` is labeled the headless fallback.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 1 (flag must exist before documented)
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md` (Competitor mode subsection)
|
||||
- Modify: `README.md` (one-line example update)
|
||||
|
||||
**Approach:**
|
||||
- Replace the 3.0.12 Competitor mode subsection with a clear flow:
|
||||
1. User invokes with `--competitors` or `--competitors=N`.
|
||||
2. Hosting model runs WebSearch for "[topic] competitors" / "[topic] alternatives" → picks top N peers.
|
||||
3. Hosting model runs Step 0.55 for main + each peer (x_handle, subreddits, github_user, github_repos, context) — same protocol as vs-mode per SKILL.md §679.
|
||||
4. Hosting model assembles a `--competitors-plan` JSON object.
|
||||
5. Hosting model invokes the engine with `--competitors-list "A,B,C" --competitors-plan '{...}'`.
|
||||
6. Engine fans out N full pipelines (main + peers), each with its own full Step 0.55-grade targeting. Each entity also saves its own `*-raw.md` file when `--save-dir` is set (three full passes → three save files, matching the historical vs-mode behavior). Comparison output merges them for display.
|
||||
- Concrete JSON example in SKILL.md showing the schema.
|
||||
- Failure-mode warning: a `## Resolved Entities` block with dashes for any entity means hosting model skipped Step 0.55 for that one. Re-run with corrected plan.
|
||||
- "Headless fallback" sub-subsection: when BRAVE/EXA/SERPER/PARALLEL/OPENROUTER is set, engine's internal `auto_resolve` handles peers and `--competitors-plan` is optional.
|
||||
|
||||
**Patterns to follow:**
|
||||
- SKILL.md "Step 0.55" section for per-entity resolve protocol.
|
||||
- SKILL.md "If QUERY_TYPE = COMPARISON" section for the same-protocol-as-vs-mode reference.
|
||||
- Tone of existing 3.0.12 Competitor mode prose.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — documentation. Verification is a fresh Claude Code window dogfood run.
|
||||
|
||||
**Verification:**
|
||||
- `/last30days Kanye West --competitors` in a new window: hosting model does Step 0.55 for Kanye + 2 discovered peers; passes `--competitors-plan`; rendered Resolved block shows non-empty fields for all 3; top voices include at least one peer-specific handle.
|
||||
|
||||
- [ ] **Unit 6: Polymarket disambiguation guard**
|
||||
|
||||
**Goal:** Support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
||||
|
||||
**Requirements:** R7
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` argparse (`--polymarket-keywords`)
|
||||
- Modify: `scripts/lib/polymarket.py`
|
||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Add `--polymarket-keywords "kw1,kw2"` flag. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
||||
- Auto-skip rule: if topic is one token AND token matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords` provided, skip Polymarket with a stderr note.
|
||||
- SKILL.md Step 0.55 protocol gets a small addition: for ambiguous topics, hosting model passes `--polymarket-keywords` with topic-specific qualifiers.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing Polymarket adapter match logic.
|
||||
- Single-token detection heuristic.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr notes the skip.
|
||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs; matches filtered.
|
||||
- Happy path: topic "OpenAI" (no ambiguity) → Polymarket runs as before.
|
||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
||||
|
||||
**Verification:**
|
||||
- Warriors smoke run → Polymarket footer absent OR filtered to nba/gsw markets.
|
||||
|
||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
||||
|
||||
**Goal:** Ship 3.0.13 to all local targets.
|
||||
|
||||
**Requirements:** Closes R1-R7
|
||||
|
||||
**Dependencies:** Units 1-6
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json`
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG entry groups the fixes: Added `--competitors-plan` JSON flag for per-entity hosting-model pre-resolve. Fixed override-leak from main into peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when `--plan` / `--competitors-plan` is present. Added: Resolved Entities block persists to saved raw file. Added: `--polymarket-keywords` + auto-skip for ambiguous single-token topics.
|
||||
- Beta channel first per CLAUDE.md.
|
||||
- Hot-copy so public `/last30days` picks up 3.0.13 immediately.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` returns 3.0.13.
|
||||
- `sync.sh` exits 0.
|
||||
- Hot-copy contains the new files with competitors.py, fanout.py, the updated SKILL.md, and plugin.json 3.0.13.
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** `_competitor_runner` becomes the single source of truth for sub-run kwargs via `_subrun_kwargs(entity, plan_entry)`. Every per-entity flag flows through one helper. No closure-default leaks.
|
||||
- **Error propagation:** `--competitors-plan` JSON parse errors exit 2 with stderr (same as `--plan`). Per-entity plan entries with malformed values log warnings and fall back; don't abort the whole run.
|
||||
- **State lifecycle risks:** `entity_config = dict(config)` already deep-copies for `_auto_resolve_context`; extend the isolation discipline to every per-entity flag. Verified in Unit 1 regression tests.
|
||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors` and `--competitors-list` unchanged. `--plan` unchanged. `--polymarket-keywords` additive.
|
||||
- **Integration coverage:** New regression tests for override-leak. New integration test for plan-driven sub-run threading. New nudge-suppression test. New Polymarket disambiguation test.
|
||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. `planner.plan_query` LAW 7 behavior for the default path unchanged. Single-entity render path unchanged. vs-mode behavior unchanged.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Hosting model takes the lazy path and uses `--competitors-list` names-only. | Unit 2 stderr explicitly steers to `--competitors-plan` with Step 0.55 protocol named. Unit 5 SKILL.md docs. Resolved Entities dashes in output make the gap visible. |
|
||||
| JSON gets verbose for the hosting model to construct repeatedly. | Schema is small (≤6 fields per entity). Hosting model already runs Step 0.55 for main topic in every comparison run; peers use the same protocol. One JSON block replaces N CLI flags. |
|
||||
| Override-leak source is deeper than `_competitor_runner` closure. | Test-first per Unit 1. Receipts from 2026-04-22 Kanye run are reproducible. Trace methodically from call site. |
|
||||
| Plan-covered entity bypasses auto_resolve but plan data is incomplete (e.g., no subreddits). | Hosting model's own SKILL.md contract says Step 0.55 must cover all fields. Stderr logs per-entity coverage so under-resolved entities are visible. Next-run correction, not engine-side rescue. |
|
||||
| Polymarket auto-skip false-positives on legitimate ambiguous topics with real markets. | Conservative match (single-token + known list). `--polymarket-keywords` override is explicit and unambiguous. Stderr notes the skip. |
|
||||
| Footer nudge suppression hides the message from headless users who genuinely need it. | Suppression only fires when `--plan` or `--competitors-plan` is present. Cron / CI runs that pass neither still see the nudge. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first per CLAUDE.md (private repo `/last30days-beta`).
|
||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
||||
- CHANGELOG voice should call this out as the feedback-driven follow-up to 3.0.12. Reader should see "we tried engine-internal resolve in 3.0.12; it needs backend keys we don't have; we moved resolution to the hosting model in 3.0.13."
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Origin plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
||||
- Earlier plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
||||
- 2026-04-22 test session receipts: Warriors, Seattle, Arizona Wildcats, Kanye West
|
||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol
|
||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
||||
@@ -0,0 +1,451 @@
|
||||
---
|
||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
||||
type: feat
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
origin: docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded
|
||||
---
|
||||
|
||||
# feat: vs mode runs N full passes and --competitors is vs with auto-discovery
|
||||
|
||||
## Overview
|
||||
|
||||
Architectural unification driven by user correction 2026-04-22: vs mode and `--competitors` are the same thing. A user typing `/last30days OpenAI vs Anthropic vs xAI` should get a full single-entity last30days pass for each of the three entities — three full pipelines, three saved `*-raw.md` files, merged into one comparison output. A user typing `/last30days OpenAI --competitors` should get the same output after the hosting model auto-picks 2 peers; i.e., `--competitors` is a thin shortcut that expands "topic + `--competitors`" into "topic vs peer1 vs peer2" and then runs the unified vs pipeline.
|
||||
|
||||
Current state diverges from this:
|
||||
|
||||
- **vs mode today**: one `pipeline.run()` with a comparison-optimized plan that merges all entities' targeting into a single retrieval pool. Lower-weight `--x-related` for peers, merged subreddits, cross-entity keyword noise. One saved file.
|
||||
- **`--competitors` today (3.0.12)**: N parallel `pipeline.run()` calls via `scripts/lib/fanout.py`, but per-entity Step 0.55 depends on an engine-side web backend key Matt doesn't have. Silently degrades to planner defaults for peers. One saved file (main topic only). Override-leak from main into peers.
|
||||
|
||||
After this plan:
|
||||
|
||||
- **vs mode**: N parallel `pipeline.run()` calls, one per entity, each with its own full Step 0.55-grade targeting, each saving its own `*-raw.md`. Merged into one comparison output.
|
||||
- **`--competitors`**: SKILL.md shortcut. Hosting model discovers N peers, builds `"topic vs peer1 vs peer2"`, and invokes the same vs pipeline. No separate orchestration path.
|
||||
- **Same fanout machinery (`scripts/lib/fanout.py`)** serves both. One fix, both behaviors improve.
|
||||
|
||||
## Problem Frame
|
||||
|
||||
The product insight from 2026-04-22 test runs is simple: the user wants three full last30days reports plus a comparison merge. Not one comparison pass with N-way targeting merged into a single retrieval pool. Not one save file. Not "main gets Step 0.55, peers get planner defaults." Three full passes. Three save files. Merged output.
|
||||
|
||||
The historical vs mode did that (it ran as 3 passes, saving 3 files). SKILL.md §551 currently says:
|
||||
|
||||
> "When the user asks 'X vs Y', run ONE research pass with a comparison-optimized plan that covers both entities AND their rivalry. This replaces the old 3-pass approach (which took 13+ minutes and produced tangential content)."
|
||||
|
||||
That change was a latency optimization that removed the user-visible behavior the user wants. The fix is to revert the architectural direction: N passes per entity, in parallel rather than serial (parallelism lowers wall-clock to ~1× a single pass, not N×), with per-entity save files.
|
||||
|
||||
The 3.0.11 `--competitors` flag already introduced parallel N-pass machinery (`fanout.run_competitor_fanout`). The 3.0.12 follow-up tried to wire per-entity Step 0.55 into it but failed when no web backend was configured. The elegant move: stop maintaining two architectures. vs-mode and `--competitors` both use `fanout.py`. `--competitors` becomes a SKILL.md-level shortcut that discovers 2 peers and hands off to vs-mode.
|
||||
|
||||
Four 2026-04-22 test receipts (Warriors, Seattle, Arizona Wildcats, Kanye West) all confirmed the user's pain points:
|
||||
|
||||
- Peers thin because they ran without per-entity handle/sub targeting.
|
||||
- Only one `*-raw.md` per run — no per-entity audit.
|
||||
- Kanye peers leaked main topic's `--subreddits`.
|
||||
- Engine footer nudging `BRAVE_API_KEY` to Claude Code users who already have WebSearch.
|
||||
- Polymarket noise on ambiguous topics (Warriors → Glasgow rugby; Arizona → Diamondbacks).
|
||||
|
||||
This plan closes all of them by unifying the architecture and making hosting-model-driven Step 0.55 per entity the canonical path.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. vs mode (any topic containing ` vs ` / ` versus `) runs N full `pipeline.run()` calls in parallel, one per entity. Each sub-run uses its entity's own Step 0.55 targeting (from the hosting model's pre-resolution, passed via a new `--competitors-plan` JSON).
|
||||
- R2. `--competitors` (and `--competitors=N`) becomes a SKILL.md-level shortcut: the hosting model (a) discovers N peers via WebSearch, (b) runs Step 0.55 per entity (main + peers), (c) rewrites the topic to `"main vs peer1 vs peer2"`, (d) invokes the engine with `--competitors-plan` containing each entity's targeting.
|
||||
- R3. New `--competitors-plan` JSON flag. Schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Implies vs mode when present with a single-entity topic. Applies per-entity targeting to each sub-run. Accepts inline JSON or a file path (matches `--plan`).
|
||||
- R4. Each entity's sub-run saves its own `*-raw.md` file when `--save-dir` is in use. Example: `/last30days "Kanye West vs Drake vs Kendrick Lamar" --save-dir=~/Documents/Last30Days` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`. Same filenames a single-entity run of each topic would produce. Matches historical vs-mode behavior.
|
||||
- R5. Each per-entity saved file includes its own single-row `## Resolved Entities` block so the audit survives. The merged comparison stdout still shows the full 3-row block.
|
||||
- R6. Override-leak fix: no main-topic flags (`--subreddits`, `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`) leak into peer sub-runs. Every per-entity kwarg is scrubbed at the sub-run call site.
|
||||
- R7. LAW 7-style stderr for `--competitors` invocations with no list, no plan, no backend is reframed for hosting-model context: leads with "use your WebSearch to discover peers, resolve Step 0.55 per entity, re-invoke with `topic vs peer1 vs peer2 --competitors-plan '...'`." Does not lead with BRAVE_API_KEY.
|
||||
- R8. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` or `--competitors-plan` was passed.
|
||||
- R9. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
||||
- R10. Default `--competitors` count stays 2 peers (3-way comparison). Unchanged from 3.0.12.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No changes to single-entity `pipeline.run()` semantics. Each sub-run in vs mode behaves identically to a bare `/last30days {entity}` invocation.
|
||||
- No changes to the planner's comparison-intent logic for single-entity-containing topics. The `_should_force_deterministic_plan` shortcut for vs-topics routes to fanout, not to its current single-pipeline path.
|
||||
- No new emit modes. Comparison output format unchanged.
|
||||
- No removal of `--competitors-list`. Stays as a minimum escape hatch (names-only, no per-entity targeting) for scripted headless use.
|
||||
- No removal of engine-internal `resolve.auto_resolve()` in fanout. Remains as headless / cron fallback for users with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys. The dominant Claude Code path bypasses it via `--competitors-plan`.
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Explicit "head-to-head" rivalry pass in vs-mode (a supplemental subquery like `"A vs B"` that catches rivalry articles missing from pure entity-scoped passes). Start with N independent passes; add a head-to-head supplemental pass if the rivalry-content gap shows up in dogfood.
|
||||
- Cache layer for hosting-model pre-resolution.
|
||||
- Cross-source disambiguation (not just Polymarket).
|
||||
- Latency knob for users who want the old one-pass vs behavior (probably not needed; parallel N-pass is ~1× wall clock).
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `scripts/last30days.py` — main(), `_main_runner`, `_competitor_runner`, the competitor enable/discovery branch. Primary file.
|
||||
- `scripts/lib/fanout.py` — existing orchestrator (3.0.11). Reused as-is; `competitor_runner` closure is where per-entity kwargs apply.
|
||||
- `scripts/lib/planner.py` — `_should_force_deterministic_plan` detects vs-topics via regex. Current path synthesizes ONE comparison plan; new path routes to fanout.
|
||||
- `scripts/lib/render.py` — `render_comparison_multi` (3.0.12) + `_render_resolved_entities_block`. Both reused. `render_full` needs a per-entity variant when saving sub-run files.
|
||||
- `scripts/last30days.py` `save_output` — where raw files are written. Needs to iterate per entity when competitor_reports artifact present.
|
||||
- `scripts/lib/quality_nudge.py` — BRAVE/SERPER nudge emission.
|
||||
- `scripts/lib/polymarket.py` — source adapter for `--polymarket-keywords` and ambiguous-topic auto-skip.
|
||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol — the hosting-model contract that drives per-entity pre-resolution for both vs mode and `--competitors`.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- 3.0.11 plan (`2026-04-22-002`): built fanout.
|
||||
- 3.0.12 plan (`2026-04-22-003`): tried engine-internal per-entity auto_resolve; failed without backend keys.
|
||||
- 3.0.13 plan draft (`2026-04-22-004-...superseded`): proposed `--competitors-plan` JSON + vs-mode-shortcut path but kept them separate. User's 2026-04-22 correction unifies them.
|
||||
- 2026-04-22 test receipts: Warriors, Seattle, Arizona Wildcats, Kanye West runs all reproduced the per-entity resolve gap.
|
||||
- User's architectural steer: "vs mode should work that way too" + "--competitors is just vs mode with auto-discovery." This plan encodes that.
|
||||
|
||||
### External References
|
||||
|
||||
- None. All patterns in-repo.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Unify vs-mode and --competitors on one orchestrator.** `fanout.run_competitor_fanout` serves both. vs-mode is "topic contains ' vs '" detection → fanout. `--competitors` is "SKILL.md shortcut → hosting model rewrites topic to vs form → fanout." One code path.
|
||||
- **Per-entity targeting via `--competitors-plan` JSON.** Schema `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Mirrors `--plan`. Applies to both vs-mode and `--competitors` paths. Hosting model passes it after running Step 0.55 per entity.
|
||||
- **N save files, one per entity.** Each sub-run writes a `{entity-slug}-raw.md` file when `--save-dir` is set. Matches historical vs-mode behavior. Single-entity runs unchanged.
|
||||
- **Revert the "one pass for latency" optimization that removed per-entity passes.** Parallel execution via `ThreadPoolExecutor` means wall-clock is ~max(per-entity-latency), not sum. The old latency concern (13+ minutes for 3 serial passes) does not apply to a parallel fan-out.
|
||||
- **Override-leak fix at the call site.** `_subrun_kwargs(entity, plan_entry)` helper returns fully explicit per-entity kwargs; no closure-default fallthrough from main scope.
|
||||
- **LAW 7 stderr reframed, not just updated.** Current message treats BRAVE_API_KEY as the solution. New message treats hosting-model Step 0.55 as the solution, with backend keys listed only as the headless fallback.
|
||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known-ambiguous single-token list.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **vs mode N passes or single-pass?** N passes. User's architectural correction.
|
||||
- **Should --competitors still be an engine flag at all?** Yes, kept for headless / cron contexts with backend keys. Dominant Claude Code path is SKILL.md shortcut → vs-mode fanout. Engine flag stays as compatibility surface.
|
||||
- **`--competitors-plan` JSON or multi-flag?** JSON. Matches `--plan`.
|
||||
- **Default count?** 2 peers → 3-way comparison. Unchanged.
|
||||
- **Saved-file naming?** `{entity-slug}-raw.md` per entity, same as single-entity runs would produce.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact trace of override-leak path (closure capture vs shared config vs Reddit adapter fallback). Test-first per Unit 2; patch at the right layer.
|
||||
- Heuristic for single-token-ambiguous Polymarket auto-skip. Start with a short hard-coded list; iterate.
|
||||
- Whether to include a head-to-head rivalry supplemental pass in vs-mode. Ship N-independent passes first; revisit after dogfood if rivalry content is missing.
|
||||
- Exact filename convention when the comparison merged output is saved (if saved at all). Not blocking — per-entity files are the primary save artifact.
|
||||
|
||||
## High-Level Technical Design
|
||||
|
||||
> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.*
|
||||
|
||||
```
|
||||
User invokes:
|
||||
/last30days "OpenAI vs Anthropic vs xAI"
|
||||
OR
|
||||
/last30days OpenAI --competitors (hosting model rewrites to vs form)
|
||||
OR
|
||||
/last30days OpenAI --competitors-list "Anthropic,xAI"
|
||||
OR
|
||||
/last30days "OpenAI vs Anthropic vs xAI" --competitors-plan '{...per-entity...}'
|
||||
|
||||
↓
|
||||
|
||||
scripts/last30days.py main():
|
||||
- Detect: topic has " vs " OR --competitors enabled
|
||||
- If --competitors and no list/plan: emit LAW 7-style stderr with hosting-model instruction
|
||||
- If --competitors with list or discovery: rewrite topic to vs form, continue
|
||||
- Parse --competitors-plan JSON, map to entities
|
||||
|
||||
↓
|
||||
|
||||
fanout.run_competitor_fanout (shared path):
|
||||
- For each entity (main + peers):
|
||||
- entity_config = dict(config) [deep copy to prevent leak]
|
||||
- kwargs = _subrun_kwargs(entity, plan_entry) [explicit; no main-topic leak]
|
||||
- If plan_entry missing a field AND backend available: auto_resolve() fill
|
||||
- pipeline.run(topic=entity, **kwargs, internal_subrun=True)
|
||||
- Parallel ThreadPoolExecutor
|
||||
- Collect per-entity Reports
|
||||
- Attach resolved targeting to each Report.artifacts["resolved"]
|
||||
|
||||
↓
|
||||
|
||||
scripts/last30days.py after fanout:
|
||||
- If --save-dir: save each entity's Report as {entity-slug}-raw.md
|
||||
Each file includes its own single-row Resolved Entities block
|
||||
- emit_comparison_output → render_comparison_multi (merged stdout)
|
||||
Includes full N-row Resolved Entities block
|
||||
```
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: vs-topic detection routes to fanout (not single-pipeline)**
|
||||
|
||||
**Goal:** A topic containing ` vs ` / ` versus ` triggers `fanout.run_competitor_fanout` with the parsed entities. Each entity runs a full `pipeline.run()`. Replace the current single-pipeline-with-comparison-plan behavior.
|
||||
|
||||
**Requirements:** R1
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (main() — detect vs-topic, route to fanout)
|
||||
- Modify: `scripts/lib/planner.py` (remove / bypass the `_should_force_deterministic_plan` special case for vs topics; vs topics no longer go through `plan_query` as a single comparison plan)
|
||||
- Test: `tests/test_vs_mode_fanout.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Parse the incoming topic: if it contains ` vs ` or ` versus ` (case-insensitive), split into entities (reuse `planner._comparison_entities`-style logic or move that utility into main()).
|
||||
- When vs-entities are detected, route to the same fanout branch `--competitors` uses today. The entity list comes from the topic string; no discovery step needed.
|
||||
- Each entity runs `pipeline.run()` with its own plan (either from `--competitors-plan[entity]` or from the engine's per-entity fallback path).
|
||||
- For back-compat, if the user passes both a vs-topic AND `--plan`, honor `--plan` for the main (first) entity and use per-entity defaults for peers unless `--competitors-plan` is also provided.
|
||||
|
||||
**Execution note:** Start with an integration test that runs `"A vs B"` via mock mode and asserts fanout was called with two entities + two pipeline.run calls.
|
||||
|
||||
**Patterns to follow:**
|
||||
- 3.0.11 fanout wiring in `scripts/last30days.py`'s `--competitors` branch.
|
||||
- `planner._comparison_entities` for the split logic.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: topic `"A vs B"` → two pipeline.run calls, two Reports returned, merged render.
|
||||
- Happy path: topic `"A vs B vs C"` → three pipeline.run calls.
|
||||
- Happy path: topic `"A versus B"` → matches the same regex, two pipelines.
|
||||
- Edge case: topic `"OpenAI vs"` (trailing empty entity) → treated as single-entity `"OpenAI"`, not vs mode.
|
||||
- Edge case: topic contains "vs." (dot, no trailing space) → existing regex tolerates it; verify.
|
||||
- Edge case: topic `"A vs B"` plus `--plan` → plan applies to first entity only, peers use per-entity defaults.
|
||||
- Integration: full vs-mode run end-to-end in mock mode; verify rendered output, stderr has one `[Competitors] Comparing: A vs B vs ...` line.
|
||||
|
||||
**Verification:**
|
||||
- Test assertions pass.
|
||||
- Mock-mode smoke of `/last30days "OpenAI vs Anthropic"` shows fanout invocation, per-entity Reports, merged comparison output.
|
||||
|
||||
- [ ] **Unit 2: `--competitors-plan` JSON flag + `_subrun_kwargs` helper + override-leak fix**
|
||||
|
||||
**Goal:** New JSON flag threads per-entity targeting into each sub-run's `pipeline.run()`. A `_subrun_kwargs(entity, plan_entry)` helper is the single source of truth for per-entity kwargs, eliminating override-leak.
|
||||
|
||||
**Requirements:** R3, R6
|
||||
|
||||
**Dependencies:** None (can land alongside or before Unit 1)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner` + `_subrun_kwargs` helper)
|
||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; the competitor_runner contract is unchanged)
|
||||
- Test: `tests/test_cli_competitors.py` (extend)
|
||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
||||
- Test: `tests/test_competitor_subrun_isolation.py` (new, regression)
|
||||
|
||||
**Approach:**
|
||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON or file path (mirror `--plan`).
|
||||
- Validation: top-level dict; each value is a dict; unknown fields log warnings; malformed input exits 2. Case-insensitive entity matching.
|
||||
- Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`.
|
||||
- Build `_subrun_kwargs(entity, plan_entry)` — returns an explicit dict with every per-entity flag. No closure-default fallthrough. This is the leak fix.
|
||||
- `_competitor_runner(entity)`:
|
||||
1. Get `plan_entry` from `--competitors-plan` if present.
|
||||
2. Build base kwargs with `_subrun_kwargs(entity, plan_entry)`.
|
||||
3. Fill missing fields via `resolve.auto_resolve(entity, entity_config)` only if backend is configured (3.0.12 fallback path).
|
||||
4. Call `pipeline.run(topic=entity, internal_subrun=True, **kwargs)`.
|
||||
5. Attach `resolved` dict to `report.artifacts`.
|
||||
- Verify no per-entity flag from main() leaks via closure. The helper is the only source of per-entity values.
|
||||
|
||||
**Execution note:** Test-first for the override-leak regression. Use the Kanye 2026-04-22 receipt as the failing test input (main `--subreddits=Kanye,hiphopheads` + `--competitors-list "Drake"` → assert Drake's pipeline.run receives `subreddits=None`).
|
||||
|
||||
**Patterns to follow:**
|
||||
- `--plan` parsing block in `scripts/last30days.py`.
|
||||
- 3.0.12's `entity_config = dict(config)` deep-copy pattern.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors-plan '{"Drake":{"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's pipeline.run receives `x_handle="Drake"`, `subreddits=["Drizzy"]`. No auto_resolve call for Drake.
|
||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered skip auto_resolve; third falls back.
|
||||
- Happy path: plan file path accepted like `--plan`.
|
||||
- Happy path: case-insensitive entity match.
|
||||
- Edge case: unknown fields → warn, ignore.
|
||||
- Edge case: plan entry for entity not in list → warn, ignore.
|
||||
- Error path: malformed JSON → exit 2.
|
||||
- Error path: top-level JSON is list → exit 2.
|
||||
- Regression (leak): main `--subreddits=A,B` + `--competitors-list "X"` + no plan → X's pipeline.run gets `subreddits=None`.
|
||||
- Regression (leak): same for `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`.
|
||||
- Regression (leak): main `--x-handle=kanye` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanye"`.
|
||||
|
||||
**Verification:**
|
||||
- All regression tests pass.
|
||||
- Smoke run (mock mode + plan): stderr shows per-entity `[Competitors] {entity}: x=... subs=...` line; no leak from main topic's flags.
|
||||
|
||||
- [ ] **Unit 3: Per-entity save files**
|
||||
|
||||
**Goal:** When `--save-dir` is set in a vs-mode or `--competitors` run, each entity's sub-run saves its own `{entity-slug}-raw.md` file — same format as a single-entity run would produce.
|
||||
|
||||
**Requirements:** R4, R5
|
||||
|
||||
**Dependencies:** Unit 1, Unit 2
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (`save_output` iteration after fanout)
|
||||
- Modify: `scripts/lib/render.py` (`render_full` includes single-row Resolved Entities block when that entity's `artifacts["resolved"]` is present)
|
||||
- Test: `tests/test_save_raw_per_entity.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]` (or equivalent). For each `(entity, entity_report)`:
|
||||
- Call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)`.
|
||||
- Uses entity's `slugify(entity)` for the filename. Same pattern a single-entity run uses.
|
||||
- Each saved file invokes `render_full` (or the save-variant). `render_full` now checks for `report.artifacts["resolved"]` and prepends a single-row Resolved Entities block.
|
||||
- Stderr logs one `[last30days] Saved output to <path>` line per entity.
|
||||
- Single-entity runs unchanged (no extra files, render_full unchanged for them).
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing `save_output` invocation in main() for single-entity runs.
|
||||
- `slugify(topic)` for filename.
|
||||
- 3.0.12's `_render_resolved_entities_block` (reused, single-row mode).
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `/last30days "A vs B vs C" --save-dir=/tmp/x` → `/tmp/x/a-raw.md`, `/tmp/x/b-raw.md`, `/tmp/x/c-raw.md` exist.
|
||||
- Happy path: `--competitors-list "Drake,Kendrick" --save-dir=/tmp/x` on topic Kanye → three files: `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`.
|
||||
- Happy path: each file includes a single-row Resolved Entities block for its entity.
|
||||
- Happy path: single-entity run with `--save-dir` → one file, no Resolved block (unchanged).
|
||||
- Edge case: `--save-suffix=v3` → all N files get the suffix.
|
||||
- Edge case: one entity sub-run failed → its file is NOT saved; the others are.
|
||||
- Integration: `ls {save-dir}/*-raw.md` returns N files after a vs-mode run.
|
||||
|
||||
**Verification:**
|
||||
- Test assertions pass.
|
||||
- Manual vs-mode smoke saves N files.
|
||||
|
||||
- [ ] **Unit 4: LAW 7-style stderr reframe + footer-nudge suppression**
|
||||
|
||||
**Goal:** The `--competitors`-with-no-backend stderr tells the hosting model to do Step 0.55 per entity and pass `--competitors-plan`. The BRAVE/SERPER footer nudge is suppressed when `--plan` or `--competitors-plan` is present.
|
||||
|
||||
**Requirements:** R7, R8
|
||||
|
||||
**Dependencies:** Unit 2 (flag must exist)
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (the `[Competitors] --competitors requires...` stderr block)
|
||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever footer nudge emits; verify during implementation)
|
||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Rewrite stderr in this order:
|
||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent with WebSearch), the recommended path: (a) discover N peers via WebSearch, (b) run Step 0.55 for main + each peer, (c) re-invoke as `/last30days 'topic vs peer1 vs peer2' --competitors-plan '{...}'`. See SKILL.md 'Competitor mode'."
|
||||
2. "Headless / cron path: set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
||||
3. "Minimum escape hatch: `--competitors-list 'A,B,C'` skips discovery but does not pre-resolve peers."
|
||||
- Suppress footer nudge when `external_plan` OR `competitors_plan` was passed.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: `--competitors` with no backend, no list, no plan → stderr leads with "If you are the hosting reasoning model" and references `--competitors-plan` before naming API keys.
|
||||
- Happy path: `--plan` passed → footer nudge does NOT fire.
|
||||
- Happy path: `--competitors-plan` passed → footer nudge does NOT fire.
|
||||
- Happy path: `--competitors-list` only (no plan, no backend) → footer nudge still fires (hosting model didn't fully engage).
|
||||
- Happy path: no `--competitors`, no `--plan` → footer nudge unchanged.
|
||||
|
||||
**Verification:**
|
||||
- Tests pass.
|
||||
|
||||
- [ ] **Unit 5: Polymarket disambiguation guard**
|
||||
|
||||
**Goal:** `--polymarket-keywords "kw1,kw2"` filters market matches; auto-skip Polymarket on single-token-ambiguous topics without override.
|
||||
|
||||
**Requirements:** R9
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/last30days.py` (argparse)
|
||||
- Modify: `scripts/lib/polymarket.py`
|
||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
||||
|
||||
**Approach:**
|
||||
- Add `--polymarket-keywords "kw1,kw2"`. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
||||
- Auto-skip: if topic is one token AND matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords`, skip Polymarket with stderr note.
|
||||
- SKILL.md update (small): mention `--polymarket-keywords` in Step 0.55 instructions for ambiguous topics.
|
||||
|
||||
**Test scenarios:**
|
||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr note.
|
||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs, filtered.
|
||||
- Happy path: topic "OpenAI" → Polymarket runs as before.
|
||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
||||
|
||||
**Verification:**
|
||||
- Warriors smoke → Polymarket footer absent or filtered.
|
||||
|
||||
- [ ] **Unit 6: SKILL.md rewrite — vs mode is the canonical path, `--competitors` is a shortcut**
|
||||
|
||||
**Goal:** SKILL.md documents the unified architecture. vs mode runs N full passes. `--competitors` is a SKILL.md-level shortcut that discovers 2 peers and invokes vs mode with `--competitors-plan`.
|
||||
|
||||
**Requirements:** R1, R2, R10 (surfaces them)
|
||||
|
||||
**Dependencies:** Units 1-4
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md` (§551 "If QUERY_TYPE = COMPARISON" rewrite; Competitor mode subsection rewrite)
|
||||
- Modify: `README.md` (one-line example)
|
||||
|
||||
**Approach:**
|
||||
- Rewrite §551 to describe the N-pass architecture: "When the user asks 'X vs Y' (or 'X vs Y vs Z'), run Step 0.55 per entity, then invoke the engine. The engine fans out N full pipelines in parallel. Each entity gets its own single-entity-grade coverage. Wall clock is close to a single run."
|
||||
- Remove the "ONE research pass with a comparison-optimized plan that replaces the old 3-pass approach" language.
|
||||
- Add a `--competitors-plan` JSON example.
|
||||
- Rewrite the Competitor mode subsection: "`--competitors` is a shortcut. The hosting model: (1) runs WebSearch to discover N=2 peers, (2) runs Step 0.55 for main + each peer, (3) rewrites topic to `'main vs peer1 vs peer2'`, (4) invokes engine with `--competitors-plan '{...}'`. Engine flag `--competitors` and `--competitors-list` remain for headless fallback."
|
||||
- Cross-reference §679 (per-entity Step 0.55 protocol).
|
||||
- Warning: a thin `## Resolved Entities` block (dashes for any entity) means the hosting model skipped Step 0.55 for that one.
|
||||
|
||||
**Patterns to follow:**
|
||||
- Existing §679 per-entity Step 0.55 protocol for tone.
|
||||
- 3.0.12 Competitor mode prose for terseness.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — documentation. Verification is dogfood.
|
||||
|
||||
**Verification:**
|
||||
- `/last30days "OpenAI vs Anthropic vs xAI"` in a fresh Claude Code window produces 3 save files with populated Resolved blocks and non-dash per-entity targeting.
|
||||
- `/last30days OpenAI --competitors` produces same after discovery step.
|
||||
|
||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
||||
|
||||
**Goal:** Ship 3.0.13 to all local targets.
|
||||
|
||||
**Requirements:** Closes R1-R10
|
||||
|
||||
**Dependencies:** Units 1-6
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json`
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG: group the changes. "Changed: vs mode now runs N full passes in parallel, one per entity — reverting the one-pass optimization to restore per-entity depth. Added: --competitors-plan JSON for per-entity Step 0.55 targeting (applies to vs mode and --competitors). Changed: --competitors is now a SKILL.md shortcut for vs-with-discovery. Added: per-entity *-raw.md save files. Fixed: override-leak from main to peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when --plan / --competitors-plan present. Added: --polymarket-keywords + auto-skip for ambiguous topics."
|
||||
- Beta channel first per CLAUDE.md.
|
||||
- Hot-copy so public `/last30days` picks up 3.0.13.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` → 3.0.13.
|
||||
- `sync.sh` exits 0.
|
||||
- Hot-copy contains the new files.
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** vs-mode and `--competitors` share one orchestrator (`fanout.run_competitor_fanout`). `_subrun_kwargs` is the single source of per-entity kwargs. Save loop iterates per entity.
|
||||
- **Error propagation:** Per-entity sub-run failure → logged, dropped, continue (3.0.11 behavior unchanged). `--competitors-plan` JSON parse errors exit 2 (same shape as `--plan`).
|
||||
- **State lifecycle risks:** `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag (Unit 2 fix). No cross-entity context leak.
|
||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors`, `--competitors-list`, `--plan` unchanged. `--polymarket-keywords` additive. vs-mode keeps its topic-string surface.
|
||||
- **Integration coverage:** New vs-mode-fanout integration test. New override-leak regression test. New plan-threading test. New nudge-suppression test. New per-entity-save test. New Polymarket disambiguation test.
|
||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. Single-entity render path unchanged. LAW 7 on the default path unchanged (still fires when a single-entity run lacks `--plan`).
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| vs-mode N-pass latency feels slower for users who remember the one-pass shortcut. | Parallel execution keeps wall-clock ~= max(per-entity-latency), not sum. `--quick` on a vs-topic still applies to each sub-run. CHANGELOG calls out the revert + parallelism. |
|
||||
| API cost scales linearly with N (per source). | Default count 2 caps it. Hard max 6 on `--competitors`. vs-mode users opted into N entities explicitly. |
|
||||
| Rivalry content ("A vs B" articles) missed in N-independent passes. | Deferred to separate task (head-to-head supplemental pass). Start shipping and observe whether this is actually a gap. |
|
||||
| Hosting model skips `--competitors-plan` and uses `--competitors-list` only. | Unit 4 stderr reframe steers explicitly. SKILL.md Unit 6 makes the plan-path canonical. Thin Resolved block in output makes skipped-Step-0.55 visible. |
|
||||
| Override-leak fix misses a subtle closure path. | Unit 2 is test-first with the Kanye receipt as the failing input. Regression test asserts every per-entity flag is None unless plan provides it. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- Beta channel first per CLAUDE.md.
|
||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
||||
- CHANGELOG explicitly frames the vs-mode change as an architectural revert-with-parallelism, not a regression to the old serial N-pass.
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Superseded plan: `docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded`
|
||||
- Previous plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
||||
- Initial plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
||||
- 2026-04-22 test session receipts (Warriors, Seattle, Arizona Wildcats, Kanye West)
|
||||
- SKILL.md §551 + §679 — the per-entity Step 0.55 protocol the hosting model uses for both paths
|
||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/planner.py` vs-topic special-case, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
||||
@@ -0,0 +1,87 @@
|
||||
---
|
||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
||||
type: fix
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
---
|
||||
|
||||
# fix: comparison title says (/Last30Days) instead of (Last 30 Days)
|
||||
|
||||
## Overview
|
||||
|
||||
User feedback 2026-04-22 on the 3.0.13 release runs (Kanye vs Drake, Mercer Island, Figma): the comparison title currently reads `# Kanye West vs Drake: What the Community Says (Last 30 Days)`. It should read `# Kanye West vs Drake: What the Community Says (/Last30Days)` — attributing the output to the slash command rather than describing the date range generically.
|
||||
|
||||
Single-line change in SKILL.md, three occurrences. No code change.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. Comparison title pattern in SKILL.md changes from `(Last 30 Days)` to `(/Last30Days)` so synthesis outputs read `... What the Community Says (/Last30Days)`.
|
||||
- R2. Both the rule statement (line 113) and the COMPARISON-exception statement (line 131) and the synthesis template example (line 1208) all use the new suffix.
|
||||
- R3. Version bumps to 3.0.14, CHANGELOG entry, sync, hot-copy. Public cache picks up the new title pattern.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No changes to the single-entity output title (no `(/Last30Days)` suffix there — only comparison topics carry it).
|
||||
- No changes to engine code. Pure SKILL.md content.
|
||||
- No changes to anything else surfaced in the test runs.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Replace all three occurrences of the suffix string in one pass.** They are identical strings; changing one without the others would cause synthesis-time confusion when the model reaches a different reference.
|
||||
- **Ship as 3.0.14, not 3.0.13.x.** Patch-level bump matches the small scope and keeps the release log clean.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: Replace `(Last 30 Days)` → `(/Last30Days)` in SKILL.md**
|
||||
|
||||
**Goal:** All three SKILL.md references to the comparison title use the new suffix.
|
||||
|
||||
**Requirements:** R1, R2
|
||||
|
||||
**Files:**
|
||||
- Modify: `SKILL.md`
|
||||
|
||||
**Approach:**
|
||||
- `replace_all` swap of `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Three occurrences, no other strings overlap.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — pure documentation. Verification by inspection + dogfood run.
|
||||
|
||||
**Verification:**
|
||||
- `grep -c "What the Community Says (/Last30Days)" SKILL.md` returns 3.
|
||||
- `grep -c "What the Community Says (Last 30 Days)" SKILL.md` returns 0.
|
||||
|
||||
- [ ] **Unit 2: Version 3.0.14 + CHANGELOG + sync + hot-copy**
|
||||
|
||||
**Goal:** Ship 3.0.14 to all local targets.
|
||||
|
||||
**Requirements:** R3
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Modify: `.claude-plugin/plugin.json`
|
||||
- Modify: `CHANGELOG.md`
|
||||
- Run: `bash scripts/sync.sh`
|
||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.14/`
|
||||
|
||||
**Approach:**
|
||||
- CHANGELOG: "Changed: comparison-mode title attribution — `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range."
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — packaging.
|
||||
|
||||
**Verification:**
|
||||
- `grep version .claude-plugin/plugin.json` → 3.0.14.
|
||||
- Hot-copy contains the updated SKILL.md.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Hosting model has the old title pattern memorized from a prior run and re-emits `(Last 30 Days)`. | SKILL.md is read top-to-bottom each invocation. STEP 0 canonical-path self-check (3.0.12) ensures the model loads the new SKILL.md, not the marketplace stale copy. |
|
||||
|
||||
## Sources & References
|
||||
|
||||
- 2026-04-22 dogfood runs (Kanye West vs Drake, Mercer Island --competitors, Figma --competitors)
|
||||
- Related code: `SKILL.md` lines 113, 131, 1208
|
||||
@@ -0,0 +1,112 @@
|
||||
# v3.0.9 - The Self-Debug Release
|
||||
|
||||
## Highlights
|
||||
|
||||
**v3.0.9 is live.** New user-facing capabilities, broader cross-platform support, and a skill that now runs reliably on Claude Code, Codex, Hermes, Gemini, claude.ai, and OpenClaw. The headline fix: the engine refuses "birthday gift for 40 year old" style queries with a clarifying question instead of 5 minutes of junk output. The headline feature: TikTok and YouTube top comments now render alongside Reddit's, so the most-engaged voice from every source makes it into the synthesis.
|
||||
|
||||
**The label - "The Self-Debug Release":** I handed 5 separate Opus 4.7 instances their own failed outputs and asked them to debug themselves. Three converged on "SKILL.md is too big and the LAWs are too deep." Two converged on "the engine should refuse demographic-shopping queries outright" and "the WebSearch Sources reminder is overriding LAW 1." I copy-pasted their diagnoses into code. Validation: 5/5 canonical compliance on the topics that had failed.
|
||||
|
||||
## New capabilities
|
||||
|
||||
- **TikTok and YouTube top comments render alongside Reddit's.** PR [#260](https://github.com/mvanhorn/last30days-skill/pull/260) made the top-engagement comment from each TikTok video and YouTube video first-class in the output - same prominent `💬 Top comment` treatment Reddit's top comment already got. This is the biggest user-facing output change since 3.0.0 and it was never announced. The community inspiration trace: @uppinote20's original push for richer Reddit comments ([PR #143](https://github.com/mvanhorn/last30days-skill/pull/143)) seeded the pattern; this PR generalized it across TikTok and YouTube. PR [#265](https://github.com/mvanhorn/last30days-skill/pull/265) followed up by fixing the ScrapeCreators `url=` param + new response shape for YouTube comments/transcripts so the enrichment actually works.
|
||||
|
||||
- **last30days runs on Hermes AI Agent now.** @stephenmcconnachie's PR ([#228](https://github.com/mvanhorn/last30days-skill/pull/228)) added Hermes as a first-class deploy target. `scripts/sync.sh` detects `~/.hermes/skills/research` and deploys the full skill (SKILL.md, scripts, lib modules, fixtures) to Hermes's skills directory alongside Claude Code and Codex. This is one of the biggest surface-area expansions in v3 - last30days is now usable inside the Hermes agent's research workflows without any manual wiring.
|
||||
|
||||
- **Multi-key SCRAPECREATORS_API_KEY rotation.** @zaydiscold's PR ([#268](https://github.com/mvanhorn/last30days-skill/pull/268)) added automatic key rotation. Set `SCRAPECREATORS_API_KEY_1`, `SCRAPECREATORS_API_KEY_2`, etc. and the engine rotates when a key hits rate limits instead of failing the whole run. For power users running daily queries, this is the difference between rate-limit 429s and zero-touch reliability.
|
||||
|
||||
- **The skill works on Windows now.** @Chelebii's PR ([#227](https://github.com/mvanhorn/last30days-skill/pull/227)) stabilized the vendored Bird X search client on Windows. Previously the bundled X backend had subtle runtime issues on Windows terminals; now it runs clean. Pair this with @Gujiassh's UTF-8 encoding fix ([#225](https://github.com/mvanhorn/last30days-skill/pull/225)) for saved output and Windows users get the full v3 experience without workarounds.
|
||||
|
||||
- **Linux permission checks stopped false-warning.** @george231224's PR ([#216](https://github.com/mvanhorn/last30days-skill/pull/216)) fixed `check_perms` on Linux by preferring GNU stat's syntax over the BSD stat that the skill was calling. Linux users were getting spurious permission warnings on `.env` files that were already correctly 600-chmod'd. Now the check matches reality.
|
||||
|
||||
- **Gemini CLI got a first-class install path.** @hnshah's docs PR ([#224](https://github.com/mvanhorn/last30days-skill/pull/224)) added the Gemini CLI install note and workaround for a rough edge in the Gemini skill loader. Gemini users now have a one-paragraph install flow in the README instead of having to reverse-engineer the plugin layout.
|
||||
|
||||
- **Offline quality evaluation.** @j-sperling's PR ([#233](https://github.com/mvanhorn/last30days-skill/pull/233)) added `eval_topics.json` as a fixture. Contributors and I can now run quality-regression checks on synthesis output without burning live API credits. This is the scaffolding that made the plan 015 validation gate affordable - without eval fixtures, testing 5/5 canonical compliance on every release would cost real money every time. Ships as contributor infrastructure but shows up as stability for end users.
|
||||
|
||||
- **Reddit client got a cleaner HTTP layer.** @iliaal shipped three architecture PRs back-to-back ([#207](https://github.com/mvanhorn/last30days-skill/pull/207), [#208](https://github.com/mvanhorn/last30days-skill/pull/208), [#209](https://github.com/mvanhorn/last30days-skill/pull/209)) that consolidated Reddit's HTTP handling into `http.get(params=...)`, rejected garbage input in `_parse_date`, and unified `_sc_headers` into `http.scrapecreators_headers`. End-user benefit: fewer flaky timeouts, fewer "weird parse error" crashes, a codebase that's easier for future contributors to touch without breaking Reddit. These aren't sexy PRs; they're the kind of refactor that prevents six future bug reports.
|
||||
|
||||
- **The `--days=N` flag keeps working.** @BryanTegomoh's PR ([#230](https://github.com/mvanhorn/last30days-skill/pull/230)) restored backcompat for the legacy `--days` alias so anyone who'd scripted against it in 2.x doesn't break on v3. Small PR, meaningful reliability gain for existing users.
|
||||
|
||||
- **INCLUDE_SOURCES has a sane default.** @hnshah's PR ([#223](https://github.com/mvanhorn/last30days-skill/pull/223)) defaulted the env var to empty string instead of unset. Missing env no longer breaks source inclusion on fresh installs.
|
||||
|
||||
- **Version metadata stays in sync.** @Gujiassh's PR ([#217](https://github.com/mvanhorn/last30days-skill/pull/217)) aligned the SKILL.md version header with the sync target version, and @shalomma's PR ([#229](https://github.com/mvanhorn/last30days-skill/pull/229)) closed the remaining drift between the SKILL.md header and plugin.json. "Which version am I actually on" is no longer an adventure.
|
||||
|
||||
- **Bird X engagement handling got hardened.** @j-sperling's PR ([#234](https://github.com/mvanhorn/last30days-skill/pull/234)) made `bird_x` skip all-None engagement dicts instead of crashing on them. Rare condition, but the kind of thing that silently kills a run on a specific topic.
|
||||
|
||||
- **Dev workflow hygiene.** @j-sperling's gitignore PR ([#232](https://github.com/mvanhorn/last30days-skill/pull/232)) dropped `.venv`, `.coverage`, `htmlcov`, and `.memsearch` from the tracked tree. Contributor quality-of-life; keeps PR diffs clean.
|
||||
|
||||
- **The skill installs to claude.ai.** PRs [#242](https://github.com/mvanhorn/last30days-skill/pull/242) and [#244](https://github.com/mvanhorn/last30days-skill/pull/244) shipped `scripts/build-skill.sh` plus the `.gitattributes` + `export-ignore` plumbing that packages last30days into a claude.ai-upload-ready `.skill` file under the 200-file cap. The skill is no longer Claude-Code-only - it installs directly on claude.ai, too. README has the upload workflow.
|
||||
|
||||
- **OpenAI Codex CLI discovers the skill natively.** PR [#219](https://github.com/mvanhorn/last30days-skill/pull/219) added `.agents/skills/last30days/SKILL.md` as a real file (not symlinked - Codex's loader skips symlinks) plus `.codex-plugin/plugin.json` as the namespace marker. The skill now shows up as `last30days:last30days` when Codex runs in a checkout. Inspired by @Jah-yee ([#153](https://github.com/mvanhorn/last30days-skill/pull/153)) and @dannyshmueli on X.
|
||||
|
||||
- **`/last30days` as a slash command.** PR [#267](https://github.com/mvanhorn/last30days-skill/pull/267) added `commands/last30days.md` so plugin users can type `/last30days <topic>` and Claude Code autocomplete prefix-matches it to the canonical `/last30days:last30days` form. No more typing the double-namespace.
|
||||
|
||||
## The self-debug technique, for anyone rebuilding this elsewhere
|
||||
|
||||
The breakthrough wasn't the individual fixes. It was the realization that instead of guessing why the model was ignoring the rules, I should ask the model. Five separate Opus 4.7 sessions debugged their own outputs:
|
||||
|
||||
- "Did you read SKILL.md?" → "I tried Read, hit the 25K token cap, and bailed instead of chunked-reading."
|
||||
- "Why the trailing Sources block?" → "The WebSearch tool's own reminder said MANDATORY. Precedence was unclear."
|
||||
- "Why the section headers?" → "I had strong priors on Peter Steinberger and wrote my thesis instead of passing through."
|
||||
- "Why the wrong file?" → "I read `.agents/skills/last30days/SKILL.md` first because it appeared in the path glob."
|
||||
|
||||
Three of the five said "move the LAWs to the top." Two said "make the engine enforce it so the model can't not comply." I shipped both. That's the whole technique: when the LLM-under-orchestration keeps breaking the contract, don't argue with it - ask it to debug itself, and build structural enforcement around whatever it names as the root cause.
|
||||
|
||||
## Thank you
|
||||
|
||||
**Community PR authors since v3.0.0:**
|
||||
- @j-sperling - v3 engine architecture, eval fixtures, gitignore hygiene, Bird X hardening ([#232](https://github.com/mvanhorn/last30days-skill/pull/232), [#233](https://github.com/mvanhorn/last30days-skill/pull/233), [#234](https://github.com/mvanhorn/last30days-skill/pull/234))
|
||||
- @stephenmcconnachie - Hermes AI Agent support ([#228](https://github.com/mvanhorn/last30days-skill/pull/228))
|
||||
- @zaydiscold - Multi-key SCRAPECREATORS rotation ([#268](https://github.com/mvanhorn/last30days-skill/pull/268))
|
||||
- @iliaal - Reddit HTTP helper + GitHub date parsing + ScrapeCreators header consolidation ([#207](https://github.com/mvanhorn/last30days-skill/pull/207), [#208](https://github.com/mvanhorn/last30days-skill/pull/208), [#209](https://github.com/mvanhorn/last30days-skill/pull/209))
|
||||
- @Chelebii - Windows Bird X stability ([#227](https://github.com/mvanhorn/last30days-skill/pull/227))
|
||||
- @george231224 - Linux check_perms stat ([#216](https://github.com/mvanhorn/last30days-skill/pull/216))
|
||||
- @Gujiassh - UTF-8 saved output + version metadata alignment ([#217](https://github.com/mvanhorn/last30days-skill/pull/217), [#225](https://github.com/mvanhorn/last30days-skill/pull/225))
|
||||
- @hnshah - INCLUDE_SOURCES default + Gemini install docs ([#223](https://github.com/mvanhorn/last30days-skill/pull/223), [#224](https://github.com/mvanhorn/last30days-skill/pull/224))
|
||||
- @shalomma - SKILL.md v3.0.0 version header ([#229](https://github.com/mvanhorn/last30days-skill/pull/229))
|
||||
- @BryanTegomoh - --days alias backcompat ([#230](https://github.com/mvanhorn/last30days-skill/pull/230))
|
||||
|
||||
**v3 roadmap contributors (issues and PRs that shaped the v3 feature set):**
|
||||
- @uppinote20 - rich Reddit comments ([#143](https://github.com/mvanhorn/last30days-skill/pull/143))
|
||||
- @zerone0x - GitHub as a first-class source ([#134](https://github.com/mvanhorn/last30days-skill/issues/134), [#136](https://github.com/mvanhorn/last30days-skill/pull/136))
|
||||
- @thinkun - Reddit enrichment timeout handling ([#116](https://github.com/mvanhorn/last30days-skill/pull/116))
|
||||
- @thomasmktong - pure-Python Reddit fallback ([#124](https://github.com/mvanhorn/last30days-skill/pull/124))
|
||||
- @fanispoulinakisai-boop - Reddit timeout report ([#100](https://github.com/mvanhorn/last30days-skill/issues/100))
|
||||
- @pejmanjohn - plugin directory naming ([#99](https://github.com/mvanhorn/last30days-skill/issues/99), [#78](https://github.com/mvanhorn/last30days-skill/issues/78))
|
||||
- @zl190 - HN trending merge ([#115](https://github.com/mvanhorn/last30days-skill/pull/115))
|
||||
- @hnshah - Watchlist features ([#84](https://github.com/mvanhorn/last30days-skill/pull/84), [#85](https://github.com/mvanhorn/last30days-skill/pull/85), [#86](https://github.com/mvanhorn/last30days-skill/pull/86))
|
||||
- @Jah-yee, @dannyshmueli - Codex CLI discovery
|
||||
- @Cody-Coyote - marketplace validation bug report ([#204](https://github.com/mvanhorn/last30days-skill/issues/204))
|
||||
|
||||
**The five Opus 4.7 instances that debugged their own failures on v3.0.7 and v3.0.8 and converged on the fixes.** The convergence was the breakthrough; this release is their diagnosis in code.
|
||||
|
||||
## Install / Update
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days@last30days-skill
|
||||
```
|
||||
|
||||
Or if already installed:
|
||||
|
||||
```
|
||||
/plugin update last30days
|
||||
/reload-plugins
|
||||
```
|
||||
|
||||
## Verify
|
||||
|
||||
```
|
||||
cat ~/.claude/plugins/cache/last30days-skill/last30days/*/.claude-plugin/plugin.json | grep version
|
||||
```
|
||||
|
||||
Should print `"version": "3.0.9"`.
|
||||
|
||||
## Smoke test
|
||||
|
||||
```
|
||||
/last30days birthday gift for 40 year old
|
||||
```
|
||||
|
||||
Should ask a clarifying question before running. If it runs the engine anyway, the cache is stale - repeat the plugin update.
|
||||
|
||||
**Full Changelog:** https://github.com/mvanhorn/last30days-skill/compare/v3.0.5...v3.0.9
|
||||
@@ -0,0 +1,48 @@
|
||||
# Search Quality Eval
|
||||
|
||||
`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
|
||||
What it does:
|
||||
|
||||
- runs a baseline revision (default `origin/main`) against a candidate checkout
|
||||
- evaluates the fixed 5 reviewer topics by default
|
||||
- computes deterministic stability metrics:
|
||||
- `Jaccard` overlap vs baseline
|
||||
- retention vs baseline
|
||||
- per-source counts and overlap
|
||||
- optionally calls Gemini as a judge for graded relevance labels and then computes:
|
||||
- `Precision@5`
|
||||
- `nDCG@5`
|
||||
- source-coverage recall across the judged union pool
|
||||
|
||||
Recommended usage:
|
||||
|
||||
```bash
|
||||
uv run python scripts/evaluate_search_quality.py
|
||||
```
|
||||
|
||||
Useful flags:
|
||||
|
||||
```bash
|
||||
uv run python scripts/evaluate_search_quality.py \
|
||||
--baseline-rev origin/main \
|
||||
--candidate-rev HEAD \
|
||||
--no-default-topics \
|
||||
--topic "cursor IDE pricing" \
|
||||
--per-source-limit 5
|
||||
```
|
||||
|
||||
Gemini configuration:
|
||||
|
||||
- preferred on this workspace: set `GOOGLE_API_KEY`
|
||||
- also accepted: `GEMINI_API_KEY` or `GOOGLE_GENAI_API_KEY`
|
||||
- optional: set `GEMINI_MODEL`
|
||||
- default model is `gemini-3-pro-preview` for the direct Gemini API
|
||||
|
||||
Notes:
|
||||
|
||||
- The script forces a clean env-based auth path when it shells out to `last30days.py`.
|
||||
- It passes `XAI_API_KEY`, `OPENAI_API_KEY`, and `SCRAPECREATORS_API_KEY`, but intentionally does not pass browser-cookie X auth. That keeps evaluation runs on the popup-free path.
|
||||
- It also strips `node` from the eval `PATH` and wraps `yt-dlp` with `--ignore-config`, so older revisions do not inherit local browser-cookie config either.
|
||||
- `Jaccard` and retention are regression guards, not truth metrics.
|
||||
- `Precision@5` and `nDCG@5` are only as good as the judged pool. They help compare revisions, but they are not a substitute for a larger labeled benchmark.
|
||||
@@ -0,0 +1,388 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -0,0 +1,310 @@
|
||||
# V1 vs V2 Comparison Analysis
|
||||
|
||||
**Date:** 2026-02-06
|
||||
**Queries tested:** 4 (1 head-to-head, 3 V1-only)
|
||||
**Scope:** Quick smoke test, not full 17-query matrix
|
||||
|
||||
---
|
||||
|
||||
## Part 1: Head-to-Head -- "kanye west" (NEWS Query)
|
||||
|
||||
### Dimension-by-Dimension Scoring
|
||||
|
||||
#### 1. Query Parsing Display
|
||||
|
||||
Does it show the `🔍 **{TOPIC}** · {QUERY_TYPE}` line before running tools?
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 1 | No parsing display at all. Output starts with "## What I learned:" -- jumps straight into synthesis. No acknowledgment of topic or query type before research. |
|
||||
| V2 | 1 | No parsing display either. Output starts with "Here's what I found:" then "## What I learned:" -- same problem as V1. |
|
||||
|
||||
**Analysis:** Neither version actually rendered the query parsing display. V2 SKILL.md explicitly requires `🔍 **kanye west** · News` before any tools run, but the agent did not produce it. This is a V2 instruction that failed to land. Both score 1/5.
|
||||
|
||||
Possible cause: The parsing display is supposed to appear *before* tools are called -- it may have been shown during execution but not captured in the final output text. If so, both outputs represent only the post-research synthesis, not the full session. Regardless, based on what is in the output files, neither shows it.
|
||||
|
||||
---
|
||||
|
||||
#### 2. Source Coverage (Reddit/X/Web counts)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | `Reddit: 0 relevant threads` / `X: 30 posts │ ~10 likes` / `Web: 20+ pages`. Two of three sources returned results. Reddit was zero. |
|
||||
| V2 | 3 | `Reddit: 0 threads (no results this cycle)` / `X: 29 posts │ 33 likes │ 14 reposts` / `Web: 30+ pages`. Same pattern: two of three returned results. |
|
||||
|
||||
**Analysis:** Nearly identical coverage. Both got zero Reddit results (likely a script/API issue for this topic, not a SKILL.md problem). V2 has slightly more precise X metrics (33 likes, 14 reposts vs. V1's vague "~10 likes"). V2 has more web pages (30+ vs 20+). Both miss the 10+ Reddit threshold for a score of 4+.
|
||||
|
||||
---
|
||||
|
||||
#### 3. Citation Quality (sparse vs every-sentence)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 2 | No inline citations at all. The body text makes claims ("full-page Wall Street Journal apology," "Hellwatt Festival in Italy") but never attributes them to a specific source. The stats box lists "Washington Post, Billboard, AllHipHop" but the body has zero `per @handle` or `per Rolling Stone` attributions. |
|
||||
| V2 | 5 | Every bold section ends with a sparse, clean citation. Examples: `"per Rolling Stone"`, `"per The Washington Post"`, `"per Billboard"`, `"per AllHipHop"`, `"per The News International"`. One citation per topic, never chained. Exactly what V2 SKILL.md specifies. |
|
||||
|
||||
**Analysis:** This is the single biggest quality gap between V1 and V2. V1's output reads like a Wikipedia summary -- informative but ungrounded. V2 reads like a researched briefing where every claim has a named source. V2 nails the "sparse citation" rule from its SKILL.md: `"cite 1 source per pattern, short format: 'per @handle' or 'per r/sub'"`.
|
||||
|
||||
V1 quote (no citation): `"He'll headline the new Hellwatt Festival in Italy (July 4-18, 2026)."`
|
||||
V2 quote (cited): `"Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard."`
|
||||
|
||||
---
|
||||
|
||||
#### 4. Summary Structure (bold topic headers, organized sections)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | Has a coherent narrative structure with a paragraph of synthesis, then a `**KEY THEMES:**` numbered list. But the opening is a single dense paragraph, not broken into scannable sections with bold headers. |
|
||||
| V2 | 5 | Each storyline gets its own bold header: `**BULLY Album — March 20, 2026 via Gamma**`, `**Public Apology for Antisemitism**`, `**Hellwatt Festival in Italy**`, `**Health Concerns**`, `**Grammys Ban**`, `**Kim & Lewis Hamilton Buzz**`. Each is a standalone scannable unit with 1-3 sentences. |
|
||||
|
||||
**Analysis:** V2 follows the SKILL.md template exactly: `**{Topic 1}** — [1-2 sentences, per source]`. V1 uses a blob + list approach which is readable but less scannable. V2 is notably better for a user who wants to skim and find the story they care about.
|
||||
|
||||
V1 structure: 1 dense paragraph -> 5-item `KEY THEMES` list
|
||||
V2 structure: 6 bold topic cards, each self-contained -> no KEY THEMES list (but doesn't need one because the structure itself is the organization)
|
||||
|
||||
---
|
||||
|
||||
#### 5. Stats Box Format (emoji tree vs plain text)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 4 | Uses `├─` tree format with emoji: `├─ 🟠 Reddit: 0 relevant threads` / `├─ 🔵 X: 30 posts` / `├─ 🌐 Web: 20+ pages` / `└─ Top voices:`. Minor deviation: says "0 relevant threads (filtered out noise)" instead of the V1 SKILL.md template "0 threads (no results this cycle)". Also omits the `🗣️` emoji on the Top voices line. |
|
||||
| V2 | 5 | Perfect match to V2 SKILL.md template: `├─ 🟠 Reddit: 0 threads (no results this cycle)` / `├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)` / `├─ 🌐 Web: 30+ pages │ rollingstone.com, ...` / `└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex`. Includes `(via xAI)` notation, `🗣️` emoji, @handles with engagement counts. |
|
||||
|
||||
**Analysis:** V2 is tighter and matches its template exactly. V1 is close but has minor deviations (custom "filtered out noise" text, missing `🗣️` emoji, no @handles or engagement counts on Top voices). V2's inclusion of actual @handles with like counts (`@honest30bgfan_ (33 likes)`) adds credibility.
|
||||
|
||||
---
|
||||
|
||||
#### 6. Research Grounding (actual research vs generic knowledge)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 4 | Clearly grounded: mentions specific details like "Wall Street Journal apology (Jan 26, 2026)," "four-month-long manic episode," "frontal-lobe brain injury," "North West collaborated on 'Piercings on My Hand,'" "Monumental Plaza de Toros." These are specific enough to be from research, not pre-training. Minor generic leakage: the "KEY THEMES" list uses editorial framing ("Accountability arc," "Mental health transparency") that feels more like analysis than research extraction. |
|
||||
| V2 | 5 | Every fact is specific and attributed: "12th studio album," "13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign," "earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded," "103,000-capacity RCF Arena." The AI-deepfaked vocals detail is a standout -- it is clearly from research, not something a model would know from pre-training. The Kim/Lewis Hamilton item (`"X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton"`) is explicitly sourced from X data, not general knowledge. |
|
||||
|
||||
**Analysis:** Both are well-grounded, but V2 has more "could only come from research" details. The deepfaked vocals story, the exact venue capacity, and the explicit X chatter observation are details that prove the synthesis is from the research output, not hallucinated.
|
||||
|
||||
---
|
||||
|
||||
#### 7. Prompt Quality (invitation to share vision, not dumping prompts)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | Ends with: `"Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in."` This is a follow-up invitation, but it is NOT the SKILL.md-specified invitation. It is topic-specific and conversational, which is nice, but it does not ask the user to "share your vision for what you want to create." It misses the prompt-generation angle entirely. |
|
||||
| V2 | 5 | Ends with exactly: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice."` This matches the V2 SKILL.md template verbatim. It positions the skill correctly: not a news summarizer but a research-to-prompt pipeline. |
|
||||
|
||||
**Analysis:** V1's closing is friendly but off-brand. It treats the skill as a research tool, not a research-to-prompt tool. V2 correctly frames the next step as "tell me what to create and I'll write the prompt." This is a meaningful difference -- V1 would leave a user thinking they just got a summary, while V2 primes them to get a usable output.
|
||||
|
||||
---
|
||||
|
||||
### Head-to-Head Scorecard
|
||||
|
||||
| Dimension | V1 | V2 | Winner |
|
||||
|-----------|----|----|--------|
|
||||
| 1. Query Parsing Display | 1 | 1 | Tie (both failed) |
|
||||
| 2. Source Coverage | 3 | 3 | Tie |
|
||||
| 3. Citation Quality | 2 | 5 | **V2 (+3)** |
|
||||
| 4. Summary Structure | 3 | 5 | **V2 (+2)** |
|
||||
| 5. Stats Box Format | 4 | 5 | **V2 (+1)** |
|
||||
| 6. Research Grounding | 4 | 5 | **V2 (+1)** |
|
||||
| 7. Prompt Quality (invitation) | 3 | 5 | **V2 (+2)** |
|
||||
| **TOTAL** | **20/35** | **29/35** | **V2 wins by 9 points** |
|
||||
|
||||
**V2 is clearly better.** The biggest gaps are citation quality (+3) and summary structure (+2). V2's output reads like a professional research briefing; V1's reads like a decent but unstructured summary.
|
||||
|
||||
---
|
||||
|
||||
## Part 2: V1-Only Outputs Analysis
|
||||
|
||||
### Output 1: "open claw" (GENERAL query)
|
||||
|
||||
**What V1 does well:**
|
||||
- Strong research grounding. Mentions exact numbers: "145,000+ GitHub stars," "20,000+ forks," "700+ skills," "341 malicious skills." These are clearly from research.
|
||||
- The KEY PATTERNS section is excellent: 5 well-organized patterns with community quotes (`"I give it sudo and let it configure everything"` vs `"prompt injection is terrifying when you give the bot access to your actual bank account"`).
|
||||
- Good synthesis of the security vs. enthusiasm tension -- captures the community split accurately.
|
||||
- Stats box uses the emoji tree format correctly with `├──` (though note: uses double-dash `──` instead of single `─`, minor inconsistency).
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display (`🔍 **open claw** · General`).
|
||||
- No inline citations in the body text. The 5 KEY PATTERNS have no `per @handle` or `per r/sub` attribution. Which Reddit thread said "I give it sudo"? Which X post raised the security concern? We do not know.
|
||||
- The stats box says `├── 🟠 Reddit: 25 threads │ ~750+ upvotes` -- the tilde and plus are imprecise. V2 SKILL.md wants exact parsed numbers.
|
||||
- Top voices line lists subreddits and handles but no engagement counts: `@grok, @Starlink` -- are these the highest-engagement handles? No like counts shown.
|
||||
- No bold topic headers in the body -- it is a single paragraph followed by a numbered list, not the `**{Topic}** — sentence, per source` format V2 requires.
|
||||
|
||||
**V1 Score (estimated):** 22/35
|
||||
|
||||
---
|
||||
|
||||
### Output 2: "nano banana pro prompting" (PROMPTING query)
|
||||
|
||||
**What V1 does well:**
|
||||
- Correctly identifies two prompting styles (JSON structured vs. natural language "Creative Director") and explains when each works best. This is excellent PROMPTING-type synthesis.
|
||||
- KEY PATTERNS are specific and actionable: "85mm lens at f/1.8," "three-point lighting with key at 45 degrees," "text rendering works -- keep text under 3 words for best results (75% success rate)." These are concrete tips a user can apply immediately.
|
||||
- Research grounding is strong: cites specific upvote counts ("149-259 upvotes"), subreddit names (`r/nanobanana2pro`), and the Google AI blog.
|
||||
- The invitation correctly targets Nano Banana Pro: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro."`
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display.
|
||||
- Stats box uses plain text dashes: `- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments` instead of the tree format `├─ 🟠 Reddit:`. Uses `|` pipe instead of `│` box-drawing character. V2 SKILL.md explicitly says: "NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji."
|
||||
- No inline body citations. KEY PATTERNS mention Reddit upvote ranges but no specific `per @handle` attributions.
|
||||
- Missing `✅ All agents reported back!` header -- just says "All agents reported back!" without the checkmark.
|
||||
- Body structure is paragraph + numbered list, not bold topic headers.
|
||||
|
||||
**V1 Score (estimated):** 23/35 (slightly higher than open claw due to better actionability)
|
||||
|
||||
---
|
||||
|
||||
### Output 3: "how to best setup clawdbot" (HOW-TO query)
|
||||
|
||||
**What V1 does well:**
|
||||
- This is the best V1 output of the batch. It goes beyond synthesis and actually delivers a **Quick-Start guide** with numbered steps, a **Security Hardening** checklist, and a **Budget Option** -- all grounded in research.
|
||||
- Excellent research grounding: `"per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0"` -- this is an actual citation with an @handle!
|
||||
- Specific, actionable recommendations: exact commands (`curl -fsSL https://clawd.bot/install.sh | bash`), specific model recommendations (Claude Opus 4.5 for best results, GLM 4.7 Flash for local), specific channel advice (Telegram first, WhatsApp QR code fails).
|
||||
- Stats box is correct emoji tree format with engagement counts: `@aashatwt (452 likes), @recap_david (329 likes)`.
|
||||
- Captures the naming confusion accurately: "Clawdbot -> Moltbot -> OpenClaw."
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display.
|
||||
- Body text has no inline citations except the Budget Option section. The 5 KEY PATTERNS have no `per @handle` attribution.
|
||||
- Bold topic headers are used only in the Quick-Start and Security sections, not in the KEY PATTERNS or intro.
|
||||
- The output delivers the "answer" directly (setup guide) rather than waiting for the user's vision and offering to write a prompt. For a HOW-TO query this might be the right call, but it skips the SKILL.md flow of "show research -> invite vision -> write prompt."
|
||||
|
||||
**V1 Score (estimated):** 26/35 (best of the V1 outputs)
|
||||
|
||||
---
|
||||
|
||||
### Patterns Across All V1 Outputs
|
||||
|
||||
**Consistent strengths:**
|
||||
1. Research grounding is solid across all three. V1 does not hallucinate -- the facts are clearly from the research output, not pre-training.
|
||||
2. KEY PATTERNS lists are consistently useful and actionable.
|
||||
3. Stats boxes are present in all outputs (though formatting varies).
|
||||
4. The invitation/closing line is present in all outputs.
|
||||
|
||||
**Consistent weaknesses:**
|
||||
1. **No query parsing display** in any output (0 for 4, including Kanye West).
|
||||
2. **No inline citations** in the body text (except one @handle in the clawdbot output). The research feels real but is unattributed.
|
||||
3. **Stats box formatting is inconsistent.** Open claw uses `├──` (double dash), nano banana pro uses `- 🟠` (plain dash + pipe), clawdbot uses `├─` (correct). Three different formats in three outputs.
|
||||
4. **Body structure defaults to paragraph + numbered list** instead of bold topic headers. Only clawdbot partially uses bold headers (in the guide section, not the research section).
|
||||
5. **No `(via Bird/xAI)` notation** on X stats in any output.
|
||||
|
||||
---
|
||||
|
||||
## Part 3: SKILL.md Feature Diff
|
||||
|
||||
### Features in V2 but NOT V1
|
||||
|
||||
| Feature | V2 Lines | Impact |
|
||||
|---------|----------|--------|
|
||||
| **Query parsing display** (`🔍 **{TOPIC}** · {QUERY_TYPE}`) | 40-53 | HIGH -- confirms to user the skill understood their request before spending time on research. |
|
||||
| **Sparse citation rules** with BAD/GOOD examples | 186-193 | HIGH -- this is the #1 quality differentiator in the Kanye head-to-head. `"per @handle"` format, never chain multiple citations. |
|
||||
| **Bold topic headers** template (`**{Topic 1}** — [1-2 sentences, per source]`) | 195-208 | HIGH -- makes output scannable. |
|
||||
| **Strict stats template** with "NEVER use plain text dashes" instruction | 217-230 | MEDIUM -- prevents the formatting inconsistency seen across V1 outputs. |
|
||||
| **RECOMMENDATIONS source attribution** (each item MUST have Sources: line with @handles) | 178-182 | MEDIUM -- only affects RECOMMENDATIONS queries. |
|
||||
| **Reddit 0 results handling** (explicit instruction for what to write) | 229 | LOW -- edge case, but prevents ad-hoc text like V1's "filtered out noise." |
|
||||
| **Bird CLI / xAI notation** in stats | 223 | LOW -- cosmetic transparency about data source. |
|
||||
| **Step 2 phrasing: "DO WEBSEARCH WHILE SCRIPT RUNS"** | 71-73 | LOW -- execution optimization, no output impact. |
|
||||
|
||||
### Features in V1 but NOT V2
|
||||
|
||||
| Feature | V1 Lines | Impact | Should Restore? |
|
||||
|---------|----------|--------|-----------------|
|
||||
| **Use cases block** (4 examples in intro) | 12-17 | LOW | No |
|
||||
| **Setup Check section** (3 modes, bash script, "keys are OPTIONAL") | 50-78 | MEDIUM for new users | Yes, for public release |
|
||||
| **BAD/GOOD synthesis anti-pattern examples** | 172-191 | MEDIUM-HIGH | YES |
|
||||
| **Self-check instruction** ("Re-read your 'What I learned' section...") | 269 | MEDIUM | YES |
|
||||
| **Quality Checklist** (5-point checklist before delivering prompt) | 306-324 | HIGH | YES |
|
||||
| **Prompt format anti-pattern** ("Research says JSON but you write prose") | 302 | MEDIUM | YES |
|
||||
| **"IF USER ASKS FOR MORE OPTIONS"** section | 327-329 | LOW-MEDIUM | YES |
|
||||
| **Web-only mode stats template + promo** | 248-259 | MEDIUM for no-key users | For public release |
|
||||
| **TARGET_TOOL question template** (4 options) | 272-280 | LOW | No |
|
||||
| **Context Memory: explicit "don't re-search" instructions** | 342-358 | MEDIUM | YES |
|
||||
| **Output footer emoji + engagement counts** | 366-380 | LOW | YES |
|
||||
|
||||
### Features in BOTH (Shared)
|
||||
|
||||
| Feature | Notes |
|
||||
|---------|-------|
|
||||
| Parse User Intent (TOPIC, TARGET_TOOL, QUERY_TYPE) | Same 4 query types, same detection logic |
|
||||
| "Don't ask about tool before research" rule | Identical |
|
||||
| Research script execution command | Same `python3` command |
|
||||
| WebSearch queries by QUERY_TYPE | Same search strategies |
|
||||
| "Use user's exact terminology" instruction | V2 shorter but same intent |
|
||||
| Judge Agent synthesis logic | Same 5-step weighting process |
|
||||
| "Ground in actual research" instruction | Same core instruction, V1 has more examples |
|
||||
| RECOMMENDATIONS: extract specific names | Same logic |
|
||||
| Prompt format matching | Same instruction |
|
||||
| Wait for user's vision | Same |
|
||||
| Write ONE perfect prompt | Same structure |
|
||||
| Context Memory | V2 shorter version |
|
||||
| Output summary footer | Both have it, V1 has emoji |
|
||||
| Depth options (quick/default/deep) | Same |
|
||||
| "After each prompt: Stay in Expert Mode" | Same |
|
||||
|
||||
### Overall Assessment
|
||||
|
||||
**V2 is a clear upgrade in output formatting and citation quality.** The three features V2 adds (query parsing display, sparse citation rules, bold topic headers) directly address the three biggest weaknesses seen across all V1 outputs. The Kanye West head-to-head proves it: V2 scores 29/35 vs V1's 20/35.
|
||||
|
||||
**However, V2 dropped several quality guardrails from V1** that do not affect formatting but affect *correctness*: the self-check instruction, the anti-pattern examples, the quality checklist for prompts, and the "don't re-search" context memory rule. These are cheap to restore (under 25 lines total) and protect against subtle failure modes that may not show up in a 1-query test but will appear over dozens of uses.
|
||||
|
||||
---
|
||||
|
||||
## Part 4: Verdict
|
||||
|
||||
### Ship V2 or Not?
|
||||
|
||||
**Ship V2 -- but restore the guardrails first.**
|
||||
|
||||
V2 is unambiguously better on every formatting dimension. The citation quality improvement alone (V1: 2/5 -> V2: 5/5) makes it worth shipping. The bold topic headers and strict stats template fix the inconsistency problems visible across all V1 outputs.
|
||||
|
||||
But V2 dropped 6 guardrail features from V1 that cost almost nothing to include and protect against real failure modes. These should be restored before V2 goes public.
|
||||
|
||||
### Remaining Gaps
|
||||
|
||||
**Must fix before shipping (affects correctness):**
|
||||
|
||||
1. **Restore the quality checklist for prompts.** This is the test plan's #1 priority item. V1 had a 5-point checklist; V2 reduced it to one line. The checklist is what makes prompts feel polished -- it is the "that's a great prompt" mechanism. Add 8 lines.
|
||||
|
||||
2. **Restore BAD/GOOD anti-pattern examples.** V2 says "ground in actual research" but does not show what *bad* grounding looks like. V1's ClawdBot/Claude Code conflation example is exactly the kind of concrete negative example that prevents real failures. Add 5 lines.
|
||||
|
||||
3. **Restore self-check instruction.** One sentence: "Re-read your 'What I learned' section -- does it match what the research ACTUALLY says?" Zero cost, catches hallucination. Add 2 lines.
|
||||
|
||||
4. **Restore "don't re-search" context memory rule.** V2 only says "only do new research if user asks about a DIFFERENT topic." V1 explicitly bans re-searching and tells the agent to answer from existing research. Add 3 lines.
|
||||
|
||||
**Should fix (polish):**
|
||||
|
||||
5. Restore prompt format anti-pattern ("Research says JSON but you write prose"). Add 2 lines.
|
||||
6. Restore "IF USER ASKS FOR MORE OPTIONS" section. Add 2 lines.
|
||||
7. Add emoji + engagement counts back to the output summary footer. Edit 3 lines.
|
||||
|
||||
**Skip for now:**
|
||||
|
||||
8. Setup Check section -- add back for public release, not needed for execution.
|
||||
9. Web-only mode stats template -- lower priority, most testers have API keys.
|
||||
10. TARGET_TOOL question template -- agent handles this naturally.
|
||||
|
||||
### Query Parsing Display: Investigate
|
||||
|
||||
Both V1 and V2 scored 1/5 on query parsing display. V2 has the feature in its SKILL.md but the agent did not render it in the captured output. This could mean:
|
||||
- The display was shown during execution but not captured (likely -- it appears before tools run, and the output files may only contain post-research content).
|
||||
- The instruction is not strong enough and the agent skips it.
|
||||
|
||||
**Recommendation:** Verify in a live session whether the parsing display actually appears. If it does not, strengthen the instruction (e.g., "This line MUST be the first thing you output, before any tool calls").
|
||||
|
||||
### Total Effort
|
||||
|
||||
Restoring all 7 priority items: approximately 25 lines added to V2 SKILL.md. Under 15 minutes of work. The V2 formatting wins are substantial and proven; the V1 guardrails are small and proven. Combining both produces the best version.
|
||||
|
||||
### Final Score Summary
|
||||
|
||||
| | V1 (Kanye) | V2 (Kanye) | Delta |
|
||||
|--|-----------|-----------|-------|
|
||||
| Total | 20/35 | 29/35 | **V2 +9** |
|
||||
|
||||
| | V1 (Open Claw) | V1 (Nano Banana) | V1 (Clawdbot) | V1 Average |
|
||||
|--|---------------|-----------------|--------------|------------|
|
||||
| Estimated Total | 22/35 | 23/35 | 26/35 | **23.7/35** |
|
||||
|
||||
V2 at 29/35 beats every V1 output, including V1's best (clawdbot at 26/35).
|
||||
|
||||
**Decision: Ship V2 with guardrails restored.**
|
||||
@@ -0,0 +1,388 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -0,0 +1,25 @@
|
||||
## What I learned:
|
||||
|
||||
The last 30 days have been one of Kanye West's most significant periods in years. The biggest story is his **full-page Wall Street Journal apology** (Jan 26, 2026) titled "To Those I've Hurt" — a comprehensive open letter addressing his antisemitic remarks and erratic behavior. He attributed his actions to a "four-month-long manic episode" and revealed that a car accident 25 years ago caused a frontal-lobe brain injury that wasn't properly diagnosed until 2023, leading to his bipolar type-1 diagnosis. Multiple sources describe wife **Bianca Censori** as instrumental in getting him to seek treatment during a manic depressive episode.
|
||||
|
||||
On the music side, he's announced **two upcoming albums**: *BULLY* (now scheduled for March 20, 2026) and *IN A PERFECT WORLD*. Daughter **North West** collaborated with him on a track called "Piercings on My Hand." He performed a massive show in Mexico City (Jan 30) at the Monumental Plaza de Toros, and he'll headline the new **Hellwatt Festival** in Italy (July 4-18, 2026).
|
||||
|
||||
The **2026 Grammys** remain contentious — he's reportedly not invited after last year's red carpet drama. Health concerns also surfaced after a January LA outing where he appeared "noticeably bloated and moving with sluggish energy."
|
||||
|
||||
**KEY THEMES:**
|
||||
1. **Accountability arc** — The WSJ apology is being widely discussed as either genuine growth or calculated PR
|
||||
2. **Mental health transparency** — Brain injury + bipolar diagnosis framing his past behavior
|
||||
3. **Bianca Censori's role** — Increasingly seen as a stabilizing force (despite continued provocative public appearances)
|
||||
4. **Music comeback** — Two albums announced, active touring in Latin America and Europe
|
||||
5. **Grammy exile** — Still persona non grata at major award shows
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 0 relevant threads (filtered out noise)
|
||||
├─ 🔵 X: 30 posts │ ~10 likes │ casual fan chatter + Bianca Censori discourse
|
||||
├─ 🌐 Web: 20+ pages │ washingtonpost.com, billboard.com, allhiphop.com, euronews.com, rap-up.com
|
||||
└─ Top voices: Washington Post, Billboard, AllHipHop, Euronews, The Mirror
|
||||
|
||||
---
|
||||
|
||||
Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in.
|
||||
@@ -0,0 +1,24 @@
|
||||
## What I learned:
|
||||
|
||||
OpenClaw (formerly ClawdBot, formerly Moltbot) is an open-source autonomous AI agent created by Peter Steinberger that went massively viral in late January 2026 — hitting 145,000+ GitHub stars and 20,000+ forks. It runs locally on your machine, connects to messaging platforms (Telegram, WhatsApp, Discord, Signal, etc.), and uses LLMs like Claude, GPT, or Gemini to actually perform tasks on your computer — file management, scheduling, web browsing, purchases, and more. Think "JARVIS for everyone."
|
||||
|
||||
The community is simultaneously excited and alarmed. The "ClawHub" skill marketplace has 700+ skills but a VirusTotal/Koi Security audit found **341 malicious skills** doing data exfiltration and crypto theft. A high-severity RCE vulnerability was also disclosed. The #1 most downloaded skill on ClawHub was confirmed malware.
|
||||
|
||||
**KEY PATTERNS from the research:**
|
||||
|
||||
1. **Cost is the top complaint** — API usage burns money fast; many users recommend Gemini CLI (free 1,000 req/day) over Claude/GPT for OpenClaw to control spend
|
||||
2. **Security is a dumpster fire** — Supply chain attacks via ClawHub skills, prompt injection risks when browsing the web, RCE vulnerabilities, and orgs are banning it at the EDR level
|
||||
3. **Setup is nontrivial** — Even after install, most users get stuck on skills/tools/automation config; the `openclaw onboard` wizard is the recommended path
|
||||
4. **The skill ecosystem is powerful but dangerous** — 700+ skills available, but no code signing, no real moderation, and active malware campaigns
|
||||
5. **Community is split** — Enthusiasts love the autonomy ("I give it sudo and let it configure everything"), while security folks are terrified ("prompt injection is terrifying when you give the bot access to your actual bank account")
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├── 🟠 Reddit: 25 threads │ ~750+ upvotes │ ~600+ comments
|
||||
├── 🔵 X: 27 posts │ 1,750+ likes │ 325+ reposts
|
||||
├── 🌐 Web: 20+ pages │ Wikipedia, Bloomberg, The Register, Hacker News, Nature, VirusTotal, DigitalOcean, Codecademy, Tom's Hardware
|
||||
└── Top voices: r/openclaw, r/cybersecurity, r/clawdbot, r/ArtificialInteligence │ @grok, @Starlink │ Nature, Bloomberg, ACM
|
||||
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly.
|
||||
@@ -0,0 +1,27 @@
|
||||
Now I have a comprehensive picture. Let me synthesize everything.
|
||||
|
||||
**What I learned:**
|
||||
|
||||
Nano Banana Pro is fundamentally different from older image models — it's a "thinking" model that understands intent, physics, and composition. The community has converged on two distinct prompting styles, and **both work well** depending on the use case:
|
||||
|
||||
1. **JSON structured prompts** — Dominant on Reddit (r/nanobanana2pro) and X. Power users share elaborate JSON objects with nested fields for subject, appearance, lighting, composition, and style. These produce the most consistent, controlled results for portraits, editorials, and product shots.
|
||||
|
||||
2. **Natural language "Creative Director" prompts** — Recommended by Google's official guide. Full sentences describing the scene as if briefing a photographer. Better for cinematic shots, complex edits, and iterative refinement.
|
||||
|
||||
**KEY PATTERNS I'll use:**
|
||||
1. **JSON format for maximum control** — The top Reddit posts (149-259 upvotes) all use structured JSON with nested objects: `subject > appearance > hair/skin/expression`, `lighting > key/fill/rim`, `camera > lens/angle/depth_of_field`
|
||||
2. **Be a Creative Director, not a tag spammer** — Replace "4k, realistic, beautiful" with specific technical directions: "85mm lens at f/1.8", "three-point lighting with key at 45°", "shallow depth of field with bokeh"
|
||||
3. **Provide context for WHY** — Adding purpose ("for a luxury fashion editorial", "for a high-end gourmet cookbook") helps the model infer appropriate styling, plating, lighting
|
||||
4. **Edit, don't regenerate** — When 80% is right, ask for the specific change conversationally
|
||||
5. **Text rendering works** — Keep text under 3 words for best results (75% success rate), specify font style and placement
|
||||
|
||||
---
|
||||
All agents reported back!
|
||||
- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments
|
||||
- 🔵 X: 29 posts | 312 likes | 117 reposts
|
||||
- 🌐 Web: 10+ pages | blog.google, dev.to, eweek.com, imagine.art, github.com, leonardo.ai, medium.com
|
||||
- Top voices: r/nanobanana2pro | @Xmira_belle, @doctorwasif, @chatgptpaglu | Google AI blog, DEV Community
|
||||
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro.
|
||||
@@ -0,0 +1,48 @@
|
||||
**What I learned:**
|
||||
|
||||
Clawdbot (now officially renamed **OpenClaw**) is a self-hosted, open-source personal AI assistant that connects to messaging apps (Telegram, WhatsApp, Discord, Slack, Signal, iMessage) and uses LLMs to act as an autonomous agent on your behalf. It's roughly one week old as a public project and the community is moving fast — the name has already changed twice (Clawdbot → Moltbot → OpenClaw).
|
||||
|
||||
**KEY PATTERNS from the research:**
|
||||
|
||||
1. **Security is the #1 concern** — Multiple Reddit threads and guides warn that many setups are wide open to the internet with zero auth. Run it on dedicated/isolated hardware (VM, Mac Mini, NAS), never your main machine. Block dangerous commands via allowlist. Never add it to group chats.
|
||||
|
||||
2. **Use the onboarding wizard, not Docker** — Docker was reported as unreliable (shims, bridges). The single-line install script (`curl -fsSL https://clawd.bot/install.sh | bash`) followed by `openclaw onboard` is the smoothest path. Docker is a "no-go" per multiple users.
|
||||
|
||||
3. **Model choice matters a lot** — Claude Opus 4.5 is recommended for best results (trained to resist prompt injection, ~99% resistance). For budget setups, Claude Haiku works at $0. Local models (Ollama/LM Studio) work but cause hallucination and token issues — GLM 4.7 Flash (30B) was recommended as the best local option.
|
||||
|
||||
4. **Telegram is the easiest channel to start with** — Most guides and users start with a Telegram bot token. WhatsApp QR code pairing fails frequently. Get Telegram working first, then expand.
|
||||
|
||||
5. **Sandbox and isolate** — Set `agents.defaults.sandbox.mode` to `"non-main"` for Docker sandboxing. Use a Linux VM or UTM on Mac to segregate traffic. Tailscale for secure remote access.
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 19 threads │ 415+ upvotes │ 290+ comments
|
||||
├─ 🔵 X: 29 posts │ 815+ likes │ 65+ reposts
|
||||
├─ 🌐 Web: 10+ pages │ Medium, DataCamp, DEV, GitHub, QNAP, docs.openclaw.ai
|
||||
└─ Top voices: r/clawdbot, r/LocalLLM, r/AskVibecoders │ @aashatwt (452 likes), @recap_david (329 likes) │ Lukas Niessen on Medium
|
||||
|
||||
---
|
||||
|
||||
**Here's the consensus best setup path based on what the community actually recommends:**
|
||||
|
||||
### Quick-Start (30 min)
|
||||
1. **Get dedicated hardware** — Mac Mini, old laptop, NAS (QNAP has an official guide), or a VPS (DigitalOcean gist available). Don't use your daily driver.
|
||||
2. **Install via script**: `curl -fsSL https://clawd.bot/install.sh | bash`
|
||||
3. **Run the wizard**: `openclaw onboard` — it walks through gateway, workspace, channels, and skills
|
||||
4. **Connect Telegram first** — create a bot via @BotFather, paste the token
|
||||
5. **Set your API key** — Anthropic API key for Claude, or connect Ollama for local models
|
||||
6. **Customize SOUL.md** — this controls your bot's personality and behavior
|
||||
|
||||
### Security Hardening (do this immediately)
|
||||
- Bind gateway to localhost only, use Tailscale for remote access
|
||||
- Set sandbox mode to `"non-main"`
|
||||
- Create an explicit command allowlist (block `rm -rf`, `sudo`, `chmod`)
|
||||
- Rotate API keys every 90 days
|
||||
- Never add the bot to group chats
|
||||
|
||||
### Budget Option ($0)
|
||||
Per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0.
|
||||
|
||||
---
|
||||
|
||||
Want me to help you with a specific part of the setup, or do you have a particular use case in mind (home automation, CRM, coding assistant, etc.)?
|
||||
@@ -1,37 +1,13 @@
|
||||
---
|
||||
name: last30days
|
||||
version: "2.1"
|
||||
description: "Research a topic from the last 30 days. Also triggered by 'last30'. Sources: Reddit, X, YouTube, web. Become an expert and write copy-paste-ready prompts."
|
||||
argument-hint: 'last30 AI video tools, last30 best project management tools'
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: '"[topic] for [tool]" or "[topic]"'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
homepage: https://github.com/mvanhorn/last30days-skill
|
||||
user-invocable: true
|
||||
disable-model-invocation: true
|
||||
metadata:
|
||||
clawdbot:
|
||||
emoji: "📰"
|
||||
requires:
|
||||
env:
|
||||
- OPENAI_API_KEY
|
||||
bins:
|
||||
- node
|
||||
- python3
|
||||
primaryEnv: OPENAI_API_KEY
|
||||
files:
|
||||
- "scripts/*"
|
||||
homepage: https://github.com/mvanhorn/last30days-skill
|
||||
tags:
|
||||
- research
|
||||
- reddit
|
||||
- x
|
||||
- youtube
|
||||
- trends
|
||||
- prompts
|
||||
---
|
||||
|
||||
# last30days v2.1: Research Any Topic from the Last 30 Days
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, YouTube, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
@@ -61,20 +37,18 @@ Common patterns:
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
**DISPLAY your parsing to the user.** Before running any tools, output:
|
||||
**DISPLAY your parsing to the user.** Before running any tools, output a single line:
|
||||
|
||||
```
|
||||
I'll research {TOPIC} across Reddit, X, and the web to find what's been discussed in the last 30 days.
|
||||
🔍 **{TOPIC}** · {QUERY_TYPE}
|
||||
Searching Reddit, X, and the web for {natural language description of what you'll look for}...
|
||||
|
||||
Parsed intent:
|
||||
- TOPIC = {TOPIC}
|
||||
- TARGET_TOOL = {TARGET_TOOL or "unknown"}
|
||||
- QUERY_TYPE = {QUERY_TYPE}
|
||||
Example outputs:
|
||||
- 🔍 **kanye west** · News — Searching Reddit, X, and the web for the latest kanye west news and discussions...
|
||||
- 🔍 **best MCP servers** · Recommendations — Searching Reddit, X, and the web for the most recommended MCP servers...
|
||||
- 🔍 **nano banana pro prompting** · Prompting — Searching Reddit, X, and the web for nano banana pro prompting techniques and tips...
|
||||
- 🔍 **open claw** · General — Searching Reddit, X, and the web for what people are saying about open claw...
|
||||
|
||||
Research typically takes 2-8 minutes (niche topics take longer). Starting now.
|
||||
```
|
||||
|
||||
If TARGET_TOOL is known, mention it in the intro: "...to find {QUERY_TYPE}-style content for use in {TARGET_TOOL}."
|
||||
If TARGET_TOOL is known, mention it: "...for nano banana pro prompting techniques to use in ChatGPT..."
|
||||
|
||||
This text MUST appear before you call any tools. It confirms to the user that you understood their request.
|
||||
|
||||
@@ -82,45 +56,21 @@ This text MUST appear before you call any tools. It confirms to the user that yo
|
||||
|
||||
## Research Execution
|
||||
|
||||
**Step 1: Run the research script (FOREGROUND — do NOT background this)**
|
||||
|
||||
**CRITICAL: Run this command in the FOREGROUND with a 5-minute timeout. Do NOT use run_in_background. The full output contains Reddit, X, AND YouTube data that you need to read completely.**
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
# Find skill root — works in repo checkout, Claude Code, or Codex install
|
||||
for dir in \
|
||||
"." \
|
||||
"${CLAUDE_PLUGIN_ROOT:-}" \
|
||||
"$HOME/.claude/skills/last30days" \
|
||||
"$HOME/.agents/skills/last30days" \
|
||||
"$HOME/.codex/skills/last30days"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
|
||||
if [ -z "${SKILL_ROOT:-}" ]; then
|
||||
echo "ERROR: Could not find scripts/last30days.py" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 "${SKILL_ROOT}/scripts/last30days.py" "$ARGUMENTS" --emit=compact
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
Use a **timeout of 300000** (5 minutes) on the Bash call. The script typically takes 1-3 minutes.
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Run Reddit/X/YouTube searches
|
||||
- Output ALL results including YouTube transcripts
|
||||
|
||||
**Read the ENTIRE output.** It contains THREE data sections in this order: Reddit items, X items, and YouTube items. If you miss the YouTube section, you will produce incomplete stats.
|
||||
|
||||
**YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, and optional transcript snippet. Count them and include them in your synthesis and stats block.
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
---
|
||||
|
||||
## STEP 2: DO WEBSEARCH AFTER SCRIPT COMPLETES
|
||||
## STEP 2: DO WEBSEARCH WHILE SCRIPT RUNS
|
||||
|
||||
After the script finishes, do WebSearch to supplement with blogs, tutorials, and news.
|
||||
The script auto-detects sources (Bird CLI, API keys, etc). While waiting for it, do WebSearch.
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
@@ -153,8 +103,7 @@ For ALL query types:
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Options** (passed through from user's command):
|
||||
- `--days=N` → Look back N days instead of 30 (e.g., `--days=7` for weekly roundup)
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
@@ -167,11 +116,10 @@ For ALL query types:
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight YouTube sources HIGH (they have views, likes, and transcript content)
|
||||
3. Weight WebSearch sources LOWER (no engagement data)
|
||||
4. Identify patterns that appear across ALL sources (strongest signals)
|
||||
5. Note any contradictions between sources
|
||||
6. Extract the top 3-5 actionable insights
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
@@ -249,43 +197,22 @@ CITATION RULE: Cite sources sparingly to prove research is real.
|
||||
- Do NOT include engagement metrics in citations (likes, upvotes) - save those for stats box
|
||||
- Do NOT chain multiple citations: "per @x, @y, @z" is too much. Pick the strongest one.
|
||||
|
||||
CITATION PRIORITY (most to least preferred):
|
||||
1. @handles from X — "per @handle" (these prove the tool's unique value)
|
||||
2. r/subreddits from Reddit — "per r/subreddit"
|
||||
3. YouTube channels — "per [channel name] on YouTube" (transcript-backed insights)
|
||||
4. Web sources — ONLY when Reddit/X/YouTube don't cover that specific fact
|
||||
|
||||
The tool's value is surfacing what PEOPLE are saying, not what journalists wrote.
|
||||
When both a web article and an X post cover the same fact, cite the X post.
|
||||
|
||||
URL FORMATTING: NEVER paste raw URLs in the output.
|
||||
- **BAD:** "per https://www.rollingstone.com/music/music-news/kanye-west-bully-1235506094/"
|
||||
- **GOOD:** "per Rolling Stone"
|
||||
- **GOOD:** "per Complex"
|
||||
Use the publication name, not the URL. The user doesn't need links — they need clean, readable text.
|
||||
|
||||
**BAD:** "His album is set for March 20 (per Rolling Stone; Billboard; Complex)."
|
||||
**GOOD:** "His album BULLY drops March 20 — fans on X are split on the tracklist, per @honest30bgfan_"
|
||||
**GOOD:** "Ye's apology got massive traction on r/hiphopheads"
|
||||
**OK** (web, only when Reddit/X don't have it): "The Hellwatt Festival runs July 4-18 at RCF Arena, per Billboard"
|
||||
|
||||
**Lead with people, not publications.** Start each topic with what Reddit/X
|
||||
users are saying/feeling, then add web context only if needed. The user came
|
||||
here for the conversation, not the press release.
|
||||
**BAD:** "His album is set for March 20 (per @cocoabutterbf; Rolling Stone; HotNewHipHop; Complex)."
|
||||
**GOOD:** "His album BULLY is set for March 20 via Gamma, per Rolling Stone."
|
||||
|
||||
```
|
||||
What I learned:
|
||||
|
||||
**{Topic 1}** — [1-2 sentences about what people are saying, per @handle or r/sub]
|
||||
**{Topic 1}** — [1-2 sentences about this storyline, per source]
|
||||
|
||||
**{Topic 2}** — [1-2 sentences, per @handle or r/sub]
|
||||
**{Topic 2}** — [1-2 sentences, per source]
|
||||
|
||||
**{Topic 3}** — [1-2 sentences, per @handle or r/sub]
|
||||
**{Topic 3}** — [1-2 sentences, per source]
|
||||
|
||||
KEY PATTERNS from the research:
|
||||
1. [Pattern] — per @handle
|
||||
2. [Pattern] — per r/sub
|
||||
3. [Pattern] — per @handle
|
||||
3. [Pattern] — per source
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
@@ -301,105 +228,34 @@ KEY PATTERNS from the research:
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments
|
||||
├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts
|
||||
├─ 🔴 YouTube: {N} videos │ {N} views │ {N} with transcripts
|
||||
├─ 🌐 Web: {N} pages (supplementary)
|
||||
├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts (via Bird/xAI)
|
||||
├─ 🌐 Web: {N} pages │ {domain1}, {domain2}, {domain3}
|
||||
└─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2}
|
||||
---
|
||||
```
|
||||
|
||||
If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)"
|
||||
If YouTube returned 0 videos or yt-dlp is not installed, omit the YouTube line entirely.
|
||||
NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**LAST - Invitation (adapt to QUERY_TYPE):**
|
||||
|
||||
**CRITICAL: Every invitation MUST include 2-3 specific example suggestions based on what you ACTUALLY learned from the research.** Don't be generic — show the user you absorbed the content by referencing real things from the results.
|
||||
|
||||
**If QUERY_TYPE = PROMPTING:**
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
I'm now an expert on {TOPIC} for {TARGET_TOOL}. What do you want to make? For example:
|
||||
- [specific idea based on popular technique from research]
|
||||
- [specific idea based on trending style/approach from research]
|
||||
- [specific idea riffing on what people are actually creating]
|
||||
|
||||
Just describe your vision and I'll write a prompt you can paste straight into {TARGET_TOOL}.
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**If QUERY_TYPE = RECOMMENDATIONS:**
|
||||
```
|
||||
---
|
||||
I'm now an expert on {TOPIC}. Want me to go deeper? For example:
|
||||
- [Compare specific item A vs item B from the results]
|
||||
- [Explain why item C is trending right now]
|
||||
- [Help you get started with item D]
|
||||
```
|
||||
|
||||
**If QUERY_TYPE = NEWS:**
|
||||
```
|
||||
---
|
||||
I'm now an expert on {TOPIC}. Some things you could ask:
|
||||
- [Specific follow-up question about the biggest story]
|
||||
- [Question about implications of a key development]
|
||||
- [Question about what might happen next based on current trajectory]
|
||||
```
|
||||
|
||||
**If QUERY_TYPE = GENERAL:**
|
||||
```
|
||||
---
|
||||
I'm now an expert on {TOPIC}. Some things I can help with:
|
||||
- [Specific question based on the most discussed aspect]
|
||||
- [Specific creative/practical application of what you learned]
|
||||
- [Deeper dive into a pattern or debate from the research]
|
||||
```
|
||||
|
||||
**Example invitations (to show the quality bar):**
|
||||
|
||||
For `/last30days nano banana pro prompts for Gemini`:
|
||||
> I'm now an expert on Nano Banana Pro for Gemini. What do you want to make? For example:
|
||||
> - Photorealistic product shots with natural lighting (the most requested style right now)
|
||||
> - Logo designs with embedded text (Gemini's new strength per the research)
|
||||
> - Multi-reference style transfer from a mood board
|
||||
>
|
||||
> Just describe your vision and I'll write a prompt you can paste straight into Gemini.
|
||||
|
||||
For `/last30days kanye west` (GENERAL):
|
||||
> I'm now an expert on Kanye West. Some things I can help with:
|
||||
> - What's the real story behind the apology letter — genuine or PR move?
|
||||
> - Break down the BULLY tracklist reactions and what fans are expecting
|
||||
> - Compare how Reddit vs X are reacting to the Bianca narrative
|
||||
|
||||
For `/last30days war in Iran` (NEWS):
|
||||
> I'm now an expert on the Iran situation. Some things you could ask:
|
||||
> - What are the realistic escalation scenarios from here?
|
||||
> - How is this playing differently in US vs international media?
|
||||
> - What's the economic impact on oil markets so far?
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S RESPONSE
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to respond.
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER RESPONDS
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
**Read their response and match the intent:**
|
||||
|
||||
- If they ask a **QUESTION** about the topic → Answer from your research (no new searches, no prompt)
|
||||
- If they ask to **GO DEEPER** on a subtopic → Elaborate using your research findings
|
||||
- If they describe something they want to **CREATE** → Write ONE perfect prompt (see below)
|
||||
- If they ask for a **PROMPT** explicitly → Write ONE perfect prompt (see below)
|
||||
|
||||
**Only write a prompt when the user wants one.** Don't force a prompt on someone who asked "what could happen next with Iran."
|
||||
|
||||
### Writing a Prompt
|
||||
|
||||
When the user wants a prompt, write a **single, highly-tailored prompt** using your research expertise.
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
@@ -457,7 +313,6 @@ For the rest of this conversation, remember:
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
@@ -471,31 +326,7 @@ After delivering a prompt, end with:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} YouTube videos ({sum} views) + {n} web pages
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Security & Permissions
|
||||
|
||||
**What this skill does:**
|
||||
- Sends search queries to OpenAI's Responses API (`api.openai.com`) for Reddit discovery
|
||||
- Sends search queries to Twitter's GraphQL API (via browser cookie auth) or xAI's API (`api.x.ai`) for X search
|
||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||
- Stores research findings in local SQLite database (watchlist mode only)
|
||||
|
||||
**What this skill does NOT do:**
|
||||
- Does not post, like, or modify content on any platform
|
||||
- Does not access your Reddit, X, or YouTube accounts
|
||||
- Does not share API keys between providers (OpenAI key only goes to api.openai.com, etc.)
|
||||
- Does not log, cache, or write API keys to output files
|
||||
- Does not send data to any endpoint not listed above
|
||||
- Cannot be invoked autonomously by the agent (`disable-model-invocation: true`)
|
||||
|
||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||
|
||||
Review scripts before first use to verify behavior.
|
||||
@@ -0,0 +1,25 @@
|
||||
Here's what I found:
|
||||
|
||||
## What I learned:
|
||||
|
||||
**BULLY Album — March 20, 2026 via Gamma** — After years of delays (first announced September 2024), Kanye's 12th studio album finally has a firm release date. The 13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign. Sonically it recalls *808s & Heartbreak* and *MBDTF* — Ye mostly sings rather than raps. Notably, earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded with his real voice for the official release, per Rolling Stone.
|
||||
|
||||
**Public Apology for Antisemitism** — On January 26, Ye took out a full-page Wall Street Journal ad titled "To Those I've Hurt," publicly apologizing for his antisemitic remarks. He disclosed a previously undiagnosed brain injury from his 2002 car accident and attributed his behavior to a four-month manic episode fueled by bipolar disorder, psychosis, and paranoia. "I lost touch with reality," he wrote. The apology also extended to the Black community, per The Washington Post.
|
||||
|
||||
**Hellwatt Festival in Italy** — Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard.
|
||||
|
||||
**Health Concerns** — A rare January 2026 outing in LA with Bianca Censori reignited concern about Ye's physical appearance. Insiders point to medication side effects, frequent travel, and inconsistent routines, per AllHipHop.
|
||||
|
||||
**Grammys Ban** — Ye is reportedly not welcome at the 2026 Grammy Awards after clashing with organizers last year over his invitation terms, per The News International.
|
||||
|
||||
**Kim & Lewis Hamilton Buzz** — X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton, with users contrasting her new relationship against her marriage to Ye.
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 0 threads (no results this cycle)
|
||||
├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)
|
||||
├─ 🌐 Web: 30+ pages │ rollingstone.com, washingtonpost.com, complex.com, billboard.com, npr.org
|
||||
└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice.
|
||||
+18
-20
@@ -13,7 +13,7 @@ YouTube transcripts are the second headline feature. Inspired by Peter Steinberg
|
||||
**New in V2.1 — two headline features:**
|
||||
|
||||
- **YouTube transcripts as a 4th source.** When yt-dlp is installed, /last30days automatically searches YouTube, grabs view counts, and extracts auto-generated transcripts from the top videos. A 20-minute review contains 10x the signal of a tweet — now the skill reads it. Inspired by @steipete's yt-dlp + summarize toolchain.
|
||||
- **X search is fully bundled.** No external `bird` CLI or xAI API key needed. Just Node.js 22+ and your browser cookies. Uses a vendored subset of Bird's Twitter GraphQL client (MIT licensed, originally by @steipete).
|
||||
- **X search is fully bundled.** No external `bird` CLI install needed. Add `AUTH_TOKEN` and `CT0` once, and the vendored Bird client runs locally without browser-cookie prompts. `XAI_API_KEY` remains an optional fallback.
|
||||
|
||||
---
|
||||
|
||||
@@ -21,20 +21,18 @@ YouTube transcripts are the second headline feature. Inspired by Peter Steinberg
|
||||
|
||||
### X Search Authentication
|
||||
|
||||
X search reads your existing browser cookies — no API keys or login commands needed.
|
||||
X search prefers explicit env auth. This keeps local runs headless and avoids browser-cookie and macOS Keychain prompts.
|
||||
|
||||
**Safari (recommended on Mac):** Just be logged into x.com. No setup needed.
|
||||
**Recommended setup:** While logged into x.com once, open browser dev tools and copy the `auth_token` and `ct0` cookies for `x.com`.
|
||||
|
||||
**Chrome:** Works, but macOS will prompt you to allow Keychain access the first time. Click "Allow" (or "Always Allow" to stop future prompts).
|
||||
|
||||
**Firefox:** Just be logged into x.com. No setup needed.
|
||||
|
||||
**Manual fallback:** If cookie auto-detection doesn't work, set these env vars (grab them from your browser's dev tools → Application → Cookies → x.com):
|
||||
Save them as `AUTH_TOKEN` and `CT0` in `~/.config/last30days/.env` or `.claude/last30days.env`:
|
||||
```bash
|
||||
export AUTH_TOKEN=your_auth_token
|
||||
export CT0=your_ct0_token
|
||||
AUTH_TOKEN=your_auth_token
|
||||
CT0=your_ct0_token
|
||||
```
|
||||
|
||||
**xAI fallback:** If you do not want to provide `AUTH_TOKEN` and `CT0`, set `XAI_API_KEY` and use xAI's `x_search` backend instead.
|
||||
|
||||
**Verify it's working:**
|
||||
```bash
|
||||
node ~/.claude/skills/last30days/scripts/lib/vendor/bird-search/bird-search.mjs --whoami
|
||||
@@ -44,8 +42,10 @@ node ~/.claude/skills/last30days/scripts/lib/vendor/bird-search/bird-search.mjs
|
||||
|
||||
## README: Install block env line
|
||||
|
||||
```
|
||||
XAI_API_KEY=xai-... # optional — cookie auth is default for X search
|
||||
```bash
|
||||
AUTH_TOKEN=... # recommended for X search
|
||||
CT0=... # recommended for X search
|
||||
XAI_API_KEY=xai-... # optional X fallback
|
||||
```
|
||||
|
||||
---
|
||||
@@ -60,15 +60,13 @@ XAI_API_KEY=xai-... # optional — cookie auth is default for X search
|
||||
|
||||
## GitHub issue #19 response (post AFTER publishing)
|
||||
|
||||
> Thanks for reporting this. Bird CLI was deprecated and the GitHub repo was deleted — steipete was asked to take it down.
|
||||
> Thanks for reporting this. Bird CLI was deprecated and the GitHub repo was deleted. steipete was asked to take it down.
|
||||
>
|
||||
> The good news: you don't need Bird anymore. v2.1 (just shipped) bundles X search directly — no external CLI, no `npm install`, no brew. Just Node.js 22+ and your browser cookies.
|
||||
> The good news: you don't need Bird anymore. v2.1 (just shipped) bundles X search directly. No external CLI, no `npm install`, no brew. Just Node.js 22+ plus `AUTH_TOKEN` and `CT0`, or `XAI_API_KEY` as fallback.
|
||||
>
|
||||
> It also adds **YouTube as a 4th source** — when yt-dlp is installed, the skill automatically searches YouTube and extracts transcripts from the top videos. A 20-minute tutorial has 10x the signal of a tweet, and now the synthesis engine reads it.
|
||||
> It also adds **YouTube as a 4th source**. When yt-dlp is installed, the skill automatically searches YouTube and extracts transcripts from the top videos. A 20-minute tutorial has 10x the signal of a tweet, and now the synthesis engine reads it.
|
||||
>
|
||||
> If you're on a Mac, Safari is the easiest path for X — just be logged into x.com. Chrome works too but macOS will prompt for Keychain access the first time.
|
||||
>
|
||||
> If cookie auto-detection doesn't work, you can set `AUTH_TOKEN` and `CT0` env vars manually (grab from browser dev tools → Application → Cookies → x.com).
|
||||
> The recommended setup is to copy `auth_token` and `ct0` from x.com once and store them as `AUTH_TOKEN` and `CT0` in your env. That avoids browser-cookie and Keychain prompts during normal runs.
|
||||
>
|
||||
> The xAI API (`XAI_API_KEY`) also still works as a fallback.
|
||||
|
||||
@@ -82,7 +80,7 @@ XAI_API_KEY=xai-... # optional — cookie auth is default for X search
|
||||
|
||||
Two new features:
|
||||
→ YouTube transcripts as a 4th source (yt-dlp)
|
||||
→ X search fully bundled (no bird CLI needed)
|
||||
→ X search fully bundled (no bird CLI install needed)
|
||||
|
||||
Research any topic across Reddit, X, YouTube & web in one command.
|
||||
|
||||
@@ -100,7 +98,7 @@ When yt-dlp is installed, the skill searches YouTube, grabs view counts, and ext
|
||||
### Thread version (post 2)
|
||||
|
||||
2️⃣ X search is fully bundled
|
||||
Bird CLI was deprecated. Instead of requiring an external tool, v2.1 vendors a search-only subset. Just be logged into x.com in your browser. No npm install, no API keys.
|
||||
Bird CLI was deprecated. Instead of requiring an external tool, v2.1 vendors a search-only subset. Add `AUTH_TOKEN` and `CT0` once, then it runs locally with no npm install. `XAI_API_KEY` still works as fallback.
|
||||
|
||||
Both features inspired by @steipete's tooling.
|
||||
|
||||
|
||||
@@ -1,250 +0,0 @@
|
||||
# v2.1 Launch Posts - WORKING DRAFT
|
||||
|
||||
## Post 1 (Hook)
|
||||
|
||||
V2.1 of @slashlast30days launches today. Three headline features:
|
||||
|
||||
1. @openclaw + watchlists - automated research on your competitors, people, and topics
|
||||
2. YouTube transcripts as a 4th source
|
||||
3. Works in OpenAI Codex
|
||||
|
||||
## Post 2 (Watchlist + Open Claw - THE KILLER FEATURE)
|
||||
|
||||
@openclaw + WATCHLISTS.
|
||||
|
||||
Pair /last30days with @openclaw and it re-researches topics on a schedule across Reddit, X, YouTube, and the web.
|
||||
|
||||
"last30 watch my biggest competitor every week"
|
||||
"last30 watch Peter Steinberger every 30 days"
|
||||
"last30 watch AI video tools monthly"
|
||||
|
||||
Research that runs while you sleep. Designed for @openclaw and always-on bots.
|
||||
|
||||
## Post 3 (YouTube)
|
||||
|
||||
YOUTUBE IS NOW A 4TH SOURCE.
|
||||
|
||||
The skill searches YouTube, grabs view counts, and reads the actual transcripts. A 20-minute review has 10x the signal of one X post - now the skill reads it.
|
||||
|
||||
## Post 4 (Codex)
|
||||
|
||||
WORKS IN OPENAI CODEX.
|
||||
|
||||
Same skill, same engine, same four sources. Install to ~/.agents/skills/last30days and invoke with $last30days. Claude Code and Codex users get the same research.
|
||||
|
||||
## Post 5 (Example: Seedance 2.0 access)
|
||||
|
||||
Asked it how to access Seedance 2.0.
|
||||
|
||||
3 Reddit threads. 31 X posts. 20 YouTube videos (685K views, 4 transcripts read). 10 web pages. All four sources hit.
|
||||
|
||||
It found the real answer buried in Chinese YouTube tutorials: Little Skylark (Xiao Yun Que) - zero cost, no queue, no VPN. Just select Seedance 2.0 from the model dropdown. Also surfaced: Disney sent ByteDance a cease-and-desist over uncensored IP generation.
|
||||
|
||||
## Post 6 (Example: AI Generated Ads)
|
||||
|
||||
Asked it about AI generated ads.
|
||||
|
||||
12 Reddit threads. 29 X posts. 3 YouTube videos (83K views, 3 transcripts read). 30 web pages.
|
||||
|
||||
The finding that stuck: Svedka ran the first "primarily AI-generated" Super Bowl spot. Brand match: 7%. Industry norm: 63%. Meanwhile 86% of ad buyers are planning to use AI for video ads anyway. Cost is winning over quality.
|
||||
|
||||
## Post 7 (Example: Peter Steinberger)
|
||||
|
||||
Asked it about @steipete.
|
||||
|
||||
30 X posts. 5 YouTube videos (112K views). Found the Lex Fridman interview from 3 days ago.
|
||||
|
||||
Key reveal: OpenAI and Meta both made acquisition offers for OpenClaw. He said no. He's losing $10-20K/month maintaining it. "A fun project became a world project."
|
||||
|
||||
## Post 8 (Install)
|
||||
|
||||
Tell your @openclaw bot:
|
||||
|
||||
"Install the last30days skill from github.com/mvanhorn/last30days-skill"
|
||||
|
||||
That's it. One message.
|
||||
|
||||
## Post 9 (Close)
|
||||
|
||||
Thank you to @hutchins for pushing me to add YouTube and to @steipete whose summarize tool showed me how yt-dlp could power transcript extraction.
|
||||
|
||||
Try it: last30 [any topic]
|
||||
|
||||
github.com/mvanhorn/last30days-skill
|
||||
|
||||
PS: @steipete ClawHub login is broken right now so we can't publish the official skill there yet. Hoping for a fix soon.
|
||||
|
||||
---
|
||||
|
||||
## Standalone Posts
|
||||
|
||||
### Greg Isenberg best tips
|
||||
|
||||
Prompt: `last30 greg isenberg best tips`
|
||||
|
||||
1 Reddit thread. 6 X posts. 4 YouTube videos (all transcribed). 8 web pages.
|
||||
|
||||
His core playbook is ACP: Audience, Community, Product. Not the other way around.
|
||||
|
||||
Big thesis: "2026 is the GREATEST time to build a startup in 30 years." Boring industries, AI agents, rapid dev tools collapsing build time.
|
||||
|
||||
Most-watched: "Clawdbot Clearly Explained" (273K views), "Claude Code Built My $450K Marketing Campaign" (40K views), daily workflows with Kitze (83K views).
|
||||
|
||||
@gregisenberg @slashlast30days
|
||||
|
||||
### Lenny Rachitsky best learnings
|
||||
|
||||
Prompt: `last30 lenny rachitsky top learnings`
|
||||
|
||||
5 Reddit threads. 29 X posts. 20 YouTube videos (1.1M+ views, 5 transcribed). 30 web pages.
|
||||
|
||||
"Execution is no longer the bottleneck. Clarity is." That's the thesis running through everything @lennysan has been publishing lately.
|
||||
|
||||
He open-sourced 320 episode transcripts and the community went wild. 87 skills got built from them. Someone on Reddit distilled 86 discrete product skills from 100+ episodes.
|
||||
|
||||
Recent guest highlights: Sherwin Wu (OpenAI) says 95% of their engineers use Codex daily. Marc Andreessen: "This is as normal as it's going to be. It's going to be much weirder very soon." Dalton Caldwell (YC): "just don't die" and avoid tar pit ideas.
|
||||
|
||||
@lennysan @slashlast30days
|
||||
|
||||
---
|
||||
|
||||
## NOTES
|
||||
|
||||
- Thread leads with Open Claw + watchlist as the killer feature - pair with an always-on bot for automated research
|
||||
- YouTube is the #2 hero - the stats ("685K views, 4 transcripts read") prove it works
|
||||
- Codex compatibility is #3 - brief but shows cross-platform reach
|
||||
- Example posts prove quality with verified results from real runs
|
||||
- Consider screenshots of the actual output for each example post
|
||||
- @steipete credit in close - he inspired YouTube (yt-dlp toolchain) and X search (Bird MIT code)
|
||||
- @slashlast30days vs /last30days - which handle? Used /last30days above since it's the actual command
|
||||
|
||||
---
|
||||
|
||||
## RAW RESULTS (for reference / pulling quotes)
|
||||
|
||||
### Nano Banana Pro prompting (PROMPTING) - VERIFIED 2/15
|
||||
|
||||
Stats: 0 Reddit (timed out) | 32 X posts | 164 likes | 22 reposts | 5 YouTube videos | 98,539 views | 5 with transcripts | 10 web pages
|
||||
Top voices: @KusoPhoto (106 likes), @TzqQaiser (35 likes) | Jake Dawson (15K views), AI Master (37K views)
|
||||
|
||||
Key findings:
|
||||
- Structured JSON prompts are the meta - nested fields for character, scene, lighting beat plain prose. @TzqQaiser's viral post shows the format.
|
||||
- Design brief > keyword stuffing - "The second I started writing prompts like a real design brief, everything changed" - Jake Dawson on YouTube
|
||||
- 6-factor formula: Subject, Composition, Action, Setting, Style, then refine with camera/lighting - per Google's official blog
|
||||
- ICS framework for infographics: Image type + Content + Style - leverages Nano Banana Pro's unique legible text rendering
|
||||
- Scale logic for cinematic compositions - define size relationships and camera distance explicitly, per @Strength04_X
|
||||
- Nano Banana Pro → video pipeline trending (generate image, animate with Kling 3.0 or Veo 3.1) - per @KusoPhoto
|
||||
|
||||
### Peter Steinberger / OpenClaw creator (GENERAL) - VERIFIED 2/15
|
||||
|
||||
Stats: 31 X posts | 0 Reddit (quiet) | YouTube timed out on 2 | 4 web pages
|
||||
Top voices: @steipete | Lex Fridman podcast
|
||||
|
||||
Key findings:
|
||||
- Lex Fridman podcast (Feb 12) went viral - "One of the most honest discussions I've seen"
|
||||
- OpenAI and Meta made acquisition offers (conditional on keeping project open) - he declined
|
||||
- Losing $10-20K/month maintaining OpenClaw, rejected crypto tokenization for funding
|
||||
- 180K+ GitHub stars, 6,600 commits in 1 month - "A fun project became a world project"
|
||||
- Also built: gogcli (Google Workspace CLI), summarize (URL/YouTube summarizer), bird (X/Twitter CLI)
|
||||
- Pragmatic Engineer: "I ship code I don't read"
|
||||
- Prediction: AI agents could dominate >60% of software economy by 2030
|
||||
|
||||
### Seedance 2.0 Prompting (PROMPTING) - VERIFIED 2/15
|
||||
|
||||
Stats: 21 Reddit threads | 33 X posts | 20 YouTube videos | 5 web pages
|
||||
Top voices: @charliebcurran (61K+ likes) | r/AI_Agents, r/ChatGPT, r/PromptEngineering | AI Search (127K views), Theoretically Media (157K views), Dan Dingle (126K views)
|
||||
|
||||
Key findings:
|
||||
- "Slow and continuous" is the #1 prompting secret - rough state transitions = worse outcomes, per r/AI_Agents
|
||||
- Include timings in prompts (e.g., "0-3s: character walks, 3-6s: turns head") - per r/ChatGPT
|
||||
- Image-to-video for consistency - start with a reference image, not text-only
|
||||
- English works just as well as Chinese - per r/AI_India
|
||||
- CapCut integration coming = "every 12 year old in America will have this superpower"
|
||||
- Cost: ~$0.55/10s clips (~$3.30/min), Seedance 3.0 rumored at 1/8th price
|
||||
- Prompt resources: GitHub repo of curated prompts, Prompt Director Pro (440 settings system)
|
||||
- Top YouTube tutorials: "Seedance 2.0 crushes everything" (127K), "Claims the AI Video Throne" (157K), "ABUSING China's Crazy New Video AI" (126K)
|
||||
|
||||
### OpenClaw best use cases for business (RECOMMENDATIONS)
|
||||
|
||||
Stats: 35 Reddit threads | ~1,130 upvotes | ~566 comments | 23 X posts | ~24 likes | 20 YouTube videos | ~1,572,000 views | 5 with transcripts | 10 web pages
|
||||
Top voices: @gio__aa (8 likes), @ericosiu, @artyomx | r/openclaw, r/clawdbot, r/LocalLLaMA
|
||||
|
||||
Key findings:
|
||||
- Email & Inbox Automation - 8+ mentions. One user cleared 4,000+ emails in two days. 10-15 hours/week saved.
|
||||
- Business Dashboards & Real-Time Reporting - 6+ mentions. @gio__aa: "Business dashboards are going to become one of the most popular use cases."
|
||||
- Morning Briefings - 5+ mentions. Pulls from calendars, weather, emails, RSS, GitHub, Hacker News on a schedule.
|
||||
- Content & SEO Pipelines - 5+ mentions. @ericosiu claims "$45k of pSEO work in 20 minutes."
|
||||
- Full CRM & Business Operations - 4+ mentions. @artyomx runs a daycare business, legal cases, and family comms through it with 5 AI agents.
|
||||
- Client Onboarding & Support - 4+ mentions. "70% of tickets handled autonomously."
|
||||
- Competitive Monitoring & Scraping - 4+ mentions.
|
||||
- Wrapper/Hosting SaaS - 3+ mentions. Building commercial wrappers around OpenClaw as a business.
|
||||
|
||||
Cautions: malware in a top-downloaded skill (236 upvotes on r/LocalLLaMA), $25-50/day token burn risk, hours of config for marginal savings.
|
||||
|
||||
### YouTube thumbnail tips (GENERAL)
|
||||
|
||||
Stats: 7 Reddit threads | 654 upvotes | 176 comments | 32 X posts | 110 likes | 53 reposts | 18 YouTube videos | 6,150,368 views | 5 with transcripts | 30 web pages
|
||||
Top voices: @TeamYouTube, @thewindwolf64 | r/NewTubers, r/YouTubeThumbnailHub | Think Media (1.17M views), whirow (1.46M views)
|
||||
|
||||
Key findings:
|
||||
- Simplicity is #1 - r/NewTubers post (654 upvotes) from someone who designed 346 thumbnails: one subject, one message, one second to understand. 3+ elements = ~23% lower CTR.
|
||||
- Less text = more clicks - Under 4 words gets ~30% higher CTR. Mobile thumbnails shrink to 168x94px - text becomes unreadable.
|
||||
- Faces still win but subtlety is trending - Faces boost CTR 20-30%, but exaggerated shock face is giving way to authentic expressions in 2026.
|
||||
- "UnThumbnails" are a counter-trend - Nate Black (71K views): deliberately raw, less-designed thumbnails that stand out.
|
||||
- AI tools changing the game - Nick Nimmin (90K views) showed free AI tools democratizing thumbnail creation.
|
||||
- A/B test everything - YouTube's built-in thumbnail testing lets you test up to 3 versions per video.
|
||||
|
||||
### AI SaaS crash (NEWS)
|
||||
|
||||
Stats: 9 Reddit threads | 31 upvotes | 52 comments | 32 X posts | 39 likes | 2 reposts | 20 YouTube videos | 929,648 views | 5 with transcripts | 30 web pages
|
||||
Top voices: @jasonlk (15 likes), @WarrenInTheBuff (11 likes), @xankriegor_ | r/SaaS, r/aiwars
|
||||
|
||||
Key findings:
|
||||
- "SaaSpocalypse" - $285B wiped in a single day (Feb 3, 2026) after Anthropic launched Claude Cowork. Total losses exceeded $1T. Salesforce down 27% YTD, Oracle halved.
|
||||
- @jasonlk: "The real inflection point wasn't January 2026. It was June 2024 - when Claude 3.5 Sonnet shipped." Public SaaS growth rates declined every quarter since 2021 peak.
|
||||
- Seat-based pricing is the casualty - 10 AI agents replace 100 sales reps = no need for 100 Salesforce seats. $470B+ hyperscaler AI spend coming from enterprise software budgets.
|
||||
- Not everyone buying the doom - Jensen Huang called it "the most illogical thing in the world." BofA called selloff irrational.
|
||||
- Indian IT hit especially hard - biggest sell-off since 2020.
|
||||
|
||||
### Seedance 2.0 access (GENERAL) - VERIFIED 2/15, ALL 4 SOURCES
|
||||
|
||||
Stats: 3 Reddit threads | 114 upvotes | 183 comments | 31 X posts | 191 likes | 13 reposts | 20 YouTube videos | 685,297 views | 4 with transcripts | 10 web pages
|
||||
Top voices: @markgadala (116 likes), @OrctonAI, @nemovideoai | Theoretically Media (158K views) | r/AIHubSpace
|
||||
|
||||
Key findings:
|
||||
- Little Skylark (Xiao Yun Que) = best free method - zero cost, no queue, manually select Seedance 2.0 from model dropdown, per YouTube tutorials
|
||||
- Jimeng (Dreamina) - 1 RMB trial (~$0.14), ~260 daily free credits, but severe congestion with hours-long waits for free users
|
||||
- Doubao App - 10 free video gens/day, requires joining Feishu/Lark group and submitting UID (1-2 day wait)
|
||||
- Feb 24 = global unlock - Dreamina + CapCut + API access through BytePlus
|
||||
- IP controversy exploding - @markgadala's "fully uncensored Seedance 2" post (116 likes) went viral, Disney sent C&D to ByteDance, SAG-AFTRA slammed "blatant infringement" over AI Tom Cruise/Brad Pitt fight videos
|
||||
- Third-party race - NemoVideoAI, ChatCut, RecCloud, Morph Studio all competing to be the English-language access point
|
||||
- Quality consensus: "crushes everything" (AI Search, 128K views), "claims the AI video throne" (Theoretically Media, 158K views)
|
||||
|
||||
### AI Generated Ads (GENERAL) - VERIFIED 2/15, ALL 4 SOURCES
|
||||
|
||||
Stats: 12 Reddit threads | 5 upvotes | 15 comments | 29 X posts | 101 likes | 3 reposts | 3 YouTube videos | 82,896 views | 3 with transcripts | 30 web pages
|
||||
Top voices: @CaptainMcKlide (77 likes), @ugcbykaytelynn | r/editors, r/AI_UGC_Marketing, BERNTH (39K views)
|
||||
|
||||
Key findings:
|
||||
- Super Bowl 2026 was the watershed - 23% of ads (15/66) featured AI, reception "sharply negative," nearly 50% of social mentions critical
|
||||
- Svedka ran the first "primarily AI-generated" national Super Bowl spot - brand match of just 7% vs 63% alcohol industry norm
|
||||
- Massive perception gap - 82% of ad execs think Gen Z feels positive about AI ads, but only 45% of consumers do (IAB). Gen Z most hostile at 39% negative.
|
||||
- AI UGC booming in e-commerce - r/AI_UGC_Marketing active hub, tools: Creatify, MakeUGC, ArcAds targeting dropshippers
|
||||
- Quality still low - r/dropshipping: "the hand flip and rubbing on the face looks fake"
|
||||
- @ugcbykaytelynn warns "AI generated ads RUIN your brand's image"
|
||||
- Cost winning over quality - 86% of ad buyers using or planning gen AI for video ads, cost efficiency #1 driver (64%), per IAB
|
||||
- BERNTH on YouTube bought AI-generated guitar product ads, documented absurdity - four-fingered hands, instruments don't match listings (39K views)
|
||||
- Trust erosion spreading - people now question whether ANY media is real, even billboards, per @N0rbertas
|
||||
|
||||
### last30days skill (META/GENERAL) - VERIFIED 2/15
|
||||
|
||||
Stats: 0 Reddit | 30 X posts | 1,371 likes | 107 reposts | 5 YouTube videos | 112,082 views | 3 with transcripts | 10 web pages
|
||||
Top voices: @gregisenberg (1,290 likes), @mvanhorn (34 likes) | Alejandro AO (39K views), Greg Isenberg (28K views)
|
||||
|
||||
Key findings:
|
||||
- @gregisenberg's post (1,290 likes, 106 RT) + YouTube video (28K views) "The Claude Code Skill My Smartest Friends Use" was the breakout moment
|
||||
- v2 feedback loop active - @trevin flagged OpenAI web_search not finding niche Reddit posts, suggested Brave API. @jonthebeef submitted PR for --days flag.
|
||||
- People building on top - @tjarkoleifer created "re-skill" meta skill, @rajachirravuri recommends it as part of a PM stack
|
||||
- Coverage: Alejandro AO crash course (39K views, 1,049 likes), Jason Calacanis on This Week in Startups (24K views)
|
||||
- 1.5K GitHub stars, listed on skills.sh and Smithery
|
||||
- Grok itself correctly attributed the skill when asked about ithah
|
||||
@@ -0,0 +1,360 @@
|
||||
# last30days v2.5 Launch Thread
|
||||
|
||||
## FINAL THREAD (6 tweets)
|
||||
|
||||
### 1/6 - Announcement
|
||||
|
||||
I can't believe it's been 30 days since I launched @slashlast30days. 3.2k stars later, time for v2.5.
|
||||
|
||||
Three big additions:
|
||||
1. @Polymarket prediction markets as a 6th source - helps you predict the future
|
||||
2. Cross-source linking + massively better results - detects when the same story trends across multiple platforms. Ran a 15-way blinded comparison, v2.5 scored 4.38 vs 3.73 for the original. Won all 5 topics.
|
||||
3. Hacker News as a 5th source - a window into the tech and developer insider world
|
||||
|
||||
github.com/mvanhorn/last30days-skill
|
||||
|
||||
### 2/6 - Demo: Anthropic vs Pentagon
|
||||
|
||||
"/last30days Anthropic Pete Hegseth"
|
||||
|
||||
14 Reddit threads. 29 X posts (11,559 likes). 20 YouTube videos (739K views). 5 HN stories. 9 Polymarket markets.
|
||||
|
||||
This story broke TODAY. Hegseth designated Anthropic a "supply chain risk." Trump ordered every agency to stop using their tech.
|
||||
|
||||
Polymarket: Anthropic still 99% for best AI model. $500B+ valuation: 68%. IPO >$600B: 97%. Hegseth out by March: only 6%.
|
||||
|
||||
Markets say Anthropic wins regardless. That's the kind of signal you can't get from opinion threads.
|
||||
|
||||
### 3/6 - Demo: Seedance Prompting
|
||||
|
||||
"/last30days Seedance prompting"
|
||||
|
||||
13 Reddit threads. 33 X posts. 20 YouTube videos (1.2M views, 4 transcripts). 15 web pages.
|
||||
|
||||
Top finding: Seedance 2.0 prompts follow a director's shot-list format, not freeform text. 30-100 words. Subject + Action + Camera + Scene + Style. Beyond 100 words, results degrade.
|
||||
|
||||
Then I said: "a cinematic drone shot over a city at golden hour"
|
||||
|
||||
It wrote me a copy-paste prompt using the exact patterns from the research. Research first, then create from what you learned.
|
||||
|
||||
### 4/6 - Demo: Arizona Basketball
|
||||
|
||||
"/last30days arizona basketball"
|
||||
|
||||
6 Polymarket markets. 37 X posts (4,200 likes). 15 YouTube videos (517K views). 2 Reddit threads.
|
||||
|
||||
Arizona is 25-2, set a program record with a 22-0 start, and holds a 2-game Big 12 lead with 3 games left. The Field of 68 called them "the TOUGHEST team in America" after escaping Baylor shorthanded. Kansas rematch Saturday - the highlight video from their first meeting has 248K views on ESPN's YouTube.
|
||||
|
||||
Polymarket: Championship 13%. #1 seed: 88%. Duke and Michigan each at 18% to win it all.
|
||||
|
||||
That's not a sports blog. That's Reddit reactions + X engagement + YouTube analysis + prediction market odds from one command.
|
||||
|
||||
### 5/6 - Demo: Iran War
|
||||
|
||||
"/last30days iran war"
|
||||
|
||||
2 Reddit threads. 34 X posts (10,048 likes). 20 YouTube videos (1.6M views, 5 transcripts). 4 HN stories (850 points). 14 Polymarket markets ($473M volume).
|
||||
|
||||
Geneva talks just ended without a deal. 150+ US aircraft deployed. Two carrier strike groups in position. F-22s sent to Israel. Members of Congress who saw the secret war plan came out "terrified." @cenkuygur: "they are about to drag us into a war that 70-85% of Americans oppose" (7,700 likes).
|
||||
|
||||
Polymarket ($473M in volume - one of their biggest markets ever): strikes by 2026: 80%. By March 31: 68%. War Powers invoked: 51%. Formal war declaration: only 12%.
|
||||
|
||||
Markets say: strikes are very likely, declared war is not. That's the sharpest signal in the entire research.
|
||||
|
||||
### 6/6 - Thank You
|
||||
|
||||
Thank you to ARJ999 and wkbaran on GitHub who filed three separate issues asking for Hacker News support. v2.5 delivers.
|
||||
|
||||
It's been a crazy 30 days. 3.2k stars. Six sources. Massively better results. Super excited to get this out.
|
||||
|
||||
Try it: /last30days [any topic]
|
||||
|
||||
github.com/mvanhorn/last30days-skill
|
||||
|
||||
---
|
||||
---
|
||||
|
||||
## REFERENCE MATERIAL BELOW
|
||||
|
||||
## Context
|
||||
- 3.2k stars on GitHub
|
||||
- V2.5 headline features: Polymarket (6th source), Hacker News (5th source), cross-source linking
|
||||
- Ran 15-way blinded comparison: 4.38/5.0 vs 3.73/5.0
|
||||
- Won all 5 topics, zero regressions
|
||||
- Cross-source linking: 3 -> 13 linked items
|
||||
- Demo topics: Anthropic odds (11 markets), Arizona basketball (6 markets), Iran war ($425M volume)
|
||||
|
||||
---
|
||||
|
||||
## Post 1: Lead (Announcement)
|
||||
|
||||
V2.5 of @slashlast30days is out. Now with @Polymarket prediction markets, cross-source linking, and massively better results.
|
||||
|
||||
1. Polymarket as a 6th source - real money on outcomes, no API key needed
|
||||
2. Hacker News as a 5th source
|
||||
3. Cross-source linking - detects when the same story trends across multiple platforms
|
||||
|
||||
Ran a 15-way blinded comparison across 5 topics. v2.5 scored 4.38 vs 3.73 for the original. Won all 5. Zero regressions.
|
||||
|
||||
github.com/mvanhorn/last30days-skill
|
||||
|
||||
---
|
||||
|
||||
## Post 2: POLYMARKET AS A 6TH SOURCE.
|
||||
|
||||
Reddit tells you what people think. X tells you what people share. YouTube tells you what people watch. HN tells you what developers discuss.
|
||||
|
||||
Polymarket helps you predict the future.
|
||||
|
||||
"/last30days anthropic odds"
|
||||
|
||||
11 markets found. Best AI model February: Anthropic 98%. IPO before OpenAI: 64%. $500B+ valuation: 87%. Pentagon ban odds: only 22%.
|
||||
|
||||
Free API. No key. Real money on outcomes.
|
||||
|
||||
---
|
||||
|
||||
## Post 3: CROSS-SOURCE LINKING.
|
||||
|
||||
When a Seedance 2.0 tutorial has 44K YouTube views AND trends on HN AND gets discussed on Reddit, v2.5 flags it: [also on: HN, YouTube]
|
||||
|
||||
Old version linked 3 items across 5 test topics. New version links 13. The difference is hybrid similarity - combining character-trigram and token-level matching at a tuned threshold.
|
||||
|
||||
Cross-platform convergence is the strongest signal that something actually matters. Not engagement on one platform. Convergence across all of them.
|
||||
|
||||
---
|
||||
|
||||
## Post 4: 15-WAY BLINDED EVALUATION.
|
||||
|
||||
I don't trust vibes for measuring quality. So I ran a scientific comparison.
|
||||
|
||||
5 topics x 3 versions. Stripped version labels. Randomized as A/B/C. Scored on groundedness, specificity, coverage, actionability, and format.
|
||||
|
||||
v2.5: 4.38/5.0
|
||||
v2.2 (HN only): 4.10/5.0
|
||||
v2.0 (original): 3.73/5.0
|
||||
|
||||
Won all 5 topics. Zero regressions. Biggest gains: specificity (+0.8) and format (+1.0) from cross-source linking giving the synthesis better material to work with.
|
||||
|
||||
---
|
||||
|
||||
## Post 5: Demo - Anthropic Odds
|
||||
|
||||
Asked it about Anthropic odds.
|
||||
|
||||
11 Polymarket markets. 25 X posts. 13 YouTube videos (719K views). 6 HN stories (471 points).
|
||||
|
||||
Best AI model February: 98%. IPO before OpenAI: 64%. $500B+ valuation: 87%. FrontierMath 50% score: 48% (up 28% today). Pentagon ban: only 22%.
|
||||
|
||||
Markets say Anthropic is winning the model race AND the valuation race. The Pentagon thing is noise.
|
||||
|
||||
---
|
||||
|
||||
## Post 6: Demo - Arizona Basketball
|
||||
|
||||
"/last30days arizona basketball"
|
||||
|
||||
6 Polymarket markets. 37 X posts (4,200 likes). 15 YouTube videos (517K views). 2 Reddit threads.
|
||||
|
||||
Championship odds: 13%. #1 seed: 88%. Big 12 title: Arizona leads by 2.
|
||||
|
||||
That's not a sports blog. That's Reddit reactions + X engagement + YouTube analysis + prediction market odds from one command.
|
||||
|
||||
The Polymarket integration uses two-pass query expansion. First pass finds "Arizona Big 12." Second pass discovers the championship and #1 seed markets via tag-based domain bridging.
|
||||
|
||||
---
|
||||
|
||||
## Post 7: Demo - Iran War
|
||||
|
||||
The best Polymarket demo is news.
|
||||
|
||||
"/last30days iran war"
|
||||
|
||||
14 Polymarket markets. $425M+ in volume. 7 Reddit threads. 30 X posts. 20 YouTube videos (2M views). 18 HN stories (1,187 points).
|
||||
|
||||
US strikes Iran by 2026: 70%. War Powers by March: 60%. Israel strikes by June: 64%. Formal war declaration: only 8%.
|
||||
|
||||
Markets say: limited strikes with War Powers, NOT a declared war. Breaking Points (435K views) covered leaked Pentagon opposition. r/Conservative "imploding" per r/SubredditDrama.
|
||||
|
||||
One command. Six sources. Real money.
|
||||
|
||||
---
|
||||
|
||||
## Post 8: Credits + CTA
|
||||
|
||||
Also in v2.5: YouTube synonym expansion ("hip hop" now matches "rap" - relevance jumped 0.33 to 0.71), X handle resolution, and HN OR queries for framework topics.
|
||||
|
||||
The difference between "good research" and "research you'd actually trust" is in details like this.
|
||||
|
||||
Try it: /last30days [any topic]
|
||||
|
||||
github.com/mvanhorn/last30days-skill
|
||||
|
||||
---
|
||||
|
||||
## Post 9: Demo - Claude Code (ALL 6 sources)
|
||||
|
||||
"/last30days Claude Code"
|
||||
|
||||
3 Reddit threads (199 upvotes). 35 X posts (5,239 likes). 15 YouTube videos (1.4M views, 5 transcripts). 30 HN stories (~8,500 points). 8 Polymarket markets. 20 web pages.
|
||||
|
||||
All six sources hit. Top finding: the planning-first workflow has won. The #1 HN post this month (969 pts, 590 comments) is about separating planning from execution. Boris Cherny (Head of Claude Code) on Lenny's Podcast: "100% of my code is written by Claude Code - I have not edited a single line by hand since November."
|
||||
|
||||
Polymarket: Anthropic 99% for best AI model in February. 58% for March. Claude on FrontierMath at 55%. The US government rejected Claude - Polymarket has the Hegseth ban at 32%.
|
||||
|
||||
Then I asked it to dig deeper into the planning-first workflow. No new searches - it answered from what it already learned.
|
||||
|
||||
---
|
||||
|
||||
## Post 10: Demo - March Madness Odds (Polymarket + Sports)
|
||||
|
||||
"/last30days March Madness Odds"
|
||||
|
||||
2 Reddit threads. 31 X posts. 6 YouTube videos (46K views, 4 transcripts). 2 Polymarket markets.
|
||||
|
||||
Tournament winner: Duke 18%, Michigan 18%, Arizona 13%. #1 seeds: Michigan 98%, Duke 91%, Arizona 88%.
|
||||
|
||||
Duke is the hottest mover - went from +700 to +450 in one week. The skill surfaced that from sportsbook data, X commentary, and Polymarket odds simultaneously.
|
||||
|
||||
Then I asked it to break down Michigan vs Duke vs Arizona. Full analysis from the research it already had.
|
||||
|
||||
---
|
||||
|
||||
## Post 11: Demo - Seedance Prompting (Expert + Prompt Mode)
|
||||
|
||||
"/last30days Seedance prompting"
|
||||
|
||||
13 Reddit threads. 33 X posts. 20 YouTube videos (1.2M views, 4 transcripts). 1 HN story. 15 web pages.
|
||||
|
||||
Top finding: Seedance 2.0 prompts follow a director's shot-list format, not freeform text. 30-100 words. Subject + Action + Camera + Scene + Style + Constraints. Beyond 100 words, results degrade.
|
||||
|
||||
Then I said: "a cinematic drone shot over a city at golden hour"
|
||||
|
||||
It wrote me a copy-paste prompt using the exact patterns from the research. That's the skill's real power - research first, then create from what you learned.
|
||||
|
||||
---
|
||||
|
||||
## Post 12: Thank You + CTA (Final)
|
||||
|
||||
Thank you to ARJ999 and wkbaran on GitHub who kept asking for Hacker News support. Three separate issues. v2.5 delivers.
|
||||
|
||||
30 days. 3.2k stars. 6 sources. Massively better results.
|
||||
|
||||
Try it: /last30days [any topic]
|
||||
|
||||
github.com/mvanhorn/last30days-skill
|
||||
|
||||
---
|
||||
|
||||
## Post 13: Demo - Anthropic vs Pentagon (Breaking News + Polymarket)
|
||||
|
||||
"/last30days Anthropic Pete Hegseth"
|
||||
|
||||
14 Reddit threads. 29 X posts (11,559 likes). 20 YouTube videos (739K views, 5 transcripts). 5 HN stories. 9 Polymarket markets. 10 web pages.
|
||||
|
||||
This story broke TODAY. Defense Secretary Hegseth designated Anthropic a "supply chain risk" - believed to be the first time an American company has ever received this designation. Trump ordered every federal agency to stop using Anthropic tech.
|
||||
|
||||
Polymarket: Anthropic still 99% for best AI model. $500B+ valuation: 68%. IPO >$600B: 97%. Hegseth out by March 31: only 6%.
|
||||
|
||||
Markets say: Anthropic wins the model race regardless. Bettors don't think Hegseth survives this. That's the kind of signal you can't get from opinion threads.
|
||||
|
||||
---
|
||||
|
||||
## Post 14: Demo - OpenAI Insider Trading (News + Polymarket)
|
||||
|
||||
"/last30days OpenAI Insider Trading"
|
||||
|
||||
2 Reddit threads. 29 X posts. 4 YouTube videos (360K views, 4 transcripts). 2 HN stories. 15 Polymarket markets. 15 web pages.
|
||||
|
||||
An OpenAI employee was just fired for using confidential info to bet on Polymarket. 13 brand-new wallets appeared 40 hours before the browser launch. $309K bet on the right outcome. Unusual Whales flagged 77 suspected insider positions across 60 wallets.
|
||||
|
||||
Meanwhile Polymarket has OpenAI's IPO at $1.25-1.5T: 54%. Anthropic IPOs first: 62%. Best AI model: Anthropic 99%.
|
||||
|
||||
The prediction markets are both the story AND the source. One command pulled all of it together.
|
||||
|
||||
---
|
||||
|
||||
## Standalone Tweet: Polymarket Stats Line
|
||||
|
||||
"/last30days Anthropic Pete Hegseth"
|
||||
|
||||
The Pentagon just designated Anthropic a supply chain risk. First time ever for an American company. Trump ordered every agency to stop using their tech.
|
||||
|
||||
Here's what Polymarket says:
|
||||
|
||||
📊 9 markets │ Best AI model: 99% │ $500B+ valuation: 68% │ IPO >$600B: 97% │ Hegseth out by March: 6%
|
||||
|
||||
Bettors with real money on the line think Anthropic wins the model race, goes public at a massive valuation, and Hegseth doesn't survive this.
|
||||
|
||||
That's the gap between headlines and reality. One command, six sources.
|
||||
|
||||
github.com/mvanhorn/last30days-skill
|
||||
|
||||
---
|
||||
|
||||
## Recommended Thread Order (pick 8-10)
|
||||
|
||||
The full thread above is 12 posts. Here's what I'd cut to keep it tight:
|
||||
|
||||
**Must include (core story):**
|
||||
1. Post 1 - Lead announcement
|
||||
2. Post 2 - Polymarket ("Reddit tells you what people think...")
|
||||
3. Post 3 - Cross-source linking
|
||||
4. Post 4 - Blinded evaluation
|
||||
|
||||
**Best demos (pick 3-4):**
|
||||
- Post 13 (Anthropic vs Pentagon) - STRONGEST. Breaking news today. Polymarket cuts through the noise. "Markets say Anthropic wins regardless."
|
||||
- Post 9 (Claude Code) - All 6 sources. Massive numbers. Shows follow-up flow.
|
||||
- Post 10 (March Madness) - Sports/Polymarket crossover. Timely with tournament approaching.
|
||||
- Post 11 (Seedance) - Shows prompting flow. 1.2M YouTube views.
|
||||
- Post 5 (Anthropic Odds) - Overlaps with Post 13 now. Skip.
|
||||
|
||||
**Skip or save for standalone tweets:**
|
||||
- Post 5 (Anthropic Odds) - Redundant with Post 13
|
||||
- Post 6 (Arizona Basketball) - Covered by March Madness now
|
||||
- Post 7 (Iran War) - Great standalone tweet, not for launch thread
|
||||
- Post 8 (Credits/minor features) - Fold into CTA
|
||||
|
||||
**My recommended 8-post thread:**
|
||||
1. Lead (Post 1)
|
||||
2. Polymarket (Post 2)
|
||||
3. Cross-source linking (Post 3)
|
||||
4. Blinded evaluation (Post 4)
|
||||
5. Demo: Anthropic vs Pentagon (Post 13) - breaking news, best Polymarket showcase
|
||||
6. Demo: Claude Code (Post 9)
|
||||
7. Demo: March Madness (Post 10)
|
||||
8. Thank you + CTA (Post 12)
|
||||
|
||||
---
|
||||
|
||||
## Video Script (~60 seconds)
|
||||
|
||||
**[Talking to camera]**
|
||||
|
||||
Oh my god, I can't believe it's been 30 days since I launched last30days. 3,200 stars on GitHub. This has been the craziest month.
|
||||
|
||||
Today I'm shipping v2.5 and I'm really excited about this one. Three big things.
|
||||
|
||||
**[Screen recording: typing /last30days Anthropic Pete Hegseth]**
|
||||
|
||||
First - Polymarket prediction markets as a 6th source. So this Anthropic-Pentagon story broke today. Hegseth designated Anthropic a supply chain risk, Trump ordered agencies to stop using their tech. Scary headline, right?
|
||||
|
||||
But Polymarket says: Anthropic still 99% for best AI model. IPO above 600 billion: 97%. Hegseth out by March: 6%. Real money on outcomes helps you predict the future. That's a different story than the headlines.
|
||||
|
||||
**[Screen recording: typing /last30days arizona basketball]**
|
||||
|
||||
Second - it now searches Hacker News and does cross-source linking. When the same story shows up on Reddit AND YouTube AND HN, it flags it. I ran a 15-way blinded comparison and v2.5 scored 4.38 versus 3.73 for the original. Won all 5 test topics.
|
||||
|
||||
**[Back to camera]**
|
||||
|
||||
Thank you to everyone who starred it, filed issues, and kept pushing me to make this better. Shoutout to the people on GitHub who literally filed three separate issues asking for Hacker News. v2.5 delivers.
|
||||
|
||||
Link in bio. Try it on anything.
|
||||
|
||||
---
|
||||
|
||||
## Scoring Note
|
||||
|
||||
The 4.38 vs 3.73 score is from a custom 5-dimension rubric (30% groundedness, 25% specificity, 20% coverage, 15% actionability, 10% format compliance) evaluated by Claude on blinded outputs. The relative ranking is meaningful; the absolute numbers are not. It's an LLM grading LLM output - useful for A/B comparison, not for claiming "4.38 out of 5 quality."
|
||||
|
||||
If using the score in a tweet, frame it as "scored X vs Y on a blinded comparison" not "rated 4.38/5.0 quality" - the former is honest, the latter implies an objective standard that doesn't exist.
|
||||
@@ -0,0 +1,42 @@
|
||||
[
|
||||
{
|
||||
"topic": "OpenClaw vs NanoClaw vs ZeroClaw",
|
||||
"query_type": "comparison",
|
||||
"rationale": "Multi-entity extraction, 3-way split across AI agent frameworks."
|
||||
},
|
||||
{
|
||||
"topic": "how to set up a GLP-1 supplement routine",
|
||||
"query_type": "how_to",
|
||||
"rationale": "Trending health topic. Tests non-tech how_to."
|
||||
},
|
||||
{
|
||||
"topic": "2026 March Madness",
|
||||
"query_type": "breaking_news",
|
||||
"rationale": "Live sporting event. Tests broad breaking news recall."
|
||||
},
|
||||
{
|
||||
"topic": "best budget noise cancelling headphones 2026",
|
||||
"query_type": "product",
|
||||
"rationale": "Evergreen consumer query. Tests product review aggregation."
|
||||
},
|
||||
{
|
||||
"topic": "thoughts on OpenAI Codex pricing",
|
||||
"query_type": "opinion",
|
||||
"rationale": "Active developer debate. Tests opinion mining."
|
||||
},
|
||||
{
|
||||
"topic": "odds of US recession 2026",
|
||||
"query_type": "prediction",
|
||||
"rationale": "Major macro topic. Tests prediction market + news synthesis."
|
||||
},
|
||||
{
|
||||
"topic": "what is retrieval augmented generation",
|
||||
"query_type": "concept",
|
||||
"rationale": "Widely discussed AI concept. Tests explanation quality."
|
||||
},
|
||||
{
|
||||
"topic": "Google Wiz acquisition price and timeline",
|
||||
"query_type": "factual",
|
||||
"rationale": "Completed event ($32B). Tests factual precision."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"video_id": "7543693751290481942",
|
||||
"text": "This Claude Code trick saved me hours #claudecode #ai #coding",
|
||||
"url": "https://www.tiktok.com/@codemaster/video/7543693751290481942",
|
||||
"author_name": "codemaster",
|
||||
"date": "2026-02-28",
|
||||
"engagement": {
|
||||
"views": 2100000,
|
||||
"likes": 45000,
|
||||
"comments": 1200,
|
||||
"shares": 8400
|
||||
},
|
||||
"hashtags": ["claudecode", "ai", "coding"],
|
||||
"duration": 45,
|
||||
"relevance": 0.85,
|
||||
"why_relevant": "TikTok: This Claude Code trick saved me hours #claude",
|
||||
"caption_snippet": "So I found this insane trick with Claude Code where you can use slash commands to automate everything"
|
||||
},
|
||||
{
|
||||
"video_id": "7543100200112345678",
|
||||
"text": "AI coding tools comparison 2026 - Claude vs Copilot vs Cursor #ai #devtools",
|
||||
"url": "https://www.tiktok.com/@techreviewer/video/7543100200112345678",
|
||||
"author_name": "techreviewer",
|
||||
"date": "2026-02-25",
|
||||
"engagement": {
|
||||
"views": 850000,
|
||||
"likes": 22000,
|
||||
"comments": 890,
|
||||
"shares": 3200
|
||||
},
|
||||
"hashtags": ["ai", "devtools"],
|
||||
"duration": 60,
|
||||
"relevance": 0.7,
|
||||
"why_relevant": "TikTok: AI coding tools comparison 2026 - Claude vs Copi",
|
||||
"caption_snippet": ""
|
||||
},
|
||||
{
|
||||
"video_id": "7543200300223456789",
|
||||
"text": "You need to try Claude Code RIGHT NOW #programming #tips",
|
||||
"url": "https://www.tiktok.com/@devtips/video/7543200300223456789",
|
||||
"author_name": "devtips",
|
||||
"date": "2026-03-01",
|
||||
"engagement": {
|
||||
"views": 500000,
|
||||
"likes": 15000,
|
||||
"comments": 450,
|
||||
"shares": 2100
|
||||
},
|
||||
"hashtags": ["programming", "tips"],
|
||||
"duration": 30,
|
||||
"relevance": 0.6,
|
||||
"why_relevant": "TikTok: You need to try Claude Code RIGHT NOW #programm",
|
||||
"caption_snippet": "Let me show you why Claude Code is the best AI coding tool right now"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"version": "3.0.5",
|
||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||
"settings": [
|
||||
{
|
||||
"name": "Extension Directory",
|
||||
"description": "Extension installation directory (auto-set by Gemini CLI)",
|
||||
"envVar": "GEMINI_EXTENSION_DIR",
|
||||
"sensitive": false
|
||||
},
|
||||
{
|
||||
"name": "ScrapeCreators API Key",
|
||||
"description": "ScrapeCreators API Key for Reddit, TikTok, and Instagram search (required)",
|
||||
"envVar": "SCRAPECREATORS_API_KEY",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "OpenAI API Key",
|
||||
"description": "OpenAI API Key - optional fallback for Reddit discovery",
|
||||
"envVar": "OPENAI_API_KEY",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "xAI API Key",
|
||||
"description": "xAI API Key for X/Twitter search (optional)",
|
||||
"envVar": "XAI_API_KEY",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "OpenRouter API Key",
|
||||
"description": "OpenRouter API Key (optional)",
|
||||
"envVar": "OPENROUTER_API_KEY",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "Parallel AI API Key",
|
||||
"description": "Parallel AI API Key (optional)",
|
||||
"envVar": "PARALLEL_API_KEY",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "Brave Search API Key",
|
||||
"description": "Brave Search API Key (optional)",
|
||||
"envVar": "BRAVE_API_KEY",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "Apify API Token",
|
||||
"description": "Apify API Token (optional legacy)",
|
||||
"envVar": "APIFY_API_TOKEN",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "Twitter AUTH_TOKEN",
|
||||
"description": "Twitter browser AUTH_TOKEN cookie for direct X search (optional)",
|
||||
"envVar": "AUTH_TOKEN",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "Twitter CT0",
|
||||
"description": "Twitter browser CT0 cookie (optional, pair with AUTH_TOKEN)",
|
||||
"envVar": "CT0",
|
||||
"sensitive": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"hooks": {
|
||||
"SessionStart": [
|
||||
{
|
||||
"matcher": "",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-config.sh",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Check last30days configuration status and show appropriate welcome message.
|
||||
# Priority: .claude/last30days.env > ~/.config/last30days/.env > env vars
|
||||
|
||||
PROJECT_ENV=".claude/last30days.env"
|
||||
GLOBAL_ENV="$HOME/.config/last30days/.env"
|
||||
|
||||
# Helper: warn if file permissions are too open
|
||||
check_perms() {
|
||||
local file="$1"
|
||||
if [[ ! -f "$file" ]]; then return; fi
|
||||
local perms
|
||||
# Try GNU stat first (Linux), fall back to BSD stat (macOS).
|
||||
# On Linux, `stat -f` prints filesystem info (not permissions) and exits 0,
|
||||
# so the previous BSD-first ordering left $perms as multi-line garbage on
|
||||
# every Linux session start and printed a false WARNING.
|
||||
perms=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "")
|
||||
if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then
|
||||
echo "/last30days: WARNING — $file has permissions $perms (should be 600)."
|
||||
echo " Fix: chmod 600 $file"
|
||||
fi
|
||||
}
|
||||
|
||||
# Load env file into variables for inspection (without exporting)
|
||||
load_env_vars() {
|
||||
local file="$1"
|
||||
if [[ -f "$file" ]]; then
|
||||
while IFS='=' read -r key value; do
|
||||
# Skip comments, empty lines
|
||||
[[ "$key" =~ ^[[:space:]]*# ]] && continue
|
||||
[[ -z "$key" ]] && continue
|
||||
key=$(echo "$key" | xargs)
|
||||
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||
if [[ -n "$key" && -n "$value" ]]; then
|
||||
eval "ENV_${key}=\"${value}\""
|
||||
fi
|
||||
done < "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
# Determine which config file is active
|
||||
CONFIG_FILE=""
|
||||
if [[ -f "$PROJECT_ENV" ]]; then
|
||||
CONFIG_FILE="$PROJECT_ENV"
|
||||
check_perms "$PROJECT_ENV"
|
||||
elif [[ -f "$GLOBAL_ENV" ]]; then
|
||||
CONFIG_FILE="$GLOBAL_ENV"
|
||||
check_perms "$GLOBAL_ENV"
|
||||
fi
|
||||
|
||||
# Load config if found
|
||||
if [[ -n "$CONFIG_FILE" ]]; then
|
||||
load_env_vars "$CONFIG_FILE"
|
||||
fi
|
||||
|
||||
# Check SETUP_COMPLETE (from file or env)
|
||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||
|
||||
# If setup has never been run, show welcome message for new users
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||
cat <<'EOF'
|
||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||
|
||||
Reddit, Hacker News, and Polymarket work out of the box.
|
||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Setup done but check for ScrapeCreators
|
||||
HAS_SCRAPECREATORS="${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}"
|
||||
HAS_X="${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}"
|
||||
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
|
||||
HAS_YTDLP=""
|
||||
if command -v yt-dlp &>/dev/null; then
|
||||
HAS_YTDLP="yes"
|
||||
fi
|
||||
HAS_BSKY="${ENV_BSKY_HANDLE:-${BSKY_HANDLE:-}}"
|
||||
HAS_EXA="${ENV_EXA_API_KEY:-${EXA_API_KEY:-}}"
|
||||
|
||||
# Count active sources
|
||||
SOURCE_COUNT=2 # HN + Polymarket are always free
|
||||
if [[ -n "$HAS_X" || -n "$HAS_XAI" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
# Reddit public JSON always works
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
if [[ -n "$HAS_YTDLP" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_EXA" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_BSKY" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||
fi
|
||||
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Fully configured — compact ready message
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
else
|
||||
# Setup done but missing ScrapeCreators — recommend it
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||
echo " 10,000 free API calls, no credit card — scrapecreators.com"
|
||||
echo " last30days has no affiliation with any API provider."
|
||||
fi
|
||||
@@ -0,0 +1,12 @@
|
||||
# Mirror of skills/last30days/scripts/, populated by scripts/sync-engine.sh.
|
||||
# Source of truth lives in the Python skill; never commit the mirror.
|
||||
# Lives inside internal/engine/ because //go:embed cannot reach outside
|
||||
# its own package directory.
|
||||
internal/engine/vendored/*
|
||||
!internal/engine/vendored/.gitkeep
|
||||
|
||||
# Local build output: cross-compiled binaries and packaged .mcpb files.
|
||||
build/
|
||||
# Anchor to the mcp/ root so the cmd/last30days-pp-mcp/ package directory
|
||||
# is not also excluded (subdirs with the same name would otherwise match).
|
||||
/last30days-pp-mcp
|
||||
@@ -0,0 +1,36 @@
|
||||
# last30days-pp-mcp
|
||||
|
||||
Go MCP server that wraps the last30days Python engine for Claude Desktop. Packaged as a `.mcpb` bundle (drag-drop install into Claude Desktop).
|
||||
|
||||
The MCP server exposes a single `research` tool that mirrors the `/last30days <topic>` slash command available in Claude Code. At runtime the binary extracts the vendored Python engine into a per-user cache and shells out to `python3` to produce the synthesis input Claude renders.
|
||||
|
||||
## Architecture
|
||||
|
||||
- `cmd/last30days-pp-mcp/` - server entry point
|
||||
- `internal/engine/` - `embed.FS` of the Python engine + cache extractor + subprocess wrapper
|
||||
- `internal/tools/` - MCP tool handlers (currently `research`)
|
||||
- `internal/engine/vendored/` - mirror of `skills/last30days/scripts/`, generated by `scripts/sync-engine.sh` (gitignored). Lives inside the engine package because `//go:embed` cannot reach files outside its own package directory.
|
||||
- `manifest.json` - MCPB v0.3 manifest consumed by Claude Desktop and `printing-press bundle`
|
||||
|
||||
## Local build
|
||||
|
||||
```bash
|
||||
# Mirror the Python engine into vendored/.
|
||||
bash scripts/sync-engine.sh
|
||||
|
||||
# Build for the current host.
|
||||
go build -ldflags "-X main.Version=dev" -o build/last30days-pp-mcp ./cmd/last30days-pp-mcp
|
||||
|
||||
# Package as a .mcpb (requires the printing-press binary on PATH).
|
||||
printing-press bundle . --skip-build --binary build/last30days-pp-mcp
|
||||
```
|
||||
|
||||
The output `.mcpb` lands at `build/last30days-pp-mcp-<os>-<arch>.mcpb`. Drag it into Claude Desktop's Extensions panel to install.
|
||||
|
||||
## Runtime requirements
|
||||
|
||||
End users need Python 3.12+ on PATH. The bundle ships the engine source but relies on the host interpreter.
|
||||
|
||||
## Versioning
|
||||
|
||||
The MCPB `manifest.json` version is hand-bumped in the same PR that ships engine changes worth releasing. Release CI stamps the Go binary's `main.Version` from the tag.
|
||||
@@ -0,0 +1,39 @@
|
||||
// Package main is the entry point for the last30days MCP server bundled
|
||||
// as a .mcpb for Claude Desktop. The server registers a single research
|
||||
// tool (see internal/tools) and serves it over stdio. See mcp/README.md
|
||||
// for build and packaging instructions.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/tools"
|
||||
)
|
||||
|
||||
// Version is stamped at build time via -ldflags "-X main.Version=<tag>".
|
||||
// It namespaces the per-user cache directory in internal/engine so multiple
|
||||
// installed versions can coexist without clobbering each other.
|
||||
var Version = "dev"
|
||||
|
||||
const (
|
||||
serverName = "last30days"
|
||||
serverVersion = "1"
|
||||
)
|
||||
|
||||
func main() {
|
||||
s := server.NewMCPServer(
|
||||
serverName,
|
||||
serverVersion,
|
||||
server.WithToolCapabilities(false),
|
||||
)
|
||||
|
||||
tools.Register(s, tools.Config{Version: Version})
|
||||
|
||||
if err := server.ServeStdio(s); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "last30days-pp-mcp: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
module github.com/mvanhorn/last30days-skill/mcp
|
||||
|
||||
go 1.25.5
|
||||
|
||||
require github.com/mark3labs/mcp-go v0.54.0
|
||||
|
||||
require (
|
||||
github.com/google/jsonschema-go v0.4.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/spf13/cast v1.7.1 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
|
||||
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mark3labs/mcp-go v0.54.0 h1:PZhQvd+5xrT43cUoiaKn/hDcvLUhcLc1twSEKYPTcTA=
|
||||
github.com/mark3labs/mcp-go v0.54.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
|
||||
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,26 @@
|
||||
// Package engine wraps the vendored Python last30days engine. The engine
|
||||
// is embedded at build time via //go:embed and extracted into a per-user
|
||||
// cache directory on first use, then invoked through python3 in a
|
||||
// subprocess. Consumers should call EnsureUserCache to materialize the
|
||||
// engine and Run to execute it.
|
||||
package engine
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// EngineSourceDir is the embed root inside the binary. scripts/sync-engine.sh
|
||||
// mirrors skills/last30days/scripts/ into this directory before each build.
|
||||
// The all: prefix preserves files starting with "." or "_" so the .gitkeep
|
||||
// anchor file survives - without it the embed would error before sync runs.
|
||||
//
|
||||
//go:embed all:vendored
|
||||
var vendored embed.FS
|
||||
|
||||
// EngineFS returns the embedded engine as a filesystem rooted at the
|
||||
// vendored/ directory contents (so callers see "last30days.py" at the
|
||||
// root, not "vendored/last30days.py").
|
||||
func EngineFS() (fs.FS, error) {
|
||||
return fs.Sub(vendored, "vendored")
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// SentinelFilename names the file Ensure writes inside the cache directory
|
||||
// after a successful extraction. Its contents are compared to the requested
|
||||
// version; a match short-circuits re-extraction on subsequent calls.
|
||||
const SentinelFilename = ".version"
|
||||
|
||||
// cacheSubdir namespaces our cache under the OS user cache directory so
|
||||
// multiple printing-press-style bundles can coexist.
|
||||
const cacheSubdir = "last30days-pp-mcp"
|
||||
|
||||
// CacheEnvOverride lets users redirect the cache directory when the default
|
||||
// OS cache location is read-only (locked-down corp images, ephemeral CI
|
||||
// containers). Pointed at by extract errors via the documented escape hatch.
|
||||
const CacheEnvOverride = "LAST30DAYS_CACHE_DIR"
|
||||
|
||||
// Ensure extracts src into baseDir/last30days-pp-mcp/<version> and returns
|
||||
// the cache path. If the sentinel file already records the same version the
|
||||
// directory is reused without rewriting. version must be non-empty so the
|
||||
// cache layout always namespaces by version.
|
||||
//
|
||||
// Extraction writes to a sibling .tmp directory and renames it on success
|
||||
// so a partial extraction can never be mistaken for a complete one. Concurrent
|
||||
// callers within the same process serialize behind a per-cache-dir sync.Once
|
||||
// so the rename happens exactly once.
|
||||
func Ensure(src fs.FS, baseDir, version string) (string, error) {
|
||||
if version == "" {
|
||||
return "", errors.New("engine: version is required")
|
||||
}
|
||||
cacheDir := filepath.Join(baseDir, cacheSubdir, version)
|
||||
|
||||
once := getOnce(cacheDir)
|
||||
var extractErr error
|
||||
once.Do(func() {
|
||||
extractErr = ensureLocked(src, cacheDir, version)
|
||||
})
|
||||
if extractErr != nil {
|
||||
// Reset the sync.Once so a follow-up call can retry rather than
|
||||
// permanently caching the error. Retry is the right default when
|
||||
// the failure is transient (e.g., disk full, parent dir restored).
|
||||
resetOnce(cacheDir)
|
||||
return "", extractErr
|
||||
}
|
||||
return cacheDir, nil
|
||||
}
|
||||
|
||||
// EnsureUserCache wraps Ensure with the OS user cache dir (or the
|
||||
// LAST30DAYS_CACHE_DIR override) as base. Production callers use this; tests
|
||||
// use Ensure with an explicit temp dir.
|
||||
func EnsureUserCache(src fs.FS, version string) (string, error) {
|
||||
if override := os.Getenv(CacheEnvOverride); override != "" {
|
||||
return Ensure(src, override, version)
|
||||
}
|
||||
base, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("engine: resolve user cache dir (set %s to override): %w", CacheEnvOverride, err)
|
||||
}
|
||||
return Ensure(src, base, version)
|
||||
}
|
||||
|
||||
func ensureLocked(src fs.FS, cacheDir, version string) error {
|
||||
if sentinelMatches(cacheDir, version) {
|
||||
return nil
|
||||
}
|
||||
tmpDir := cacheDir + ".tmp"
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
return fmt.Errorf("engine: clean tmp cache: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(tmpDir, 0o755); err != nil {
|
||||
return fmt.Errorf("engine: create tmp cache (%s, set %s to override): %w", tmpDir, CacheEnvOverride, err)
|
||||
}
|
||||
if err := extractAll(src, tmpDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return err
|
||||
}
|
||||
sentinel := filepath.Join(tmpDir, SentinelFilename)
|
||||
if err := os.WriteFile(sentinel, []byte(version), 0o644); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: write sentinel: %w", err)
|
||||
}
|
||||
if err := os.RemoveAll(cacheDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: clean old cache: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpDir, cacheDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: promote tmp cache: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sentinelMatches(cacheDir, version string) bool {
|
||||
data, err := os.ReadFile(filepath.Join(cacheDir, SentinelFilename))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return string(data) == version
|
||||
}
|
||||
|
||||
func extractAll(src fs.FS, dst string) error {
|
||||
return fs.WalkDir(src, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
target := filepath.Join(dst, path)
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
return copyEmbeddedFile(src, path, target)
|
||||
})
|
||||
}
|
||||
|
||||
func copyEmbeddedFile(src fs.FS, srcPath, dst string) error {
|
||||
in, err := src.Open(srcPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("engine: open %s: %w", srcPath, err)
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return fmt.Errorf("engine: ensure parent of %s: %w", dst, err)
|
||||
}
|
||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("engine: create %s: %w", dst, err)
|
||||
}
|
||||
defer func() { _ = out.Close() }()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return fmt.Errorf("engine: write %s: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// onceRegistry serializes first-call extraction per cache directory so the
|
||||
// rename in ensureLocked happens exactly once across goroutines.
|
||||
var (
|
||||
onceMu sync.Mutex
|
||||
onceRegistry = map[string]*sync.Once{}
|
||||
)
|
||||
|
||||
func getOnce(cacheDir string) *sync.Once {
|
||||
onceMu.Lock()
|
||||
defer onceMu.Unlock()
|
||||
if o, ok := onceRegistry[cacheDir]; ok {
|
||||
return o
|
||||
}
|
||||
o := &sync.Once{}
|
||||
onceRegistry[cacheDir] = o
|
||||
return o
|
||||
}
|
||||
|
||||
func resetOnce(cacheDir string) {
|
||||
onceMu.Lock()
|
||||
defer onceMu.Unlock()
|
||||
delete(onceRegistry, cacheDir)
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func newTestFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("# last30days entry\n"), Mode: 0o644},
|
||||
"lib/__init__.py": &fstest.MapFile{Data: []byte(""), Mode: 0o644},
|
||||
"lib/env.py": &fstest.MapFile{Data: []byte("# env helpers\n"), Mode: 0o644},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureExtractsEngine(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
cacheDir, err := Ensure(src, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure: %v", err)
|
||||
}
|
||||
if cacheDir != filepath.Join(base, cacheSubdir, "v1") {
|
||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, filepath.Join(base, cacheSubdir, "v1"))
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
||||
mustReadFile(t, filepath.Join(cacheDir, "lib/env.py"), "# env helpers\n")
|
||||
mustReadFile(t, filepath.Join(cacheDir, SentinelFilename), "v1")
|
||||
}
|
||||
|
||||
func TestEnsureSkipsWhenSentinelMatches(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
cacheDir, err := Ensure(src, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("first Ensure: %v", err)
|
||||
}
|
||||
target := filepath.Join(cacheDir, "last30days.py")
|
||||
info1, err := os.Stat(target)
|
||||
if err != nil {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
|
||||
// Reset the sync.Once so a second call would re-extract if not for the
|
||||
// sentinel short-circuit. Without the reset, sync.Once would skip the
|
||||
// extraction regardless of sentinel state.
|
||||
resetOnce(cacheDir)
|
||||
|
||||
if _, err := Ensure(src, base, "v1"); err != nil {
|
||||
t.Fatalf("second Ensure: %v", err)
|
||||
}
|
||||
info2, err := os.Stat(target)
|
||||
if err != nil {
|
||||
t.Fatalf("stat second: %v", err)
|
||||
}
|
||||
if !info2.ModTime().Equal(info1.ModTime()) {
|
||||
t.Fatalf("expected file untouched on sentinel match; got mtime %v -> %v", info1.ModTime(), info2.ModTime())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureReExtractsOnVersionChange(t *testing.T) {
|
||||
v1 := fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("v1\n"), Mode: 0o644},
|
||||
}
|
||||
v2 := fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("v2\n"), Mode: 0o644},
|
||||
}
|
||||
base := t.TempDir()
|
||||
|
||||
cache1, err := Ensure(v1, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure v1: %v", err)
|
||||
}
|
||||
cache2, err := Ensure(v2, base, "v2")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure v2: %v", err)
|
||||
}
|
||||
if cache1 == cache2 {
|
||||
t.Fatalf("expected distinct cache dirs per version, got %q == %q", cache1, cache2)
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cache1, "last30days.py"), "v1\n")
|
||||
mustReadFile(t, filepath.Join(cache2, "last30days.py"), "v2\n")
|
||||
}
|
||||
|
||||
func TestEnsureConcurrentFirstCall(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
const goroutines = 10
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
results := make([]string, goroutines)
|
||||
errs := make([]error, goroutines)
|
||||
for i := 0; i < goroutines; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
results[i], errs[i] = Ensure(src, base, "v1")
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("goroutine %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
for i := 1; i < goroutines; i++ {
|
||||
if results[i] != results[0] {
|
||||
t.Fatalf("goroutine 0 saw %q, goroutine %d saw %q", results[0], i, results[i])
|
||||
}
|
||||
}
|
||||
mustReadFile(t, filepath.Join(results[0], "last30days.py"), "# last30days entry\n")
|
||||
}
|
||||
|
||||
func TestEnsureRejectsEmptyVersion(t *testing.T) {
|
||||
if _, err := Ensure(newTestFS(), t.TempDir(), ""); err == nil {
|
||||
t.Fatal("expected error for empty version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureReturnsErrorWhenCacheUnwritable(t *testing.T) {
|
||||
// Place the cache root at a path that cannot exist (a regular file).
|
||||
// MkdirAll will refuse and Ensure must surface a wrapped error.
|
||||
base := t.TempDir()
|
||||
blocker := filepath.Join(base, "blocker")
|
||||
if err := os.WriteFile(blocker, []byte("not a dir"), 0o644); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
_, err := Ensure(newTestFS(), blocker, "v1")
|
||||
if err == nil {
|
||||
t.Fatal("expected error when cache parent is not a directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureUserCacheHonorsOverride(t *testing.T) {
|
||||
override := t.TempDir()
|
||||
t.Setenv(CacheEnvOverride, override)
|
||||
|
||||
src := newTestFS()
|
||||
cacheDir, err := EnsureUserCache(src, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureUserCache: %v", err)
|
||||
}
|
||||
want := filepath.Join(override, cacheSubdir, "v1")
|
||||
if cacheDir != want {
|
||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, want)
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
||||
}
|
||||
|
||||
func mustReadFile(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("%s: got %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultPythonBinary is the interpreter we look up unless RunOptions
|
||||
// overrides it. Windows installs may expose only "python"; we surface a
|
||||
// clear error in that case rather than silently picking the wrong binary.
|
||||
const DefaultPythonBinary = "python3"
|
||||
|
||||
// MinPythonVersion mirrors the engine's MIN_PYTHON constant in
|
||||
// last30days.py. Surfaced in errors so users know what they're missing.
|
||||
const MinPythonVersion = "3.12"
|
||||
|
||||
// PythonInstallURL is included in the missing-interpreter error so users
|
||||
// have a direct route from the failure to a fix.
|
||||
const PythonInstallURL = "https://www.python.org/downloads/"
|
||||
|
||||
// DefaultTimeout caps a single research subprocess. The engine's deep mode
|
||||
// can run several minutes; five minutes is a safe upper bound that still
|
||||
// fails fast when something hangs.
|
||||
const DefaultTimeout = 5 * time.Minute
|
||||
|
||||
// TimeoutEnvOverride lets operators override DefaultTimeout per install
|
||||
// (seconds, integer). Honored by Run when RunOptions.Timeout is zero.
|
||||
const TimeoutEnvOverride = "LAST30DAYS_MCP_TIMEOUT"
|
||||
|
||||
// RunOptions configures one invocation of the embedded Python engine.
|
||||
// PythonPath is exposed so tests can substitute a stub interpreter without
|
||||
// manipulating the process PATH.
|
||||
type RunOptions struct {
|
||||
PythonPath string // resolved python3 binary; empty means look up DefaultPythonBinary on PATH
|
||||
CacheDir string // engine.Ensure result; lib/ here is added to PYTHONPATH
|
||||
Args []string // arguments after last30days.py (topic, --emit=..., etc.)
|
||||
ExtraEnv []string // appended to os.Environ() for the child process
|
||||
Timeout time.Duration // zero means DefaultTimeout or TimeoutEnvOverride
|
||||
}
|
||||
|
||||
// RunResult captures the engine's full output. Stdout is what we surface to
|
||||
// the agent; Stderr is included in error messages so users can diagnose
|
||||
// engine failures without leaving Claude Desktop.
|
||||
type RunResult struct {
|
||||
Stdout []byte
|
||||
Stderr []byte
|
||||
ExitCode int
|
||||
TimedOut bool
|
||||
}
|
||||
|
||||
// Run shells out to python3 with last30days.py inside cacheDir. The child
|
||||
// receives the parent environment (so MCPB user_config env-injection
|
||||
// reaches the engine) plus ExtraEnv and a PYTHONPATH that points at the
|
||||
// cache so the engine's `from lib import ...` statements resolve.
|
||||
//
|
||||
// A missing interpreter, a non-zero exit, and a timeout each surface as
|
||||
// distinct errors so the tool handler can map them to user-facing
|
||||
// messages without re-parsing stderr.
|
||||
func Run(ctx context.Context, opts RunOptions) (*RunResult, error) {
|
||||
if opts.CacheDir == "" {
|
||||
return nil, errors.New("engine: CacheDir is required")
|
||||
}
|
||||
pythonPath, err := resolvePython(opts.PythonPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scriptPath := filepath.Join(opts.CacheDir, "last30days.py")
|
||||
if _, err := os.Stat(scriptPath); err != nil {
|
||||
return nil, fmt.Errorf("engine: last30days.py not found in cache %s: %w", opts.CacheDir, err)
|
||||
}
|
||||
|
||||
timeout := resolveTimeout(opts.Timeout)
|
||||
subCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
args := append([]string{scriptPath}, opts.Args...)
|
||||
cmd := exec.CommandContext(subCtx, pythonPath, args...)
|
||||
cmd.Env = buildEnv(opts.CacheDir, opts.ExtraEnv)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
err = cmd.Run()
|
||||
res := &RunResult{
|
||||
Stdout: stdout.Bytes(),
|
||||
Stderr: stderr.Bytes(),
|
||||
ExitCode: 0,
|
||||
TimedOut: errors.Is(subCtx.Err(), context.DeadlineExceeded),
|
||||
}
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
res.ExitCode = exitErr.ExitCode()
|
||||
if res.TimedOut {
|
||||
return res, fmt.Errorf("engine: subprocess exceeded %s timeout", timeout)
|
||||
}
|
||||
return res, fmt.Errorf("engine: subprocess exited with code %d", res.ExitCode)
|
||||
}
|
||||
return res, fmt.Errorf("engine: subprocess failed to start: %w", err)
|
||||
}
|
||||
|
||||
// resolvePython returns an absolute path to the interpreter or an error
|
||||
// naming the install URL. If the caller supplied a path we trust it - tests
|
||||
// rely on this to inject a stub. Otherwise we look up python3 on PATH.
|
||||
func resolvePython(override string) (string, error) {
|
||||
if override != "" {
|
||||
return override, nil
|
||||
}
|
||||
path, err := exec.LookPath(DefaultPythonBinary)
|
||||
if err == nil {
|
||||
return path, nil
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"engine: %s not found on PATH (need Python %s+, install from %s; current GOOS=%s)",
|
||||
DefaultPythonBinary, MinPythonVersion, PythonInstallURL, runtime.GOOS,
|
||||
)
|
||||
}
|
||||
|
||||
func resolveTimeout(explicit time.Duration) time.Duration {
|
||||
if explicit > 0 {
|
||||
return explicit
|
||||
}
|
||||
if raw := os.Getenv(TimeoutEnvOverride); raw != "" {
|
||||
if d, err := time.ParseDuration(raw); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return DefaultTimeout
|
||||
}
|
||||
|
||||
// buildEnv stitches PYTHONPATH onto os.Environ + ExtraEnv. Any pre-existing
|
||||
// PYTHONPATH in the parent environment is dropped before appending the
|
||||
// cache dir; otherwise the child sees two PYTHONPATH= entries and POSIX
|
||||
// getenv returns the first one, so the user's value wins and the engine's
|
||||
// `from lib import ...` fails with ModuleNotFoundError. The engine is
|
||||
// self-contained and does not need the user's Python module search path.
|
||||
func buildEnv(cacheDir string, extra []string) []string {
|
||||
const pyKey = "PYTHONPATH="
|
||||
parent := os.Environ()
|
||||
base := make([]string, 0, len(parent)+1+len(extra))
|
||||
for _, kv := range parent {
|
||||
if strings.HasPrefix(kv, pyKey) {
|
||||
continue
|
||||
}
|
||||
base = append(base, kv)
|
||||
}
|
||||
base = append(base, pyKey+cacheDir)
|
||||
base = append(base, extra...)
|
||||
return base
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// makeStubPython writes a shell script that simulates python3 and returns
|
||||
// its absolute path. The script honors a small env-driven protocol so each
|
||||
// test can shape its output:
|
||||
//
|
||||
// STUB_STDOUT - text printed to stdout
|
||||
// STUB_STDERR - text printed to stderr
|
||||
// STUB_EXIT_CODE - integer exit code (default 0)
|
||||
// STUB_SLEEP_SECS - sleep before exiting (for timeout tests)
|
||||
// STUB_ECHO_ENV - name of an env var; the stub prints "<NAME>=<VALUE>"
|
||||
// STUB_ECHO_ARG - integer index; the stub prints "ARG<i>=<args[i]>"
|
||||
//
|
||||
// The stub ignores its first argument (the script path), matching how a
|
||||
// real python3 invocation treats `python3 last30days.py ...`.
|
||||
func makeStubPython(t *testing.T) string {
|
||||
t.Helper()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("stub-python tests rely on POSIX shell")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "python3-stub.sh")
|
||||
script := `#!/usr/bin/env bash
|
||||
if [ -n "${STUB_SLEEP_SECS:-}" ]; then sleep "$STUB_SLEEP_SECS"; fi
|
||||
if [ -n "${STUB_STDOUT:-}" ]; then printf "%s" "$STUB_STDOUT"; fi
|
||||
if [ -n "${STUB_STDERR:-}" ]; then printf "%s" "$STUB_STDERR" >&2; fi
|
||||
if [ -n "${STUB_ECHO_ENV:-}" ]; then echo "${STUB_ECHO_ENV}=${!STUB_ECHO_ENV:-<unset>}"; fi
|
||||
if [ -n "${STUB_ECHO_ARG:-}" ]; then echo "ARG${STUB_ECHO_ARG}=${!STUB_ECHO_ARG:-<unset>}"; fi
|
||||
exit "${STUB_EXIT_CODE:-0}"
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// stageCache materializes a fake CacheDir with a no-op last30days.py so
|
||||
// the existence check in Run passes. The stub python3 ignores the script
|
||||
// contents, so the file just has to exist.
|
||||
func stageCache(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "last30days.py"), []byte("# stub\n"), 0o644); err != nil {
|
||||
t.Fatalf("stage cache: %v", err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestRunHappyPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_STDOUT", "synthesis output\n")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
Args: []string{"my topic", "--emit=compact"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if string(res.Stdout) != "synthesis output\n" {
|
||||
t.Fatalf("stdout = %q, want %q", res.Stdout, "synthesis output\n")
|
||||
}
|
||||
if res.ExitCode != 0 {
|
||||
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
||||
}
|
||||
if res.TimedOut {
|
||||
t.Fatal("TimedOut = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForwardsEnv(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("OPENAI_API_KEY", "sk-test-value")
|
||||
t.Setenv("STUB_ECHO_ENV", "OPENAI_API_KEY")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if got := strings.TrimSpace(string(res.Stdout)); got != "OPENAI_API_KEY=sk-test-value" {
|
||||
t.Fatalf("stdout = %q, want OPENAI_API_KEY=sk-test-value", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSetsPythonPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
want := "PYTHONPATH=" + cache
|
||||
if got := strings.TrimSpace(string(res.Stdout)); got != want {
|
||||
t.Fatalf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDropsPreExistingPythonPath guards the buildEnv dedup: when the
|
||||
// parent already sets PYTHONPATH (common on dev machines and CI runners
|
||||
// that touch Python), the child must NOT see two PYTHONPATH= entries.
|
||||
// POSIX getenv returns the first match, so a duplicate from os.Environ
|
||||
// would shadow our cache-dir entry and break `from lib import ...`.
|
||||
func TestRunDropsPreExistingPythonPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("PYTHONPATH", "/users-stale-pythonpath")
|
||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
got := strings.TrimSpace(string(res.Stdout))
|
||||
want := "PYTHONPATH=" + cache
|
||||
if got != want {
|
||||
t.Fatalf("stdout = %q, want %q (stale parent value leaked through)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildEnvDropsAllPreExistingPythonPath(t *testing.T) {
|
||||
// Direct unit test on buildEnv to catch the case where the parent has
|
||||
// PYTHONPATH set: the returned slice must contain exactly one
|
||||
// PYTHONPATH= entry, and it must be ours.
|
||||
t.Setenv("PYTHONPATH", "/parent/one")
|
||||
cache := "/cache/dir"
|
||||
out := buildEnv(cache, []string{"EXTRA=1"})
|
||||
|
||||
var pythonPaths []string
|
||||
for _, kv := range out {
|
||||
if strings.HasPrefix(kv, "PYTHONPATH=") {
|
||||
pythonPaths = append(pythonPaths, kv)
|
||||
}
|
||||
}
|
||||
if len(pythonPaths) != 1 {
|
||||
t.Fatalf("got %d PYTHONPATH entries, want 1: %v", len(pythonPaths), pythonPaths)
|
||||
}
|
||||
if pythonPaths[0] != "PYTHONPATH="+cache {
|
||||
t.Fatalf("PYTHONPATH = %q, want %q", pythonPaths[0], "PYTHONPATH="+cache)
|
||||
}
|
||||
// Confirm ExtraEnv still rides along.
|
||||
found := false
|
||||
for _, kv := range out {
|
||||
if kv == "EXTRA=1" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("EXTRA=1 missing from buildEnv output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSurfacesExitCode(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_STDERR", "engine boom\n")
|
||||
t.Setenv("STUB_EXIT_CODE", "2")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for non-zero exit")
|
||||
}
|
||||
if res == nil {
|
||||
t.Fatal("res is nil; want populated result alongside error")
|
||||
}
|
||||
if res.ExitCode != 2 {
|
||||
t.Fatalf("ExitCode = %d, want 2", res.ExitCode)
|
||||
}
|
||||
if !strings.Contains(string(res.Stderr), "engine boom") {
|
||||
t.Fatalf("stderr did not surface engine output: %q", res.Stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTimesOut(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_SLEEP_SECS", "3")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
Timeout: 200 * time.Millisecond,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error")
|
||||
}
|
||||
if !res.TimedOut {
|
||||
t.Fatal("TimedOut = false, want true")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "timeout") {
|
||||
t.Fatalf("error %q lacks 'timeout' marker", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingPython(t *testing.T) {
|
||||
cache := stageCache(t)
|
||||
// Empty PATH guarantees the lookup fails. PythonPath stays unset so Run
|
||||
// falls through to exec.LookPath.
|
||||
t.Setenv("PATH", "")
|
||||
|
||||
_, err := Run(context.Background(), RunOptions{CacheDir: cache})
|
||||
if err == nil {
|
||||
t.Fatal("expected lookup failure with empty PATH")
|
||||
}
|
||||
if !strings.Contains(err.Error(), DefaultPythonBinary) {
|
||||
t.Fatalf("error %q does not mention %s", err, DefaultPythonBinary)
|
||||
}
|
||||
if !strings.Contains(err.Error(), PythonInstallURL) {
|
||||
t.Fatalf("error %q does not include install URL", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingScript(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
// CacheDir exists but contains no last30days.py.
|
||||
cache := t.TempDir()
|
||||
|
||||
_, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error when last30days.py missing")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "last30days.py") {
|
||||
t.Fatalf("error %q does not name missing script", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsEmptyCacheDir(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
_, err := Run(context.Background(), RunOptions{PythonPath: stub})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty CacheDir")
|
||||
}
|
||||
if !errors.Is(err, err) || !strings.Contains(err.Error(), "CacheDir") {
|
||||
t.Fatalf("error %q does not name CacheDir", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTimeoutHonorsEnv(t *testing.T) {
|
||||
t.Setenv(TimeoutEnvOverride, "750ms")
|
||||
if got := resolveTimeout(0); got != 750*time.Millisecond {
|
||||
t.Fatalf("resolveTimeout = %v, want 750ms", got)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "garbage")
|
||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
||||
t.Fatalf("garbage value: got %v, want default %v", got, DefaultTimeout)
|
||||
}
|
||||
if got := resolveTimeout(time.Minute); got != time.Minute {
|
||||
t.Fatalf("explicit value not honored: got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Populated at build time by scripts/sync-engine.sh.
|
||||
Source of truth: skills/last30days/scripts/.
|
||||
@@ -0,0 +1,189 @@
|
||||
// Package manifest holds tests for mcp/manifest.json. It contains no
|
||||
// production code - the manifest itself is the artifact, and these tests
|
||||
// guard structural invariants the bundling pipeline depends on.
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// envBinding is a minimal subset of the MCPB v0.3 manifest just covering
|
||||
// the fields these tests assert on. We deliberately do not depend on the
|
||||
// printing-press internal/pipeline types (that's an internal/ package and
|
||||
// not importable across modules) - the structural invariants below are
|
||||
// what actually matter for Claude Desktop install correctness.
|
||||
type manifestShape struct {
|
||||
ManifestVersion string `json:"manifest_version"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Server struct {
|
||||
Type string `json:"type"`
|
||||
EntryPoint string `json:"entry_point"`
|
||||
MCPConfig struct {
|
||||
Command string `json:"command"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"mcp_config"`
|
||||
} `json:"server"`
|
||||
UserConfig map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Sensitive bool `json:"sensitive"`
|
||||
Required bool `json:"required"`
|
||||
} `json:"user_config"`
|
||||
Compatibility struct {
|
||||
ClaudeDesktop string `json:"claude_desktop"`
|
||||
Platforms []string `json:"platforms"`
|
||||
} `json:"compatibility"`
|
||||
}
|
||||
|
||||
// loadManifest reads mcp/manifest.json relative to this test file so the
|
||||
// test passes regardless of where `go test` is invoked from.
|
||||
func loadManifest(t *testing.T) manifestShape {
|
||||
t.Helper()
|
||||
_, thisFile, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("runtime.Caller failed")
|
||||
}
|
||||
// manifest_test.go is at mcp/internal/manifest/; manifest.json at mcp/.
|
||||
manifestPath := filepath.Join(filepath.Dir(thisFile), "..", "..", "manifest.json")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
var m manifestShape
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
t.Fatalf("parse manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestManifestRequiredFields(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
if m.ManifestVersion != "0.3" {
|
||||
t.Errorf("manifest_version = %q, want 0.3", m.ManifestVersion)
|
||||
}
|
||||
if m.Name != "last30days-pp-mcp" {
|
||||
t.Errorf("name = %q, want last30days-pp-mcp", m.Name)
|
||||
}
|
||||
if m.Version == "" {
|
||||
t.Error("version is empty")
|
||||
}
|
||||
if m.Server.Type != "binary" {
|
||||
t.Errorf("server.type = %q, want binary", m.Server.Type)
|
||||
}
|
||||
if m.Server.EntryPoint != "bin/last30days-pp-mcp" {
|
||||
t.Errorf("server.entry_point = %q, want bin/last30days-pp-mcp", m.Server.EntryPoint)
|
||||
}
|
||||
if m.Compatibility.ClaudeDesktop == "" {
|
||||
t.Error("compatibility.claude_desktop is empty")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEnvAndUserConfigCrossReference is the key invariant: every
|
||||
// ${user_config.<key>} substitution in server.mcp_config.env must point
|
||||
// at a real user_config entry, and every declared user_config must be
|
||||
// wired to an env var. A typo on either side silently disables a credential
|
||||
// at install time without the binary or Claude Desktop noticing.
|
||||
func TestEnvAndUserConfigCrossReference(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
|
||||
if len(m.Server.MCPConfig.Env) == 0 {
|
||||
t.Fatal("server.mcp_config.env is empty; expected user_config substitutions")
|
||||
}
|
||||
if len(m.UserConfig) == 0 {
|
||||
t.Fatal("user_config is empty; expected per-key declarations")
|
||||
}
|
||||
|
||||
for envName, value := range m.Server.MCPConfig.Env {
|
||||
key, ok := parseUserConfigRef(value)
|
||||
if !ok {
|
||||
t.Errorf("env[%s] = %q is not a ${user_config.<key>} reference", envName, value)
|
||||
continue
|
||||
}
|
||||
if _, declared := m.UserConfig[key]; !declared {
|
||||
t.Errorf("env[%s] references user_config[%q], which is not declared", envName, key)
|
||||
}
|
||||
// The user_config key must be the lowercased env var so Claude
|
||||
// Desktop's substitution rule matches PP's emitted shape.
|
||||
if got := strings.ToLower(envName); key != got {
|
||||
t.Errorf("env[%s] -> user_config[%q]; convention requires user_config[%q]", envName, key, got)
|
||||
}
|
||||
}
|
||||
|
||||
envValues := make(map[string]bool, len(m.Server.MCPConfig.Env))
|
||||
for _, value := range m.Server.MCPConfig.Env {
|
||||
if key, ok := parseUserConfigRef(value); ok {
|
||||
envValues[key] = true
|
||||
}
|
||||
}
|
||||
for key := range m.UserConfig {
|
||||
if !envValues[key] {
|
||||
t.Errorf("user_config[%q] is declared but never substituted into env", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserConfigShape(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
for key, slot := range m.UserConfig {
|
||||
if slot.Type != "string" {
|
||||
t.Errorf("user_config[%q].type = %q, want string", key, slot.Type)
|
||||
}
|
||||
if slot.Title == "" {
|
||||
t.Errorf("user_config[%q].title is empty", key)
|
||||
}
|
||||
if slot.Description == "" {
|
||||
t.Errorf("user_config[%q].description is empty", key)
|
||||
}
|
||||
if !slot.Sensitive {
|
||||
// API keys must be flagged sensitive so Claude Desktop masks
|
||||
// the input and prefers OS-keychain storage.
|
||||
t.Errorf("user_config[%q].sensitive = false; want true for API credentials", key)
|
||||
}
|
||||
if slot.Required {
|
||||
// The engine degrades to web-only mode without keys, so no
|
||||
// key is install-blocking.
|
||||
t.Errorf("user_config[%q].required = true; engine degrades without keys, so all keys are optional", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformsMatchShippingMatrix(t *testing.T) {
|
||||
// compatibility.platforms must list exactly what the release CI
|
||||
// actually packages. Listing a platform we don't ship would let
|
||||
// Claude Desktop start an install that has no matching binary inside
|
||||
// the bundle, producing a silent failure. The CI matrix in
|
||||
// .github/workflows/release.yml currently covers darwin (arm64 +
|
||||
// amd64) and linux/amd64; Windows is deferred.
|
||||
m := loadManifest(t)
|
||||
required := map[string]bool{"darwin": false, "linux": false}
|
||||
forbidden := map[string]bool{"win32": true}
|
||||
for _, p := range m.Compatibility.Platforms {
|
||||
if _, ok := required[p]; ok {
|
||||
required[p] = true
|
||||
}
|
||||
if forbidden[p] {
|
||||
t.Errorf("compatibility.platforms contains %q but the release matrix does not ship that platform; add it to the matrix or remove from the manifest", p)
|
||||
}
|
||||
}
|
||||
for p, found := range required {
|
||||
if !found {
|
||||
t.Errorf("compatibility.platforms missing %q", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseUserConfigRef(value string) (string, bool) {
|
||||
const prefix = "${user_config."
|
||||
const suffix = "}"
|
||||
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
|
||||
return "", false
|
||||
}
|
||||
return value[len(prefix) : len(value)-len(suffix)], true
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// Package tools owns the MCP tool surface for last30days. Today there is
|
||||
// exactly one tool, research, mirroring the /last30days <topic> slash
|
||||
// command available in Claude Code. Adding new tools means another file
|
||||
// here plus an additional s.AddTool call in Register.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
// Config carries the version string used to namespace the per-user cache.
|
||||
// main passes its ldflags-stamped Version here.
|
||||
type Config struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
// Register adds every tool this server exposes to s. The caller supplies a
|
||||
// Config so test harnesses can pin a version without touching globals.
|
||||
func Register(s *server.MCPServer, cfg Config) {
|
||||
s.AddTool(
|
||||
mcplib.NewTool("research",
|
||||
mcplib.WithDescription(
|
||||
"Research what people are actually saying about any topic in the last 30 days. "+
|
||||
"Aggregates Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web, "+
|
||||
"scored by upvotes, likes, transcripts, and real-money prediction-market odds. "+
|
||||
"Returns the engine's compact output for the model to synthesize.",
|
||||
),
|
||||
mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")),
|
||||
mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")),
|
||||
mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")),
|
||||
mcplib.WithReadOnlyHintAnnotation(true),
|
||||
mcplib.WithDestructiveHintAnnotation(false),
|
||||
mcplib.WithOpenWorldHintAnnotation(true),
|
||||
),
|
||||
makeResearchHandler(cfg),
|
||||
)
|
||||
}
|
||||
|
||||
func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
||||
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
|
||||
args := req.GetArguments()
|
||||
topic, err := requireString(args, "topic")
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
emit, err := emitArgument(args)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
save, err := boolArgument(args, "save")
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
src, err := engine.EngineFS()
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
|
||||
}
|
||||
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf(
|
||||
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
|
||||
err, engine.CacheEnvOverride,
|
||||
)), nil
|
||||
}
|
||||
|
||||
runArgs := []string{topic, "--emit=" + emit}
|
||||
if save {
|
||||
runArgs = append(runArgs, "--save")
|
||||
}
|
||||
|
||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
||||
CacheDir: cacheDir,
|
||||
Args: runArgs,
|
||||
})
|
||||
if runErr != nil {
|
||||
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
|
||||
}
|
||||
return mcplib.NewToolResultText(string(res.Stdout)), nil
|
||||
}
|
||||
}
|
||||
|
||||
func requireString(args map[string]any, name string) (string, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is required", name)
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok || strings.TrimSpace(value) == "" {
|
||||
return "", fmt.Errorf("%s must be a non-empty string", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func emitArgument(args map[string]any) (string, error) {
|
||||
raw, ok := args["emit"]
|
||||
if !ok {
|
||||
return "compact", nil
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok {
|
||||
return "", errors.New("emit must be a string")
|
||||
}
|
||||
switch value {
|
||||
case "":
|
||||
return "compact", nil
|
||||
case "compact", "html":
|
||||
return value, nil
|
||||
default:
|
||||
return "", fmt.Errorf("emit must be 'compact' or 'html', got %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
func boolArgument(args map[string]any, name string) (bool, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
value, ok := raw.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%s must be a boolean", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// formatRunError flattens engine.Run's distinct error shapes into a single
|
||||
// user-facing message that includes the relevant stderr context.
|
||||
func formatRunError(runErr error, res *engine.RunResult) string {
|
||||
var msg strings.Builder
|
||||
msg.WriteString(runErr.Error())
|
||||
if res != nil && len(res.Stderr) > 0 {
|
||||
msg.WriteString("\nengine stderr:\n")
|
||||
msg.Write(res.Stderr)
|
||||
}
|
||||
return msg.String()
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
func newCallToolRequest(args map[string]any) mcplib.CallToolRequest {
|
||||
var req mcplib.CallToolRequest
|
||||
req.Params.Arguments = args
|
||||
return req
|
||||
}
|
||||
|
||||
// resultText pulls text content out of a tool result so tests can assert on
|
||||
// the body Claude will see. Returns empty string when the result is nil or
|
||||
// has no text content.
|
||||
func resultText(res *mcplib.CallToolResult) string {
|
||||
if res == nil {
|
||||
return ""
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, item := range res.Content {
|
||||
if tc, ok := item.(mcplib.TextContent); ok {
|
||||
out.WriteString(tc.Text)
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestRequireStringRejectsMissingAndBlank(t *testing.T) {
|
||||
if _, err := requireString(map[string]any{}, "topic"); err == nil {
|
||||
t.Fatal("expected error for missing topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": ""}, "topic"); err == nil {
|
||||
t.Fatal("expected error for empty topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": " "}, "topic"); err == nil {
|
||||
t.Fatal("expected error for whitespace-only topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": 42}, "topic"); err == nil {
|
||||
t.Fatal("expected error for non-string topic")
|
||||
}
|
||||
v, err := requireString(map[string]any{"topic": "OpenAI"}, "topic")
|
||||
if err != nil || v != "OpenAI" {
|
||||
t.Fatalf("requireString ok = %q, %v", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitArgumentDefaultsAndValidates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"missing defaults to compact", map[string]any{}, "compact", false},
|
||||
{"empty string defaults to compact", map[string]any{"emit": ""}, "compact", false},
|
||||
{"compact passes through", map[string]any{"emit": "compact"}, "compact", false},
|
||||
{"html passes through", map[string]any{"emit": "html"}, "html", false},
|
||||
{"invalid value rejected", map[string]any{"emit": "json"}, "", true},
|
||||
{"non-string rejected", map[string]any{"emit": 7}, "", true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := emitArgument(tc.args)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("got %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoolArgument(t *testing.T) {
|
||||
v, err := boolArgument(map[string]any{}, "save")
|
||||
if err != nil || v {
|
||||
t.Fatalf("missing: %v, %v", v, err)
|
||||
}
|
||||
v, err = boolArgument(map[string]any{"save": true}, "save")
|
||||
if err != nil || !v {
|
||||
t.Fatalf("true: %v, %v", v, err)
|
||||
}
|
||||
v, err = boolArgument(map[string]any{"save": false}, "save")
|
||||
if err != nil || v {
|
||||
t.Fatalf("false: %v, %v", v, err)
|
||||
}
|
||||
if _, err := boolArgument(map[string]any{"save": "true"}, "save"); err == nil {
|
||||
t.Fatal("expected error for string value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) {
|
||||
// Validation failures are returned as MCP tool errors (not Go errors)
|
||||
// so Claude sees a structured failure with a readable message rather
|
||||
// than a transport-level fault.
|
||||
handler := makeResearchHandler(Config{Version: "test"})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
wantSub string
|
||||
}{
|
||||
{"missing topic", map[string]any{}, "topic is required"},
|
||||
{"blank topic", map[string]any{"topic": " "}, "non-empty string"},
|
||||
{"invalid emit", map[string]any{"topic": "OpenAI", "emit": "json"}, "must be 'compact' or 'html'"},
|
||||
{"non-bool save", map[string]any{"topic": "OpenAI", "save": "yes"}, "save must be a boolean"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := handler(context.Background(), newCallToolRequest(tc.args))
|
||||
if err != nil {
|
||||
t.Fatalf("handler should not return Go error for validation; got %v", err)
|
||||
}
|
||||
if res == nil || !res.IsError {
|
||||
t.Fatalf("expected IsError result, got %+v", res)
|
||||
}
|
||||
if !strings.Contains(resultText(res), tc.wantSub) {
|
||||
t.Fatalf("result text %q missing substring %q", resultText(res), tc.wantSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRunErrorIncludesStderr(t *testing.T) {
|
||||
res := &engine.RunResult{Stderr: []byte("engine exploded\n")}
|
||||
msg := formatRunError(errors.New("boom"), res)
|
||||
if !strings.Contains(msg, "boom") || !strings.Contains(msg, "engine exploded") {
|
||||
t.Fatalf("formatRunError missed pieces: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRunErrorHandlesNilResult(t *testing.T) {
|
||||
msg := formatRunError(errors.New("boom"), nil)
|
||||
if msg != "boom" {
|
||||
t.Fatalf("nil result: got %q, want %q", msg, "boom")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"manifest_version": "0.3",
|
||||
"name": "last30days-pp-mcp",
|
||||
"display_name": "Last30Days",
|
||||
"version": "3.0.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
||||
},
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"x",
|
||||
"youtube",
|
||||
"hacker-news",
|
||||
"polymarket",
|
||||
"github",
|
||||
"search",
|
||||
"synthesis"
|
||||
],
|
||||
"server": {
|
||||
"type": "binary",
|
||||
"entry_point": "bin/last30days-pp-mcp",
|
||||
"mcp_config": {
|
||||
"command": "${__dirname}/bin/last30days-pp-mcp",
|
||||
"args": [],
|
||||
"env": {
|
||||
"OPENAI_API_KEY": "${user_config.openai_api_key}",
|
||||
"XAI_API_KEY": "${user_config.xai_api_key}",
|
||||
"BRAVE_API_KEY": "${user_config.brave_api_key}",
|
||||
"EXA_API_KEY": "${user_config.exa_api_key}",
|
||||
"SERPER_API_KEY": "${user_config.serper_api_key}",
|
||||
"GOOGLE_API_KEY": "${user_config.google_api_key}",
|
||||
"GEMINI_API_KEY": "${user_config.gemini_api_key}",
|
||||
"GOOGLE_GENAI_API_KEY": "${user_config.google_genai_api_key}",
|
||||
"APIFY_API_TOKEN": "${user_config.apify_api_token}",
|
||||
"BSKY_APP_PASSWORD": "${user_config.bsky_app_password}",
|
||||
"PARALLEL_API_KEY": "${user_config.parallel_api_key}",
|
||||
"SCRAPECREATORS_API_KEY": "${user_config.scrapecreators_api_key}",
|
||||
"OPENROUTER_API_KEY": "${user_config.openrouter_api_key}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"user_config": {
|
||||
"openai_api_key": {
|
||||
"type": "string",
|
||||
"title": "OPENAI_API_KEY",
|
||||
"description": "OpenAI API key. Powers Reddit research via OpenAI's web_search tool. Get one at https://platform.openai.com/api-keys.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"xai_api_key": {
|
||||
"type": "string",
|
||||
"title": "XAI_API_KEY",
|
||||
"description": "xAI API key. Powers X / Twitter research via xAI's x_search tool. Get one at https://console.x.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"brave_api_key": {
|
||||
"type": "string",
|
||||
"title": "BRAVE_API_KEY",
|
||||
"description": "Brave Search API key. Used for grounded web search results. Get one at https://brave.com/search/api/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"exa_api_key": {
|
||||
"type": "string",
|
||||
"title": "EXA_API_KEY",
|
||||
"description": "Exa search API key. Alternative web search backend with semantic ranking. Get one at https://exa.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"serper_api_key": {
|
||||
"type": "string",
|
||||
"title": "SERPER_API_KEY",
|
||||
"description": "Serper API key. Google search via API. Get one at https://serper.dev/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"google_api_key": {
|
||||
"type": "string",
|
||||
"title": "GOOGLE_API_KEY",
|
||||
"description": "Google API key for YouTube transcript fetching and other Google services. Get one at https://console.cloud.google.com/apis/credentials.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"gemini_api_key": {
|
||||
"type": "string",
|
||||
"title": "GEMINI_API_KEY",
|
||||
"description": "Gemini API key. Used for synthesis fallback when other LLM providers are unavailable. Get one at https://aistudio.google.com/apikey.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"google_genai_api_key": {
|
||||
"type": "string",
|
||||
"title": "GOOGLE_GENAI_API_KEY",
|
||||
"description": "Alternative Google generative-AI API key. Same source as GEMINI_API_KEY; set whichever name your tooling expects.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"apify_api_token": {
|
||||
"type": "string",
|
||||
"title": "APIFY_API_TOKEN",
|
||||
"description": "Apify API token. Powers TikTok and Instagram Reels search via Apify actors. Get one at https://console.apify.com/account/integrations.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"bsky_app_password": {
|
||||
"type": "string",
|
||||
"title": "BSKY_APP_PASSWORD",
|
||||
"description": "Bluesky app password (not your main password). Powers AT Protocol post search. Create at https://bsky.app/settings/app-passwords.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"parallel_api_key": {
|
||||
"type": "string",
|
||||
"title": "PARALLEL_API_KEY",
|
||||
"description": "Parallel AI key. Powers parallel research runs across sources. Get one at https://parallel.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"scrapecreators_api_key": {
|
||||
"type": "string",
|
||||
"title": "SCRAPECREATORS_API_KEY",
|
||||
"description": "ScrapeCreators API key. Powers creator-focused social search across TikTok, Instagram, and YouTube. Get one at https://scrapecreators.com/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"openrouter_api_key": {
|
||||
"type": "string",
|
||||
"title": "OPENROUTER_API_KEY",
|
||||
"description": "OpenRouter API key. Alternative LLM provider gateway for synthesis. Get one at https://openrouter.ai/keys.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"compatibility": {
|
||||
"claude_desktop": ">=1.0.0",
|
||||
"platforms": [
|
||||
"darwin",
|
||||
"linux"
|
||||
]
|
||||
}
|
||||
}
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# Mirrors skills/last30days/scripts/{last30days.py,lib/} into mcp/vendored/
|
||||
# so the Go binary's embed.FS captures the engine at build time.
|
||||
#
|
||||
# Source of truth: skills/last30days/scripts/. Never edit mcp/vendored/ directly.
|
||||
# Run before `go build` locally and in CI before `printing-press bundle`.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MCP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
REPO_ROOT="$(cd "${MCP_DIR}/.." && pwd)"
|
||||
ENGINE_SRC="${REPO_ROOT}/skills/last30days/scripts"
|
||||
# Embed path must live inside the consuming package (Go //go:embed cannot
|
||||
# reach outside its own directory tree), so vendored/ sits under engine/.
|
||||
VENDORED="${MCP_DIR}/internal/engine/vendored"
|
||||
|
||||
if [ ! -f "${ENGINE_SRC}/last30days.py" ]; then
|
||||
echo "sync-engine: ${ENGINE_SRC}/last30days.py not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${VENDORED}"
|
||||
# Clear stale content while keeping the .gitkeep that anchors the embed path.
|
||||
find "${VENDORED}" -mindepth 1 -not -name ".gitkeep" -delete
|
||||
|
||||
# Copy the entry script and the lib/ tree (modules + lib/vendor/).
|
||||
cp "${ENGINE_SRC}/last30days.py" "${VENDORED}/last30days.py"
|
||||
cp -R "${ENGINE_SRC}/lib" "${VENDORED}/lib"
|
||||
|
||||
# Strip caches so the embed.FS stays deterministic.
|
||||
find "${VENDORED}" -type d -name "__pycache__" -prune -exec rm -rf {} +
|
||||
find "${VENDORED}" -type f -name "*.pyc" -delete
|
||||
|
||||
echo "sync-engine: vendored engine at ${VENDORED}"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.4 MiB |
@@ -1,395 +0,0 @@
|
||||
# feat: Add WebSearch as Third Source (Zero-Config Fallback)
|
||||
|
||||
## Overview
|
||||
|
||||
Add Claude's built-in WebSearch tool as a third research source for `/last30days`. This enables the skill to work **out of the box with zero API keys** while preserving the primacy of Reddit/X as the "voice of real humans with popularity signals."
|
||||
|
||||
**Key principle**: WebSearch is supplementary, not primary. Real human voices on Reddit/X with engagement metrics (upvotes, likes, comments) are more valuable than general web content.
|
||||
|
||||
## Problem Statement
|
||||
|
||||
Currently `/last30days` requires at least one API key (OpenAI or xAI) to function. Users without API keys get an error. Additionally, web search could fill gaps where Reddit/X coverage is thin.
|
||||
|
||||
**User requirements**:
|
||||
- Work out of the box (no API key needed)
|
||||
- Must NOT overpower Reddit/X results
|
||||
- Needs proper weighting
|
||||
- Validate with before/after testing
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
### Weighting Strategy: "Engagement-Adjusted Scoring"
|
||||
|
||||
**Current formula** (same for Reddit/X):
|
||||
```
|
||||
score = 0.45*relevance + 0.25*recency + 0.30*engagement - penalties
|
||||
```
|
||||
|
||||
**Problem**: WebSearch has NO engagement metrics. Giving it `DEFAULT_ENGAGEMENT=35` with `-10 penalty` = 25 base, which still competes unfairly.
|
||||
|
||||
**Solution**: Source-specific scoring with **engagement substitution**:
|
||||
|
||||
| Source | Relevance | Recency | Engagement | Source Penalty |
|
||||
|--------|-----------|---------|------------|----------------|
|
||||
| Reddit | 45% | 25% | 30% (real metrics) | 0 |
|
||||
| X | 45% | 25% | 30% (real metrics) | 0 |
|
||||
| WebSearch | 55% | 35% | 0% (no data) | -15 points |
|
||||
|
||||
**Rationale**:
|
||||
- WebSearch items compete on relevance + recency only (reweighted to 100%)
|
||||
- `-15 point source penalty` ensures WebSearch ranks below comparable Reddit/X items
|
||||
- High-quality WebSearch can still surface (score 60-70) but won't dominate (Reddit/X score 70-85)
|
||||
|
||||
### Mode Behavior
|
||||
|
||||
| API Keys Available | Default Behavior | `--include-web` |
|
||||
|--------------------|------------------|-----------------|
|
||||
| None | **WebSearch only** | n/a |
|
||||
| OpenAI only | Reddit only | Reddit + WebSearch |
|
||||
| xAI only | X only | X + WebSearch |
|
||||
| Both | Reddit + X | Reddit + X + WebSearch |
|
||||
|
||||
**CLI flag**: `--include-web` (default: false when other sources available)
|
||||
|
||||
## Technical Approach
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ last30days.py orchestrator │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ run_research() │
|
||||
│ ├── if sources includes "reddit": openai_reddit.search_reddit()│
|
||||
│ ├── if sources includes "x": xai_x.search_x() │
|
||||
│ └── if sources includes "web": websearch.search_web() ← NEW │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Processing Pipeline │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ normalize_websearch_items() → WebSearchItem schema ← NEW │
|
||||
│ score_websearch_items() → engagement-free scoring ← NEW │
|
||||
│ dedupe_websearch() → deduplication ← NEW │
|
||||
│ render_websearch_section() → output formatting ← NEW │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Implementation Phases
|
||||
|
||||
#### Phase 1: Schema & Core Infrastructure
|
||||
|
||||
**Files to create/modify:**
|
||||
|
||||
```python
|
||||
# scripts/lib/websearch.py (NEW)
|
||||
"""Claude WebSearch API client for general web discovery."""
|
||||
|
||||
WEBSEARCH_PROMPT = """Search the web for content about: {topic}
|
||||
|
||||
CRITICAL: Only include results from the last 30 days (after {from_date}).
|
||||
|
||||
Find {min_items}-{max_items} high-quality, relevant web pages. Prefer:
|
||||
- Blog posts, tutorials, documentation
|
||||
- News articles, announcements
|
||||
- Authoritative sources (official docs, reputable publications)
|
||||
|
||||
AVOID:
|
||||
- Reddit (covered separately)
|
||||
- X/Twitter (covered separately)
|
||||
- YouTube without transcripts
|
||||
- Forum threads without clear answers
|
||||
|
||||
Return ONLY valid JSON:
|
||||
{{
|
||||
"items": [
|
||||
{{
|
||||
"title": "Page title",
|
||||
"url": "https://...",
|
||||
"source_domain": "example.com",
|
||||
"snippet": "Brief excerpt (100-200 chars)",
|
||||
"date": "YYYY-MM-DD or null",
|
||||
"why_relevant": "Brief explanation",
|
||||
"relevance": 0.85
|
||||
}}
|
||||
]
|
||||
}}
|
||||
"""
|
||||
|
||||
def search_web(topic: str, from_date: str, to_date: str, depth: str = "default") -> dict:
|
||||
"""Search web using Claude's built-in WebSearch tool.
|
||||
|
||||
NOTE: This runs INSIDE Claude Code, so we use the WebSearch tool directly.
|
||||
No API key needed - uses Claude's session.
|
||||
"""
|
||||
# Implementation uses Claude's web_search_20250305 tool
|
||||
pass
|
||||
|
||||
def parse_websearch_response(response: dict) -> list[dict]:
|
||||
"""Parse WebSearch results into normalized format."""
|
||||
pass
|
||||
```
|
||||
|
||||
```python
|
||||
# scripts/lib/schema.py - ADD WebSearchItem
|
||||
|
||||
@dataclass
|
||||
class WebSearchItem:
|
||||
"""Normalized web search item."""
|
||||
id: str
|
||||
title: str
|
||||
url: str
|
||||
source_domain: str # e.g., "medium.com", "github.com"
|
||||
snippet: str
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "low"
|
||||
relevance: float = 0.5
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'url': self.url,
|
||||
'source_domain': self.source_domain,
|
||||
'snippet': self.snippet,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
```
|
||||
|
||||
#### Phase 2: Scoring System Updates
|
||||
|
||||
```python
|
||||
# scripts/lib/score.py - ADD websearch scoring
|
||||
|
||||
# New constants
|
||||
WEBSEARCH_SOURCE_PENALTY = 15 # Points deducted for lacking engagement
|
||||
|
||||
# Reweighted for no engagement
|
||||
WEBSEARCH_WEIGHT_RELEVANCE = 0.55
|
||||
WEBSEARCH_WEIGHT_RECENCY = 0.45
|
||||
|
||||
def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
|
||||
"""Score WebSearch items WITHOUT engagement metrics.
|
||||
|
||||
Uses reweighted formula: 55% relevance + 45% recency - 15pt source penalty
|
||||
"""
|
||||
for item in items:
|
||||
rel_score = int(item.relevance * 100)
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=0, # Explicitly zero - no engagement data
|
||||
)
|
||||
|
||||
overall = (
|
||||
WEBSEARCH_WEIGHT_RELEVANCE * rel_score +
|
||||
WEBSEARCH_WEIGHT_RECENCY * rec_score
|
||||
)
|
||||
|
||||
# Apply source penalty (WebSearch < Reddit/X)
|
||||
overall -= WEBSEARCH_SOURCE_PENALTY
|
||||
|
||||
# Apply date confidence penalty (same as other sources)
|
||||
if item.date_confidence == "low":
|
||||
overall -= 10
|
||||
elif item.date_confidence == "med":
|
||||
overall -= 5
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
```
|
||||
|
||||
#### Phase 3: Orchestrator Integration
|
||||
|
||||
```python
|
||||
# scripts/last30days.py - UPDATE run_research()
|
||||
|
||||
def run_research(...) -> tuple:
|
||||
"""Run the research pipeline.
|
||||
|
||||
Returns: (reddit_items, x_items, web_items, raw_openai, raw_xai,
|
||||
raw_websearch, reddit_error, x_error, web_error)
|
||||
"""
|
||||
# ... existing Reddit/X code ...
|
||||
|
||||
# WebSearch (new)
|
||||
web_items = []
|
||||
raw_websearch = None
|
||||
web_error = None
|
||||
|
||||
if sources in ("all", "web", "reddit-web", "x-web"):
|
||||
if progress:
|
||||
progress.start_web()
|
||||
|
||||
try:
|
||||
raw_websearch = websearch.search_web(topic, from_date, to_date, depth)
|
||||
web_items = websearch.parse_websearch_response(raw_websearch)
|
||||
except Exception as e:
|
||||
web_error = f"{type(e).__name__}: {e}"
|
||||
|
||||
if progress:
|
||||
progress.end_web(len(web_items))
|
||||
|
||||
return (reddit_items, x_items, web_items, raw_openai, raw_xai,
|
||||
raw_websearch, reddit_error, x_error, web_error)
|
||||
```
|
||||
|
||||
#### Phase 4: CLI & Environment Updates
|
||||
|
||||
```python
|
||||
# scripts/last30days.py - ADD CLI flag
|
||||
|
||||
parser.add_argument(
|
||||
"--include-web",
|
||||
action="store_true",
|
||||
help="Include general web search alongside Reddit/X (lower weighted)",
|
||||
)
|
||||
|
||||
# scripts/lib/env.py - UPDATE get_available_sources()
|
||||
|
||||
def get_available_sources(config: dict) -> str:
|
||||
"""Determine available sources. WebSearch always available (no API key)."""
|
||||
has_openai = bool(config.get('OPENAI_API_KEY'))
|
||||
has_xai = bool(config.get('XAI_API_KEY'))
|
||||
|
||||
if has_openai and has_xai:
|
||||
return 'both' # WebSearch available but not default
|
||||
elif has_openai:
|
||||
return 'reddit'
|
||||
elif has_xai:
|
||||
return 'x'
|
||||
else:
|
||||
return 'web' # Fallback: WebSearch only (no keys needed)
|
||||
```
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
- [x] Skill works with zero API keys (WebSearch-only mode)
|
||||
- [x] `--include-web` flag adds WebSearch to Reddit/X searches
|
||||
- [x] WebSearch items have lower average scores than Reddit/X items with similar relevance
|
||||
- [x] WebSearch results exclude Reddit/X URLs (handled separately)
|
||||
- [x] Date filtering uses natural language ("last 30 days") in prompt
|
||||
- [x] Output clearly labels source type: `[WEB]`, `[Reddit]`, `[X]`
|
||||
|
||||
### Non-Functional Requirements
|
||||
|
||||
- [x] WebSearch adds <10s latency to total research time (0s - deferred to Claude)
|
||||
- [x] Graceful degradation if WebSearch fails
|
||||
- [ ] Cache includes WebSearch results appropriately
|
||||
|
||||
### Quality Gates
|
||||
|
||||
- [x] Before/after testing shows WebSearch doesn't dominate rankings (via -15pt penalty)
|
||||
- [x] Test: 10 Reddit + 10 X + 10 WebSearch → WebSearch avg score 15-20pts lower (scoring formula verified)
|
||||
- [x] Test: WebSearch-only mode produces useful results for common topics
|
||||
|
||||
## Testing Plan
|
||||
|
||||
### Before/After Comparison Script
|
||||
|
||||
```python
|
||||
# tests/test_websearch_weighting.py
|
||||
|
||||
"""
|
||||
Test harness to validate WebSearch doesn't overpower Reddit/X.
|
||||
|
||||
Run same queries with:
|
||||
1. Reddit + X only (baseline)
|
||||
2. Reddit + X + WebSearch (comparison)
|
||||
|
||||
Verify: WebSearch items rank lower on average.
|
||||
"""
|
||||
|
||||
TEST_QUERIES = [
|
||||
"best practices for react server components",
|
||||
"AI coding assistants comparison",
|
||||
"typescript 5.5 new features",
|
||||
]
|
||||
|
||||
def test_websearch_weighting():
|
||||
for query in TEST_QUERIES:
|
||||
# Run without WebSearch
|
||||
baseline = run_research(query, sources="both")
|
||||
baseline_scores = [item.score for item in baseline.reddit + baseline.x]
|
||||
|
||||
# Run with WebSearch
|
||||
with_web = run_research(query, sources="both", include_web=True)
|
||||
web_scores = [item.score for item in with_web.web]
|
||||
reddit_x_scores = [item.score for item in with_web.reddit + with_web.x]
|
||||
|
||||
# Assertions
|
||||
avg_reddit_x = sum(reddit_x_scores) / len(reddit_x_scores)
|
||||
avg_web = sum(web_scores) / len(web_scores) if web_scores else 0
|
||||
|
||||
assert avg_web < avg_reddit_x - 10, \
|
||||
f"WebSearch avg ({avg_web}) too close to Reddit/X avg ({avg_reddit_x})"
|
||||
|
||||
# Check top 5 aren't all WebSearch
|
||||
top_5 = sorted(with_web.reddit + with_web.x + with_web.web,
|
||||
key=lambda x: -x.score)[:5]
|
||||
web_in_top_5 = sum(1 for item in top_5 if isinstance(item, WebSearchItem))
|
||||
assert web_in_top_5 <= 2, f"Too many WebSearch items in top 5: {web_in_top_5}"
|
||||
```
|
||||
|
||||
### Manual Test Scenarios
|
||||
|
||||
| Scenario | Expected Outcome |
|
||||
|----------|------------------|
|
||||
| No API keys, run `/last30days AI tools` | WebSearch-only results, useful output |
|
||||
| Both keys + `--include-web`, run `/last30days react` | Mix of all 3 sources, Reddit/X dominate top 10 |
|
||||
| Niche topic (no Reddit/X coverage) | WebSearch fills gap, becomes primary |
|
||||
| Popular topic (lots of Reddit/X) | WebSearch present but lower-ranked |
|
||||
|
||||
## Dependencies & Prerequisites
|
||||
|
||||
- Claude Code's WebSearch tool (`web_search_20250305`) - already available
|
||||
- No new API keys required
|
||||
- Existing test infrastructure in `tests/`
|
||||
|
||||
## Risk Analysis & Mitigation
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|------------|--------|------------|
|
||||
| WebSearch returns stale content | Medium | Medium | Enforce date in prompt, apply low-confidence penalty |
|
||||
| WebSearch dominates rankings | Low | High | Source penalty (-15pts), testing validates |
|
||||
| WebSearch adds spam/low-quality | Medium | Medium | Exclude social media domains, domain filtering |
|
||||
| Date parsing unreliable | High | Medium | Accept "low" confidence as normal for WebSearch |
|
||||
|
||||
## Future Considerations
|
||||
|
||||
1. **Domain authority scoring**: Could proxy engagement with domain reputation
|
||||
2. **User-configurable weights**: Let users adjust WebSearch penalty
|
||||
3. **Domain whitelist/blacklist**: Filter WebSearch to trusted sources
|
||||
4. **Parallel execution**: Run all 3 sources concurrently for speed
|
||||
|
||||
## References
|
||||
|
||||
### Internal References
|
||||
- Scoring algorithm: `scripts/lib/score.py:8-15`
|
||||
- Source detection: `scripts/lib/env.py:57-72`
|
||||
- Schema patterns: `scripts/lib/schema.py:76-138`
|
||||
- Orchestrator: `scripts/last30days.py:54-164`
|
||||
|
||||
### External References
|
||||
- Claude WebSearch docs: https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool
|
||||
- WebSearch pricing: $10/1K searches + token costs
|
||||
- Date filtering limitation: No explicit date params, use natural language
|
||||
|
||||
### Research Findings
|
||||
- Reddit upvotes are ~12% of ranking value in SEO (strong signal)
|
||||
- E-E-A-T framework: Engagement metrics = trust signal
|
||||
- MSA2C2 approach: Dynamic weight learning for multi-source aggregation
|
||||
@@ -1,328 +0,0 @@
|
||||
# fix: Enforce Strict 30-Day Date Filtering
|
||||
|
||||
## Overview
|
||||
|
||||
The `/last30days` skill is returning content older than 30 days, violating its core promise. Analysis shows:
|
||||
- **Reddit**: Only 40% of results within 30 days (9/15 were older, some from 2022!)
|
||||
- **X**: 100% within 30 days (working correctly)
|
||||
- **WebSearch**: 90% had unknown dates (can't verify freshness)
|
||||
|
||||
## Problem Statement
|
||||
|
||||
The skill's name is "last30days" - users expect ONLY content from the last 30 days. Currently:
|
||||
|
||||
1. **Reddit search prompt** says "prefer recent threads, but include older relevant ones if recent ones are scarce" - this is too permissive
|
||||
2. **X search prompt** explicitly includes `from_date` and `to_date` - this is why it works
|
||||
3. **WebSearch** returns pages without publication dates - we can't verify they're recent
|
||||
4. **Scoring penalties** (-10 for low date confidence) don't prevent old content from appearing
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
### Strategy: "Hard Filter, Not Soft Penalty"
|
||||
|
||||
Instead of penalizing old content, **exclude it entirely**. If it's not from the last 30 days, it shouldn't appear.
|
||||
|
||||
| Source | Current Behavior | New Behavior |
|
||||
|--------|------------------|--------------|
|
||||
| Reddit | Weak "prefer recent" | Explicit date range + hard filter |
|
||||
| X | Explicit date range (working) | No change needed |
|
||||
| WebSearch | No date awareness | Require recent markers OR exclude |
|
||||
|
||||
## Technical Approach
|
||||
|
||||
### Phase 1: Fix Reddit Date Filtering
|
||||
|
||||
**File: `scripts/lib/openai_reddit.py`**
|
||||
|
||||
Current prompt (line 33):
|
||||
```
|
||||
Find {min_items}-{max_items} relevant Reddit discussion threads.
|
||||
Prefer recent threads, but include older relevant ones if recent ones are scarce.
|
||||
```
|
||||
|
||||
New prompt:
|
||||
```
|
||||
Find {min_items}-{max_items} relevant Reddit discussion threads from {from_date} to {to_date}.
|
||||
|
||||
CRITICAL: Only include threads posted within the last 30 days (after {from_date}).
|
||||
Do NOT include threads older than {from_date}, even if they seem relevant.
|
||||
If you cannot find enough recent threads, return fewer results rather than older ones.
|
||||
```
|
||||
|
||||
**Changes needed:**
|
||||
1. Add `from_date` and `to_date` parameters to `search_reddit()` function
|
||||
2. Inject dates into `REDDIT_SEARCH_PROMPT` like X does
|
||||
3. Update caller in `last30days.py` to pass dates
|
||||
|
||||
### Phase 2: Add Hard Date Filtering (Post-Processing)
|
||||
|
||||
**File: `scripts/lib/normalize.py`**
|
||||
|
||||
Add a filter step that DROPS items with dates before `from_date`:
|
||||
|
||||
```python
|
||||
def filter_by_date_range(
|
||||
items: List[Union[RedditItem, XItem, WebSearchItem]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
require_date: bool = False,
|
||||
) -> List:
|
||||
"""Hard filter: Remove items outside the date range.
|
||||
|
||||
Args:
|
||||
items: List of items to filter
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
require_date: If True, also remove items with no date
|
||||
|
||||
Returns:
|
||||
Filtered list with only items in range
|
||||
"""
|
||||
result = []
|
||||
for item in items:
|
||||
if item.date is None:
|
||||
if not require_date:
|
||||
result.append(item) # Keep unknown dates (with penalty)
|
||||
continue
|
||||
|
||||
# Hard filter: if date is before from_date, exclude
|
||||
if item.date < from_date:
|
||||
continue # DROP - too old
|
||||
|
||||
if item.date > to_date:
|
||||
continue # DROP - future date (likely parsing error)
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
```
|
||||
|
||||
### Phase 3: WebSearch Date Intelligence
|
||||
|
||||
WebSearch CAN find recent content - Medium posts have dates, GitHub has commit timestamps, news sites have publication dates. We should **extract and prioritize** these signals.
|
||||
|
||||
**Strategy: "Date Detective"**
|
||||
|
||||
1. **Extract dates from URLs**: Many sites embed dates in URLs
|
||||
- Medium: `medium.com/@author/title-abc123` (no date) vs news sites
|
||||
- GitHub: Look for commit dates, release dates in snippets
|
||||
- News: `/2026/01/24/article-title`
|
||||
- Blogs: `/blog/2026/01/title`
|
||||
|
||||
2. **Extract dates from snippets**: Look for date markers
|
||||
- "January 24, 2026", "Jan 2026", "yesterday", "this week"
|
||||
- "Published:", "Posted:", "Updated:"
|
||||
- Relative markers: "2 days ago", "last week"
|
||||
|
||||
3. **Prioritize results with verifiable dates**:
|
||||
- Results with recent dates (within 30 days): Full score
|
||||
- Results with old dates: EXCLUDE
|
||||
- Results with no date signals: Heavy penalty (-20) but keep as supplementary
|
||||
|
||||
**File: `scripts/lib/websearch.py`**
|
||||
|
||||
Add date extraction functions:
|
||||
|
||||
```python
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Patterns for date extraction
|
||||
URL_DATE_PATTERNS = [
|
||||
r'/(\d{4})/(\d{2})/(\d{2})/', # /2026/01/24/
|
||||
r'/(\d{4})-(\d{2})-(\d{2})/', # /2026-01-24/
|
||||
r'/(\d{4})(\d{2})(\d{2})/', # /20260124/
|
||||
]
|
||||
|
||||
SNIPPET_DATE_PATTERNS = [
|
||||
r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* (\d{1,2}),? (\d{4})',
|
||||
r'(\d{1,2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* (\d{4})',
|
||||
r'(\d{4})-(\d{2})-(\d{2})',
|
||||
r'Published:?\s*(\d{4}-\d{2}-\d{2})',
|
||||
r'(\d{1,2}) (days?|hours?|minutes?) ago', # Relative dates
|
||||
]
|
||||
|
||||
def extract_date_from_url(url: str) -> Optional[str]:
|
||||
"""Try to extract a date from URL path."""
|
||||
for pattern in URL_DATE_PATTERNS:
|
||||
match = re.search(pattern, url)
|
||||
if match:
|
||||
# Parse and return YYYY-MM-DD format
|
||||
...
|
||||
return None
|
||||
|
||||
def extract_date_from_snippet(snippet: str) -> Optional[str]:
|
||||
"""Try to extract a date from text snippet."""
|
||||
for pattern in SNIPPET_DATE_PATTERNS:
|
||||
match = re.search(pattern, snippet, re.IGNORECASE)
|
||||
if match:
|
||||
# Parse and return YYYY-MM-DD format
|
||||
...
|
||||
return None
|
||||
|
||||
def extract_date_signals(url: str, snippet: str, title: str) -> tuple[Optional[str], str]:
|
||||
"""Extract date from any available signal.
|
||||
|
||||
Returns: (date_string, confidence)
|
||||
- date from URL: 'high' confidence
|
||||
- date from snippet: 'med' confidence
|
||||
- no date found: None, 'low' confidence
|
||||
"""
|
||||
# Try URL first (most reliable)
|
||||
url_date = extract_date_from_url(url)
|
||||
if url_date:
|
||||
return url_date, 'high'
|
||||
|
||||
# Try snippet
|
||||
snippet_date = extract_date_from_snippet(snippet)
|
||||
if snippet_date:
|
||||
return snippet_date, 'med'
|
||||
|
||||
# Try title
|
||||
title_date = extract_date_from_snippet(title)
|
||||
if title_date:
|
||||
return title_date, 'med'
|
||||
|
||||
return None, 'low'
|
||||
```
|
||||
|
||||
**Update WebSearch parsing to use date extraction:**
|
||||
|
||||
```python
|
||||
def parse_websearch_results(results, topic, from_date, to_date):
|
||||
items = []
|
||||
for result in results:
|
||||
url = result.get('url', '')
|
||||
snippet = result.get('snippet', '')
|
||||
title = result.get('title', '')
|
||||
|
||||
# Extract date signals
|
||||
extracted_date, confidence = extract_date_signals(url, snippet, title)
|
||||
|
||||
# Hard filter: if we found a date and it's too old, skip
|
||||
if extracted_date and extracted_date < from_date:
|
||||
continue # DROP - verified old content
|
||||
|
||||
item = {
|
||||
'date': extracted_date,
|
||||
'date_confidence': confidence,
|
||||
...
|
||||
}
|
||||
items.append(item)
|
||||
|
||||
return items
|
||||
```
|
||||
|
||||
**File: `scripts/lib/score.py`**
|
||||
|
||||
Update WebSearch scoring to reward date-verified results:
|
||||
|
||||
```python
|
||||
# WebSearch date confidence adjustments
|
||||
WEBSEARCH_NO_DATE_PENALTY = 20 # Heavy penalty for no date (was 10)
|
||||
WEBSEARCH_VERIFIED_BONUS = 10 # Bonus for URL-verified recent date
|
||||
|
||||
def score_websearch_items(items):
|
||||
for item in items:
|
||||
...
|
||||
# Date confidence adjustments
|
||||
if item.date_confidence == 'high':
|
||||
overall += WEBSEARCH_VERIFIED_BONUS # Reward verified dates
|
||||
elif item.date_confidence == 'low':
|
||||
overall -= WEBSEARCH_NO_DATE_PENALTY # Heavy penalty for unknown
|
||||
...
|
||||
```
|
||||
|
||||
**Result**: WebSearch results with verifiable recent dates rank well. Results with no dates are heavily penalized but still appear as supplementary context. Old verified content is excluded entirely.
|
||||
|
||||
### Phase 4: Update Statistics Display
|
||||
|
||||
Only count Reddit and X in "from the last 30 days" claim. WebSearch should be clearly labeled as supplementary.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
- [x] Reddit search prompt includes explicit `from_date` and `to_date`
|
||||
- [x] Items with dates before `from_date` are EXCLUDED, not just penalized
|
||||
- [x] X search continues working (no regression)
|
||||
- [x] WebSearch extracts dates from URLs (e.g., `/2026/01/24/`)
|
||||
- [x] WebSearch extracts dates from snippets (e.g., "January 24, 2026")
|
||||
- [x] WebSearch with verified recent dates gets +10 bonus
|
||||
- [x] WebSearch with no date signals gets -20 penalty (but still appears)
|
||||
- [x] WebSearch with verified OLD dates is EXCLUDED
|
||||
|
||||
### Non-Functional Requirements
|
||||
|
||||
- [ ] No increase in API latency
|
||||
- [ ] Graceful handling when few recent results exist (return fewer, not older)
|
||||
- [ ] Clear user messaging when results are limited due to strict filtering
|
||||
|
||||
### Quality Gates
|
||||
|
||||
- [ ] Test: Reddit search returns 0% results older than 30 days
|
||||
- [ ] Test: X search continues to return 100% recent results
|
||||
- [ ] Test: WebSearch is clearly differentiated in output
|
||||
- [ ] Test: Edge case - topic with no recent content shows helpful message
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. **Phase 1**: Fix Reddit prompt (highest impact, simple change)
|
||||
2. **Phase 2**: Add hard date filter in normalize.py (safety net)
|
||||
3. **Phase 3**: Add WebSearch date extraction (URL + snippet parsing)
|
||||
4. **Phase 4**: Update WebSearch scoring (bonus for verified, heavy penalty for unknown)
|
||||
5. **Phase 5**: Update output display to show date confidence
|
||||
|
||||
## Testing Plan
|
||||
|
||||
### Before/After Test
|
||||
|
||||
Run same query before and after fix:
|
||||
```
|
||||
/last30days remotion launch videos
|
||||
```
|
||||
|
||||
**Expected Before:**
|
||||
- Reddit: 40% within 30 days
|
||||
|
||||
**Expected After:**
|
||||
- Reddit: 100% within 30 days (or fewer results if not enough recent content)
|
||||
|
||||
### Edge Case Tests
|
||||
|
||||
| Scenario | Expected Behavior |
|
||||
|----------|-------------------|
|
||||
| Topic with no recent content | Return 0 results + helpful message |
|
||||
| Topic with 5 recent results | Return 5 results (not pad with old ones) |
|
||||
| Mixed old/new results | Only return new ones |
|
||||
|
||||
### WebSearch Date Extraction Tests
|
||||
|
||||
| URL/Snippet | Expected Date | Confidence |
|
||||
|-------------|---------------|------------|
|
||||
| `medium.com/blog/2026/01/15/title` | 2026-01-15 | high |
|
||||
| `github.com/repo` + "Released Jan 20, 2026" | 2026-01-20 | med |
|
||||
| `docs.example.com/guide` (no date signals) | None | low |
|
||||
| `news.site.com/2024/05/old-article` | 2024-05-XX | EXCLUDE (too old) |
|
||||
| Snippet: "Updated 3 days ago" | calculated | med |
|
||||
|
||||
## Risk Analysis
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|------|------------|--------|------------|
|
||||
| Fewer results for niche topics | High | Medium | Explain why in output |
|
||||
| User confusion about reduced results | Medium | Low | Clear messaging |
|
||||
| Date parsing errors exclude valid content | Low | Medium | Keep items with unknown dates, just label clearly |
|
||||
|
||||
## References
|
||||
|
||||
### Internal References
|
||||
- Reddit search: `scripts/lib/openai_reddit.py:25-63`
|
||||
- X search (working example): `scripts/lib/xai_x.py:26-55`
|
||||
- Date confidence: `scripts/lib/dates.py:62-90`
|
||||
- Scoring penalties: `scripts/lib/score.py:149-153`
|
||||
- Normalization: `scripts/lib/normalize.py:49,99`
|
||||
|
||||
### External References
|
||||
- OpenAI Responses API lacks native date filtering
|
||||
- Must rely on prompt engineering + post-processing
|
||||
@@ -0,0 +1,37 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.2.3"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
addopts = [
|
||||
"-q",
|
||||
"--tb=short",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["skills/last30days/scripts", "tests"]
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
[tool.coverage.report]
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
+63
-50
@@ -1,73 +1,86 @@
|
||||
The AI world reinvents itself every month. This skill keeps you current.
|
||||
|
||||
`/last30days` researches your topic across **Reddit, X, YouTube, and the web** from the last 30 days, finds what the community is actually upvoting, sharing, and saying on camera, and writes you a prompt that works today, not six months ago.
|
||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||
|
||||
## Three Headline Features
|
||||
## v3 is the intelligent search release
|
||||
|
||||
**1. Open-class skill with watchlists.** Add any topic to a watchlist -- your competitors, specific people, emerging technologies -- and /last30days re-researches it on demand or via cron. Designed for always-on environments like [Open Claw](https://github.com/openclaw/openclaw). SQLite-backed with FTS5 full-text search.
|
||||
v3 is a ground-up engine rewrite by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
|
||||
**2. YouTube transcripts as a 4th source.** When yt-dlp is installed, /last30days automatically searches YouTube, grabs view counts, and extracts auto-generated transcripts from the top videos. A 20-minute review contains 10x the signal of a single post -- now the skill reads it. Inspired by [@steipete](https://x.com/steipete)'s yt-dlp + [summarize](https://github.com/steipete/summarize) toolchain.
|
||||
Type "OpenClaw" and v3 resolves @steipete, r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags before a single API call fires. Type "Peter Steinberger" and it resolves his X handle and GitHub profile, switches to person mode, and shows what he shipped this month at 85% merge rate across 22 PRs. None of that was on Google.
|
||||
|
||||
**3. Works in OpenAI Codex CLI.** Same skill, same engine, same four sources. Install to `~/.agents/skills/last30days` and invoke with `$last30days`.
|
||||
## Headline features
|
||||
|
||||
Plus: **Bundled X search** -- vendored Bird GraphQL client (MIT). No external CLI, no npm install, no API keys needed. Just Node.js 22+ and your browser cookies.
|
||||
### Intelligent pre-research
|
||||
|
||||
## Real Results (verified Feb 15)
|
||||
The killer feature. A new Python pre-research brain resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags, all resolved before a single API call.
|
||||
|
||||
| Topic | Reddit | X | YouTube | Web |
|
||||
|-------|--------|---|---------|-----|
|
||||
| Nano Banana Pro | -- | 32 posts, 164 likes | 5 videos, 98K views, 5 transcripts | 10 pages |
|
||||
| Seedance 2.0 access | 3 threads, 114 upvotes | 31 posts, 191 likes | 20 videos, 685K views, 4 transcripts | 10 pages |
|
||||
| OpenClaw use cases | 35 threads, 1,130 upvotes | 23 posts | 20 videos, 1.57M views, 5 transcripts | 10 pages |
|
||||
| YouTube thumbnails | 7 threads, 654 upvotes | 32 posts, 110 likes | 18 videos, 6.15M views, 5 transcripts | 30 pages |
|
||||
| AI generated ads | 12 threads | 29 posts, 101 likes | 3 videos, 83K views, 3 transcripts | 30 pages |
|
||||
### Best Takes
|
||||
|
||||
## What's New
|
||||
A second LLM judge scores every result for humor, wit, and virality alongside relevance. Every brief now ends with a Best Takes section surfacing the cleverest one-liners and most viral quotes. The Reddit and X people are funny, and the old engine buried their best stuff.
|
||||
|
||||
### Added
|
||||
- Open-class skill with watchlist, briefing, and history modes
|
||||
- YouTube search + transcript extraction via yt-dlp
|
||||
- OpenAI Codex CLI compatibility
|
||||
- Bundled Twitter/X search (vendored Bird GraphQL, MIT)
|
||||
- Native web search backends (Parallel AI, Brave, OpenRouter/Perplexity Sonar Pro)
|
||||
- `--diagnose` flag for source status checking
|
||||
- `--store` flag for SQLite accumulation
|
||||
- Conversational first-run experience (NUX)
|
||||
### Cross-source cluster merging
|
||||
|
||||
### Changed
|
||||
- Two-phase search architecture (entity-aware drill-down)
|
||||
- Reddit JSON enrichment for real engagement metrics
|
||||
- Smarter query construction with auto-retry on 0 results
|
||||
- Engagement-weighted scoring (relevance 45%, recency 25%, engagement 30%)
|
||||
- `--days=N` configurable lookback (thanks @jonthebeef)
|
||||
When the same story hits Reddit, X, and YouTube, v3 merges them into one cluster instead of three duplicates. Entity-based overlap detection catches matches even when the titles use different words.
|
||||
|
||||
### Fixed
|
||||
- YouTube/Reddit timeout resilience
|
||||
- Reddit 429 rate limit fail-fast
|
||||
- Eager import crash in Codex environments
|
||||
- X search returning 0 results on popular topics
|
||||
- Windows Unicode crash (thanks @JosephOIbrahim)
|
||||
### Single-pass comparisons
|
||||
|
||||
## New Contributors
|
||||
"X vs Y" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides at once. Same depth, 3 minutes.
|
||||
|
||||
- @JosephOIbrahim -- Windows Unicode fix ([#17](https://github.com/mvanhorn/last30days-skill/pull/17))
|
||||
- @levineam -- Model fallback for unverified orgs ([#16](https://github.com/mvanhorn/last30days-skill/pull/16))
|
||||
- @jonthebeef -- `--days=N` configurable lookback ([#18](https://github.com/mvanhorn/last30days-skill/pull/18))
|
||||
### GitHub person-mode and project-mode
|
||||
|
||||
## Credits
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. PR velocity, top repos by stars, release notes for what shipped this month, woven into the narrative alongside X posts and Reddit threads.
|
||||
|
||||
- [@steipete](https://github.com/steipete) -- Bird CLI (vendored X search) and yt-dlp/summarize inspiration for YouTube transcripts
|
||||
- [@galligan](https://github.com/galligan) -- Marketplace plugin inspiration
|
||||
- [@hutchins](https://x.com/hutchins) -- Pushed for YouTube feature
|
||||
When the topic is a project, it pulls live star counts, READMEs, releases, and top issues from the GitHub API. No stale blog posts.
|
||||
|
||||
### ELI5 mode
|
||||
|
||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations, just clearer. Say "eli5 off" to go back.
|
||||
|
||||
### 13+ sources
|
||||
|
||||
v3 adds Threads, Pinterest, Perplexity, Bluesky, and Parallel AI grounding to the existing Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and Web lineup. Perplexity Deep Research (`--deep-research`) gives you 50+ citation reports for serious investigation.
|
||||
|
||||
### Per-author cap and entity disambiguation
|
||||
|
||||
Max 3 items per author prevents single-voice dominance. Synthesis trusts resolved handles over fuzzy keyword matches.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# Claude Code
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||
Claude Code:
|
||||
|
||||
# Codex CLI
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.agents/skills/last30days
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
30 days of research. 30 seconds of work. Four sources. Zero stale prompts.
|
||||
OpenClaw:
|
||||
|
||||
```
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`.
|
||||
|
||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
## v3 Community
|
||||
|
||||
v3 was shaped by community contributors whose PRs and issues inspired core features. Their code wasn't merged directly (v3 was a ground-up rewrite), but their ideas drove what shipped.
|
||||
|
||||
Thanks to @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, and @hnshah. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list.
|
||||
|
||||
Contributors who shaped the release itself:
|
||||
|
||||
- @Jah-yee (#153) surfaced the need for a real Codex CLI integration, which shipped in #219
|
||||
- @Cody-Coyote (#204) reported the marketplace validation bug that needed fixing before v3 could ship cleanly
|
||||
- @dannyshmueli pushed for v3 and Codex family support publicly on X
|
||||
|
||||
Full Added / Changed / Fixed detail lives in [CHANGELOG.md](CHANGELOG.md) under `[3.0.0]`.
|
||||
|
||||
## Earlier contributors
|
||||
|
||||
From the v1 and v2 lineage:
|
||||
|
||||
- [@galligan](https://github.com/galligan) for marketplace plugin inspiration
|
||||
- [@hutchins](https://x.com/hutchins) for pushing the YouTube feature
|
||||
|
||||
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,213 +0,0 @@
|
||||
"""Brave Search web search for last30days skill.
|
||||
|
||||
Uses the Brave Search API as a fallback web search backend.
|
||||
Simple, cheap (free tier: 2,000 queries/month), widely available.
|
||||
|
||||
API docs: https://api-dashboard.search.brave.com/app/documentation/web-search/get-started
|
||||
"""
|
||||
|
||||
import html
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from . import http
|
||||
|
||||
ENDPOINT = "https://api.search.brave.com/res/v1/web/search"
|
||||
|
||||
# Freshness codes: pd=24h, pw=7d, pm=31d
|
||||
FRESHNESS_MAP = {1: "pd", 7: "pw", 31: "pm"}
|
||||
|
||||
# Domains to exclude (handled by Reddit/X search)
|
||||
EXCLUDED_DOMAINS = {
|
||||
"reddit.com", "www.reddit.com", "old.reddit.com",
|
||||
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
|
||||
}
|
||||
|
||||
|
||||
def search_web(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
api_key: str,
|
||||
depth: str = "default",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web via Brave Search API.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
api_key: Brave Search API key
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
|
||||
|
||||
Raises:
|
||||
http.HTTPError: On API errors
|
||||
"""
|
||||
count = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
|
||||
|
||||
# Calculate days for freshness filter
|
||||
days = _days_between(from_date, to_date)
|
||||
freshness = _brave_freshness(days)
|
||||
|
||||
params = {
|
||||
"q": topic,
|
||||
"result_filter": "web,news",
|
||||
"count": count,
|
||||
"safesearch": "strict",
|
||||
"text_decorations": 0,
|
||||
"spellcheck": 0,
|
||||
}
|
||||
if freshness:
|
||||
params["freshness"] = freshness
|
||||
|
||||
url = f"{ENDPOINT}?{urlencode(params)}"
|
||||
|
||||
sys.stderr.write(f"[Web] Searching Brave for: {topic}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
response = http.request(
|
||||
"GET",
|
||||
url,
|
||||
headers={"X-Subscription-Token": api_key},
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
return _normalize_results(response, from_date, to_date)
|
||||
|
||||
|
||||
def _days_between(from_date: str, to_date: str) -> int:
|
||||
"""Calculate days between two YYYY-MM-DD dates."""
|
||||
try:
|
||||
d1 = datetime.strptime(from_date, "%Y-%m-%d")
|
||||
d2 = datetime.strptime(to_date, "%Y-%m-%d")
|
||||
return max(1, (d2 - d1).days)
|
||||
except (ValueError, TypeError):
|
||||
return 30
|
||||
|
||||
|
||||
def _brave_freshness(days: Optional[int]) -> Optional[str]:
|
||||
"""Convert days to Brave freshness parameter.
|
||||
|
||||
Uses canned codes for <=31d, explicit date range for longer periods.
|
||||
"""
|
||||
if days is None:
|
||||
return None
|
||||
code = next((v for d, v in sorted(FRESHNESS_MAP.items()) if days <= d), None)
|
||||
if code:
|
||||
return code
|
||||
start = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
end = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
return f"{start}to{end}"
|
||||
|
||||
|
||||
def _normalize_results(
|
||||
response: Dict[str, Any],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convert Brave Search response to websearch item schema.
|
||||
|
||||
Merges news + web results, cleans HTML entities, filters excluded domains.
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Merge news results (tend to be more recent) with web results
|
||||
raw_results = (
|
||||
response.get("news", {}).get("results", []) +
|
||||
response.get("web", {}).get("results", [])
|
||||
)
|
||||
|
||||
for i, result in enumerate(raw_results):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
|
||||
url = result.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
title = _clean_html(str(result.get("title", "")).strip())
|
||||
snippet = _clean_html(str(result.get("description", "")).strip())
|
||||
|
||||
if not title and not snippet:
|
||||
continue
|
||||
|
||||
# Parse date from Brave's 'age' field or 'page_age'
|
||||
date = _parse_brave_date(result.get("age"), result.get("page_age"))
|
||||
date_confidence = "med" if date else "low"
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": snippet[:500],
|
||||
"date": date,
|
||||
"date_confidence": date_confidence,
|
||||
"relevance": 0.6, # Brave doesn't provide relevance scores
|
||||
"why_relevant": "",
|
||||
})
|
||||
|
||||
sys.stderr.write(f"[Web] Brave: {len(items)} results\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _clean_html(text: str) -> str:
|
||||
"""Remove HTML tags and decode entities."""
|
||||
text = re.sub(r"<[^>]*>", "", text)
|
||||
text = html.unescape(text)
|
||||
return text
|
||||
|
||||
|
||||
def _parse_brave_date(age: Optional[str], page_age: Optional[str]) -> Optional[str]:
|
||||
"""Parse Brave's age/page_age fields to YYYY-MM-DD.
|
||||
|
||||
Brave returns dates like "3 hours ago", "2 days ago", "January 24, 2026".
|
||||
"""
|
||||
text = age or page_age
|
||||
if not text:
|
||||
return None
|
||||
|
||||
text_lower = text.lower().strip()
|
||||
now = datetime.now()
|
||||
|
||||
# "X hours ago" -> today
|
||||
if re.search(r'\d+\s*hours?\s*ago', text_lower):
|
||||
return now.strftime("%Y-%m-%d")
|
||||
|
||||
# "X days ago"
|
||||
match = re.search(r'(\d+)\s*days?\s*ago', text_lower)
|
||||
if match:
|
||||
days = int(match.group(1))
|
||||
if days <= 60:
|
||||
return (now - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
|
||||
# "X weeks ago"
|
||||
match = re.search(r'(\d+)\s*weeks?\s*ago', text_lower)
|
||||
if match:
|
||||
weeks = int(match.group(1))
|
||||
return (now - timedelta(weeks=weeks)).strftime("%Y-%m-%d")
|
||||
|
||||
# ISO format: 2026-01-24T...
|
||||
match = re.search(r'(\d{4}-\d{2}-\d{2})', text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
@@ -1,165 +0,0 @@
|
||||
"""Caching utilities for last30days skill."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
CACHE_DIR = Path.home() / ".cache" / "last30days"
|
||||
DEFAULT_TTL_HOURS = 24
|
||||
MODEL_CACHE_TTL_DAYS = 7
|
||||
MODEL_CACHE_FILE = CACHE_DIR / "model_selection.json"
|
||||
|
||||
|
||||
def ensure_cache_dir():
|
||||
"""Ensure cache directory exists. Supports env override and sandbox fallback."""
|
||||
global CACHE_DIR, MODEL_CACHE_FILE
|
||||
env_dir = os.environ.get("LAST30DAYS_CACHE_DIR")
|
||||
if env_dir:
|
||||
CACHE_DIR = Path(env_dir)
|
||||
MODEL_CACHE_FILE = CACHE_DIR / "model_selection.json"
|
||||
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
except PermissionError:
|
||||
CACHE_DIR = Path(tempfile.gettempdir()) / "last30days" / "cache"
|
||||
MODEL_CACHE_FILE = CACHE_DIR / "model_selection.json"
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def get_cache_key(topic: str, from_date: str, to_date: str, sources: str) -> str:
|
||||
"""Generate a cache key from query parameters."""
|
||||
key_data = f"{topic}|{from_date}|{to_date}|{sources}"
|
||||
return hashlib.sha256(key_data.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def get_cache_path(cache_key: str) -> Path:
|
||||
"""Get path to cache file."""
|
||||
return CACHE_DIR / f"{cache_key}.json"
|
||||
|
||||
|
||||
def is_cache_valid(cache_path: Path, ttl_hours: int = DEFAULT_TTL_HOURS) -> bool:
|
||||
"""Check if cache file exists and is within TTL."""
|
||||
if not cache_path.exists():
|
||||
return False
|
||||
|
||||
try:
|
||||
stat = cache_path.stat()
|
||||
mtime = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||
now = datetime.now(timezone.utc)
|
||||
age_hours = (now - mtime).total_seconds() / 3600
|
||||
return age_hours < ttl_hours
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def load_cache(cache_key: str, ttl_hours: int = DEFAULT_TTL_HOURS) -> Optional[dict]:
|
||||
"""Load data from cache if valid."""
|
||||
cache_path = get_cache_path(cache_key)
|
||||
|
||||
if not is_cache_valid(cache_path, ttl_hours):
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(cache_path, 'r') as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def get_cache_age_hours(cache_path: Path) -> Optional[float]:
|
||||
"""Get age of cache file in hours."""
|
||||
if not cache_path.exists():
|
||||
return None
|
||||
try:
|
||||
stat = cache_path.stat()
|
||||
mtime = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||
now = datetime.now(timezone.utc)
|
||||
return (now - mtime).total_seconds() / 3600
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def load_cache_with_age(cache_key: str, ttl_hours: int = DEFAULT_TTL_HOURS) -> tuple:
|
||||
"""Load data from cache with age info.
|
||||
|
||||
Returns:
|
||||
Tuple of (data, age_hours) or (None, None) if invalid
|
||||
"""
|
||||
cache_path = get_cache_path(cache_key)
|
||||
|
||||
if not is_cache_valid(cache_path, ttl_hours):
|
||||
return None, None
|
||||
|
||||
age = get_cache_age_hours(cache_path)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'r') as f:
|
||||
return json.load(f), age
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None, None
|
||||
|
||||
|
||||
def save_cache(cache_key: str, data: dict):
|
||||
"""Save data to cache."""
|
||||
ensure_cache_dir()
|
||||
cache_path = get_cache_path(cache_key)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'w') as f:
|
||||
json.dump(data, f)
|
||||
except OSError:
|
||||
pass # Silently fail on cache write errors
|
||||
|
||||
|
||||
def clear_cache():
|
||||
"""Clear all cache files."""
|
||||
if CACHE_DIR.exists():
|
||||
for f in CACHE_DIR.glob("*.json"):
|
||||
try:
|
||||
f.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# Model selection cache (longer TTL) — MODEL_CACHE_FILE is set at module level
|
||||
# and updated by ensure_cache_dir() if env override or fallback is needed.
|
||||
|
||||
|
||||
def load_model_cache() -> dict:
|
||||
"""Load model selection cache."""
|
||||
if not is_cache_valid(MODEL_CACHE_FILE, MODEL_CACHE_TTL_DAYS * 24):
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(MODEL_CACHE_FILE, 'r') as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def save_model_cache(data: dict):
|
||||
"""Save model selection cache."""
|
||||
ensure_cache_dir()
|
||||
try:
|
||||
with open(MODEL_CACHE_FILE, 'w') as f:
|
||||
json.dump(data, f)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def get_cached_model(provider: str) -> Optional[str]:
|
||||
"""Get cached model selection for a provider."""
|
||||
cache = load_model_cache()
|
||||
return cache.get(provider)
|
||||
|
||||
|
||||
def set_cached_model(provider: str, model: str):
|
||||
"""Cache model selection for a provider."""
|
||||
cache = load_model_cache()
|
||||
cache[provider] = model
|
||||
cache['updated_at'] = datetime.now(timezone.utc).isoformat()
|
||||
save_model_cache(cache)
|
||||
@@ -1,130 +0,0 @@
|
||||
"""Near-duplicate detection for last30days skill."""
|
||||
|
||||
import re
|
||||
from typing import List, Set, Tuple, Union
|
||||
|
||||
from . import schema
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""Normalize text for comparison.
|
||||
|
||||
- Lowercase
|
||||
- Remove punctuation
|
||||
- Collapse whitespace
|
||||
"""
|
||||
text = text.lower()
|
||||
text = re.sub(r'[^\w\s]', ' ', text)
|
||||
text = re.sub(r'\s+', ' ', text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def get_ngrams(text: str, n: int = 3) -> Set[str]:
|
||||
"""Get character n-grams from text."""
|
||||
text = normalize_text(text)
|
||||
if len(text) < n:
|
||||
return {text}
|
||||
return {text[i:i+n] for i in range(len(text) - n + 1)}
|
||||
|
||||
|
||||
def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
|
||||
"""Compute Jaccard similarity between two sets."""
|
||||
if not set1 or not set2:
|
||||
return 0.0
|
||||
intersection = len(set1 & set2)
|
||||
union = len(set1 | set2)
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def get_item_text(item: Union[schema.RedditItem, schema.XItem, schema.YouTubeItem]) -> str:
|
||||
"""Get comparable text from an item."""
|
||||
if isinstance(item, schema.RedditItem):
|
||||
return item.title
|
||||
elif isinstance(item, schema.YouTubeItem):
|
||||
return f"{item.title} {item.channel_name}"
|
||||
else:
|
||||
return item.text
|
||||
|
||||
|
||||
def find_duplicates(
|
||||
items: List[Union[schema.RedditItem, schema.XItem]],
|
||||
threshold: float = 0.7,
|
||||
) -> List[Tuple[int, int]]:
|
||||
"""Find near-duplicate pairs in items.
|
||||
|
||||
Args:
|
||||
items: List of items to check
|
||||
threshold: Similarity threshold (0-1)
|
||||
|
||||
Returns:
|
||||
List of (i, j) index pairs where i < j and items are similar
|
||||
"""
|
||||
duplicates = []
|
||||
|
||||
# Pre-compute n-grams
|
||||
ngrams = [get_ngrams(get_item_text(item)) for item in items]
|
||||
|
||||
for i in range(len(items)):
|
||||
for j in range(i + 1, len(items)):
|
||||
similarity = jaccard_similarity(ngrams[i], ngrams[j])
|
||||
if similarity >= threshold:
|
||||
duplicates.append((i, j))
|
||||
|
||||
return duplicates
|
||||
|
||||
|
||||
def dedupe_items(
|
||||
items: List[Union[schema.RedditItem, schema.XItem]],
|
||||
threshold: float = 0.7,
|
||||
) -> List[Union[schema.RedditItem, schema.XItem]]:
|
||||
"""Remove near-duplicates, keeping highest-scored item.
|
||||
|
||||
Args:
|
||||
items: List of items (should be pre-sorted by score descending)
|
||||
threshold: Similarity threshold
|
||||
|
||||
Returns:
|
||||
Deduplicated items
|
||||
"""
|
||||
if len(items) <= 1:
|
||||
return items
|
||||
|
||||
# Find duplicate pairs
|
||||
dup_pairs = find_duplicates(items, threshold)
|
||||
|
||||
# Mark indices to remove (always remove the lower-scored one)
|
||||
# Since items are pre-sorted by score, the second index is always lower
|
||||
to_remove = set()
|
||||
for i, j in dup_pairs:
|
||||
# Keep the higher-scored one (lower index in sorted list)
|
||||
if items[i].score >= items[j].score:
|
||||
to_remove.add(j)
|
||||
else:
|
||||
to_remove.add(i)
|
||||
|
||||
# Return items not marked for removal
|
||||
return [item for idx, item in enumerate(items) if idx not in to_remove]
|
||||
|
||||
|
||||
def dedupe_reddit(
|
||||
items: List[schema.RedditItem],
|
||||
threshold: float = 0.7,
|
||||
) -> List[schema.RedditItem]:
|
||||
"""Dedupe Reddit items."""
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_x(
|
||||
items: List[schema.XItem],
|
||||
threshold: float = 0.7,
|
||||
) -> List[schema.XItem]:
|
||||
"""Dedupe X items."""
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_youtube(
|
||||
items: List[schema.YouTubeItem],
|
||||
threshold: float = 0.7,
|
||||
) -> List[schema.YouTubeItem]:
|
||||
"""Dedupe YouTube items."""
|
||||
return dedupe_items(items, threshold)
|
||||
@@ -1,276 +0,0 @@
|
||||
"""Environment and API key management for last30days skill."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
# Allow override via environment variable for testing
|
||||
# Set LAST30DAYS_CONFIG_DIR="" for clean/no-config mode
|
||||
# Set LAST30DAYS_CONFIG_DIR="/path/to/dir" for custom config location
|
||||
_config_override = os.environ.get('LAST30DAYS_CONFIG_DIR')
|
||||
if _config_override == "":
|
||||
# Empty string = no config file (clean mode)
|
||||
CONFIG_DIR = None
|
||||
CONFIG_FILE = None
|
||||
elif _config_override:
|
||||
CONFIG_DIR = Path(_config_override)
|
||||
CONFIG_FILE = CONFIG_DIR / ".env"
|
||||
else:
|
||||
CONFIG_DIR = Path.home() / ".config" / "last30days"
|
||||
CONFIG_FILE = CONFIG_DIR / ".env"
|
||||
|
||||
|
||||
def load_env_file(path: Path) -> Dict[str, str]:
|
||||
"""Load environment variables from a file."""
|
||||
env = {}
|
||||
if not path.exists():
|
||||
return env
|
||||
|
||||
with open(path, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
if '=' in line:
|
||||
key, _, value = line.partition('=')
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
# Remove quotes if present
|
||||
if value and value[0] in ('"', "'") and value[-1] == value[0]:
|
||||
value = value[1:-1]
|
||||
if key and value:
|
||||
env[key] = value
|
||||
return env
|
||||
|
||||
|
||||
def get_config() -> Dict[str, Any]:
|
||||
"""Load configuration from ~/.config/last30days/.env and environment."""
|
||||
# Load from config file first (if configured)
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||
|
||||
# Build config: process.env > .env file
|
||||
keys = [
|
||||
('OPENAI_API_KEY', None),
|
||||
('XAI_API_KEY', None),
|
||||
('OPENROUTER_API_KEY', None),
|
||||
('PARALLEL_API_KEY', None),
|
||||
('BRAVE_API_KEY', None),
|
||||
('OPENAI_MODEL_POLICY', 'auto'),
|
||||
('OPENAI_MODEL_PIN', None),
|
||||
('XAI_MODEL_POLICY', 'latest'),
|
||||
('XAI_MODEL_PIN', None),
|
||||
]
|
||||
|
||||
config = {}
|
||||
for key, default in keys:
|
||||
config[key] = os.environ.get(key) or file_env.get(key, default)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def config_exists() -> bool:
|
||||
"""Check if configuration file exists."""
|
||||
return CONFIG_FILE.exists()
|
||||
|
||||
|
||||
def get_available_sources(config: Dict[str, Any]) -> str:
|
||||
"""Determine which sources are available based on API keys.
|
||||
|
||||
Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
|
||||
"""
|
||||
has_openai = bool(config.get('OPENAI_API_KEY'))
|
||||
has_xai = bool(config.get('XAI_API_KEY'))
|
||||
has_web = has_web_search_keys(config)
|
||||
|
||||
if has_openai and has_xai:
|
||||
return 'all' if has_web else 'both'
|
||||
elif has_openai:
|
||||
return 'reddit-web' if has_web else 'reddit'
|
||||
elif has_xai:
|
||||
return 'x-web' if has_web else 'x'
|
||||
elif has_web:
|
||||
return 'web'
|
||||
else:
|
||||
return 'web' # Fallback: assistant WebSearch (no API keys needed)
|
||||
|
||||
|
||||
def has_web_search_keys(config: Dict[str, Any]) -> bool:
|
||||
"""Check if any web search API keys are configured."""
|
||||
return bool(config.get('OPENROUTER_API_KEY') or config.get('PARALLEL_API_KEY') or config.get('BRAVE_API_KEY'))
|
||||
|
||||
|
||||
def get_web_search_source(config: Dict[str, Any]) -> Optional[str]:
|
||||
"""Determine the best available web search backend.
|
||||
|
||||
Priority: Parallel AI > Brave > OpenRouter/Sonar Pro
|
||||
|
||||
Returns: 'parallel', 'brave', 'openrouter', or None
|
||||
"""
|
||||
if config.get('PARALLEL_API_KEY'):
|
||||
return 'parallel'
|
||||
if config.get('BRAVE_API_KEY'):
|
||||
return 'brave'
|
||||
if config.get('OPENROUTER_API_KEY'):
|
||||
return 'openrouter'
|
||||
return None
|
||||
|
||||
|
||||
def get_missing_keys(config: Dict[str, Any]) -> str:
|
||||
"""Determine which sources are missing (accounting for Bird).
|
||||
|
||||
Returns: 'all', 'both', 'reddit', 'x', 'web', or 'none'
|
||||
"""
|
||||
has_openai = bool(config.get('OPENAI_API_KEY'))
|
||||
has_xai = bool(config.get('XAI_API_KEY'))
|
||||
has_web = has_web_search_keys(config)
|
||||
|
||||
# Check if Bird provides X access (import here to avoid circular dependency)
|
||||
from . import bird_x
|
||||
has_bird = bird_x.is_bird_installed() and bird_x.is_bird_authenticated()
|
||||
|
||||
has_x = has_xai or has_bird
|
||||
|
||||
if has_openai and has_x and has_web:
|
||||
return 'none'
|
||||
elif has_openai and has_x:
|
||||
return 'web' # Missing web search keys
|
||||
elif has_openai:
|
||||
return 'x' # Missing X source (and possibly web)
|
||||
elif has_x:
|
||||
return 'reddit' # Missing OpenAI key (and possibly web)
|
||||
else:
|
||||
return 'all' # Missing everything
|
||||
|
||||
|
||||
def validate_sources(requested: str, available: str, include_web: bool = False) -> tuple[str, Optional[str]]:
|
||||
"""Validate requested sources against available keys.
|
||||
|
||||
Args:
|
||||
requested: 'auto', 'reddit', 'x', 'both', or 'web'
|
||||
available: Result from get_available_sources()
|
||||
include_web: If True, add WebSearch to available sources
|
||||
|
||||
Returns:
|
||||
Tuple of (effective_sources, error_message)
|
||||
"""
|
||||
# No API keys at all
|
||||
if available == 'none':
|
||||
if requested == 'auto':
|
||||
return 'web', "No API keys configured. The assistant can still search the web if it has a search tool."
|
||||
elif requested == 'web':
|
||||
return 'web', None
|
||||
else:
|
||||
return 'web', f"No API keys configured. Add keys to ~/.config/last30days/.env for Reddit/X."
|
||||
|
||||
# Web-only mode (only web search API keys)
|
||||
if available == 'web':
|
||||
if requested == 'auto':
|
||||
return 'web', None
|
||||
elif requested == 'web':
|
||||
return 'web', None
|
||||
else:
|
||||
return 'web', f"Only web search keys configured. Add OPENAI_API_KEY for Reddit, XAI_API_KEY for X."
|
||||
|
||||
if requested == 'auto':
|
||||
# Add web to sources if include_web is set
|
||||
if include_web:
|
||||
if available == 'both':
|
||||
return 'all', None # reddit + x + web
|
||||
elif available == 'reddit':
|
||||
return 'reddit-web', None
|
||||
elif available == 'x':
|
||||
return 'x-web', None
|
||||
return available, None
|
||||
|
||||
if requested == 'web':
|
||||
return 'web', None
|
||||
|
||||
if requested == 'both':
|
||||
if available not in ('both',):
|
||||
missing = 'xAI' if available == 'reddit' else 'OpenAI'
|
||||
return 'none', f"Requested both sources but {missing} key is missing. Use --sources=auto to use available keys."
|
||||
if include_web:
|
||||
return 'all', None
|
||||
return 'both', None
|
||||
|
||||
if requested == 'reddit':
|
||||
if available == 'x':
|
||||
return 'none', "Requested Reddit but only xAI key is available."
|
||||
if include_web:
|
||||
return 'reddit-web', None
|
||||
return 'reddit', None
|
||||
|
||||
if requested == 'x':
|
||||
if available == 'reddit':
|
||||
return 'none', "Requested X but only OpenAI key is available."
|
||||
if include_web:
|
||||
return 'x-web', None
|
||||
return 'x', None
|
||||
|
||||
return requested, None
|
||||
|
||||
|
||||
def get_x_source(config: Dict[str, Any]) -> Optional[str]:
|
||||
"""Determine the best available X/Twitter source.
|
||||
|
||||
Priority: Bird (free) → xAI (paid API)
|
||||
|
||||
Args:
|
||||
config: Configuration dict from get_config()
|
||||
|
||||
Returns:
|
||||
'bird' if Bird is installed and authenticated,
|
||||
'xai' if XAI_API_KEY is configured,
|
||||
None if no X source available.
|
||||
"""
|
||||
# Import here to avoid circular dependency
|
||||
from . import bird_x
|
||||
|
||||
# Check Bird first (free option)
|
||||
if bird_x.is_bird_installed():
|
||||
username = bird_x.is_bird_authenticated()
|
||||
if username:
|
||||
return 'bird'
|
||||
|
||||
# Fall back to xAI if key exists
|
||||
if config.get('XAI_API_KEY'):
|
||||
return 'xai'
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_ytdlp_available() -> bool:
|
||||
"""Check if yt-dlp is installed for YouTube search."""
|
||||
from . import youtube_yt
|
||||
return youtube_yt.is_ytdlp_installed()
|
||||
|
||||
|
||||
def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Get detailed X source status for UI decisions.
|
||||
|
||||
Returns:
|
||||
Dict with keys: source, bird_installed, bird_authenticated,
|
||||
bird_username, xai_available, can_install_bird
|
||||
"""
|
||||
from . import bird_x
|
||||
|
||||
bird_status = bird_x.get_bird_status()
|
||||
xai_available = bool(config.get('XAI_API_KEY'))
|
||||
|
||||
# Determine active source
|
||||
if bird_status["authenticated"]:
|
||||
source = 'bird'
|
||||
elif xai_available:
|
||||
source = 'xai'
|
||||
else:
|
||||
source = None
|
||||
|
||||
return {
|
||||
"source": source,
|
||||
"bird_installed": bird_status["installed"],
|
||||
"bird_authenticated": bird_status["authenticated"],
|
||||
"bird_username": bird_status["username"],
|
||||
"xai_available": xai_available,
|
||||
"can_install_bird": bird_status["can_install"],
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
"""Model auto-selection for last30days skill."""
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from . import cache, http
|
||||
|
||||
# OpenAI API
|
||||
OPENAI_MODELS_URL = "https://api.openai.com/v1/models"
|
||||
OPENAI_FALLBACK_MODELS = ["gpt-5.2", "gpt-5.1", "gpt-5", "gpt-4.1", "gpt-4o"]
|
||||
|
||||
# xAI API - Agent Tools API requires grok-4 family
|
||||
XAI_MODELS_URL = "https://api.x.ai/v1/models"
|
||||
XAI_ALIASES = {
|
||||
"latest": "grok-4-1-fast", # Required for x_search tool
|
||||
"stable": "grok-4-1-fast",
|
||||
}
|
||||
|
||||
|
||||
def parse_version(model_id: str) -> Optional[Tuple[int, ...]]:
|
||||
"""Parse semantic version from model ID.
|
||||
|
||||
Examples:
|
||||
gpt-5 -> (5,)
|
||||
gpt-5.2 -> (5, 2)
|
||||
gpt-5.2.1 -> (5, 2, 1)
|
||||
"""
|
||||
match = re.search(r'(\d+(?:\.\d+)*)', model_id)
|
||||
if match:
|
||||
return tuple(int(x) for x in match.group(1).split('.'))
|
||||
return None
|
||||
|
||||
|
||||
def is_mainline_openai_model(model_id: str) -> bool:
|
||||
"""Check if model is a mainline GPT model (not mini/nano/chat/codex/pro)."""
|
||||
model_lower = model_id.lower()
|
||||
|
||||
# Must be gpt-4o, gpt-4.1+, or gpt-5+ series (mainline, not mini/nano/etc)
|
||||
if not re.match(r'^gpt-(?:4o|4\.1|5)(\.\d+)*$', model_lower):
|
||||
return False
|
||||
|
||||
# Exclude variants
|
||||
excludes = ['mini', 'nano', 'chat', 'codex', 'pro', 'preview', 'turbo']
|
||||
for exc in excludes:
|
||||
if exc in model_lower:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def select_openai_model(
|
||||
api_key: str,
|
||||
policy: str = "auto",
|
||||
pin: Optional[str] = None,
|
||||
mock_models: Optional[List[Dict]] = None,
|
||||
) -> str:
|
||||
"""Select the best OpenAI model based on policy.
|
||||
|
||||
Args:
|
||||
api_key: OpenAI API key
|
||||
policy: 'auto' or 'pinned'
|
||||
pin: Model to use if policy is 'pinned'
|
||||
mock_models: Mock model list for testing
|
||||
|
||||
Returns:
|
||||
Selected model ID
|
||||
"""
|
||||
if policy == "pinned" and pin:
|
||||
return pin
|
||||
|
||||
# Check cache first
|
||||
cached = cache.get_cached_model("openai")
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
# Fetch model list
|
||||
if mock_models is not None:
|
||||
models = mock_models
|
||||
else:
|
||||
try:
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
response = http.get(OPENAI_MODELS_URL, headers=headers)
|
||||
models = response.get("data", [])
|
||||
except http.HTTPError:
|
||||
# Fall back to known models
|
||||
return OPENAI_FALLBACK_MODELS[0]
|
||||
|
||||
# Filter to mainline models
|
||||
candidates = [m for m in models if is_mainline_openai_model(m.get("id", ""))]
|
||||
|
||||
if not candidates:
|
||||
# No gpt-5 models found, use fallback
|
||||
return OPENAI_FALLBACK_MODELS[0]
|
||||
|
||||
# Sort by version (descending), then by created timestamp
|
||||
def sort_key(m):
|
||||
version = parse_version(m.get("id", "")) or (0,)
|
||||
created = m.get("created", 0)
|
||||
return (version, created)
|
||||
|
||||
candidates.sort(key=sort_key, reverse=True)
|
||||
selected = candidates[0]["id"]
|
||||
|
||||
# Cache the selection
|
||||
cache.set_cached_model("openai", selected)
|
||||
|
||||
return selected
|
||||
|
||||
|
||||
def select_xai_model(
|
||||
api_key: str,
|
||||
policy: str = "latest",
|
||||
pin: Optional[str] = None,
|
||||
mock_models: Optional[List[Dict]] = None,
|
||||
) -> str:
|
||||
"""Select the best xAI model based on policy.
|
||||
|
||||
Args:
|
||||
api_key: xAI API key
|
||||
policy: 'latest', 'stable', or 'pinned'
|
||||
pin: Model to use if policy is 'pinned'
|
||||
mock_models: Mock model list for testing
|
||||
|
||||
Returns:
|
||||
Selected model ID
|
||||
"""
|
||||
if policy == "pinned" and pin:
|
||||
return pin
|
||||
|
||||
# Use alias system
|
||||
if policy in XAI_ALIASES:
|
||||
alias = XAI_ALIASES[policy]
|
||||
|
||||
# Check cache first
|
||||
cached = cache.get_cached_model("xai")
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
# Cache the alias
|
||||
cache.set_cached_model("xai", alias)
|
||||
return alias
|
||||
|
||||
# Default to latest
|
||||
return XAI_ALIASES["latest"]
|
||||
|
||||
|
||||
def get_models(
|
||||
config: Dict,
|
||||
mock_openai_models: Optional[List[Dict]] = None,
|
||||
mock_xai_models: Optional[List[Dict]] = None,
|
||||
) -> Dict[str, Optional[str]]:
|
||||
"""Get selected models for both providers.
|
||||
|
||||
Returns:
|
||||
Dict with 'openai' and 'xai' keys
|
||||
"""
|
||||
result = {"openai": None, "xai": None}
|
||||
|
||||
if config.get("OPENAI_API_KEY"):
|
||||
result["openai"] = select_openai_model(
|
||||
config["OPENAI_API_KEY"],
|
||||
config.get("OPENAI_MODEL_POLICY", "auto"),
|
||||
config.get("OPENAI_MODEL_PIN"),
|
||||
mock_openai_models,
|
||||
)
|
||||
|
||||
if config.get("XAI_API_KEY"):
|
||||
result["xai"] = select_xai_model(
|
||||
config["XAI_API_KEY"],
|
||||
config.get("XAI_MODEL_POLICY", "latest"),
|
||||
config.get("XAI_MODEL_PIN"),
|
||||
mock_xai_models,
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -1,205 +0,0 @@
|
||||
"""Normalization of raw API data to canonical schema."""
|
||||
|
||||
from typing import Any, Dict, List, TypeVar, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem)
|
||||
|
||||
|
||||
def filter_by_date_range(
|
||||
items: List[T],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
require_date: bool = False,
|
||||
) -> List[T]:
|
||||
"""Hard filter: Remove items outside the date range.
|
||||
|
||||
This is the safety net - even if the prompt lets old content through,
|
||||
this filter will exclude it.
|
||||
|
||||
Args:
|
||||
items: List of items to filter
|
||||
from_date: Start date (YYYY-MM-DD) - exclude items before this
|
||||
to_date: End date (YYYY-MM-DD) - exclude items after this
|
||||
require_date: If True, also remove items with no date
|
||||
|
||||
Returns:
|
||||
Filtered list with only items in range (or unknown dates if not required)
|
||||
"""
|
||||
result = []
|
||||
for item in items:
|
||||
if item.date is None:
|
||||
if not require_date:
|
||||
result.append(item) # Keep unknown dates (with scoring penalty)
|
||||
continue
|
||||
|
||||
# Hard filter: if date is before from_date, exclude
|
||||
if item.date < from_date:
|
||||
continue # DROP - too old
|
||||
|
||||
# Hard filter: if date is after to_date, exclude (likely parsing error)
|
||||
if item.date > to_date:
|
||||
continue # DROP - future date
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def normalize_reddit_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.RedditItem]:
|
||||
"""Normalize raw Reddit items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw Reddit items from API
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of RedditItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for item in items:
|
||||
# Parse engagement
|
||||
engagement = None
|
||||
eng_raw = item.get("engagement")
|
||||
if isinstance(eng_raw, dict):
|
||||
engagement = schema.Engagement(
|
||||
score=eng_raw.get("score"),
|
||||
num_comments=eng_raw.get("num_comments"),
|
||||
upvote_ratio=eng_raw.get("upvote_ratio"),
|
||||
)
|
||||
|
||||
# Parse comments
|
||||
top_comments = []
|
||||
for c in item.get("top_comments", []):
|
||||
top_comments.append(schema.Comment(
|
||||
score=c.get("score", 0),
|
||||
date=c.get("date"),
|
||||
author=c.get("author", ""),
|
||||
excerpt=c.get("excerpt", ""),
|
||||
url=c.get("url", ""),
|
||||
))
|
||||
|
||||
# Determine date confidence
|
||||
date_str = item.get("date")
|
||||
date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
|
||||
|
||||
normalized.append(schema.RedditItem(
|
||||
id=item.get("id", ""),
|
||||
title=item.get("title", ""),
|
||||
url=item.get("url", ""),
|
||||
subreddit=item.get("subreddit", ""),
|
||||
date=date_str,
|
||||
date_confidence=date_confidence,
|
||||
engagement=engagement,
|
||||
top_comments=top_comments,
|
||||
comment_insights=item.get("comment_insights", []),
|
||||
relevance=item.get("relevance", 0.5),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_x_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.XItem]:
|
||||
"""Normalize raw X items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw X items from API
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of XItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for item in items:
|
||||
# Parse engagement
|
||||
engagement = None
|
||||
eng_raw = item.get("engagement")
|
||||
if isinstance(eng_raw, dict):
|
||||
engagement = schema.Engagement(
|
||||
likes=eng_raw.get("likes"),
|
||||
reposts=eng_raw.get("reposts"),
|
||||
replies=eng_raw.get("replies"),
|
||||
quotes=eng_raw.get("quotes"),
|
||||
)
|
||||
|
||||
# Determine date confidence
|
||||
date_str = item.get("date")
|
||||
date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
|
||||
|
||||
normalized.append(schema.XItem(
|
||||
id=item.get("id", ""),
|
||||
text=item.get("text", ""),
|
||||
url=item.get("url", ""),
|
||||
author_handle=item.get("author_handle", ""),
|
||||
date=date_str,
|
||||
date_confidence=date_confidence,
|
||||
engagement=engagement,
|
||||
relevance=item.get("relevance", 0.5),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_youtube_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.YouTubeItem]:
|
||||
"""Normalize raw YouTube items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw YouTube items from yt-dlp
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of YouTubeItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for item in items:
|
||||
# Parse engagement
|
||||
eng_raw = item.get("engagement") or {}
|
||||
engagement = schema.Engagement(
|
||||
views=eng_raw.get("views"),
|
||||
likes=eng_raw.get("likes"),
|
||||
num_comments=eng_raw.get("comments"),
|
||||
)
|
||||
|
||||
# YouTube dates are reliable (always YYYY-MM-DD from yt-dlp)
|
||||
date_str = item.get("date")
|
||||
|
||||
normalized.append(schema.YouTubeItem(
|
||||
id=item.get("video_id", ""),
|
||||
title=item.get("title", ""),
|
||||
url=item.get("url", ""),
|
||||
channel_name=item.get("channel_name", ""),
|
||||
date=date_str,
|
||||
date_confidence="high",
|
||||
engagement=engagement,
|
||||
transcript_snippet=item.get("transcript_snippet", ""),
|
||||
relevance=item.get("relevance", 0.7),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
|
||||
"""Convert schema items to dicts for JSON serialization."""
|
||||
return [item.to_dict() for item in items]
|
||||
@@ -1,374 +0,0 @@
|
||||
"""OpenAI Responses API client for Reddit discovery."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import http
|
||||
|
||||
# Fallback models when the selected model isn't accessible (e.g., org not verified for GPT-5)
|
||||
MODEL_FALLBACK_ORDER = ["gpt-4.1", "gpt-4o", "gpt-4o-mini"]
|
||||
|
||||
|
||||
def _log_error(msg: str):
|
||||
"""Log error to stderr."""
|
||||
sys.stderr.write(f"[REDDIT ERROR] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def _log_info(msg: str):
|
||||
"""Log info to stderr."""
|
||||
sys.stderr.write(f"[REDDIT] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def _is_model_access_error(error: http.HTTPError) -> bool:
|
||||
"""Check if error is due to model access/verification issues."""
|
||||
if error.status_code not in (400, 403):
|
||||
return False
|
||||
if not error.body:
|
||||
return False
|
||||
body_lower = error.body.lower()
|
||||
# Check for common access/verification error messages
|
||||
return any(phrase in body_lower for phrase in [
|
||||
"verified",
|
||||
"organization must be",
|
||||
"does not have access",
|
||||
"not available",
|
||||
"not found",
|
||||
])
|
||||
|
||||
|
||||
OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
||||
|
||||
# Depth configurations: (min, max) threads to request
|
||||
# Request MORE than needed since many get filtered by date
|
||||
DEPTH_CONFIG = {
|
||||
"quick": (15, 25),
|
||||
"default": (30, 50),
|
||||
"deep": (70, 100),
|
||||
}
|
||||
|
||||
REDDIT_SEARCH_PROMPT = """Find Reddit discussion threads about: {topic}
|
||||
|
||||
STEP 1: EXTRACT THE CORE SUBJECT
|
||||
Get the MAIN NOUN/PRODUCT/TOPIC:
|
||||
- "best nano banana prompting practices" → "nano banana"
|
||||
- "killer features of clawdbot" → "clawdbot"
|
||||
- "top Claude Code skills" → "Claude Code"
|
||||
DO NOT include "best", "top", "tips", "practices", "features" in your search.
|
||||
|
||||
STEP 2: SEARCH BROADLY
|
||||
Search for the core subject:
|
||||
1. "[core subject] site:reddit.com"
|
||||
2. "reddit [core subject]"
|
||||
3. "[core subject] reddit"
|
||||
|
||||
Return as many relevant threads as you find. We filter by date server-side.
|
||||
|
||||
STEP 3: INCLUDE ALL MATCHES
|
||||
- Include ALL threads about the core subject
|
||||
- Set date to "YYYY-MM-DD" if you can determine it, otherwise null
|
||||
- We verify dates and filter old content server-side
|
||||
- DO NOT pre-filter aggressively - include anything relevant
|
||||
|
||||
REQUIRED: URLs must contain "/r/" AND "/comments/"
|
||||
REJECT: developers.reddit.com, business.reddit.com
|
||||
|
||||
Find {min_items}-{max_items} threads. Return MORE rather than fewer.
|
||||
|
||||
Return JSON:
|
||||
{{
|
||||
"items": [
|
||||
{{
|
||||
"title": "Thread title",
|
||||
"url": "https://www.reddit.com/r/sub/comments/xyz/title/",
|
||||
"subreddit": "subreddit_name",
|
||||
"date": "YYYY-MM-DD or null",
|
||||
"why_relevant": "Why relevant",
|
||||
"relevance": 0.85
|
||||
}}
|
||||
]
|
||||
}}"""
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for retry."""
|
||||
noise = ['best', 'top', 'how to', 'tips for', 'practices', 'features',
|
||||
'killer', 'guide', 'tutorial', 'recommendations', 'advice',
|
||||
'prompting', 'using', 'for', 'with', 'the', 'of', 'in', 'on']
|
||||
words = topic.lower().split()
|
||||
result = [w for w in words if w not in noise]
|
||||
return ' '.join(result[:3]) or topic # Keep max 3 words
|
||||
|
||||
|
||||
def _build_subreddit_query(topic: str) -> str:
|
||||
"""Build a subreddit-targeted search query for fallback.
|
||||
|
||||
When standard search returns few results, try searching for the
|
||||
subreddit itself: 'r/kanye', 'r/howie', etc.
|
||||
"""
|
||||
core = _extract_core_subject(topic)
|
||||
# Remove dots and special chars for subreddit name guess
|
||||
sub_name = core.replace('.', '').replace(' ', '').lower()
|
||||
return f"r/{sub_name} site:reddit.com"
|
||||
|
||||
|
||||
def search_reddit(
|
||||
api_key: str,
|
||||
model: str,
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
mock_response: Optional[Dict] = None,
|
||||
_retry: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Search Reddit for relevant threads using OpenAI Responses API.
|
||||
|
||||
Args:
|
||||
api_key: OpenAI API key
|
||||
model: Model to use
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD) - only include threads after this
|
||||
to_date: End date (YYYY-MM-DD) - only include threads before this
|
||||
depth: Research depth - "quick", "default", or "deep"
|
||||
mock_response: Mock response for testing
|
||||
|
||||
Returns:
|
||||
Raw API response
|
||||
"""
|
||||
if mock_response is not None:
|
||||
return mock_response
|
||||
|
||||
min_items, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Adjust timeout based on depth (generous for OpenAI web_search which can be slow)
|
||||
timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
|
||||
|
||||
# Build list of models to try: requested model first, then fallbacks
|
||||
models_to_try = [model] + [m for m in MODEL_FALLBACK_ORDER if m != model]
|
||||
|
||||
# Note: allowed_domains accepts base domain, not subdomains
|
||||
# We rely on prompt to filter out developers.reddit.com, etc.
|
||||
input_text = REDDIT_SEARCH_PROMPT.format(
|
||||
topic=topic,
|
||||
from_date=from_date,
|
||||
to_date=to_date,
|
||||
min_items=min_items,
|
||||
max_items=max_items,
|
||||
)
|
||||
|
||||
last_error = None
|
||||
for current_model in models_to_try:
|
||||
payload = {
|
||||
"model": current_model,
|
||||
"tools": [
|
||||
{
|
||||
"type": "web_search",
|
||||
"filters": {
|
||||
"allowed_domains": ["reddit.com"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"include": ["web_search_call.action.sources"],
|
||||
"input": input_text,
|
||||
}
|
||||
|
||||
try:
|
||||
return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=timeout)
|
||||
except http.HTTPError as e:
|
||||
last_error = e
|
||||
if _is_model_access_error(e):
|
||||
_log_info(f"Model {current_model} not accessible, trying fallback...")
|
||||
continue
|
||||
# Non-access error, don't retry with different model
|
||||
raise
|
||||
|
||||
# All models failed with access errors
|
||||
if last_error:
|
||||
_log_error(f"All models failed. Last error: {last_error}")
|
||||
raise last_error
|
||||
raise http.HTTPError("No models available")
|
||||
|
||||
|
||||
def search_subreddits(
|
||||
subreddits: List[str],
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
count_per: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search specific subreddits via Reddit's free JSON endpoint.
|
||||
|
||||
No API key needed. Uses reddit.com/r/{sub}/search/.json endpoint.
|
||||
Used in Phase 2 supplemental search after entity extraction.
|
||||
|
||||
Args:
|
||||
subreddits: List of subreddit names (without r/)
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
count_per: Results to request per subreddit
|
||||
|
||||
Returns:
|
||||
List of raw item dicts (same format as parse_reddit_response output).
|
||||
"""
|
||||
all_items = []
|
||||
core = _extract_core_subject(topic)
|
||||
|
||||
for sub in subreddits:
|
||||
sub = sub.lstrip("r/")
|
||||
try:
|
||||
url = f"https://www.reddit.com/r/{sub}/search/.json"
|
||||
params = f"q={_url_encode(core)}&restrict_sr=on&sort=new&limit={count_per}&raw_json=1"
|
||||
full_url = f"{url}?{params}"
|
||||
|
||||
headers = {
|
||||
"User-Agent": http.USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
}
|
||||
|
||||
data = http.get(full_url, headers=headers, timeout=15, retries=1)
|
||||
|
||||
# Reddit search returns {"data": {"children": [...]}}
|
||||
children = data.get("data", {}).get("children", [])
|
||||
for i, child in enumerate(children):
|
||||
if child.get("kind") != "t3": # t3 = link/submission
|
||||
continue
|
||||
post = child.get("data", {})
|
||||
permalink = post.get("permalink", "")
|
||||
if not permalink:
|
||||
continue
|
||||
|
||||
item = {
|
||||
"id": f"RS{len(all_items)+1}",
|
||||
"title": str(post.get("title", "")).strip(),
|
||||
"url": f"https://www.reddit.com{permalink}",
|
||||
"subreddit": str(post.get("subreddit", sub)).strip(),
|
||||
"date": None,
|
||||
"why_relevant": f"Found in r/{sub} supplemental search",
|
||||
"relevance": 0.65, # Slightly lower default for supplemental
|
||||
}
|
||||
|
||||
# Parse date from created_utc
|
||||
created_utc = post.get("created_utc")
|
||||
if created_utc:
|
||||
from . import dates as dates_mod
|
||||
item["date"] = dates_mod.timestamp_to_date(created_utc)
|
||||
|
||||
all_items.append(item)
|
||||
|
||||
except http.HTTPError as e:
|
||||
_log_info(f"Subreddit search failed for r/{sub}: {e}")
|
||||
if e.status_code == 429:
|
||||
_log_info("Reddit rate-limited (429) — skipping remaining subreddits")
|
||||
break
|
||||
except Exception as e:
|
||||
_log_info(f"Subreddit search error for r/{sub}: {e}")
|
||||
|
||||
return all_items
|
||||
|
||||
|
||||
def _url_encode(text: str) -> str:
|
||||
"""Simple URL encoding for query parameters."""
|
||||
import urllib.parse
|
||||
return urllib.parse.quote_plus(text)
|
||||
|
||||
|
||||
def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Parse OpenAI response to extract Reddit items.
|
||||
|
||||
Args:
|
||||
response: Raw API response
|
||||
|
||||
Returns:
|
||||
List of item dicts
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Check for API errors first
|
||||
if "error" in response and response["error"]:
|
||||
error = response["error"]
|
||||
err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error)
|
||||
_log_error(f"OpenAI API error: {err_msg}")
|
||||
if http.DEBUG:
|
||||
_log_error(f"Full error response: {json.dumps(response, indent=2)[:1000]}")
|
||||
return items
|
||||
|
||||
# Try to find the output text
|
||||
output_text = ""
|
||||
if "output" in response:
|
||||
output = response["output"]
|
||||
if isinstance(output, str):
|
||||
output_text = output
|
||||
elif isinstance(output, list):
|
||||
for item in output:
|
||||
if isinstance(item, dict):
|
||||
if item.get("type") == "message":
|
||||
content = item.get("content", [])
|
||||
for c in content:
|
||||
if isinstance(c, dict) and c.get("type") == "output_text":
|
||||
output_text = c.get("text", "")
|
||||
break
|
||||
elif "text" in item:
|
||||
output_text = item["text"]
|
||||
elif isinstance(item, str):
|
||||
output_text = item
|
||||
if output_text:
|
||||
break
|
||||
|
||||
# Also check for choices (older format)
|
||||
if not output_text and "choices" in response:
|
||||
for choice in response["choices"]:
|
||||
if "message" in choice:
|
||||
output_text = choice["message"].get("content", "")
|
||||
break
|
||||
|
||||
if not output_text:
|
||||
print(f"[REDDIT WARNING] No output text found in OpenAI response. Keys present: {list(response.keys())}", flush=True)
|
||||
return items
|
||||
|
||||
# Extract JSON from the response
|
||||
json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
|
||||
if json_match:
|
||||
try:
|
||||
data = json.loads(json_match.group())
|
||||
items = data.get("items", [])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Validate and clean items
|
||||
clean_items = []
|
||||
for i, item in enumerate(items):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
url = item.get("url", "")
|
||||
if not url or "reddit.com" not in url:
|
||||
continue
|
||||
|
||||
clean_item = {
|
||||
"id": f"R{i+1}",
|
||||
"title": str(item.get("title", "")).strip(),
|
||||
"url": url,
|
||||
"subreddit": str(item.get("subreddit", "")).strip().lstrip("r/"),
|
||||
"date": item.get("date"),
|
||||
"why_relevant": str(item.get("why_relevant", "")).strip(),
|
||||
"relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))),
|
||||
}
|
||||
|
||||
# Validate date format
|
||||
if clean_item["date"]:
|
||||
if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])):
|
||||
clean_item["date"] = None
|
||||
|
||||
clean_items.append(clean_item)
|
||||
|
||||
return clean_items
|
||||
@@ -1,216 +0,0 @@
|
||||
"""Perplexity Sonar Pro web search via OpenRouter for last30days skill.
|
||||
|
||||
Uses OpenRouter's chat completions API with Perplexity's Sonar Pro model,
|
||||
which has built-in web search and returns citations with URLs, titles, and dates.
|
||||
This is the recommended web search backend -- highest quality results.
|
||||
|
||||
API docs: https://openrouter.ai/docs/quickstart
|
||||
Model: perplexity/sonar-pro
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import http
|
||||
|
||||
ENDPOINT = "https://openrouter.ai/api/v1/chat/completions"
|
||||
MODEL = "perplexity/sonar-pro"
|
||||
|
||||
# Domains to exclude (handled by Reddit/X search)
|
||||
EXCLUDED_DOMAINS = {
|
||||
"reddit.com", "www.reddit.com", "old.reddit.com",
|
||||
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
|
||||
}
|
||||
|
||||
|
||||
def search_web(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
api_key: str,
|
||||
depth: str = "default",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web via Perplexity Sonar Pro on OpenRouter.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
api_key: OpenRouter API key
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
|
||||
|
||||
Raises:
|
||||
http.HTTPError: On API errors
|
||||
"""
|
||||
max_tokens = {"quick": 1024, "default": 2048, "deep": 4096}.get(depth, 2048)
|
||||
|
||||
prompt = (
|
||||
f"Find recent blog posts, news articles, tutorials, and discussions "
|
||||
f"about {topic} published between {from_date} and {to_date}. "
|
||||
f"Exclude results from reddit.com, x.com, and twitter.com. "
|
||||
f"For each result, provide the title, URL, publication date, "
|
||||
f"and a brief summary of why it's relevant."
|
||||
)
|
||||
|
||||
payload = {
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
|
||||
sys.stderr.write(f"[Web] Searching Sonar Pro via OpenRouter for: {topic}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
response = http.post(
|
||||
ENDPOINT,
|
||||
json_data=payload,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"HTTP-Referer": "https://github.com/mvanhorn/last30days-openclaw",
|
||||
"X-Title": "last30days",
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
return _normalize_results(response)
|
||||
|
||||
|
||||
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Convert Sonar Pro response to websearch item schema.
|
||||
|
||||
Sonar Pro returns:
|
||||
- search_results: [{title, url, date}] -- structured source metadata
|
||||
- citations: [url, ...] -- flat list of cited URLs
|
||||
- choices[0].message.content -- the synthesized text with [N] references
|
||||
|
||||
We prefer search_results (richer metadata), fall back to citations.
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Try search_results first (has title, url, date)
|
||||
search_results = response.get("search_results", [])
|
||||
if isinstance(search_results, list) and search_results:
|
||||
items = _parse_search_results(search_results)
|
||||
|
||||
# Fall back to citations if no search_results
|
||||
if not items:
|
||||
citations = response.get("citations", [])
|
||||
content = _get_content(response)
|
||||
if isinstance(citations, list) and citations:
|
||||
items = _parse_citations(citations, content)
|
||||
|
||||
sys.stderr.write(f"[Web] Sonar Pro: {len(items)} results\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _parse_search_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Parse the search_results array from Sonar Pro."""
|
||||
items = []
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
|
||||
url = result.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
title = str(result.get("title", "")).strip()
|
||||
if not title:
|
||||
continue
|
||||
|
||||
# Sonar Pro provides dates in search_results
|
||||
date = result.get("date")
|
||||
date_confidence = "med" if date else "low"
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": str(result.get("snippet", result.get("description", ""))).strip()[:500],
|
||||
"date": date,
|
||||
"date_confidence": date_confidence,
|
||||
"relevance": 0.7, # Sonar Pro results are generally high quality
|
||||
"why_relevant": "",
|
||||
})
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _parse_citations(citations: List[str], content: str) -> List[Dict[str, Any]]:
|
||||
"""Parse the flat citations array, enriching with content context."""
|
||||
items = []
|
||||
|
||||
for i, url in enumerate(citations):
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
# Try to extract title from content references like [1] Title...
|
||||
title = _extract_title_for_citation(content, i + 1) or domain
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": "",
|
||||
"date": None,
|
||||
"date_confidence": "low",
|
||||
"relevance": 0.6,
|
||||
"why_relevant": "",
|
||||
})
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _get_content(response: Dict[str, Any]) -> str:
|
||||
"""Extract the text content from the chat completion response."""
|
||||
try:
|
||||
return response["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_title_for_citation(content: str, index: int) -> Optional[str]:
|
||||
"""Try to extract a title near a citation reference [N] in the content."""
|
||||
if not content:
|
||||
return None
|
||||
|
||||
# Look for patterns like [1] Title or [1](url) Title
|
||||
pattern = rf'\[{index}\][)\s]*([^\[\n]{{5,80}})'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
title = match.group(1).strip().rstrip('.')
|
||||
# Clean up markdown artifacts
|
||||
title = re.sub(r'[*_`]', '', title)
|
||||
return title if len(title) > 3 else None
|
||||
|
||||
return None
|
||||
@@ -1,139 +0,0 @@
|
||||
"""Parallel AI web search for last30days skill.
|
||||
|
||||
Uses the Parallel AI Search API to find web content (blogs, docs, news, tutorials).
|
||||
This is the preferred web search backend -- it returns LLM-optimized results
|
||||
with extended excerpts ranked by relevance.
|
||||
|
||||
API docs: https://docs.parallel.ai/search-api/search-quickstart
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import http
|
||||
|
||||
ENDPOINT = "https://api.parallel.ai/v1beta/search"
|
||||
|
||||
# Domains to exclude (handled by Reddit/X search)
|
||||
EXCLUDED_DOMAINS = {
|
||||
"reddit.com", "www.reddit.com", "old.reddit.com",
|
||||
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
|
||||
}
|
||||
|
||||
|
||||
def search_web(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
api_key: str,
|
||||
depth: str = "default",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web via Parallel AI Search API.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
api_key: Parallel AI API key
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
|
||||
|
||||
Raises:
|
||||
http.HTTPError: On API errors
|
||||
"""
|
||||
max_results = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
|
||||
|
||||
payload = {
|
||||
"objective": (
|
||||
f"Find recent blog posts, tutorials, news articles, and discussions "
|
||||
f"about {topic} from {from_date} to {to_date}. "
|
||||
f"Exclude reddit.com, x.com, and twitter.com."
|
||||
),
|
||||
"max_results": max_results,
|
||||
"max_chars_per_result": 500,
|
||||
}
|
||||
|
||||
sys.stderr.write(f"[Web] Searching Parallel AI for: {topic}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
response = http.post(
|
||||
ENDPOINT,
|
||||
json_data=payload,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"parallel-beta": "search-extract-2025-10-10",
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
return _normalize_results(response)
|
||||
|
||||
|
||||
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Convert Parallel AI response to websearch item schema.
|
||||
|
||||
Args:
|
||||
response: Raw API response
|
||||
|
||||
Returns:
|
||||
List of normalized result dicts
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Handle different response shapes
|
||||
results = response.get("results", [])
|
||||
if not isinstance(results, list):
|
||||
return items
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
|
||||
url = result.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
# Clean domain for display
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
title = str(result.get("title", "")).strip()
|
||||
snippet = str(result.get("excerpt", result.get("snippet", result.get("description", "")))).strip()
|
||||
|
||||
if not title and not snippet:
|
||||
continue
|
||||
|
||||
# Extract relevance score if provided
|
||||
relevance = result.get("relevance_score", result.get("relevance", 0.6))
|
||||
try:
|
||||
relevance = min(1.0, max(0.0, float(relevance)))
|
||||
except (TypeError, ValueError):
|
||||
relevance = 0.6
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": snippet[:500],
|
||||
"date": result.get("published_date", result.get("date")),
|
||||
"date_confidence": "med" if result.get("published_date") or result.get("date") else "low",
|
||||
"relevance": relevance,
|
||||
"why_relevant": str(result.get("summary", "")).strip()[:200],
|
||||
})
|
||||
|
||||
sys.stderr.write(f"[Web] Parallel AI: {len(items)} results\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
return items
|
||||
@@ -1,491 +0,0 @@
|
||||
"""Output rendering for last30days skill."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from . import schema
|
||||
|
||||
OUTPUT_DIR = Path.home() / ".local" / "share" / "last30days" / "out"
|
||||
|
||||
|
||||
def ensure_output_dir():
|
||||
"""Ensure output directory exists. Supports env override and sandbox fallback."""
|
||||
global OUTPUT_DIR
|
||||
env_dir = os.environ.get("LAST30DAYS_OUTPUT_DIR")
|
||||
if env_dir:
|
||||
OUTPUT_DIR = Path(env_dir)
|
||||
|
||||
try:
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
except PermissionError:
|
||||
OUTPUT_DIR = Path(tempfile.gettempdir()) / "last30days" / "out"
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _assess_data_freshness(report: schema.Report) -> dict:
|
||||
"""Assess how much data is actually from the last 30 days."""
|
||||
reddit_recent = sum(1 for r in report.reddit if r.date and r.date >= report.range_from)
|
||||
x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from)
|
||||
web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from)
|
||||
|
||||
total_recent = reddit_recent + x_recent + web_recent
|
||||
total_items = len(report.reddit) + len(report.x) + len(report.web)
|
||||
|
||||
return {
|
||||
"reddit_recent": reddit_recent,
|
||||
"x_recent": x_recent,
|
||||
"web_recent": web_recent,
|
||||
"total_recent": total_recent,
|
||||
"total_items": total_items,
|
||||
"is_sparse": total_recent < 5,
|
||||
"mostly_evergreen": total_items > 0 and total_recent < total_items * 0.3,
|
||||
}
|
||||
|
||||
|
||||
def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "none") -> str:
|
||||
"""Render compact output for the assistant to synthesize.
|
||||
|
||||
Args:
|
||||
report: Report data
|
||||
limit: Max items per source
|
||||
missing_keys: 'both', 'reddit', 'x', or 'none'
|
||||
|
||||
Returns:
|
||||
Compact markdown string
|
||||
"""
|
||||
lines = []
|
||||
|
||||
# Header
|
||||
lines.append(f"## Research Results: {report.topic}")
|
||||
lines.append("")
|
||||
|
||||
# Assess data freshness and add honesty warning if needed
|
||||
freshness = _assess_data_freshness(report)
|
||||
if freshness["is_sparse"]:
|
||||
lines.append("**⚠️ LIMITED RECENT DATA** - Few discussions from the last 30 days.")
|
||||
lines.append(f"Only {freshness['total_recent']} item(s) confirmed from {report.range_from} to {report.range_to}.")
|
||||
lines.append("Results below may include older/evergreen content. Be transparent with the user about this.")
|
||||
lines.append("")
|
||||
|
||||
# Web-only mode banner (when no API keys)
|
||||
if report.mode == "web-only":
|
||||
lines.append("**🌐 WEB SEARCH MODE** - assistant will search blogs, docs & news")
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("**⚡ Want better results?** Add API keys to unlock Reddit & X data:")
|
||||
lines.append("- `OPENAI_API_KEY` → Reddit threads with real upvotes & comments")
|
||||
lines.append("- `XAI_API_KEY` → X posts with real likes & reposts")
|
||||
lines.append("- Edit `~/.config/last30days/.env` to add keys")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
|
||||
# Cache indicator
|
||||
if report.from_cache:
|
||||
age_str = f"{report.cache_age_hours:.1f}h old" if report.cache_age_hours else "cached"
|
||||
lines.append(f"**⚡ CACHED RESULTS** ({age_str}) - use `--refresh` for fresh data")
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"**Date Range:** {report.range_from} to {report.range_to}")
|
||||
lines.append(f"**Mode:** {report.mode}")
|
||||
if report.openai_model_used:
|
||||
lines.append(f"**OpenAI Model:** {report.openai_model_used}")
|
||||
if report.xai_model_used:
|
||||
lines.append(f"**xAI Model:** {report.xai_model_used}")
|
||||
lines.append("")
|
||||
|
||||
# Coverage note for partial coverage
|
||||
if report.mode == "reddit-only" and missing_keys == "x":
|
||||
lines.append("*💡 Tip: Add XAI_API_KEY for X/Twitter data and better triangulation.*")
|
||||
lines.append("")
|
||||
elif report.mode == "x-only" and missing_keys == "reddit":
|
||||
lines.append("*💡 Tip: Add OPENAI_API_KEY for Reddit data and better triangulation.*")
|
||||
lines.append("")
|
||||
|
||||
# Reddit items
|
||||
if report.reddit_error:
|
||||
lines.append("### Reddit Threads")
|
||||
lines.append("")
|
||||
lines.append(f"**ERROR:** {report.reddit_error}")
|
||||
lines.append("")
|
||||
elif report.mode in ("both", "reddit-only") and not report.reddit:
|
||||
lines.append("### Reddit Threads")
|
||||
lines.append("")
|
||||
lines.append("*No relevant Reddit threads found for this topic.*")
|
||||
lines.append("")
|
||||
elif report.reddit:
|
||||
lines.append("### Reddit Threads")
|
||||
lines.append("")
|
||||
for item in report.reddit[:limit]:
|
||||
eng_str = ""
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
parts = []
|
||||
if eng.score is not None:
|
||||
parts.append(f"{eng.score}pts")
|
||||
if eng.num_comments is not None:
|
||||
parts.append(f"{eng.num_comments}cmt")
|
||||
if parts:
|
||||
eng_str = f" [{', '.join(parts)}]"
|
||||
|
||||
date_str = f" ({item.date})" if item.date else " (date unknown)"
|
||||
conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else ""
|
||||
|
||||
lines.append(f"**{item.id}** (score:{item.score}) r/{item.subreddit}{date_str}{conf_str}{eng_str}")
|
||||
lines.append(f" {item.title}")
|
||||
lines.append(f" {item.url}")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
|
||||
# Top comment insights
|
||||
if item.comment_insights:
|
||||
lines.append(f" Insights:")
|
||||
for insight in item.comment_insights[:3]:
|
||||
lines.append(f" - {insight}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# X items
|
||||
if report.x_error:
|
||||
lines.append("### X Posts")
|
||||
lines.append("")
|
||||
lines.append(f"**ERROR:** {report.x_error}")
|
||||
lines.append("")
|
||||
elif report.mode in ("both", "x-only", "all", "x-web") and not report.x:
|
||||
lines.append("### X Posts")
|
||||
lines.append("")
|
||||
lines.append("*No relevant X posts found for this topic.*")
|
||||
lines.append("")
|
||||
elif report.x:
|
||||
lines.append("### X Posts")
|
||||
lines.append("")
|
||||
for item in report.x[:limit]:
|
||||
eng_str = ""
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
parts = []
|
||||
if eng.likes is not None:
|
||||
parts.append(f"{eng.likes}likes")
|
||||
if eng.reposts is not None:
|
||||
parts.append(f"{eng.reposts}rt")
|
||||
if parts:
|
||||
eng_str = f" [{', '.join(parts)}]"
|
||||
|
||||
date_str = f" ({item.date})" if item.date else " (date unknown)"
|
||||
conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else ""
|
||||
|
||||
lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{conf_str}{eng_str}")
|
||||
lines.append(f" {item.text[:200]}...")
|
||||
lines.append(f" {item.url}")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
# YouTube items
|
||||
if report.youtube_error:
|
||||
lines.append("### YouTube Videos")
|
||||
lines.append("")
|
||||
lines.append(f"**ERROR:** {report.youtube_error}")
|
||||
lines.append("")
|
||||
elif report.youtube:
|
||||
lines.append("### YouTube Videos")
|
||||
lines.append("")
|
||||
for item in report.youtube[:limit]:
|
||||
eng_str = ""
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
parts = []
|
||||
if eng.views is not None:
|
||||
parts.append(f"{eng.views:,} views")
|
||||
if eng.likes is not None:
|
||||
parts.append(f"{eng.likes:,} likes")
|
||||
if parts:
|
||||
eng_str = f" [{', '.join(parts)}]"
|
||||
|
||||
date_str = f" ({item.date})" if item.date else ""
|
||||
|
||||
lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}")
|
||||
lines.append(f" {item.title}")
|
||||
lines.append(f" {item.url}")
|
||||
if item.transcript_snippet:
|
||||
snippet = item.transcript_snippet[:200]
|
||||
if len(item.transcript_snippet) > 200:
|
||||
snippet += "..."
|
||||
lines.append(f" Transcript: {snippet}")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
# Web items (if any - populated by the assistant)
|
||||
if report.web_error:
|
||||
lines.append("### Web Results")
|
||||
lines.append("")
|
||||
lines.append(f"**ERROR:** {report.web_error}")
|
||||
lines.append("")
|
||||
elif report.web:
|
||||
lines.append("### Web Results")
|
||||
lines.append("")
|
||||
for item in report.web[:limit]:
|
||||
date_str = f" ({item.date})" if item.date else " (date unknown)"
|
||||
conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else ""
|
||||
|
||||
lines.append(f"**{item.id}** [WEB] (score:{item.score}) {item.source_domain}{date_str}{conf_str}")
|
||||
lines.append(f" {item.title}")
|
||||
lines.append(f" {item.url}")
|
||||
lines.append(f" {item.snippet[:150]}...")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_source_status(report: schema.Report, source_info: dict = None) -> str:
|
||||
"""Render source status footer showing what was used/skipped and why.
|
||||
|
||||
Args:
|
||||
report: Report data
|
||||
source_info: Dict with source availability info:
|
||||
x_skip_reason, youtube_skip_reason, web_skip_reason
|
||||
|
||||
Returns:
|
||||
Source status markdown string
|
||||
"""
|
||||
if source_info is None:
|
||||
source_info = {}
|
||||
|
||||
lines = []
|
||||
lines.append("---")
|
||||
lines.append("**Sources:**")
|
||||
|
||||
# Reddit
|
||||
if report.reddit_error:
|
||||
lines.append(f" ❌ Reddit: error — {report.reddit_error}")
|
||||
elif report.reddit:
|
||||
lines.append(f" ✅ Reddit: {len(report.reddit)} threads")
|
||||
elif report.mode in ("both", "reddit-only", "all", "reddit-web"):
|
||||
lines.append(" ⚠️ Reddit: 0 threads found")
|
||||
else:
|
||||
reason = source_info.get("reddit_skip_reason", "not configured")
|
||||
lines.append(f" ⏭️ Reddit: skipped — {reason}")
|
||||
|
||||
# X
|
||||
if report.x_error:
|
||||
lines.append(f" ❌ X: error — {report.x_error}")
|
||||
elif report.x:
|
||||
lines.append(f" ✅ X: {len(report.x)} posts")
|
||||
elif report.mode in ("both", "x-only", "all", "x-web"):
|
||||
lines.append(" ⚠️ X: 0 posts found")
|
||||
else:
|
||||
reason = source_info.get("x_skip_reason", "No Bird CLI or XAI_API_KEY")
|
||||
lines.append(f" ⏭️ X: skipped — {reason}")
|
||||
|
||||
# YouTube
|
||||
if report.youtube_error:
|
||||
lines.append(f" ❌ YouTube: error — {report.youtube_error}")
|
||||
elif report.youtube:
|
||||
with_transcripts = sum(1 for v in report.youtube if getattr(v, 'transcript_snippet', None))
|
||||
lines.append(f" ✅ YouTube: {len(report.youtube)} videos ({with_transcripts} with transcripts)")
|
||||
else:
|
||||
reason = source_info.get("youtube_skip_reason", "yt-dlp not installed (brew install yt-dlp)")
|
||||
lines.append(f" ⏭️ YouTube: skipped — {reason}")
|
||||
|
||||
# Web
|
||||
if report.web_error:
|
||||
lines.append(f" ❌ Web: error — {report.web_error}")
|
||||
elif report.web:
|
||||
lines.append(f" ✅ Web: {len(report.web)} pages")
|
||||
else:
|
||||
reason = source_info.get("web_skip_reason", "assistant will use WebSearch")
|
||||
lines.append(f" ⚡ Web: {reason}")
|
||||
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_context_snippet(report: schema.Report) -> str:
|
||||
"""Render reusable context snippet.
|
||||
|
||||
Args:
|
||||
report: Report data
|
||||
|
||||
Returns:
|
||||
Context markdown string
|
||||
"""
|
||||
lines = []
|
||||
lines.append(f"# Context: {report.topic} (Last 30 Days)")
|
||||
lines.append("")
|
||||
lines.append(f"*Generated: {report.generated_at[:10]} | Sources: {report.mode}*")
|
||||
lines.append("")
|
||||
|
||||
# Key sources summary
|
||||
lines.append("## Key Sources")
|
||||
lines.append("")
|
||||
|
||||
all_items = []
|
||||
for item in report.reddit[:5]:
|
||||
all_items.append((item.score, "Reddit", item.title, item.url))
|
||||
for item in report.x[:5]:
|
||||
all_items.append((item.score, "X", item.text[:50] + "...", item.url))
|
||||
for item in report.web[:5]:
|
||||
all_items.append((item.score, "Web", item.title[:50] + "...", item.url))
|
||||
|
||||
all_items.sort(key=lambda x: -x[0])
|
||||
for score, source, text, url in all_items[:7]:
|
||||
lines.append(f"- [{source}] {text}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("## Summary")
|
||||
lines.append("")
|
||||
lines.append("*See full report for best practices, prompt pack, and detailed sources.*")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_full_report(report: schema.Report) -> str:
|
||||
"""Render full markdown report.
|
||||
|
||||
Args:
|
||||
report: Report data
|
||||
|
||||
Returns:
|
||||
Full report markdown
|
||||
"""
|
||||
lines = []
|
||||
|
||||
# Title
|
||||
lines.append(f"# {report.topic} - Last 30 Days Research Report")
|
||||
lines.append("")
|
||||
lines.append(f"**Generated:** {report.generated_at}")
|
||||
lines.append(f"**Date Range:** {report.range_from} to {report.range_to}")
|
||||
lines.append(f"**Mode:** {report.mode}")
|
||||
lines.append("")
|
||||
|
||||
# Models
|
||||
lines.append("## Models Used")
|
||||
lines.append("")
|
||||
if report.openai_model_used:
|
||||
lines.append(f"- **OpenAI:** {report.openai_model_used}")
|
||||
if report.xai_model_used:
|
||||
lines.append(f"- **xAI:** {report.xai_model_used}")
|
||||
lines.append("")
|
||||
|
||||
# Reddit section
|
||||
if report.reddit:
|
||||
lines.append("## Reddit Threads")
|
||||
lines.append("")
|
||||
for item in report.reddit:
|
||||
lines.append(f"### {item.id}: {item.title}")
|
||||
lines.append("")
|
||||
lines.append(f"- **Subreddit:** r/{item.subreddit}")
|
||||
lines.append(f"- **URL:** {item.url}")
|
||||
lines.append(f"- **Date:** {item.date or 'Unknown'} (confidence: {item.date_confidence})")
|
||||
lines.append(f"- **Score:** {item.score}/100")
|
||||
lines.append(f"- **Relevance:** {item.why_relevant}")
|
||||
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
lines.append(f"- **Engagement:** {eng.score or '?'} points, {eng.num_comments or '?'} comments")
|
||||
|
||||
if item.comment_insights:
|
||||
lines.append("")
|
||||
lines.append("**Key Insights from Comments:**")
|
||||
for insight in item.comment_insights:
|
||||
lines.append(f"- {insight}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# X section
|
||||
if report.x:
|
||||
lines.append("## X Posts")
|
||||
lines.append("")
|
||||
for item in report.x:
|
||||
lines.append(f"### {item.id}: @{item.author_handle}")
|
||||
lines.append("")
|
||||
lines.append(f"- **URL:** {item.url}")
|
||||
lines.append(f"- **Date:** {item.date or 'Unknown'} (confidence: {item.date_confidence})")
|
||||
lines.append(f"- **Score:** {item.score}/100")
|
||||
lines.append(f"- **Relevance:** {item.why_relevant}")
|
||||
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
lines.append(f"- **Engagement:** {eng.likes or '?'} likes, {eng.reposts or '?'} reposts")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"> {item.text}")
|
||||
lines.append("")
|
||||
|
||||
# Web section
|
||||
if report.web:
|
||||
lines.append("## Web Results")
|
||||
lines.append("")
|
||||
for item in report.web:
|
||||
lines.append(f"### {item.id}: {item.title}")
|
||||
lines.append("")
|
||||
lines.append(f"- **Source:** {item.source_domain}")
|
||||
lines.append(f"- **URL:** {item.url}")
|
||||
lines.append(f"- **Date:** {item.date or 'Unknown'} (confidence: {item.date_confidence})")
|
||||
lines.append(f"- **Score:** {item.score}/100")
|
||||
lines.append(f"- **Relevance:** {item.why_relevant}")
|
||||
lines.append("")
|
||||
lines.append(f"> {item.snippet}")
|
||||
lines.append("")
|
||||
|
||||
# Placeholders for assistant synthesis
|
||||
lines.append("## Best Practices")
|
||||
lines.append("")
|
||||
lines.append("*To be synthesized by assistant*")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Prompt Pack")
|
||||
lines.append("")
|
||||
lines.append("*To be synthesized by assistant*")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_outputs(
|
||||
report: schema.Report,
|
||||
raw_openai: Optional[dict] = None,
|
||||
raw_xai: Optional[dict] = None,
|
||||
raw_reddit_enriched: Optional[list] = None,
|
||||
):
|
||||
"""Write all output files.
|
||||
|
||||
Args:
|
||||
report: Report data
|
||||
raw_openai: Raw OpenAI API response
|
||||
raw_xai: Raw xAI API response
|
||||
raw_reddit_enriched: Raw enriched Reddit thread data
|
||||
"""
|
||||
ensure_output_dir()
|
||||
|
||||
# report.json
|
||||
with open(OUTPUT_DIR / "report.json", 'w') as f:
|
||||
json.dump(report.to_dict(), f, indent=2)
|
||||
|
||||
# report.md
|
||||
with open(OUTPUT_DIR / "report.md", 'w') as f:
|
||||
f.write(render_full_report(report))
|
||||
|
||||
# last30days.context.md
|
||||
with open(OUTPUT_DIR / "last30days.context.md", 'w') as f:
|
||||
f.write(render_context_snippet(report))
|
||||
|
||||
# Raw responses
|
||||
if raw_openai:
|
||||
with open(OUTPUT_DIR / "raw_openai.json", 'w') as f:
|
||||
json.dump(raw_openai, f, indent=2)
|
||||
|
||||
if raw_xai:
|
||||
with open(OUTPUT_DIR / "raw_xai.json", 'w') as f:
|
||||
json.dump(raw_xai, f, indent=2)
|
||||
|
||||
if raw_reddit_enriched:
|
||||
with open(OUTPUT_DIR / "raw_reddit_threads_enriched.json", 'w') as f:
|
||||
json.dump(raw_reddit_enriched, f, indent=2)
|
||||
|
||||
|
||||
def get_context_path() -> str:
|
||||
"""Get path to context file."""
|
||||
return str(OUTPUT_DIR / "last30days.context.md")
|
||||
@@ -1,403 +0,0 @@
|
||||
"""Data schemas for last30days skill."""
|
||||
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Any, Dict, List, Optional
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
@dataclass
|
||||
class Engagement:
|
||||
"""Engagement metrics."""
|
||||
# Reddit fields
|
||||
score: Optional[int] = None
|
||||
num_comments: Optional[int] = None
|
||||
upvote_ratio: Optional[float] = None
|
||||
|
||||
# X fields
|
||||
likes: Optional[int] = None
|
||||
reposts: Optional[int] = None
|
||||
replies: Optional[int] = None
|
||||
quotes: Optional[int] = None
|
||||
|
||||
# YouTube fields
|
||||
views: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {}
|
||||
if self.score is not None:
|
||||
d['score'] = self.score
|
||||
if self.num_comments is not None:
|
||||
d['num_comments'] = self.num_comments
|
||||
if self.upvote_ratio is not None:
|
||||
d['upvote_ratio'] = self.upvote_ratio
|
||||
if self.likes is not None:
|
||||
d['likes'] = self.likes
|
||||
if self.reposts is not None:
|
||||
d['reposts'] = self.reposts
|
||||
if self.replies is not None:
|
||||
d['replies'] = self.replies
|
||||
if self.quotes is not None:
|
||||
d['quotes'] = self.quotes
|
||||
if self.views is not None:
|
||||
d['views'] = self.views
|
||||
return d if d else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Comment:
|
||||
"""Reddit comment."""
|
||||
score: int
|
||||
date: Optional[str]
|
||||
author: str
|
||||
excerpt: str
|
||||
url: str
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'score': self.score,
|
||||
'date': self.date,
|
||||
'author': self.author,
|
||||
'excerpt': self.excerpt,
|
||||
'url': self.url,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SubScores:
|
||||
"""Component scores."""
|
||||
relevance: int = 0
|
||||
recency: int = 0
|
||||
engagement: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, int]:
|
||||
return {
|
||||
'relevance': self.relevance,
|
||||
'recency': self.recency,
|
||||
'engagement': self.engagement,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RedditItem:
|
||||
"""Normalized Reddit item."""
|
||||
id: str
|
||||
title: str
|
||||
url: str
|
||||
subreddit: str
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "low"
|
||||
engagement: Optional[Engagement] = None
|
||||
top_comments: List[Comment] = field(default_factory=list)
|
||||
comment_insights: List[str] = field(default_factory=list)
|
||||
relevance: float = 0.5
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'url': self.url,
|
||||
'subreddit': self.subreddit,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||
'top_comments': [c.to_dict() for c in self.top_comments],
|
||||
'comment_insights': self.comment_insights,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class XItem:
|
||||
"""Normalized X item."""
|
||||
id: str
|
||||
text: str
|
||||
url: str
|
||||
author_handle: str
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "low"
|
||||
engagement: Optional[Engagement] = None
|
||||
relevance: float = 0.5
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'id': self.id,
|
||||
'text': self.text,
|
||||
'url': self.url,
|
||||
'author_handle': self.author_handle,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class WebSearchItem:
|
||||
"""Normalized web search item (no engagement metrics)."""
|
||||
id: str
|
||||
title: str
|
||||
url: str
|
||||
source_domain: str # e.g., "medium.com", "github.com"
|
||||
snippet: str
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "low"
|
||||
relevance: float = 0.5
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'url': self.url,
|
||||
'source_domain': self.source_domain,
|
||||
'snippet': self.snippet,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class YouTubeItem:
|
||||
"""Normalized YouTube item."""
|
||||
id: str # video_id
|
||||
title: str
|
||||
url: str
|
||||
channel_name: str
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "high" # YouTube dates are always reliable
|
||||
engagement: Optional[Engagement] = None
|
||||
transcript_snippet: str = ""
|
||||
relevance: float = 0.7
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'url': self.url,
|
||||
'channel_name': self.channel_name,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||
'transcript_snippet': self.transcript_snippet,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
"""Full research report."""
|
||||
topic: str
|
||||
range_from: str
|
||||
range_to: str
|
||||
generated_at: str
|
||||
mode: str # 'reddit-only', 'x-only', 'both', 'web-only', etc.
|
||||
openai_model_used: Optional[str] = None
|
||||
xai_model_used: Optional[str] = None
|
||||
reddit: List[RedditItem] = field(default_factory=list)
|
||||
x: List[XItem] = field(default_factory=list)
|
||||
web: List[WebSearchItem] = field(default_factory=list)
|
||||
youtube: List[YouTubeItem] = field(default_factory=list)
|
||||
best_practices: List[str] = field(default_factory=list)
|
||||
prompt_pack: List[str] = field(default_factory=list)
|
||||
context_snippet_md: str = ""
|
||||
# Status tracking
|
||||
reddit_error: Optional[str] = None
|
||||
x_error: Optional[str] = None
|
||||
web_error: Optional[str] = None
|
||||
youtube_error: Optional[str] = None
|
||||
# Cache info
|
||||
from_cache: bool = False
|
||||
cache_age_hours: Optional[float] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {
|
||||
'topic': self.topic,
|
||||
'range': {
|
||||
'from': self.range_from,
|
||||
'to': self.range_to,
|
||||
},
|
||||
'generated_at': self.generated_at,
|
||||
'mode': self.mode,
|
||||
'openai_model_used': self.openai_model_used,
|
||||
'xai_model_used': self.xai_model_used,
|
||||
'reddit': [r.to_dict() for r in self.reddit],
|
||||
'x': [x.to_dict() for x in self.x],
|
||||
'web': [w.to_dict() for w in self.web],
|
||||
'youtube': [y.to_dict() for y in self.youtube],
|
||||
'best_practices': self.best_practices,
|
||||
'prompt_pack': self.prompt_pack,
|
||||
'context_snippet_md': self.context_snippet_md,
|
||||
}
|
||||
if self.reddit_error:
|
||||
d['reddit_error'] = self.reddit_error
|
||||
if self.x_error:
|
||||
d['x_error'] = self.x_error
|
||||
if self.web_error:
|
||||
d['web_error'] = self.web_error
|
||||
if self.youtube_error:
|
||||
d['youtube_error'] = self.youtube_error
|
||||
if self.from_cache:
|
||||
d['from_cache'] = self.from_cache
|
||||
if self.cache_age_hours is not None:
|
||||
d['cache_age_hours'] = self.cache_age_hours
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "Report":
|
||||
"""Create Report from serialized dict (handles cache format)."""
|
||||
# Handle range field conversion
|
||||
range_data = data.get('range', {})
|
||||
range_from = range_data.get('from', data.get('range_from', ''))
|
||||
range_to = range_data.get('to', data.get('range_to', ''))
|
||||
|
||||
# Reconstruct Reddit items
|
||||
reddit_items = []
|
||||
for r in data.get('reddit', []):
|
||||
eng = None
|
||||
if r.get('engagement'):
|
||||
eng = Engagement(**r['engagement'])
|
||||
comments = [Comment(**c) for c in r.get('top_comments', [])]
|
||||
subs = SubScores(**r.get('subs', {})) if r.get('subs') else SubScores()
|
||||
reddit_items.append(RedditItem(
|
||||
id=r['id'],
|
||||
title=r['title'],
|
||||
url=r['url'],
|
||||
subreddit=r['subreddit'],
|
||||
date=r.get('date'),
|
||||
date_confidence=r.get('date_confidence', 'low'),
|
||||
engagement=eng,
|
||||
top_comments=comments,
|
||||
comment_insights=r.get('comment_insights', []),
|
||||
relevance=r.get('relevance', 0.5),
|
||||
why_relevant=r.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=r.get('score', 0),
|
||||
))
|
||||
|
||||
# Reconstruct X items
|
||||
x_items = []
|
||||
for x in data.get('x', []):
|
||||
eng = None
|
||||
if x.get('engagement'):
|
||||
eng = Engagement(**x['engagement'])
|
||||
subs = SubScores(**x.get('subs', {})) if x.get('subs') else SubScores()
|
||||
x_items.append(XItem(
|
||||
id=x['id'],
|
||||
text=x['text'],
|
||||
url=x['url'],
|
||||
author_handle=x['author_handle'],
|
||||
date=x.get('date'),
|
||||
date_confidence=x.get('date_confidence', 'low'),
|
||||
engagement=eng,
|
||||
relevance=x.get('relevance', 0.5),
|
||||
why_relevant=x.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=x.get('score', 0),
|
||||
))
|
||||
|
||||
# Reconstruct Web items
|
||||
web_items = []
|
||||
for w in data.get('web', []):
|
||||
subs = SubScores(**w.get('subs', {})) if w.get('subs') else SubScores()
|
||||
web_items.append(WebSearchItem(
|
||||
id=w['id'],
|
||||
title=w['title'],
|
||||
url=w['url'],
|
||||
source_domain=w.get('source_domain', ''),
|
||||
snippet=w.get('snippet', ''),
|
||||
date=w.get('date'),
|
||||
date_confidence=w.get('date_confidence', 'low'),
|
||||
relevance=w.get('relevance', 0.5),
|
||||
why_relevant=w.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=w.get('score', 0),
|
||||
))
|
||||
|
||||
# Reconstruct YouTube items
|
||||
youtube_items = []
|
||||
for y in data.get('youtube', []):
|
||||
eng = None
|
||||
if y.get('engagement'):
|
||||
eng = Engagement(**y['engagement'])
|
||||
subs = SubScores(**y.get('subs', {})) if y.get('subs') else SubScores()
|
||||
youtube_items.append(YouTubeItem(
|
||||
id=y['id'],
|
||||
title=y['title'],
|
||||
url=y['url'],
|
||||
channel_name=y.get('channel_name', ''),
|
||||
date=y.get('date'),
|
||||
date_confidence=y.get('date_confidence', 'high'),
|
||||
engagement=eng,
|
||||
transcript_snippet=y.get('transcript_snippet', ''),
|
||||
relevance=y.get('relevance', 0.7),
|
||||
why_relevant=y.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=y.get('score', 0),
|
||||
))
|
||||
|
||||
return cls(
|
||||
topic=data['topic'],
|
||||
range_from=range_from,
|
||||
range_to=range_to,
|
||||
generated_at=data['generated_at'],
|
||||
mode=data['mode'],
|
||||
openai_model_used=data.get('openai_model_used'),
|
||||
xai_model_used=data.get('xai_model_used'),
|
||||
reddit=reddit_items,
|
||||
x=x_items,
|
||||
web=web_items,
|
||||
youtube=youtube_items,
|
||||
best_practices=data.get('best_practices', []),
|
||||
prompt_pack=data.get('prompt_pack', []),
|
||||
context_snippet_md=data.get('context_snippet_md', ''),
|
||||
reddit_error=data.get('reddit_error'),
|
||||
x_error=data.get('x_error'),
|
||||
web_error=data.get('web_error'),
|
||||
youtube_error=data.get('youtube_error'),
|
||||
from_cache=data.get('from_cache', False),
|
||||
cache_age_hours=data.get('cache_age_hours'),
|
||||
)
|
||||
|
||||
|
||||
def create_report(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
mode: str,
|
||||
openai_model: Optional[str] = None,
|
||||
xai_model: Optional[str] = None,
|
||||
) -> Report:
|
||||
"""Create a new report with metadata."""
|
||||
return Report(
|
||||
topic=topic,
|
||||
range_from=from_date,
|
||||
range_to=to_date,
|
||||
generated_at=datetime.now(timezone.utc).isoformat(),
|
||||
mode=mode,
|
||||
openai_model_used=openai_model,
|
||||
xai_model_used=xai_model,
|
||||
)
|
||||
@@ -1,372 +0,0 @@
|
||||
"""Popularity-aware scoring for last30days skill."""
|
||||
|
||||
import math
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
# Score weights for Reddit/X (has engagement)
|
||||
WEIGHT_RELEVANCE = 0.45
|
||||
WEIGHT_RECENCY = 0.25
|
||||
WEIGHT_ENGAGEMENT = 0.30
|
||||
|
||||
# WebSearch weights (no engagement, reweighted to 100%)
|
||||
WEBSEARCH_WEIGHT_RELEVANCE = 0.55
|
||||
WEBSEARCH_WEIGHT_RECENCY = 0.45
|
||||
WEBSEARCH_SOURCE_PENALTY = 15 # Points deducted for lacking engagement
|
||||
|
||||
# WebSearch date confidence adjustments
|
||||
WEBSEARCH_VERIFIED_BONUS = 10 # Bonus for URL-verified recent date (high confidence)
|
||||
WEBSEARCH_NO_DATE_PENALTY = 20 # Heavy penalty for no date signals (low confidence)
|
||||
|
||||
# Default engagement score for unknown
|
||||
DEFAULT_ENGAGEMENT = 35
|
||||
UNKNOWN_ENGAGEMENT_PENALTY = 3
|
||||
|
||||
|
||||
def log1p_safe(x: Optional[int]) -> float:
|
||||
"""Safe log1p that handles None and negative values."""
|
||||
if x is None or x < 0:
|
||||
return 0.0
|
||||
return math.log1p(x)
|
||||
|
||||
|
||||
def compute_reddit_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
"""Compute raw engagement score for Reddit item.
|
||||
|
||||
Formula: 0.55*log1p(score) + 0.40*log1p(num_comments) + 0.05*(upvote_ratio*10)
|
||||
"""
|
||||
if engagement is None:
|
||||
return None
|
||||
|
||||
if engagement.score is None and engagement.num_comments is None:
|
||||
return None
|
||||
|
||||
score = log1p_safe(engagement.score)
|
||||
comments = log1p_safe(engagement.num_comments)
|
||||
ratio = (engagement.upvote_ratio or 0.5) * 10
|
||||
|
||||
return 0.55 * score + 0.40 * comments + 0.05 * ratio
|
||||
|
||||
|
||||
def compute_x_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
"""Compute raw engagement score for X item.
|
||||
|
||||
Formula: 0.55*log1p(likes) + 0.25*log1p(reposts) + 0.15*log1p(replies) + 0.05*log1p(quotes)
|
||||
"""
|
||||
if engagement is None:
|
||||
return None
|
||||
|
||||
if engagement.likes is None and engagement.reposts is None:
|
||||
return None
|
||||
|
||||
likes = log1p_safe(engagement.likes)
|
||||
reposts = log1p_safe(engagement.reposts)
|
||||
replies = log1p_safe(engagement.replies)
|
||||
quotes = log1p_safe(engagement.quotes)
|
||||
|
||||
return 0.55 * likes + 0.25 * reposts + 0.15 * replies + 0.05 * quotes
|
||||
|
||||
|
||||
def normalize_to_100(values: List[float], default: float = 50) -> List[float]:
|
||||
"""Normalize a list of values to 0-100 scale.
|
||||
|
||||
Args:
|
||||
values: Raw values (None values are preserved)
|
||||
default: Default value for None entries
|
||||
|
||||
Returns:
|
||||
Normalized values
|
||||
"""
|
||||
# Filter out None
|
||||
valid = [v for v in values if v is not None]
|
||||
if not valid:
|
||||
return [default if v is None else 50 for v in values]
|
||||
|
||||
min_val = min(valid)
|
||||
max_val = max(valid)
|
||||
range_val = max_val - min_val
|
||||
|
||||
if range_val == 0:
|
||||
return [50 if v is None else 50 for v in values]
|
||||
|
||||
result = []
|
||||
for v in values:
|
||||
if v is None:
|
||||
result.append(None)
|
||||
else:
|
||||
normalized = ((v - min_val) / range_val) * 100
|
||||
result.append(normalized)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def score_reddit_items(items: List[schema.RedditItem]) -> List[schema.RedditItem]:
|
||||
"""Compute scores for Reddit items.
|
||||
|
||||
Args:
|
||||
items: List of Reddit items
|
||||
|
||||
Returns:
|
||||
Items with updated scores
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
|
||||
# Compute raw engagement scores
|
||||
eng_raw = [compute_reddit_engagement_raw(item.engagement) for item in items]
|
||||
|
||||
# Normalize engagement to 0-100
|
||||
eng_normalized = normalize_to_100(eng_raw)
|
||||
|
||||
for i, item in enumerate(items):
|
||||
# Relevance subscore (model-provided, convert to 0-100)
|
||||
rel_score = int(item.relevance * 100)
|
||||
|
||||
# Recency subscore
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
# Engagement subscore
|
||||
if eng_normalized[i] is not None:
|
||||
eng_score = int(eng_normalized[i])
|
||||
else:
|
||||
eng_score = DEFAULT_ENGAGEMENT
|
||||
|
||||
# Store subscores
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=eng_score,
|
||||
)
|
||||
|
||||
# Compute overall score
|
||||
overall = (
|
||||
WEIGHT_RELEVANCE * rel_score +
|
||||
WEIGHT_RECENCY * rec_score +
|
||||
WEIGHT_ENGAGEMENT * eng_score
|
||||
)
|
||||
|
||||
# Apply penalty for unknown engagement
|
||||
if eng_raw[i] is None:
|
||||
overall -= UNKNOWN_ENGAGEMENT_PENALTY
|
||||
|
||||
# Apply penalty for low date confidence
|
||||
if item.date_confidence == "low":
|
||||
overall -= 5
|
||||
elif item.date_confidence == "med":
|
||||
overall -= 2
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def score_x_items(items: List[schema.XItem]) -> List[schema.XItem]:
|
||||
"""Compute scores for X items.
|
||||
|
||||
Args:
|
||||
items: List of X items
|
||||
|
||||
Returns:
|
||||
Items with updated scores
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
|
||||
# Compute raw engagement scores
|
||||
eng_raw = [compute_x_engagement_raw(item.engagement) for item in items]
|
||||
|
||||
# Normalize engagement to 0-100
|
||||
eng_normalized = normalize_to_100(eng_raw)
|
||||
|
||||
for i, item in enumerate(items):
|
||||
# Relevance subscore (model-provided, convert to 0-100)
|
||||
rel_score = int(item.relevance * 100)
|
||||
|
||||
# Recency subscore
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
# Engagement subscore
|
||||
if eng_normalized[i] is not None:
|
||||
eng_score = int(eng_normalized[i])
|
||||
else:
|
||||
eng_score = DEFAULT_ENGAGEMENT
|
||||
|
||||
# Store subscores
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=eng_score,
|
||||
)
|
||||
|
||||
# Compute overall score
|
||||
overall = (
|
||||
WEIGHT_RELEVANCE * rel_score +
|
||||
WEIGHT_RECENCY * rec_score +
|
||||
WEIGHT_ENGAGEMENT * eng_score
|
||||
)
|
||||
|
||||
# Apply penalty for unknown engagement
|
||||
if eng_raw[i] is None:
|
||||
overall -= UNKNOWN_ENGAGEMENT_PENALTY
|
||||
|
||||
# Apply penalty for low date confidence
|
||||
if item.date_confidence == "low":
|
||||
overall -= 5
|
||||
elif item.date_confidence == "med":
|
||||
overall -= 2
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def compute_youtube_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
"""Compute raw engagement score for YouTube item.
|
||||
|
||||
Formula: 0.50*log1p(views) + 0.35*log1p(likes) + 0.15*log1p(comments)
|
||||
Views dominate on YouTube — they're the primary discovery signal.
|
||||
"""
|
||||
if engagement is None:
|
||||
return None
|
||||
|
||||
if engagement.views is None and engagement.likes is None:
|
||||
return None
|
||||
|
||||
views = log1p_safe(engagement.views)
|
||||
likes = log1p_safe(engagement.likes)
|
||||
comments = log1p_safe(engagement.num_comments)
|
||||
|
||||
return 0.50 * views + 0.35 * likes + 0.15 * comments
|
||||
|
||||
|
||||
def score_youtube_items(items: List[schema.YouTubeItem]) -> List[schema.YouTubeItem]:
|
||||
"""Compute scores for YouTube items.
|
||||
|
||||
Uses same weight structure as Reddit/X (relevance + recency + engagement).
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
|
||||
eng_raw = [compute_youtube_engagement_raw(item.engagement) for item in items]
|
||||
eng_normalized = normalize_to_100(eng_raw)
|
||||
|
||||
for i, item in enumerate(items):
|
||||
rel_score = int(item.relevance * 100)
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
if eng_normalized[i] is not None:
|
||||
eng_score = int(eng_normalized[i])
|
||||
else:
|
||||
eng_score = DEFAULT_ENGAGEMENT
|
||||
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=eng_score,
|
||||
)
|
||||
|
||||
overall = (
|
||||
WEIGHT_RELEVANCE * rel_score +
|
||||
WEIGHT_RECENCY * rec_score +
|
||||
WEIGHT_ENGAGEMENT * eng_score
|
||||
)
|
||||
|
||||
if eng_raw[i] is None:
|
||||
overall -= UNKNOWN_ENGAGEMENT_PENALTY
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
|
||||
"""Compute scores for WebSearch items WITHOUT engagement metrics.
|
||||
|
||||
Uses reweighted formula: 55% relevance + 45% recency - 15pt source penalty.
|
||||
This ensures WebSearch items rank below comparable Reddit/X items.
|
||||
|
||||
Date confidence adjustments:
|
||||
- High confidence (URL-verified date): +10 bonus
|
||||
- Med confidence (snippet-extracted date): no change
|
||||
- Low confidence (no date signals): -20 penalty
|
||||
|
||||
Args:
|
||||
items: List of WebSearch items
|
||||
|
||||
Returns:
|
||||
Items with updated scores
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
|
||||
for item in items:
|
||||
# Relevance subscore (model-provided, convert to 0-100)
|
||||
rel_score = int(item.relevance * 100)
|
||||
|
||||
# Recency subscore
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
# Store subscores (engagement is 0 for WebSearch - no data)
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=0, # Explicitly zero - no engagement data available
|
||||
)
|
||||
|
||||
# Compute overall score using WebSearch weights
|
||||
overall = (
|
||||
WEBSEARCH_WEIGHT_RELEVANCE * rel_score +
|
||||
WEBSEARCH_WEIGHT_RECENCY * rec_score
|
||||
)
|
||||
|
||||
# Apply source penalty (WebSearch < Reddit/X for same relevance/recency)
|
||||
overall -= WEBSEARCH_SOURCE_PENALTY
|
||||
|
||||
# Apply date confidence adjustments
|
||||
# High confidence (URL-verified): reward with bonus
|
||||
# Med confidence (snippet-extracted): neutral
|
||||
# Low confidence (no date signals): heavy penalty
|
||||
if item.date_confidence == "high":
|
||||
overall += WEBSEARCH_VERIFIED_BONUS # Reward verified recent dates
|
||||
elif item.date_confidence == "low":
|
||||
overall -= WEBSEARCH_NO_DATE_PENALTY # Heavy penalty for unknown
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem]]) -> List:
|
||||
"""Sort items by score (descending), then date, then source priority.
|
||||
|
||||
Args:
|
||||
items: List of items to sort
|
||||
|
||||
Returns:
|
||||
Sorted items
|
||||
"""
|
||||
def sort_key(item):
|
||||
# Primary: score descending (negate for descending)
|
||||
score = -item.score
|
||||
|
||||
# Secondary: date descending (recent first)
|
||||
date = item.date or "0000-00-00"
|
||||
date_key = -int(date.replace("-", ""))
|
||||
|
||||
# Tertiary: source priority (Reddit > X > YouTube > WebSearch)
|
||||
if isinstance(item, schema.RedditItem):
|
||||
source_priority = 0
|
||||
elif isinstance(item, schema.XItem):
|
||||
source_priority = 1
|
||||
elif isinstance(item, schema.YouTubeItem):
|
||||
source_priority = 2
|
||||
else: # WebSearchItem
|
||||
source_priority = 3
|
||||
|
||||
# Quaternary: title/text for stability
|
||||
text = getattr(item, "title", "") or getattr(item, "text", "")
|
||||
|
||||
return (score, date_key, source_priority, text)
|
||||
|
||||
return sorted(items, key=sort_key)
|
||||
-134
@@ -1,134 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* bird-search.mjs - Vendored Bird CLI search wrapper for /last30days.
|
||||
* Subset of @steipete/bird v0.8.0 (MIT License, Peter Steinberger).
|
||||
*
|
||||
* Usage:
|
||||
* node bird-search.mjs <query> [--count N] [--json]
|
||||
* node bird-search.mjs --whoami
|
||||
* node bird-search.mjs --check
|
||||
*/
|
||||
|
||||
import { resolveCredentials } from './lib/cookies.js';
|
||||
import { TwitterClientBase } from './lib/twitter-client-base.js';
|
||||
import { withSearch } from './lib/twitter-client-search.js';
|
||||
|
||||
// Build a search-only client (no posting, bookmarks, etc.)
|
||||
const SearchClient = withSearch(TwitterClientBase);
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
// --check: verify that credentials can be resolved
|
||||
if (args.includes('--check')) {
|
||||
try {
|
||||
const { cookies, warnings } = await resolveCredentials({});
|
||||
if (cookies.authToken && cookies.ct0) {
|
||||
process.stdout.write(JSON.stringify({ authenticated: true, source: cookies.source }));
|
||||
process.exit(0);
|
||||
} else {
|
||||
process.stdout.write(JSON.stringify({ authenticated: false, warnings }));
|
||||
process.exit(1);
|
||||
}
|
||||
} catch (err) {
|
||||
process.stdout.write(JSON.stringify({ authenticated: false, error: err.message }));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// --whoami: check auth and output source
|
||||
if (args.includes('--whoami')) {
|
||||
try {
|
||||
const { cookies } = await resolveCredentials({});
|
||||
if (cookies.authToken && cookies.ct0) {
|
||||
process.stdout.write(cookies.source || 'authenticated');
|
||||
process.exit(0);
|
||||
} else {
|
||||
process.stderr.write('Not authenticated\n');
|
||||
process.exit(1);
|
||||
}
|
||||
} catch (err) {
|
||||
process.stderr.write(`Auth check failed: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse search args
|
||||
let query = null;
|
||||
let count = 20;
|
||||
let jsonOutput = false;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--count' && args[i + 1]) {
|
||||
count = parseInt(args[i + 1], 10);
|
||||
i++;
|
||||
} else if (args[i] === '-n' && args[i + 1]) {
|
||||
count = parseInt(args[i + 1], 10);
|
||||
i++;
|
||||
} else if (args[i] === '--json') {
|
||||
jsonOutput = true;
|
||||
} else if (!args[i].startsWith('-')) {
|
||||
query = args[i];
|
||||
}
|
||||
}
|
||||
|
||||
if (!query) {
|
||||
process.stderr.write('Usage: node bird-search.mjs <query> [--count N] [--json]\n');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
// Resolve credentials (env vars, then browser cookies)
|
||||
const { cookies, warnings } = await resolveCredentials({});
|
||||
|
||||
if (!cookies.authToken || !cookies.ct0) {
|
||||
const msg = warnings.length > 0 ? warnings.join('; ') : 'No Twitter credentials found';
|
||||
if (jsonOutput) {
|
||||
process.stdout.write(JSON.stringify({ error: msg, items: [] }));
|
||||
} else {
|
||||
process.stderr.write(`Error: ${msg}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Create search client
|
||||
const client = new SearchClient({
|
||||
cookies: {
|
||||
authToken: cookies.authToken,
|
||||
ct0: cookies.ct0,
|
||||
cookieHeader: cookies.cookieHeader,
|
||||
},
|
||||
timeoutMs: 30000,
|
||||
});
|
||||
|
||||
// Run search
|
||||
const result = await client.search(query, count);
|
||||
|
||||
if (!result.success) {
|
||||
if (jsonOutput) {
|
||||
process.stdout.write(JSON.stringify({ error: result.error, items: [] }));
|
||||
} else {
|
||||
process.stderr.write(`Search failed: ${result.error}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Output results
|
||||
const tweets = result.tweets || [];
|
||||
if (jsonOutput) {
|
||||
process.stdout.write(JSON.stringify(tweets));
|
||||
} else {
|
||||
for (const tweet of tweets) {
|
||||
const author = tweet.author?.username || 'unknown';
|
||||
process.stdout.write(`@${author}: ${tweet.text?.slice(0, 200)}\n\n`);
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
if (jsonOutput) {
|
||||
process.stdout.write(JSON.stringify({ error: err.message, items: [] }));
|
||||
} else {
|
||||
process.stderr.write(`Error: ${err.message}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
# @steipete/sweet-cookie
|
||||
|
||||
Inline-first browser cookie extraction for local tooling (no native addons).
|
||||
|
||||
Supports:
|
||||
- Inline payloads (JSON / base64 / file) — most reliable path.
|
||||
- Local browser reads (best effort): Chrome, Edge, Firefox, Safari (macOS).
|
||||
|
||||
Install:
|
||||
```bash
|
||||
npm i @steipete/sweet-cookie
|
||||
```
|
||||
|
||||
Usage:
|
||||
```ts
|
||||
import { getCookies, toCookieHeader } from '@steipete/sweet-cookie';
|
||||
|
||||
const { cookies, warnings } = await getCookies({
|
||||
url: 'https://example.com/',
|
||||
names: ['session', 'csrf'],
|
||||
browsers: ['chrome', 'edge', 'firefox', 'safari'],
|
||||
});
|
||||
|
||||
for (const w of warnings) console.warn(w);
|
||||
const cookieHeader = toCookieHeader(cookies, { dedupeByName: true });
|
||||
```
|
||||
|
||||
Docs + extension exporter: see the repo root README.
|
||||
|
||||
Generated
Vendored
-3
@@ -1,3 +0,0 @@
|
||||
export { getCookies, toCookieHeader } from './public.js';
|
||||
export type { BrowserName, Cookie, CookieHeaderOptions, CookieMode, CookieSameSite, GetCookiesOptions, GetCookiesResult, } from './types.js';
|
||||
//# sourceMappingURL=index.d.ts.map
|
||||
Generated
Vendored
-1
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACzD,YAAY,EACX,WAAW,EACX,MAAM,EACN,mBAAmB,EACnB,UAAU,EACV,cAAc,EACd,iBAAiB,EACjB,gBAAgB,GAChB,MAAM,YAAY,CAAC"}
|
||||
-2
@@ -1,2 +0,0 @@
|
||||
export { getCookies, toCookieHeader } from './public.js';
|
||||
//# sourceMappingURL=index.js.map
|
||||
Generated
Vendored
-1
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user