Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5864c687a3 | |||
| 790e5bc26a |
@@ -1,20 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "last30days-skill",
|
|
||||||
"interface": {
|
|
||||||
"displayName": "Last 30 Days"
|
|
||||||
},
|
|
||||||
"plugins": [
|
|
||||||
{
|
|
||||||
"name": "last30days",
|
|
||||||
"source": {
|
|
||||||
"source": "local",
|
|
||||||
"path": "./"
|
|
||||||
},
|
|
||||||
"policy": {
|
|
||||||
"installation": "AVAILABLE",
|
|
||||||
"authentication": "ON_INSTALL"
|
|
||||||
},
|
|
||||||
"category": "Research"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,17 +1,16 @@
|
|||||||
{
|
{
|
||||||
|
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||||
"name": "last30days-skill",
|
"name": "last30days-skill",
|
||||||
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||||
"owner": {
|
"owner": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
"url": "https://github.com/mvanhorn"
|
"url": "https://github.com/mvanhorn"
|
||||||
},
|
},
|
||||||
"metadata": {
|
|
||||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
|
||||||
},
|
|
||||||
"plugins": [
|
"plugins": [
|
||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||||
"version": "3.2.3",
|
"version": "3.0.9",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
"url": "https://github.com/mvanhorn"
|
"url": "https://github.com/mvanhorn"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"version": "3.2.3",
|
"version": "3.0.9",
|
||||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
@@ -10,5 +10,6 @@
|
|||||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"],
|
||||||
|
"hooks": {}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"name": "last30days"
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
# Exclude non-runtime files from `git archive` output.
|
# Exclude non-runtime files from `git archive` output.
|
||||||
# Used by skills/last30days/scripts/build-skill.sh to produce a
|
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||||
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
|
|
||||||
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||||
|
|
||||||
# Anthropic canonical skill-packaging excludes
|
# Anthropic canonical skill-packaging excludes
|
||||||
@@ -33,8 +32,9 @@ release-notes.md export-ignore
|
|||||||
CHANGELOG.md export-ignore
|
CHANGELOG.md export-ignore
|
||||||
uv.lock export-ignore
|
uv.lock export-ignore
|
||||||
|
|
||||||
# Platform adapters are kept in git archives because Claude Code and Codex
|
# Platform adapters - skill-upload path is platform-agnostic
|
||||||
# plugin installs use the same repository archive as their source payload.
|
.agents/ export-ignore
|
||||||
|
.codex-plugin/ export-ignore
|
||||||
.hermes-plugin/ export-ignore
|
.hermes-plugin/ export-ignore
|
||||||
|
|
||||||
# CI workflows - repo-only, not needed at skill runtime
|
# CI workflows - repo-only, not needed at skill runtime
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
name: Bug Report
|
|
||||||
description: Report a bug or unexpected behavior
|
|
||||||
labels: [bug]
|
|
||||||
body:
|
|
||||||
- type: textarea
|
|
||||||
id: summary
|
|
||||||
attributes:
|
|
||||||
label: Summary
|
|
||||||
description: What happened?
|
|
||||||
placeholder: Describe the bug in 1-2 sentences.
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: repro
|
|
||||||
attributes:
|
|
||||||
label: Steps to Reproduce
|
|
||||||
description: How can we reproduce this?
|
|
||||||
placeholder: |
|
|
||||||
1. Run `python3 scripts/last30days.py "topic" --emit compact`
|
|
||||||
2. ...
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: expected
|
|
||||||
attributes:
|
|
||||||
label: Expected Behavior
|
|
||||||
description: What should have happened?
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: traceback
|
|
||||||
attributes:
|
|
||||||
label: Error / Traceback
|
|
||||||
description: Paste the full traceback or error output.
|
|
||||||
render: text
|
|
||||||
- type: dropdown
|
|
||||||
id: install
|
|
||||||
attributes:
|
|
||||||
label: Install Method
|
|
||||||
options:
|
|
||||||
- Claude Code plugin
|
|
||||||
- Gemini CLI extension
|
|
||||||
- Codex plugin
|
|
||||||
- Hermes skill
|
|
||||||
- Manual (git clone)
|
|
||||||
- Other
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: input
|
|
||||||
id: os
|
|
||||||
attributes:
|
|
||||||
label: OS
|
|
||||||
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
name: Feature Request
|
|
||||||
description: Suggest a new feature or improvement
|
|
||||||
labels: [enhancement]
|
|
||||||
body:
|
|
||||||
- type: textarea
|
|
||||||
id: problem
|
|
||||||
attributes:
|
|
||||||
label: Problem
|
|
||||||
description: What problem does this solve?
|
|
||||||
placeholder: When I try to ..., I can't ...
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: solution
|
|
||||||
attributes:
|
|
||||||
label: Proposed Solution
|
|
||||||
description: How should this work?
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: alternatives
|
|
||||||
attributes:
|
|
||||||
label: Alternatives Considered
|
|
||||||
description: Other approaches you thought of (optional).
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
## Summary
|
|
||||||
|
|
||||||
<!-- What does this PR do? 1-3 sentences. -->
|
|
||||||
|
|
||||||
## Changes
|
|
||||||
|
|
||||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
|
||||||
|
|
||||||
-
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
<!-- How did you verify this works? -->
|
|
||||||
|
|
||||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
|
||||||
|
|
||||||
## Related Issues
|
|
||||||
|
|
||||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
|
||||||
@@ -9,10 +9,7 @@ permissions:
|
|||||||
contents: write
|
contents: write
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Build the existing .skill artifact (Claude Code / Codex / Cursor install
|
build-and-release:
|
||||||
# surface). Unchanged from prior versions; just isolated into its own job
|
|
||||||
# so the .mcpb matrix can run in parallel.
|
|
||||||
build-skill:
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
@@ -22,104 +19,13 @@ jobs:
|
|||||||
|
|
||||||
- name: Build .skill artifact
|
- name: Build .skill artifact
|
||||||
run: |
|
run: |
|
||||||
bash skills/last30days/scripts/build-skill.sh
|
bash scripts/build-skill.sh
|
||||||
test -f dist/last30days.skill
|
test -f dist/last30days.skill
|
||||||
|
|
||||||
- name: Upload skill artifact
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: last30days-skill
|
|
||||||
path: dist/last30days.skill
|
|
||||||
|
|
||||||
# Cross-compile the Go MCP server for each Claude Desktop platform and
|
|
||||||
# package each as a .mcpb. printing-press bundle handles the manifest +
|
|
||||||
# zip layout; we only supply the pre-built binary via --skip-build.
|
|
||||||
build-mcpb:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- goos: darwin
|
|
||||||
goarch: arm64
|
|
||||||
platform: darwin/arm64
|
|
||||||
- goos: darwin
|
|
||||||
goarch: amd64
|
|
||||||
platform: darwin/amd64
|
|
||||||
- goos: linux
|
|
||||||
goarch: amd64
|
|
||||||
platform: linux/amd64
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version: stable
|
|
||||||
|
|
||||||
- name: Install printing-press
|
|
||||||
# Pin to a known-good PP release so the bundle command's behavior
|
|
||||||
# is deterministic across our tags. Bump deliberately when adopting
|
|
||||||
# a newer PP version. GOSUMDB=off skips the sumdb 404 some
|
|
||||||
# private-namespaced go install calls hit even when the repo is
|
|
||||||
# public; harmless here because the module path is fully qualified.
|
|
||||||
env:
|
|
||||||
GOPRIVATE: github.com/mvanhorn/*
|
|
||||||
GOSUMDB: "off"
|
|
||||||
run: go install github.com/mvanhorn/cli-printing-press/v4/cmd/printing-press@v4.8.0
|
|
||||||
|
|
||||||
- name: Sync engine into vendored/
|
|
||||||
run: bash mcp/scripts/sync-engine.sh
|
|
||||||
|
|
||||||
- name: Build MCP binary
|
|
||||||
env:
|
|
||||||
GOOS: ${{ matrix.goos }}
|
|
||||||
GOARCH: ${{ matrix.goarch }}
|
|
||||||
CGO_ENABLED: "0"
|
|
||||||
run: |
|
|
||||||
mkdir -p mcp/build
|
|
||||||
go -C mcp build \
|
|
||||||
-ldflags "-X main.Version=${{ github.ref_name }}" \
|
|
||||||
-o build/last30days-pp-mcp \
|
|
||||||
./cmd/last30days-pp-mcp
|
|
||||||
|
|
||||||
- name: Bundle .mcpb
|
|
||||||
# printing-press bundle reads manifest.json from the cli dir and
|
|
||||||
# rewrites the binary into bin/<entry_point> inside the zip. The
|
|
||||||
# --platform tag drives the output filename suffix; the binary
|
|
||||||
# itself is whatever we just cross-compiled.
|
|
||||||
run: |
|
|
||||||
printing-press bundle mcp \
|
|
||||||
--skip-build \
|
|
||||||
--binary mcp/build/last30days-pp-mcp \
|
|
||||||
--platform ${{ matrix.platform }} \
|
|
||||||
--output mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
|
||||||
|
|
||||||
- name: Upload .mcpb artifact
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }}
|
|
||||||
path: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
|
||||||
|
|
||||||
# Gather every platform artifact and attach to one GitHub release.
|
|
||||||
# release-notes generation reads commits since the prior tag.
|
|
||||||
release:
|
|
||||||
needs: [build-skill, build-mcpb]
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Download all artifacts
|
|
||||||
uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
path: dist
|
|
||||||
merge-multiple: true
|
|
||||||
|
|
||||||
- name: Create GitHub release
|
- name: Create GitHub release
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@v2
|
||||||
with:
|
with:
|
||||||
files: |
|
files: dist/last30days.skill
|
||||||
dist/last30days.skill
|
|
||||||
dist/last30days-pp-mcp-*.mcpb
|
|
||||||
generate_release_notes: true
|
generate_release_notes: true
|
||||||
draft: false
|
draft: false
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|||||||
@@ -1,26 +0,0 @@
|
|||||||
name: Validate
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
plugin-contract:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v5
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
run: uv python install 3.12
|
|
||||||
|
|
||||||
- name: Run plugin contract tests
|
|
||||||
run: uv run pytest tests/test_plugin_contract.py tests/test_version_consistency.py
|
|
||||||
@@ -26,14 +26,5 @@ htmlcov/
|
|||||||
# build artifact from scripts/build-skill.sh
|
# build artifact from scripts/build-skill.sh
|
||||||
/dist/
|
/dist/
|
||||||
|
|
||||||
# Go MCP bundle build outputs - source of truth for vendored/ stays under
|
|
||||||
# skills/last30days/scripts/; build/ holds cross-compiled binaries + .mcpb files.
|
|
||||||
# vendored/ lives inside the engine package because //go:embed cannot reach
|
|
||||||
# outside its own package directory; the .gitkeep anchor stays tracked so
|
|
||||||
# the embed pattern always finds a match even before sync-engine runs.
|
|
||||||
/mcp/internal/engine/vendored/*
|
|
||||||
!/mcp/internal/engine/vendored/.gitkeep
|
|
||||||
/mcp/build/
|
|
||||||
|
|
||||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||||
docs/plans/
|
docs/plans/
|
||||||
|
|||||||
@@ -5,114 +5,6 @@ All notable changes to this project will be documented in this file.
|
|||||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
## [Unreleased]
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- Rename "Digg AI 1000" to just "Digg" in user-facing output (footer line, source label, inline-quote suffix, why_relevant, container attribution). Internal references to the upstream Digg AI 1000 product remain in code comments and docstrings.
|
|
||||||
- Bump `POSTS_PER_CLUSTER` from 3 to 5 and the render-side display limit from 2 to 3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous 3/2 caps routinely truncated cluster context (e.g. dropped a Jason Calacanis quote tweet on a `cli-printing-press` run).
|
|
||||||
- Rewrite SKILL.md path resolution. STEP 0 narrows from a global canonical-path enforcement to a Claude-Code-marketplaces-only stale-clone guard. Step 1 SKILL_ROOT resolver walks a single precedence list (Claude plugin cache, then `~/.codex/skills/`, `~/.agents/skills/`, repo checkout, `./.skills/last30days` for `npx skills add`, CWD, Gemini). Adds SKILL.md frontmatter fallback to `render.py::_skill_version` so the badge no longer prints `v?` on installs that don't include `.claude-plugin/plugin.json`.
|
|
||||||
|
|
||||||
- Switch SKILL.md's `--plan` and `--competitors-plan` invocation templates from inline single-quoted JSON to heredoc-written tmpfiles. Apostrophes in resolved context strings ("McDonald's", "people's choice", "developer's") previously closed the outer single-quote and broke shell parsing before the engine started — observed in a Codex run during PR #400 testing. The engine's `parse_plan()` / `parse_competitors_plan()` already supported file paths (via `os.path.isfile()` probe); only the template prose changed. Fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403).
|
|
||||||
|
|
||||||
### Removed
|
|
||||||
|
|
||||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1. Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
|
||||||
- **`skills/last30days/scripts/sync.sh`.** The maintainer dev-deploy script is gone. Every job it did has a better replacement: `npx skills add . -g -y` symlinks the working tree into every detected harness's skill dir (better than sync.sh's copy model — edits propagate live), `hermes skills install mvanhorn/last30days-skill --force` handles Hermes, `clawhub install last30days-official` handles OpenClaw, and the Claude marketplace cache target was a "test against the official install path" hack we shouldn't have been recommending in the first place. The `test_sync_cache_path_uses_skill_version` test was dropped along with it. CLAUDE.md, HERMES_SETUP.md, the PR template, and a render.py docstring were updated to drop references; CHANGELOG and historical docs (release notes, plan files) keep their existing mentions as accurate history.
|
|
||||||
|
|
||||||
## [3.2.0] - 2026-05-09
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
|
|
||||||
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
|
|
||||||
|
|
||||||
## [3.1.1] - 2026-04-24
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
|
|
||||||
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
|
|
||||||
|
|
||||||
## [3.1.0] - 2026-04-22
|
|
||||||
|
|
||||||
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
|
||||||
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
|
|
||||||
|
|
||||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
|
||||||
|
|
||||||
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
|
|
||||||
|
|
||||||
## [3.0.14] - 2026-04-22
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- **Comparison-mode title attribution.** The synthesis title for vs-mode and `--competitors` outputs changes from `What the Community Says (Last 30 Days)` to `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range. Three SKILL.md occurrences updated; pure documentation change.
|
|
||||||
|
|
||||||
## [3.0.13] - 2026-04-22
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- **vs mode runs N full passes in parallel, one per entity.** Architectural revert of the 3-pass → 1-pass latency optimization from an earlier version. `/last30days "OpenAI vs Anthropic vs xAI"` now runs three full `pipeline.run()` calls in parallel via the same fanout `--competitors` uses, producing three `*-raw.md` save files plus a merged comparison output. Each entity gets its own Step 0.55-grade targeting, own primary X handle weight, own subreddit scoping — apples-to-apples depth instead of the one-pool merged retrieval the single-pass path produced. Parallel execution keeps wall clock ≈ single pass.
|
|
||||||
- **`--competitors` is now a SKILL.md-level shortcut for vs-mode with auto-discovery.** The hosting reasoning model (Claude Code, Codex, Hermes, Gemini, any agent with WebSearch) performs discovery and Step 0.55 per entity via its own WebSearch tool, then invokes the engine with a vs-topic and `--competitors-plan` JSON. The engine flag remains for headless/cron use with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys (engine-internal `auto_resolve` stays as fallback).
|
|
||||||
- **LAW 7-style stderr for `--competitors` with no backend** now leads with the hosting-model path (WebSearch + Step 0.55 + `--competitors-plan`) instead of `BRAVE_API_KEY`. API-key framing moved to a secondary "headless" section.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **`--competitors-plan` JSON flag** for per-entity Step 0.55 targeting. Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`. Accepts inline JSON or a file path (matches `--plan`). When present for an entity, skips engine-internal `auto_resolve` and uses the provided values; missing fields fall back to `auto_resolve` (if backend) or planner defaults. Case-insensitive entity matching. The `subrun_kwargs_for` helper is the single source of truth for per-entity kwargs — no closure-default fallthrough from main scope.
|
|
||||||
- **Per-entity save files** when `--save-dir` is set on a vs-mode or `--competitors` run. Each entity's sub-run produces its own `{slug}-raw.md` with a single-row Resolved Entities block — matches historical vs-mode behavior (N passes → N save files).
|
|
||||||
- **`--polymarket-keywords "kw1,kw2"`** to filter Polymarket matches for ambiguous single-token topics (e.g., "Warriors" → `nba,gsw,golden-state` kills Glasgow Warriors rugby and Honor of Kings Rogue Warriors noise).
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **BRAVE/SERPER footer nudge suppressed** when `--plan` or `--competitors-plan` is present. The nudge told Claude Code users to set an API key when they already have WebSearch via the hosting model. Nudge still fires for true headless runs (no `--plan`, no backend) where the advice is correct.
|
|
||||||
- **Override-leak regression testing.** 3.0.12 already fixed the main-topic `--subreddits` / `--x-handle` / `--github-*` from leaking into peer sub-runs via explicit per-entity kwargs scrubbing. This release adds a 4-test regression suite (`test_competitor_subrun_isolation.py`) locking in the invariant.
|
|
||||||
|
|
||||||
## [3.0.12] - 2026-04-22
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **Per-entity Step 0.55 resolution for competitor sub-runs.** In 3.0.11, only the main topic got X handle / subreddit / GitHub resolution; competitor sub-runs ran with planner defaults and produced visibly thinner evidence (Reddit 403 fallbacks, single-word queries). Each competitor sub-run now calls `resolve.auto_resolve()` inside `fanout.run_competitor_fanout` when a web backend is available, mirroring the main topic's pre-flight resolution. Per-entity X handle, subreddit list, GitHub user/repos, and news context are threaded into each sub-run's `pipeline.run()` call. Deep-copied config per sub-run prevents `_auto_resolve_context` cross-leak. Surfaces in a new `## Resolved Entities` output block so the resolution coverage is visible without reading stderr.
|
|
||||||
- **LAW 7 false-positive on internal fan-out sub-runs.** Each competitor sub-run was emitting the `[Planner] No --plan passed... YOU ARE the planner` stderr warning. LAW 7 targets the hosting-reasoning-model path, not engine-internal fan-out. New `internal_subrun=True` keyword on `planner.plan_query` and `pipeline.run` suppresses the warning for sub-runs only; the default path is unchanged.
|
|
||||||
- **Marketplace-stale SKILL.md trap.** Added a STEP 0 canonical-path self-check at the top of SKILL.md. Two of three 2026-04-22 test runs loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (Claude-Code-managed git clone pinned to origin/main, lagging the versioned cache), then ran `--help` against the same stale path, did not see `--competitors`, and fell back to a manual comparison plan. The STEP 0 block forces any reader to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` and re-read from the versioned cache if not.
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- **Default `--competitors` count is now 2 (3-way total: original + 2 peers).** Previously 3. `--competitors=N` still customizes (range 1..6). Matches the feature description's canonical example (`Kanye vs Drake vs Kendrick`).
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **`## Resolved Entities` block** in `render_comparison_multi` output. Shows per-entity X handle, subreddits, GitHub user/repos, and truncated context for every entity in the comparison. Block is omitted entirely when no entity has a resolved payload (mock mode, no backend).
|
|
||||||
|
|
||||||
## [3.0.11] - 2026-04-22
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **`--competitors` flag for auto-discovered comparison fan-out.** Pass `--competitors` on a single-entity topic and the engine discovers 2-6 peer entities via web search, then runs the full pipeline on each in parallel and emits one N-way comparison. `last30days Kanye West --competitors` resolves Drake, Kendrick Lamar, and one more peer. `last30days OpenAI --competitors` resolves Anthropic, xAI, Google Gemini. `--competitors=N` controls count, `--competitors-list="A,B,C"` skips discovery and uses the explicit list. Discovery mirrors the `auto_resolve` pattern (Brave / Exa / Serper / Parallel) with deterministic text extraction - no internal LLM call. Sub-runs inherit the main `--quick`/`--deep`/`--days`, run in a `ThreadPoolExecutor`, and degrade gracefully when at least 2 entities survive. Output reuses the existing 9-axis `## Head-to-Head` scaffold.
|
|
||||||
|
|
||||||
## [3.0.10] - 2026-04-21
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **Commenter handles on evidence lines.** Top-comment rendering now includes the commenter's handle - `u/author` for Reddit, `@handle` for TikTok/YouTube/Instagram/Bluesky/X/Threads. The enrichment adapters already captured `author`; the render layer just was not using it. Evidence lines change from `- Comment (6822 upvotes): Finally, John Apple` to `- u/Cyrisaurus (6822 upvotes): Finally, John Apple`. Person-level citations make synthesis-side inline markdown links per LAW 8 much more natural. Both the compact and full render paths are covered.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
|
||||||
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
|
|
||||||
|
|
||||||
### Behavior fallback
|
|
||||||
|
|
||||||
- When an author is empty, `[deleted]`, or `[removed]`, the render falls back to the legacy `Comment (...)` shape - no `u/` or `@` prefix with an empty handle is ever emitted.
|
|
||||||
|
|
||||||
## [3.0.9] - 2026-04-18 - The Self-Debug Release
|
## [3.0.9] - 2026-04-18 - The Self-Debug Release
|
||||||
|
|
||||||
### Highlights
|
### Highlights
|
||||||
@@ -310,15 +202,15 @@ Intelligent search, fun judge, cross-source cluster merging, single-pass compari
|
|||||||
|
|
||||||
### Highlights
|
### Highlights
|
||||||
|
|
||||||
Auto-save research briefings to the default memory directory as topic-named .md files. Every run now builds a personal research library automatically - no more manual copy-paste.
|
Auto-save research briefings to `~/Documents/Last30Days/` as topic-named .md files. Every run now builds a personal research library automatically - no more manual copy-paste.
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- Auto-save complete research briefings (synthesis, stats, follow-up suggestions) to the default memory directory after every run
|
- Auto-save complete research briefings (synthesis, stats, follow-up suggestions) to `~/Documents/Last30Days/{topic-slug}.md` after every run
|
||||||
- Kebab-case filename generation from topic (e.g., "Claude Code skills" -> `claude-code-skills.md`)
|
- Kebab-case filename generation from topic (e.g., "Claude Code skills" -> `claude-code-skills.md`)
|
||||||
- Duplicate topic handling: appends date suffix instead of overwriting (e.g., `claude-code-skills-2026-03-05.md`)
|
- Duplicate topic handling: appends date suffix instead of overwriting (e.g., `claude-code-skills-2026-03-05.md`)
|
||||||
- Agent mode (`--agent`) also saves research files
|
- Agent mode (`--agent`) also saves research files
|
||||||
- Brief confirmation after save with the saved file path
|
- Brief confirmation after save: "Saved to ~/Documents/Last30Days/{slug}.md"
|
||||||
|
|
||||||
### Credits
|
### Credits
|
||||||
|
|
||||||
|
|||||||
@@ -4,20 +4,20 @@ Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
|||||||
Python scripts with multi-source search aggregation.
|
Python scripts with multi-source search aggregation.
|
||||||
|
|
||||||
## Structure
|
## Structure
|
||||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
- `scripts/last30days.py` — main research engine
|
||||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
- `scripts/lib/` — search, enrichment, rendering modules
|
||||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
- `scripts/lib/vendor/bird-search/` — vendored X search client
|
||||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
- `SKILL.md` — skill definition (deployed to ~/.claude/skills/last30days/)
|
||||||
|
|
||||||
## Commands
|
## Commands
|
||||||
```bash
|
```bash
|
||||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
python3 scripts/last30days.py "test query" --emit=compact # Run research
|
||||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
bash scripts/sync.sh # Deploy to ~/.claude, ~/.agents, ~/.codex
|
||||||
```
|
```
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
- After edits: run `bash scripts/sync.sh` to deploy
|
||||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||||
|
|
||||||
## Beta channel
|
## Beta channel
|
||||||
|
|||||||
@@ -10,20 +10,28 @@ This guide covers installing last30days on Hermes AI Agent.
|
|||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
|
### Option 1: Via sync.sh (Recommended)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
hermes skills install mvanhorn/last30days-skill --force
|
# Clone the repo
|
||||||
|
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||||
|
cd last30days-skill
|
||||||
|
|
||||||
|
# Run the sync script
|
||||||
|
bash scripts/sync.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days/`
|
||||||
|
|
||||||
### Developer / live-edit alternative
|
### Option 2: Manual Copy
|
||||||
|
|
||||||
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
# Create directory
|
||||||
mkdir -p ~/.hermes/skills/research
|
mkdir -p ~/.hermes/skills/research/last30days
|
||||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
|
|
||||||
|
# Copy files
|
||||||
|
cp -r scripts ~/.hermes/skills/research/last30days/
|
||||||
|
cp .hermes-plugin/SKILL.md ~/.hermes/skills/research/last30days/
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
@@ -98,11 +106,13 @@ python3.12 scripts/last30days.py --diagnose
|
|||||||
|
|
||||||
## Updating
|
## Updating
|
||||||
|
|
||||||
```bash
|
To update to the latest version:
|
||||||
hermes skills install mvanhorn/last30days-skill --force
|
|
||||||
```
|
|
||||||
|
|
||||||
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step.
|
```bash
|
||||||
|
cd last30days-skill
|
||||||
|
git pull
|
||||||
|
bash scripts/sync.sh
|
||||||
|
```
|
||||||
|
|
||||||
## Support
|
## Support
|
||||||
|
|
||||||
|
|||||||
@@ -12,20 +12,23 @@
|
|||||||
|
|
||||||
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
||||||
|
|
||||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [SKILL.md](SKILL.md), which is the source of truth for the latest command and setup behavior.
|
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||||
|
|
||||||
**Claude Code (recommended — auto-updates via marketplace):**
|
Claude Code:
|
||||||
```
|
```
|
||||||
/plugin marketplace add mvanhorn/last30days-skill
|
/plugin marketplace add mvanhorn/last30days-skill
|
||||||
```
|
```
|
||||||
|
|
||||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
OpenClaw:
|
||||||
```
|
```
|
||||||
npx skills add mvanhorn/last30days-skill -g
|
clawhub install last30days-official
|
||||||
```
|
```
|
||||||
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
|
|
||||||
|
|
||||||
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
|
Hermes:
|
||||||
|
```
|
||||||
|
# The skill auto-deploys when you run sync.sh
|
||||||
|
# Or manually copy to ~/.hermes/skills/research/last30days/
|
||||||
|
```
|
||||||
|
|
||||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||||
|
|
||||||
@@ -65,7 +68,6 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
|||||||
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
||||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||||
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
|
||||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||||
@@ -94,28 +96,6 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
|||||||
|
|
||||||
## What v3 Changed
|
## What v3 Changed
|
||||||
|
|
||||||
### Shareable HTML briefs
|
|
||||||
|
|
||||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
|
||||||
|
|
||||||
```
|
|
||||||
/last30days OpenClaw --emit=html
|
|
||||||
```
|
|
||||||
|
|
||||||
or just ask in plain language:
|
|
||||||
|
|
||||||
```
|
|
||||||
/last30days OpenClaw, give me a shareable HTML brief
|
|
||||||
/last30days Cursor IDE for slack
|
|
||||||
/last30days Anthropic earnings export as html
|
|
||||||
```
|
|
||||||
|
|
||||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
|
||||||
|
|
||||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
|
||||||
|
|
||||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
|
||||||
|
|
||||||
### Intelligent search: the killer feature
|
### Intelligent search: the killer feature
|
||||||
|
|
||||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||||
@@ -134,10 +114,6 @@ When the same story appears on Reddit, X, and YouTube, v3 merges them into one c
|
|||||||
|
|
||||||
"CLI vs MCP" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides simultaneously. Same depth, 3 minutes.
|
"CLI vs MCP" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides simultaneously. Same depth, 3 minutes.
|
||||||
|
|
||||||
### Auto-discovered competitor comparisons
|
|
||||||
|
|
||||||
`/last30days OpenAI --competitors` tells the hosting reasoning model to discover the top 2 peers via WebSearch (Anthropic, xAI), run Step 0.55 per entity, and invoke the engine with `"OpenAI vs Anthropic vs xAI"` and a per-entity `--competitors-plan` JSON. The engine fans out 3 full pipelines in parallel, saves a `*-raw.md` file per entity, and merges them into a 3-way comparison. Same mechanics power `/last30days "OpenAI vs Anthropic vs xAI"` directly.
|
|
||||||
|
|
||||||
### GitHub person-mode
|
### GitHub person-mode
|
||||||
|
|
||||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. Instead of "who mentioned this name in an issue body," it answers: what are they shipping and where is it landing?
|
When the topic is a person, the engine switches from keyword search to author-scoped queries. Instead of "who mentioned this name in an issue body," it answers: what are they shipping and where is it landing?
|
||||||
@@ -165,62 +141,12 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
|||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
| Surface | Install | Updates |
|
| Surface | Install |
|
||||||
|---------|---------|---------|
|
|---------|---------|
|
||||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + |
|
||||||
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
| **Claude Code** | `/plugin marketplace add mvanhorn/last30days-skill` |
|
||||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
| **OpenClaw** | `clawhub install last30days-official` |
|
||||||
| **Claude Desktop** | [Download the `.mcpb` for your platform](https://github.com/mvanhorn/last30days-skill/releases/latest) and drag into Settings > Extensions | Re-download and drag the new bundle in |
|
| **Gemini CLI** | Clone then `gemini extensions install ./last30days-skill` (see below) |
|
||||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
|
||||||
|
|
||||||
### Claude Code (recommended)
|
|
||||||
|
|
||||||
```
|
|
||||||
/plugin marketplace add mvanhorn/last30days-skill
|
|
||||||
```
|
|
||||||
|
|
||||||
Recommended because the Claude Code marketplace handles updates for you — the plugin cache is versioned and auto-refreshes when a new release publishes. Run `claude plugin update last30days@last30days-skill` to force a check.
|
|
||||||
|
|
||||||
If you'd rather use the agent-skills install path on Claude Code, that's also supported:
|
|
||||||
|
|
||||||
```
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
|
||||||
```
|
|
||||||
|
|
||||||
The native plugin and the `npx skills` install can coexist; Claude Code dedupes the slash command.
|
|
||||||
|
|
||||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
|
||||||
|
|
||||||
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills add mvanhorn/last30days-skill -g
|
|
||||||
```
|
|
||||||
|
|
||||||
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
|
|
||||||
|
|
||||||
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
|
||||||
```
|
|
||||||
|
|
||||||
Update later with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills update last30days -g
|
|
||||||
```
|
|
||||||
|
|
||||||
Or update everything you've installed globally via `npx skills`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills update -g
|
|
||||||
```
|
|
||||||
|
|
||||||
List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
|
||||||
|
|
||||||
### claude.ai (web)
|
### claude.ai (web)
|
||||||
|
|
||||||
@@ -228,25 +154,15 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
|||||||
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
||||||
3. Click the `+` button in the Skills panel and drop the file in
|
3. Click the `+` button in the Skills panel and drop the file in
|
||||||
|
|
||||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
Enable "Code execution and file creation" under Capabilities first - skills won't run without it.
|
||||||
|
|
||||||
### Claude Desktop
|
### Claude Code
|
||||||
|
|
||||||
Claude Desktop installs `/last30days` as an MCP server via a `.mcpb` bundle (a one-click Model Context Protocol package).
|
```
|
||||||
|
/plugin marketplace add mvanhorn/last30days-skill
|
||||||
|
```
|
||||||
|
|
||||||
1. Go to the [latest release](https://github.com/mvanhorn/last30days-skill/releases/latest) and download the `.mcpb` for your platform:
|
Update later with `claude plugin update last30days@last30days-skill`.
|
||||||
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
|
|
||||||
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
|
|
||||||
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
|
|
||||||
2. Open Claude Desktop, go to Settings > Extensions, and drag the file in.
|
|
||||||
3. When prompted, paste API keys for the sources you want to enable. Every field is optional — the engine degrades to web-only mode if you skip them all. Keys are stored in your OS keychain.
|
|
||||||
4. Restart Claude Desktop. Ask Claude to "research Peter Steinberger" or any topic and it will call the `research` tool.
|
|
||||||
|
|
||||||
**Host requirement:** Python 3.12+ on PATH. The bundle ships the engine source but uses your local Python interpreter. Install from [python.org](https://www.python.org/downloads/) on Windows; macOS and most Linux distros ship a compatible version.
|
|
||||||
|
|
||||||
**Keys don't sync with the Code skill.** Claude Desktop and Claude Code maintain separate credential stores by design. If you already configured `~/.config/last30days/.env` for the Code skill, you'll re-enter the same keys here once.
|
|
||||||
|
|
||||||
Windows support is deferred until per-platform manifest entry points are sorted out; track in a follow-up issue.
|
|
||||||
|
|
||||||
### OpenClaw
|
### OpenClaw
|
||||||
|
|
||||||
@@ -254,14 +170,22 @@ Windows support is deferred until per-platform manifest entry points are sorted
|
|||||||
clawhub install last30days-official
|
clawhub install last30days-official
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Gemini CLI
|
||||||
|
|
||||||
|
Gemini CLI v0.9.0 has an upstream installer bug that can fail with `Configuration file not found at /tmp/gemini-extensionXXXXXX/gemini-extension.json` ([upstream issue](https://github.com/google-gemini/gemini-cli/issues/11452)). Workaround:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/mvanhorn/last30days-skill
|
||||||
|
gemini extensions install ./last30days-skill
|
||||||
|
```
|
||||||
|
|
||||||
### Manual (developer)
|
### Manual (developer)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
Or build the claude.ai `.skill` file from source: `bash scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||||
|
|
||||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
name: last30days
|
name: last30days
|
||||||
version: "3.2.3"
|
version: "3.0.1"
|
||||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||||
@@ -45,7 +45,6 @@ metadata:
|
|||||||
- instagram
|
- instagram
|
||||||
- hackernews
|
- hackernews
|
||||||
- polymarket
|
- polymarket
|
||||||
- digg
|
|
||||||
- bluesky
|
- bluesky
|
||||||
- truthsocial
|
- truthsocial
|
||||||
- trends
|
- trends
|
||||||
@@ -60,35 +59,6 @@ metadata:
|
|||||||
- clawhub
|
- clawhub
|
||||||
---
|
---
|
||||||
|
|
||||||
# STEP 0: STALE-CLONE SELF-CHECK — RUN BEFORE READING BELOW
|
|
||||||
|
|
||||||
Before reading anything else in this file, check whether you loaded SKILL.md from the one known stale-clone location: Claude Code's marketplaces directory.
|
|
||||||
|
|
||||||
**Run this check:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
CLAUDE_CACHE_LATEST=$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)
|
|
||||||
# Two cache layouts ship in the wild — nested ({version}/skills/last30days/SKILL.md)
|
|
||||||
# and flat ({version}/SKILL.md). Resolve to whichever shape actually exists.
|
|
||||||
CLAUDE_CACHE_SKILL_MD=""
|
|
||||||
if [ -n "$CLAUDE_CACHE_LATEST" ]; then
|
|
||||||
if [ -f "$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md" ]; then
|
|
||||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md"
|
|
||||||
elif [ -f "$CLAUDE_CACHE_LATEST/SKILL.md" ]; then
|
|
||||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/SKILL.md"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
echo "CLAUDE_CACHE_SKILL_MD=$CLAUDE_CACHE_SKILL_MD"
|
|
||||||
```
|
|
||||||
|
|
||||||
If the SKILL.md path you just Read contains `/.claude/plugins/marketplaces/` AND `$CLAUDE_CACHE_SKILL_MD` is non-empty, STOP and re-read `$CLAUDE_CACHE_SKILL_MD` before proceeding. Otherwise the SKILL.md you have is fine — continue.
|
|
||||||
|
|
||||||
**Why this specific check:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test. STEP 0 defends against that one Claude Code-specific bug.
|
|
||||||
|
|
||||||
**Other install paths are fine:** `~/.codex/skills/`, `~/.agents/skills/`, an `npx skills add` install dir, or a repo checkout are all valid load points - the resolver in Step 1 picks them up. Do NOT abort or hop on those paths.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# SKILL CONTRACT — READ BEFORE ANY TOOL CALL
|
# SKILL CONTRACT — READ BEFORE ANY TOOL CALL
|
||||||
|
|
||||||
You are inside the `/last30days` SKILL. This is a specific research tool with a 1400+ line instruction contract (the rest of this file) that defines EXACTLY how to produce the research output. It is not a generic "last 30 days of X" research prompt. Do NOT treat `/last30days` as a search keyword you can improvise against.
|
You are inside the `/last30days` SKILL. This is a specific research tool with a 1400+ line instruction contract (the rest of this file) that defines EXACTLY how to produce the research output. It is not a generic "last 30 days of X" research prompt. Do NOT treat `/last30days` as a search keyword you can improvise against.
|
||||||
@@ -97,7 +67,7 @@ You are inside the `/last30days` SKILL. This is a specific research tool with a
|
|||||||
|
|
||||||
**How v3.0.7 fixes it:** three structural anchors.
|
**How v3.0.7 fixes it:** three structural anchors.
|
||||||
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
||||||
2. **The SKILL_ROOT resolver** in the engine Bash calls walks a precedence list of known install locations and picks the highest-versioned freshest copy, never `~/.openclaw/` or other stale copies.
|
2. **The pinned SKILL_ROOT resolution** in the engine Bash calls always points to the public plugin cache, never `~/.openclaw/` or other stale copies.
|
||||||
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
||||||
|
|
||||||
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
||||||
@@ -114,13 +84,13 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
|
|||||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||||
```
|
```
|
||||||
|
|
||||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_ROOT/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||||
|
|
||||||
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
||||||
|
|
||||||
**Placement by query type:**
|
**Placement by query type:**
|
||||||
- GENERAL / NEWS / PROMPTING / RECOMMENDATIONS: badge on line 1, blank line 2, `What I learned:` on line 3, then bold-lead-in paragraphs
|
- GENERAL / NEWS / PROMPTING / RECOMMENDATIONS: badge on line 1, blank line 2, `What I learned:` on line 3, then bold-lead-in paragraphs
|
||||||
- COMPARISON: badge on line 1, blank line 2, `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (/Last30Days)` on line 3, then Quick Verdict section
|
- COMPARISON: badge on line 1, blank line 2, `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (Last 30 Days)` on line 3, then Quick Verdict section
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -138,7 +108,7 @@ These LAWs dominate every other rule in this file. If you find yourself about to
|
|||||||
|
|
||||||
**LAW 2 - NO INVENTED TITLE LINE (with COMPARISON exception).** For QUERY_TYPE GENERAL, NEWS, PROMPTING, RECOMMENDATIONS: the first line of your synthesis body (after the badge and one blank line) is the prose label `What I learned:` on its own line. Not `What I learned about {Topic}`, not `{Topic} - Last 30 Days`, not `{Topic}: What People Are Saying`, not `# {Topic}`, not `The headline`, not `Why he is everywhere this month`. Nothing above `What I learned:` except the badge. If you are tempted to write a title or a `##`-prefixed section name, the rule is: the badge IS the title, and section headers are forbidden (see LAW 4).
|
**LAW 2 - NO INVENTED TITLE LINE (with COMPARISON exception).** For QUERY_TYPE GENERAL, NEWS, PROMPTING, RECOMMENDATIONS: the first line of your synthesis body (after the badge and one blank line) is the prose label `What I learned:` on its own line. Not `What I learned about {Topic}`, not `{Topic} - Last 30 Days`, not `{Topic}: What People Are Saying`, not `# {Topic}`, not `The headline`, not `Why he is everywhere this month`. Nothing above `What I learned:` except the badge. If you are tempted to write a title or a `##`-prefixed section name, the rule is: the badge IS the title, and section headers are forbidden (see LAW 4).
|
||||||
|
|
||||||
**COMPARISON exception:** For QUERY_TYPE=COMPARISON (topics containing `vs` or `versus`), the title `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (/Last30Days)` is REQUIRED, not a violation. Comparison queries do NOT use the `What I learned:` prose label at all.
|
**COMPARISON exception:** For QUERY_TYPE=COMPARISON (topics containing `vs` or `versus`), the title `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (Last 30 Days)` is REQUIRED, not a violation. Comparison queries do NOT use the `What I learned:` prose label at all.
|
||||||
|
|
||||||
**Global-preference override:** The skill-authored template for GENERAL / NEWS / PROMPTING / RECOMMENDATIONS queries uses `**bold**` for KEY PATTERNS items and for mid-paragraph lead-ins. Do NOT strip this bold on the grounds of a personal "no bold" memory. The skill's voice contract is the formatting authority here.
|
**Global-preference override:** The skill-authored template for GENERAL / NEWS / PROMPTING / RECOMMENDATIONS queries uses `**bold**` for KEY PATTERNS items and for mid-paragraph lead-ins. Do NOT strip this bold on the grounds of a personal "no bold" memory. The skill's voice contract is the formatting authority here.
|
||||||
|
|
||||||
@@ -186,13 +156,13 @@ The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses
|
|||||||
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
|
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
|
||||||
```
|
```
|
||||||
|
|
||||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan "$QUERY_PLAN_FILE"` (the path to a tmpfile you wrote via heredoc — see Step 1 for the pattern; never inline `--plan '$JSON'`, apostrophes in search/ranking strings break shell parsing).
|
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan '$JSON'`.
|
||||||
|
|
||||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan "$QUERY_PLAN_FILE"` (or any path the engine can read). A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan '$JSON'`. A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||||
|
|
||||||
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
|
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
|
||||||
|
|
||||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan "$QUERY_PLAN_FILE"` (or another path the engine can read)? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan, then write it to a tmpfile per the Step 1 pattern. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan '$JSON'`? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||||
|
|
||||||
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
|
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
|
||||||
|
|
||||||
@@ -243,9 +213,9 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# last30days v3.2.3: Research Any Topic from the Last 30 Days
|
# last30days v3.0.1: Research Any Topic from the Last 30 Days
|
||||||
|
|
||||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||||
|
|
||||||
Research ANY topic across Reddit, X, YouTube, and other sources. Surface what people are actually discussing, recommending, betting on, and debating right now.
|
Research ANY topic across Reddit, X, YouTube, and other sources. Surface what people are actually discussing, recommending, betting on, and debating right now.
|
||||||
|
|
||||||
@@ -265,14 +235,8 @@ if [ -z "${LAST30DAYS_PYTHON:-}" ]; then
|
|||||||
echo "ERROR: last30days v3 requires Python 3.12+. Install python3.12 or python3.13 and rerun." >&2
|
echo "ERROR: last30days v3 requires Python 3.12+. Install python3.12 or python3.13 and rerun." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
LAST30DAYS_MEMORY_DIR="${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Set `LAST30DAYS_MEMORY_DIR` before invoking the skill to choose where raw research files are saved. If it is not set, the skill defaults to `~/Documents/Last30Days`.
|
|
||||||
|
|
||||||
## Step 0: First-Run Setup Wizard
|
## Step 0: First-Run Setup Wizard
|
||||||
|
|
||||||
Before proceeding to Step 1, handle first-run setup.
|
Before proceeding to Step 1, handle first-run setup.
|
||||||
@@ -327,8 +291,7 @@ Common patterns:
|
|||||||
|
|
||||||
- Always active: Reddit, Hacker News, Polymarket
|
- Always active: Reddit, Hacker News, Polymarket
|
||||||
- If gh CLI is installed (check `which gh`): add GitHub
|
- If gh CLI is installed (check `which gh`): add GitHub
|
||||||
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg
|
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set: add X
|
||||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
|
||||||
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
||||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
||||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram
|
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram
|
||||||
@@ -559,7 +522,7 @@ If `--agent` appears in ARGUMENTS (e.g., `/last30days plaud granola --agent`):
|
|||||||
5. **Skip** the follow-up invitation ("I'm now an expert on X...")
|
5. **Skip** the follow-up invitation ("I'm now an expert on X...")
|
||||||
6. **Output** the complete research report and stop - do not wait for further input
|
6. **Output** the complete research report and stop - do not wait for further input
|
||||||
|
|
||||||
Agent mode saves raw research data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) automatically via `--save-dir` (handled by the script, no extra tool calls).
|
Agent mode saves raw research data to `~/Documents/Last30Days/` automatically via `--save-dir` (handled by the script, no extra tool calls).
|
||||||
|
|
||||||
Agent mode report format:
|
Agent mode report format:
|
||||||
|
|
||||||
@@ -581,99 +544,28 @@ Generated: {date} | Sources: Reddit, X, Bluesky, YouTube, TikTok, HN, Polymarket
|
|||||||
|
|
||||||
## If QUERY_TYPE = COMPARISON
|
## If QUERY_TYPE = COMPARISON
|
||||||
|
|
||||||
When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipeline.run()` calls in parallel — one per entity — each with its own Step 0.55-grade targeting. This restored the old N-pass architecture (reverted the one-pass latency optimization that removed per-entity depth); parallel execution keeps wall clock ≈ a single pass.
|
When the user asks "X vs Y", run ONE research pass with a comparison-optimized plan that covers both entities AND their rivalry. This replaces the old 3-pass approach (which took 13+ minutes and produced tangential content).
|
||||||
|
|
||||||
**MANDATORY per-entity resolution.** For each entity, resolve the full Step 0.55 stack (X handle, subreddits, GitHub user/repos, news context). Then assemble a `--competitors-plan` JSON mapping each entity to its targeting, and invoke the engine ONCE with the vs-topic string.
|
**IMPORTANT: Include BOTH X handles (`--x-handle={TOPIC_A_HANDLE} --x-related={TOPIC_B_HANDLE},{COMPANY_HANDLES},{COMMENTATOR_HANDLES}`), `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||||
|
|
||||||
**Output shape per run:**
|
**Single pass with entity-aware subqueries:**
|
||||||
- Main topic saves to `{main-slug}-raw.md`.
|
|
||||||
- Each peer saves to `{peer-slug}-raw.md`.
|
|
||||||
- Stdout shows a merged comparison with the `## Head-to-Head` scaffold + per-entity Resolved Entities block.
|
|
||||||
|
|
||||||
**Invocation:**
|
|
||||||
```bash
|
```bash
|
||||||
# Comparison mode skips Step 1, so resolve SKILL_ROOT inline here (same precedence
|
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B}" --emit=compact --save-dir=~/Documents/Last30Days --save-suffix=v3 --plan 'COMPARISON_PLAN_JSON' --x-handle={TOPIC_A_HANDLE} --x-related={TOPIC_B_HANDLE},{COMPANY_A_HANDLE},{COMPANY_B_HANDLE},{COMMENTATOR_HANDLES} --subreddits={RESOLVED_SUBREDDITS} --tiktok-hashtags={RESOLVED_HASHTAGS} --tiktok-creators={RESOLVED_TIKTOK_CREATORS} --ig-creators={RESOLVED_IG_CREATORS}
|
||||||
# walk as Step 1 — keep the two in sync if you edit either).
|
|
||||||
SKILL_ROOT=""
|
|
||||||
CLAUDE_PLUGIN_ROOT="$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)"
|
|
||||||
if [ -n "$CLAUDE_PLUGIN_ROOT" ]; then
|
|
||||||
if [ -f "$CLAUDE_PLUGIN_ROOT/skills/last30days/scripts/last30days.py" ]; then
|
|
||||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT/skills/last30days"
|
|
||||||
elif [ -f "$CLAUDE_PLUGIN_ROOT/scripts/last30days.py" ]; then
|
|
||||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
|
||||||
for dir in \
|
|
||||||
"$HOME/.codex/skills/last30days" \
|
|
||||||
"$HOME/.agents/skills/last30days" \
|
|
||||||
"./skills/last30days" \
|
|
||||||
"./.skills/last30days" \
|
|
||||||
"." \
|
|
||||||
"${GEMINI_EXTENSION_DIR:-}"; do
|
|
||||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Write the per-entity plan to a tmpfile and pass the path to the engine.
|
|
||||||
# The engine's parse_competitors_plan() reads file paths transparently. This
|
|
||||||
# avoids the inline-single-quoted-JSON apostrophe trap (resolved context
|
|
||||||
# strings like "people's choice" or "McDonald's" otherwise close the outer
|
|
||||||
# single-quote and break shell parsing before the engine is even invoked).
|
|
||||||
# Trailing XXXXXX (no .json suffix) so BSD/macOS mktemp works the same as
|
|
||||||
# GNU; BSD only substitutes X's at the end of the template.
|
|
||||||
COMPETITORS_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-competitors.XXXXXX")
|
|
||||||
trap 'rm -f "$COMPETITORS_PLAN_FILE"' EXIT
|
|
||||||
cat > "$COMPETITORS_PLAN_FILE" <<'PLAN_EOF'
|
|
||||||
{
|
|
||||||
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
|
||||||
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
|
||||||
}
|
|
||||||
PLAN_EOF
|
|
||||||
|
|
||||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
|
||||||
--emit=compact \
|
|
||||||
--save-dir="${LAST30DAYS_MEMORY_DIR}" \
|
|
||||||
--save-suffix=v3 \
|
|
||||||
--x-handle={TOPIC_A_HANDLE} \
|
|
||||||
--subreddits={TOPIC_A_SUBS} \
|
|
||||||
--competitors-plan "$COMPETITORS_PLAN_FILE"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**The quoted heredoc marker `'PLAN_EOF'` is load-bearing** — quoting suppresses shell interpolation so apostrophes, `$`, backticks, etc. pass through verbatim. If you ever switch to an unquoted `<<PLAN_EOF`, every variable reference and apostrophe inside the JSON becomes a parse hazard.
|
**The `--plan` JSON for comparisons should include 3-4 subqueries:**
|
||||||
|
1. **Head-to-head:** `"{TOPIC_A} vs {TOPIC_B}"` - catches rivalry content, direct comparisons
|
||||||
|
2. **Entity A news:** `"{TOPIC_A} news {MONTH} {YEAR}"` - catches entity-specific developments
|
||||||
|
3. **Entity B news:** `"{TOPIC_B} news {MONTH} {YEAR}"` - catches entity-specific developments
|
||||||
|
4. (Optional) **Domain context:** `"{COMPANY_A} {COMPANY_B} {DOMAIN} news"` - catches industry context (e.g., "OpenAI Anthropic AI news")
|
||||||
|
|
||||||
Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive).
|
ALL subqueries include ALL sources. The fusion engine handles deduplication across subqueries. **At least one subquery MUST include YouTube-specific search terms** (e.g., "{PERSON} interview 2026", "{PRODUCT_A} vs {PRODUCT_B} review") to ensure YouTube content is found. Without YouTube-specific terms, the engine may only find 0-1 videos for comparison queries.
|
||||||
|
|
||||||
**Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan.
|
Then do WebSearch for: `{TOPIC_A} vs {TOPIC_B} comparison {YEAR}` and `{TOPIC_A} vs {TOPIC_B} which is better` and `{COMPANY_A} vs {COMPANY_B} news {MONTH} {YEAR}`.
|
||||||
|
|
||||||
**Then do WebSearch supplements** for: `{TOPIC_A} vs {TOPIC_B} comparison {YEAR}` and `{TOPIC_A} vs {TOPIC_B} which is better` — these catch rivalry articles that per-entity passes might not surface.
|
|
||||||
|
|
||||||
**Skip the normal Step 1 below** - go directly to the comparison synthesis format (see "If QUERY_TYPE = COMPARISON" in the synthesis section).
|
**Skip the normal Step 1 below** - go directly to the comparison synthesis format (see "If QUERY_TYPE = COMPARISON" in the synthesis section).
|
||||||
|
|
||||||
**COMPARISON TABLE SCAFFOLD (engine-emitted, pass through verbatim):** For comparison topics, the engine's compact output includes a `## Head-to-Head` block with an empty markdown table (columns = entities, rows = axes like "What it is", "Community sentiment", "Trajectory"). Your synthesis MUST include this block verbatim with filled cells, positioned between the narrative and the emoji-tree footer. Keep each cell to 5-15 words. Use ' - ' (hyphen with spaces) not em-dashes inside cells.
|
**COMPARISON TABLE SCAFFOLD (engine-emitted, pass through verbatim):** For comparison topics, the engine's compact output includes a `## Head-to-Head Comparison` block with an empty markdown table (columns = entities, rows = axes like "Core pitch", "Who it's for", "Community stance", "Trajectory") plus a "Choose X if / Choose Y if" prose block. Your synthesis MUST include this block verbatim with filled cells, positioned between the narrative and the emoji-tree footer. Keep each cell to 5-15 words. Use ' - ' (hyphen with spaces) not em-dashes inside cells. The block is the canonical comparison output shape - do not invent your own table structure.
|
||||||
|
|
||||||
### Competitor mode (`--competitors`)
|
|
||||||
|
|
||||||
`--competitors` is a SKILL.md-level shortcut for vs-mode with auto-discovery. The engine flag itself just signals intent; YOU (the hosting reasoning model) do the discovery and Step 0.55 via your own WebSearch tool, then invoke the vs-topic path above.
|
|
||||||
|
|
||||||
**The four-step protocol:**
|
|
||||||
1. **Discover peers** via WebSearch: `"{topic} competitors"` / `"{topic} alternatives"`. Pick N=2 by default (match the flag's default), N=argument value if the user passed `--competitors=N`.
|
|
||||||
2. **Run Step 0.55 for the main topic AND each peer** — same protocol you use for a single-entity topic, just N times. X handle, subreddits, GitHub, news context, per entity.
|
|
||||||
3. **Build the vs-topic string**: `"{main} vs {peer1} vs {peer2}"`.
|
|
||||||
4. **Invoke the engine** with the vs-topic, `--competitors-plan` JSON covering both peers (and the main topic if you want to override the outer flags), and the outer `--x-handle`/`--subreddits`/`--github-*` for the main topic.
|
|
||||||
|
|
||||||
**Flag surface (engine):**
|
|
||||||
- `--competitors` (bare) - signals the hosting model to discover 2 peers (3-way total).
|
|
||||||
- `--competitors=N` - N peers (1..6; out-of-range clamps with stderr warning).
|
|
||||||
- `--competitors-list="A,B,C"` - minimum escape hatch; names only, no per-entity targeting. Peer sub-runs fall back to planner defaults (visibly thinner data).
|
|
||||||
- `--competitors-plan '{entity: {x_handle, subreddits, github_user, github_repos, context}}'` - full per-entity targeting; implies vs-mode; preferred.
|
|
||||||
- `--polymarket-keywords "kw1,kw2"` - disambiguate Polymarket for ambiguous single-token topics ("Warriors" → `nba,gsw,golden-state`).
|
|
||||||
|
|
||||||
**Why --competitors-plan over --competitors-list:** without per-entity handles/subs, peer sub-runs run with deterministic single-word planner queries and produce visibly thinner evidence than the main topic. The Resolved Entities block in stdout makes the gap visible — dashes for a peer = you skipped its Step 0.55.
|
|
||||||
|
|
||||||
**Engine-internal auto-resolve (headless fallback):** if the engine detects BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY, it runs its own per-entity `resolve.auto_resolve()` before each sub-run. The hosting-model path does NOT need those keys — you are the WebSearch. The engine's auto-resolve is the cron/CI fallback for when no reasoning model is driving.
|
|
||||||
|
|
||||||
**Output:** one `{slug}-raw.md` per entity in `--save-dir` plus the merged comparison on stdout. Synthesis contract identical to the vs-mode protocol above.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -698,43 +590,6 @@ The first search finds subreddits. The second gives you current events context (
|
|||||||
|
|
||||||
Extract 3-5 subreddit names from the results. Store as `RESOLVED_SUBREDDITS` (comma-separated, no r/ prefix).
|
Extract 3-5 subreddit names from the results. Store as `RESOLVED_SUBREDDITS` (comma-separated, no r/ prefix).
|
||||||
|
|
||||||
**2a. Category-peer expansion (MANDATORY for product topics).** If the topic is a product in a recognizable category (AI image generation, AI video generation, AI coding agents, AI music, AI chat models, SaaS screen recording, prediction markets, etc.), the brand-specific subreddits that WebSearch returned are INSUFFICIENT. Add 2-3 peer subreddits from the category. Peer subs are where cross-product technique discussion actually lives. Missing them is the 2026-04-22 `GPT Image 2` failure mode: the model resolved `r/OpenAI, r/ChatGPT, r/singularity, r/ChatGPTpromptengineering` (all OpenAI-brand) and missed `r/StableDiffusion, r/midjourney, r/dalle2, r/aiArt` where prompting techniques are actually shared. The user had to manually prompt "check image generation reddits too" to get a usable run.
|
|
||||||
|
|
||||||
Canonical category peers (single source of truth; `scripts/lib/categories.py` mirrors this for the `--auto-resolve` engine path):
|
|
||||||
|
|
||||||
| Category | Trigger keywords | Peer subs (priority order) |
|
|
||||||
|----------|------------------|---------------------------|
|
|
||||||
| `ai_image_generation` | image generation, text to image, GPT Image, Nano Banana, Midjourney, Stable Diffusion, DALL-E, Flux.1, Imagen, Seedance, Ideogram, Recraft | `StableDiffusion, midjourney, dalle2, aiArt, PromptEngineering, MediaSynthesis` |
|
|
||||||
| `ai_video_generation` | video generation, text to video, Sora, Veo 3, Runway Gen, Kling, Pika Labs, Luma Dream Machine, Hailuo | `aivideo, StableDiffusion, runwayml, singularity, MediaSynthesis` |
|
|
||||||
| `ai_music_generation` | music generation, ai music, Suno, Udio, Riffusion, Stable Audio | `SunoAI, udiomusic, aimusic, artificial` |
|
|
||||||
| `ai_coding_agent` | Claude Code, Cursor IDE, GitHub Copilot, Windsurf, Aider, Cline, OpenClaw, Hermes Agent, Continue.dev, Codeium, Devin | `ChatGPTCoding, LocalLLaMA, singularity, PromptEngineering` |
|
|
||||||
| `ai_agent_framework` | agent framework, LangChain, LangGraph, CrewAI, AutoGen, LlamaIndex, DSPy, smolagents | `LangChain, LocalLLaMA, AI_Agents, MachineLearning` |
|
|
||||||
| `ai_chat_model` | GPT-5/4, Claude Opus/Sonnet/Haiku, Gemini Pro/Flash, Llama 3/4, DeepSeek, Qwen, Mistral Large, Grok | `LocalLLaMA, ChatGPT, ClaudeAI, singularity, artificial` |
|
|
||||||
| `saas_screen_recording` | screen recording, screen recorder, Loom video, Tella screen, Vidyard | `SaaS, screenrecording, productivity, Entrepreneur` |
|
|
||||||
| `saas_productivity` | Notion app, Obsidian, Linear app, Asana, ClickUp, productivity app | `productivity, SaaS, ObsidianMD, Notion` |
|
|
||||||
| `prediction_markets` | Polymarket, Kalshi, prediction market, event contracts, Manifold Markets | `Polymarket, Kalshi, predictionmarkets` |
|
|
||||||
| `crypto_defi` | DeFi protocol, yield farming, liquidity pool, stablecoin, layer 2, L2 rollup | `defi, ethfinance, CryptoCurrency, ethereum` |
|
|
||||||
|
|
||||||
**Merging rule.** Start with WebSearch-returned subs. Append 2-3 category peers in the priority order shown. Dedupe case-insensitively (don't list `midjourney` twice if WebSearch already returned it). Cap total at 10: if adding all peers would exceed the cap, keep every WebSearch-returned sub (they are the freshest signal) and drop peers from the end of the priority list.
|
|
||||||
|
|
||||||
**Extrapolation.** If the topic is a product in a category NOT listed in the table (new AI tool, niche SaaS), use the same spirit: pick the 2-3 most active cross-product communities where technique discussion happens. A new image-gen tool still gets `r/StableDiffusion, r/midjourney, r/aiArt`. A new code editor still gets `r/ChatGPTCoding, r/LocalLLaMA`.
|
|
||||||
|
|
||||||
**Worked example — the failing query.** Topic: `Prompting GPT Image 2`.
|
|
||||||
|
|
||||||
Before (the 2026-04-22 failure mode):
|
|
||||||
```
|
|
||||||
Resolved:
|
|
||||||
- Reddit: r/OpenAI, r/ChatGPT, r/singularity, r/ChatGPTpromptengineering, r/artificial
|
|
||||||
```
|
|
||||||
|
|
||||||
After (with category-peer expansion):
|
|
||||||
```
|
|
||||||
Resolved:
|
|
||||||
- Reddit: r/OpenAI, r/ChatGPT, r/singularity, r/ChatGPTpromptengineering, r/StableDiffusion, r/midjourney, r/dalle2, r/aiArt (+ ai_image_generation peers)
|
|
||||||
```
|
|
||||||
|
|
||||||
The parenthetical `(+ ai_image_generation peers)` is the observable contract of the new Resolved block format. See Step 0.55 self-check below.
|
|
||||||
|
|
||||||
**3. TikTok hashtags + creators** - **INFER these from your topic knowledge. Do NOT WebSearch for "{PERSON} TikTok account" - most people/CEOs don't have TikTok, and the search is wasted.**
|
**3. TikTok hashtags + creators** - **INFER these from your topic knowledge. Do NOT WebSearch for "{PERSON} TikTok account" - most people/CEOs don't have TikTok, and the search is wasted.**
|
||||||
|
|
||||||
- **Hashtags:** Infer 2-3 from the topic name + category. Examples: "Kanye West" → `kanyewest,ye,bully`. "Claude Code" → `claudecode,aiagent,aicoding`. "Sam Altman" → `samaltman,openai,chatgpt`.
|
- **Hashtags:** Infer 2-3 from the topic name + category. Examples: "Kanye West" → `kanyewest,ye,bully`. "Claude Code" → `claudecode,aiagent,aicoding`. "Sam Altman" → `samaltman,openai,chatgpt`.
|
||||||
@@ -797,19 +652,17 @@ Passing the resolved block visibly (per-entity, all 4 types each) is the observa
|
|||||||
|
|
||||||
**If you can't infer targeting for a platform, skip that flag -- the Python engine will fall back to keyword search.**
|
**If you can't infer targeting for a platform, skip that flag -- the Python engine will fall back to keyword search.**
|
||||||
|
|
||||||
**Step 0.55 self-check: category-peer coverage.** Before emitting the Resolved block, re-read your resolved subreddit list. Does the topic match any category in the Section 2a table (or fit the spirit of one — AI image gen, AI coding, AI music, etc.)? If YES: does your list include AT LEAST 2 peer subs from that category? If NO, widen the list NOW — do not run the engine yet. The observable contract is the `(+ {category_id} peers)` annotation on the Reddit line in the Resolved block. Its absence on a product-in-a-known-category topic is a Step 0.55 regression — the named 2026-04-22 failure mode. Person topics, music artists, news stories, and topics outside any category are exempt; omit the annotation.
|
|
||||||
|
|
||||||
**After resolving all handles and communities, display what you found before moving on.** This shows the user that intelligent pre-research happened:
|
**After resolving all handles and communities, display what you found before moving on.** This shows the user that intelligent pre-research happened:
|
||||||
|
|
||||||
```
|
```
|
||||||
Resolved:
|
Resolved:
|
||||||
- X: @{HANDLE} (+ @{COMPANY}, @{COMMENTATOR})
|
- X: @{HANDLE} (+ @{COMPANY}, @{COMMENTATOR})
|
||||||
- Reddit: r/{sub1}, r/{sub2}, r/{sub3}, r/{peer1}, r/{peer2} (+ {category_id} peers)
|
- Reddit: r/{sub1}, r/{sub2}, r/{sub3}
|
||||||
- TikTok: #{hashtag1}, #{hashtag2}
|
- TikTok: #{hashtag1}, #{hashtag2}
|
||||||
- YouTube: {query1}, {query2}
|
- YouTube: {query1}, {query2}
|
||||||
```
|
```
|
||||||
|
|
||||||
Only show lines for platforms where something was resolved. Skip empty lines. On the Reddit line, the trailing `(+ {category_id} peers)` annotation appears when Step 0.55 Section 2a added category-peer subs. Omit the annotation when the topic had no matching category. This display replaces the old "Parsed intent" block with something more useful.
|
Only show lines for platforms where something was resolved. Skip empty lines. This display replaces the old "Parsed intent" block with something more useful.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -873,7 +726,7 @@ Only show lines for platforms where something was resolved. Skip empty lines. On
|
|||||||
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
||||||
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
||||||
|
|
||||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key), digg (Digg clusters - only if `digg-pp-cli` is on PATH)
|
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key)
|
||||||
|
|
||||||
**Intent → freshness_mode mapping:**
|
**Intent → freshness_mode mapping:**
|
||||||
- breaking_news, prediction → `strict_recent`
|
- breaking_news, prediction → `strict_recent`
|
||||||
@@ -916,65 +769,34 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
|||||||
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Resolve SKILL_ROOT by walking a precedence list of known install locations.
|
# PIN SKILL_ROOT to the public plugin cache (highest-version dir wins on upgrade).
|
||||||
# Claude Code plugin cache wins when present (highest version dir picked on upgrade),
|
# DO NOT write your own path-discovery loop. The 2026-04-18 Peter Steinberger run 1
|
||||||
# then common per-harness skill dirs, then a repo checkout.
|
# regression was caused by a custom discovery loop landing on ~/.openclaw/skills/last30days/
|
||||||
SKILL_ROOT=""
|
# (a stale copy from a private-repo sync pattern). That path contains a pre-plan-007
|
||||||
|
# engine and produces non-canonical output. This pinned resolution ignores every stale
|
||||||
|
# copy (~/.openclaw/, ~/.agents/, ~/.codex/) and picks the plugin cache exclusively.
|
||||||
|
SKILL_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
|
||||||
|
SKILL_ROOT="${SKILL_ROOT%/}"
|
||||||
|
|
||||||
# 1. Claude Code plugin cache (versioned, sort -V picks freshest). Two cache layouts ship in the wild:
|
# Fallback for repo checkout / Gemini / Codex hosts where the plugin cache does not exist.
|
||||||
# nested ({cache}/{version}/skills/last30days/scripts/...) and flat ({cache}/{version}/scripts/...).
|
# Only runs if the public plugin cache is missing entirely.
|
||||||
# `find` (not `ls + glob`) because zsh errors on globs that match nothing, leaking
|
|
||||||
# noisy "no matches found" stderr in Codex/zsh sessions even with 2>/dev/null.
|
|
||||||
CLAUDE_PLUGIN_ROOT="$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)"
|
|
||||||
if [ -n "$CLAUDE_PLUGIN_ROOT" ]; then
|
|
||||||
if [ -f "$CLAUDE_PLUGIN_ROOT/skills/last30days/scripts/last30days.py" ]; then
|
|
||||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT/skills/last30days"
|
|
||||||
elif [ -f "$CLAUDE_PLUGIN_ROOT/scripts/last30days.py" ]; then
|
|
||||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 2. Common per-harness skill dirs and repo checkout (npx skills, Codex, Agents, Gemini, etc).
|
|
||||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||||
for dir in \
|
for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
|
||||||
"$HOME/.codex/skills/last30days" \
|
|
||||||
"$HOME/.agents/skills/last30days" \
|
|
||||||
"./skills/last30days" \
|
|
||||||
"./.skills/last30days" \
|
|
||||||
"." \
|
|
||||||
"${GEMINI_EXTENSION_DIR:-}"; do
|
|
||||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||||
echo "ERROR: Could not find scripts/last30days.py in any known install location" >&2
|
echo "ERROR: Could not find scripts/last30days.py in public plugin cache or repo checkout" >&2
|
||||||
echo "Searched: ~/.claude/plugins/cache/, ~/.codex/skills/, ~/.agents/skills/, ./skills/last30days, ./.skills/last30days, ." >&2
|
echo "Expected: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/scripts/last30days.py" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir=~/Documents/Last30Days --save-suffix=v3
|
||||||
```
|
```
|
||||||
|
|
||||||
**If you ran Steps 0.55 and 0.75 (agent planning), pass the plan via a tmpfile and add the targeting flags:**
|
**If you ran Steps 0.55 and 0.75 (agent planning), add these flags:**
|
||||||
|
- `--plan 'QUERY_PLAN_JSON'` (replace with actual JSON from Step 0.75)
|
||||||
```bash
|
|
||||||
# Write QUERY_PLAN_JSON to a tmpfile before the engine invocation above.
|
|
||||||
# parse_plan() reads file paths transparently; this avoids inline-JSON
|
|
||||||
# shell-quoting hazards (apostrophes in search_query / ranking_query
|
|
||||||
# strings break single-quoted command-line JSON). Trailing XXXXXX (no
|
|
||||||
# .json suffix) for BSD/macOS portability — BSD mktemp only substitutes
|
|
||||||
# X's at the end of the template.
|
|
||||||
QUERY_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-plan.XXXXXX")
|
|
||||||
trap 'rm -f "$QUERY_PLAN_FILE"' EXIT
|
|
||||||
cat > "$QUERY_PLAN_FILE" <<'PLAN_EOF'
|
|
||||||
{QUERY_PLAN_JSON_FROM_STEP_0.75}
|
|
||||||
PLAN_EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
Then add to the engine command:
|
|
||||||
|
|
||||||
- `--plan "$QUERY_PLAN_FILE"` (path to the file you just wrote)
|
|
||||||
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
|
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
|
||||||
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
|
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
|
||||||
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
|
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
|
||||||
@@ -1064,7 +886,7 @@ For ALL query types:
|
|||||||
|
|
||||||
## Step 2.5: Append WebSearch Results to Saved Raw File
|
## Step 2.5: Append WebSearch Results to Saved Raw File
|
||||||
|
|
||||||
**MANDATORY - do not skip this step.** Every post-engine WebSearch supplement you ran in Step 2 MUST be appended to the saved raw file under `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). Skipping this step is a common Opus 4.7 failure mode: the saved file ends at `## Source Coverage` with no appendix, future sessions cannot see what blog/tutorial/news sources informed the synthesis, and the user cannot trace where specific claims came from.
|
**MANDATORY - do not skip this step.** Every post-engine WebSearch supplement you ran in Step 2 MUST be appended to the saved raw file under `~/Documents/Last30Days/`. Skipping this step is a common Opus 4.7 failure mode: the saved file ends at `## Source Coverage` with no appendix, future sessions cannot see what blog/tutorial/news sources informed the synthesis, and the user cannot trace where specific claims came from.
|
||||||
|
|
||||||
**LAW 1 OVERRIDE (read before synthesizing):** the WebSearch tool description declares a "MANDATORY Sources section" in its own contract. That instruction applies to generic WebSearch usage. Inside `/last30days` it is SUPERSEDED. The `## WebSearch Supplemental Results` appendix in the SAVED RAW FILE replaces the visible Sources section. Never emit a visible `Sources:` bullet list to the user. Your user-facing response ends at the invitation block. The emoji-tree footer's `🌐 Web:` line is the only visible citation. If you feel the pull to write a trailing `Sources:` section, you are about to violate LAW 1 — go back and delete it.
|
**LAW 1 OVERRIDE (read before synthesizing):** the WebSearch tool description declares a "MANDATORY Sources section" in its own contract. That instruction applies to generic WebSearch usage. Inside `/last30days` it is SUPERSEDED. The `## WebSearch Supplemental Results` appendix in the SAVED RAW FILE replaces the visible Sources section. Never emit a visible `Sources:` bullet list to the user. Your user-facing response ends at the invitation block. The emoji-tree footer's `🌐 Web:` line is the only visible citation. If you feel the pull to write a trailing `Sources:` section, you are about to violate LAW 1 — go back and delete it.
|
||||||
|
|
||||||
@@ -1285,7 +1107,7 @@ Voice contract LAWs 1, 3, 5 apply to comparisons unchanged (no `Sources:` block,
|
|||||||
```
|
```
|
||||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||||
|
|
||||||
# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (/Last30Days)
|
# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (Last 30 Days)
|
||||||
|
|
||||||
## Quick Verdict
|
## Quick Verdict
|
||||||
|
|
||||||
@@ -1361,7 +1183,7 @@ I've compared {TOPIC_A} vs {TOPIC_B} [vs ...] using the latest community data. S
|
|||||||
- Fabricate a `## Notable Stats` block (the engine footer IS the stats block, LAW 5)
|
- Fabricate a `## Notable Stats` block (the engine footer IS the stats block, LAW 5)
|
||||||
- Produce section headers outside the six listed above (`## Quick Verdict`, `## {Entity}` per entity, `## Head-to-Head`, `## The Bottom Line`, `## The emerging stack` are the only allowed `##` headers per LAW 4 comparison exception)
|
- Produce section headers outside the six listed above (`## Quick Verdict`, `## {Entity}` per entity, `## Head-to-Head`, `## The Bottom Line`, `## The emerging stack` are the only allowed `##` headers per LAW 4 comparison exception)
|
||||||
|
|
||||||
**Reference exemplar:** `$LAST30DAYS_MEMORY_DIR/openclaw-vs-hermes-vs-paperclip-LAUNCH-VIDEO-april9-exemplar.md` preserves the April 9 canonical output with full structural analysis. Match this shape section-for-section.
|
**Reference exemplar:** `~/Documents/Last30Days/openclaw-vs-hermes-vs-paperclip-LAUNCH-VIDEO-april9-exemplar.md` preserves the April 9 canonical output with full structural analysis. Match this shape section-for-section.
|
||||||
|
|
||||||
### For all QUERY_TYPEs
|
### For all QUERY_TYPEs
|
||||||
|
|
||||||
@@ -1483,7 +1305,7 @@ Options:
|
|||||||
|
|
||||||
**THEN - Engine footer pass-through (right before invitation):**
|
**THEN - Engine footer pass-through (right before invitation):**
|
||||||
|
|
||||||
**The research output ENDS with a deterministic footer block bracketed by `---` lines, starting with `✅ All agents reported back!` and ending with `📎 Raw results saved to {resolved LAST30DAYS_MEMORY_DIR}/<slug>-raw.md`. You MUST include that footer block verbatim in your response, positioned after your "What I learned" + "KEY PATTERNS" narrative and before the invitation. Do not recompute the stats. Do not reformat the tree. Do not paraphrase. Do not skip it. Do not add your own source lines. Copy the exact bytes.**
|
**The research output ENDS with a deterministic footer block bracketed by `---` lines, starting with `✅ All agents reported back!` and ending with `📎 Raw results saved to ~/Documents/Last30Days/<slug>-raw.md`. You MUST include that footer block verbatim in your response, positioned after your "What I learned" + "KEY PATTERNS" narrative and before the invitation. Do not recompute the stats. Do not reformat the tree. Do not paraphrase. Do not skip it. Do not add your own source lines. Copy the exact bytes.**
|
||||||
|
|
||||||
- The engine already omits zero-count sources. You do not need to filter them.
|
- The engine already omits zero-count sources. You do not need to filter them.
|
||||||
- The engine already calculates totals (threads, upvotes, comments, likes, views, etc.). You do not need to add them up.
|
- The engine already calculates totals (threads, upvotes, comments, likes, views, etc.). You do not need to add them up.
|
||||||
@@ -1577,36 +1399,9 @@ Close with `I have all the links to the {N} {source list} I pulled from. Just as
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## SHAREABLE HTML BRIEF (when the user asked for one)
|
|
||||||
|
|
||||||
**This section fires if EITHER trigger is true:**
|
|
||||||
|
|
||||||
- `$ARGUMENTS` contains `--emit=html`, `--emit:html`, or `--html` as a flag
|
|
||||||
- The user's natural-language request asks for an HTML brief, shareable doc, or file for sharing (Slack, email, Notion, "export as HTML", etc). Use your judgment for phrasing variants.
|
|
||||||
|
|
||||||
**If neither trigger fires, skip this entire section and proceed to WAIT FOR USER'S RESPONSE.** No HTML save flow, no reference read needed.
|
|
||||||
|
|
||||||
**When triggered, you MUST:**
|
|
||||||
|
|
||||||
- Read `references/save-html-brief.md` BEFORE proceeding to WAIT FOR USER'S RESPONSE
|
|
||||||
- Follow that file's instructions exactly - it is the canonical source for the save flow
|
|
||||||
- Append the confirmation line (`📎 Shareable brief saved to <path>`) to your already-emitted chat response
|
|
||||||
|
|
||||||
**You MUST NOT:**
|
|
||||||
|
|
||||||
- Improvise the HTML save flow from memory or from instructions you've seen before
|
|
||||||
- Skip the reference read because the steps "look familiar"
|
|
||||||
- Save to a different path than the reference specifies
|
|
||||||
- Add data quality warnings, debug headers, or safety notes to the saved HTML
|
|
||||||
- Re-research the topic for the HTML render - the engine cache covers the second invocation
|
|
||||||
|
|
||||||
**Why the directive is forceful:** the reference file is the only source of truth for the save flow. Skipping it produces broken artifacts - wrong path conventions, missing synthesis content, leaked engine debug output, or warnings that don't belong in shareable docs.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WAIT FOR USER'S RESPONSE
|
## WAIT FOR USER'S RESPONSE
|
||||||
|
|
||||||
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.
|
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `~/Documents/Last30Days/` via `--save-dir`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -1711,7 +1506,7 @@ Want another prompt? Just tell me what you're creating next.
|
|||||||
**What this skill does:**
|
**What this skill does:**
|
||||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, and as a Reddit backup when public Reddit is unavailable (requires SCRAPECREATORS_API_KEY)
|
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, and as a Reddit backup when public Reddit is unavailable (requires SCRAPECREATORS_API_KEY)
|
||||||
- Legacy: Sends search queries to OpenAI's Responses API (`api.openai.com`) for Reddit discovery (fallback if no SCRAPECREATORS_API_KEY)
|
- Legacy: Sends search queries to OpenAI's Responses API (`api.openai.com`) for Reddit discovery (fallback if no SCRAPECREATORS_API_KEY)
|
||||||
- Sends search queries to Twitter's GraphQL API (via optional user-provided AUTH_TOKEN/CT0 env vars - no browser session access), xAI's API (`api.x.ai`), or the official X API v2 via xurl CLI (OAuth2, auto-detected when installed and authenticated) for X search
|
- Sends search queries to Twitter's GraphQL API (via optional user-provided AUTH_TOKEN/CT0 env vars - no browser session access) or xAI's API (`api.x.ai`) for X search
|
||||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||||
@@ -1719,7 +1514,7 @@ Want another prompt? Just tell me what you're creating next.
|
|||||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||||
- Stores research findings in local SQLite database (watchlist mode only)
|
- Stores research findings in local SQLite database (watchlist mode only)
|
||||||
- Saves research briefings as .md files to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`)
|
- Saves research briefings as .md files to ~/Documents/Last30Days/
|
||||||
|
|
||||||
**What this skill does NOT do:**
|
**What this skill does NOT do:**
|
||||||
- Does not post, like, or modify content on any platform
|
- Does not post, like, or modify content on any platform
|
||||||
|
Before Width: | Height: | Size: 2.7 MiB After Width: | Height: | Size: 2.7 MiB |
|
Before Width: | Height: | Size: 2.3 MiB After Width: | Height: | Size: 2.3 MiB |
|
Before Width: | Height: | Size: 3.8 MiB After Width: | Height: | Size: 3.8 MiB |
|
Before Width: | Height: | Size: 2.6 MiB After Width: | Height: | Size: 2.6 MiB |
@@ -1,303 +0,0 @@
|
|||||||
---
|
|
||||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
|
||||||
type: feat
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
---
|
|
||||||
|
|
||||||
# feat: --competitors flag for auto-discovered comparison fan-out
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Add a `--competitors` flag to the last30days engine that auto-discovers 2-4 peer entities for the topic, runs the full retrieval pipeline on each in parallel, and renders a multi-entity comparison. Invoking `last30days Kanye West --competitors` should resolve to "Kanye vs Drake vs Kendrick Lamar" and emit a comparison report covering all three. Invoking `last30days OpenAI --competitors` should resolve to "OpenAI vs Anthropic vs xAI vs Gemini" and emit a four-way comparison.
|
|
||||||
|
|
||||||
Discovery mirrors the existing `resolve.auto_resolve()` pattern used for X handles and subreddits at pipeline start — web search (Brave / Exa / Serper) plus deterministic extraction. Not an internal LLM call.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
Users who want a comparison today must type "OpenAI vs Anthropic vs xAI" themselves. The `planner._comparison_entities()` path already handles explicit multi-entity topics and `render._render_comparison_scaffold()` already emits a 9-axis comparison table. What is missing is the discovery half — a user who types a single entity with `--competitors` should get the comparison for free.
|
|
||||||
|
|
||||||
This is also the natural next step after the Step 0.55 category-peer subreddit work (PR #305, merged 2026-04-22). That feature widens the subreddit set within a single topic; this feature widens the entity set into peer entities.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. New `--competitors` boolean flag that triggers competitor discovery and multi-entity fan-out.
|
|
||||||
- R2. New `--competitors-list="A,B,C"` to explicitly skip discovery (mirrors `--plan`, `--subreddits`, `--x-handle` overrides).
|
|
||||||
- R3. New `--competitors=N` short form to set competitor count inline (N in 1..6).
|
|
||||||
- R4. Default count is 3 competitors (original + 3 = 4-way comparison).
|
|
||||||
- R5. Competitor retrieval depth inherits the main run's depth (`--quick` / `--deep`); all entities run in parallel so wall clock stays close to a single run.
|
|
||||||
- R6. Discovery mirrors `resolve.auto_resolve()`: web search for peers, deterministic text extraction. No internal LLM dependency.
|
|
||||||
- R7. If no web search backend is configured and no `--competitors-list` was passed, engine emits a LAW 7-style stderr telling the host agent to pass `--competitors-list` and exits non-zero.
|
|
||||||
- R8. Output rendering is a single comparison report covering all entities, reusing the existing 9-axis scaffold from `render._render_comparison_scaffold()` where applicable.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- Synthesis prompt changes beyond wiring N reports into the existing comparison scaffold are out of scope.
|
|
||||||
- `--competitors` does not replace the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`; both paths coexist.
|
|
||||||
- No caching layer for discovery results in v1.
|
|
||||||
- No UI/SKILL.md rewrite of the entire comparison section; only the new flag is documented.
|
|
||||||
- No new web search backend.
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Caching of competitor lookups: separate follow-up once hit rate justifies it.
|
|
||||||
- Disambiguation UX for topics with multiple common entities ("Amazon" the company vs the river): separate brainstorm.
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py:168-249` — `build_parser()` argparse definitions. Existing depth flags (`--quick`, `--deep`) and override flags (`--plan`, `--subreddits`, `--x-handle`, `--auto-resolve`) set the convention to mirror.
|
|
||||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the reference pattern: web search fan-out via `ThreadPoolExecutor`, per-query extraction functions, graceful empty-dict return when no backend is available.
|
|
||||||
- `scripts/lib/resolve.py:98-140` — `_extract_x_handle()` and sibling extractors show the deterministic text-mining style competitor extraction should mirror.
|
|
||||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature is the fan-out target. One call per entity, each returning a `schema.Report`.
|
|
||||||
- `scripts/lib/planner.py:430-564` — Existing comparison-intent handling and `_comparison_entities()` entity extraction. The new flag feeds the same mental model but populates entities from discovery instead of from the topic string.
|
|
||||||
- `scripts/lib/render.py:333-392` — `_render_comparison_scaffold()` already emits a 9-axis markdown comparison table. The new multi-report renderer should reuse this helper by assembling a synthetic "A vs B vs C" topic header for it.
|
|
||||||
- `scripts/lib/grounding.py` + `scripts/lib/providers.py` — Web search backend resolution (Brave / Exa / Serper). Reused as-is.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- No existing `docs/solutions/` entries for competitor discovery or multi-entity fan-out.
|
|
||||||
- Recent plan `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md` established the precedent of deterministic peer expansion; this plan extends that idea from subreddits to entities.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None gathered — local patterns are strong. `resolve.auto_resolve()` is a direct template.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Discovery mirrors auto_resolve, not plan_query.** Web search + regex extraction, not an LLM call. Matches the user's explicit direction ("use the python brain the same way it searches for X handles"). Cheaper, no provider credential requirement, deterministic.
|
|
||||||
- **Orchestration lives in `last30days.py` main, not inside `pipeline.run()`.** The fan-out is a top-level concern — one pipeline run per entity, each independent. Keeps `pipeline.run()` single-entity and unchanged except for sharing a `ThreadPoolExecutor` factory.
|
|
||||||
- **Sub-runs inherit main depth and run in parallel.** Wall clock ≈ single run; token cost scales linearly with N. User-controlled via the existing `--quick`/`--deep` flags.
|
|
||||||
- **New module `scripts/lib/competitors.py` instead of adding to `resolve.py`.** Keeps resolve focused on single-entity entity-bundle discovery (handles/subreddits/github); competitors.py owns peer-entity discovery. Similar shape, different responsibility.
|
|
||||||
- **Multi-report render is additive in `render.py`.** New `render_comparison_multi(reports: list[Report]) -> str` composes a synthetic "A vs B vs C" topic and delegates to the existing scaffold + synthesis path where possible. No rewrite of the single-entity render path.
|
|
||||||
- **Default count = 3 competitors (4-way comparison).** Hard cap at 6.
|
|
||||||
- **LAW 7-style stderr when no backend and no list.** Matches how `planner.plan_query()` already tells the hosting agent to pass `--plan`.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **Discovery mechanism:** Web search via `grounding.web_search()`, not an internal LLM. User confirmed the auto_resolve pattern is the target.
|
|
||||||
- **Default competitor count:** 3 (original + 3 = 4-way).
|
|
||||||
- **Sub-run depth:** Inherit main depth, parallel execution.
|
|
||||||
- **Flag naming:** `--competitors` (standard argparse double-dash). `--competitors=N` for inline count. `--competitors-list="A,B,C"` to skip discovery.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Exact extraction heuristics for competitor names across Brave / Exa / Serper result shapes. The SERP text varies (listicles, comparison pages, "vs" pages); the initial implementation will start with listicle parsing plus a "X vs Y" pattern match, and harden against real results in the test phase.
|
|
||||||
- Handling of topic ambiguity ("Amazon", "Apple"). Initial behavior: trust whatever web search returns for the topic verbatim; disambiguation is a separate concern.
|
|
||||||
- Merge strategy when two entities return overlapping URLs (e.g., an "OpenAI vs Anthropic" article shows up in both runs). Likely dedupe at the clustering step, but defer the exact policy until we see how often it happens.
|
|
||||||
- Whether to expose competitor discovery artifacts (the raw web search results) as a debug emit. Follow the existing `--debug` conventions.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: CLI flag parsing and validation**
|
|
||||||
|
|
||||||
**Goal:** Add `--competitors`, `--competitors=N`, and `--competitors-list` to the argparse surface, validate values, and thread them into the main orchestration.
|
|
||||||
|
|
||||||
**Requirements:** R1, R2, R3, R4
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py`
|
|
||||||
- Test: `tests/test_cli_competitors.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add three mutually cooperative flags near line 205 in `build_parser()`:
|
|
||||||
- `--competitors` with `nargs="?"` and `const=3` so bare `--competitors` defaults to 3, `--competitors=4` is honored, and `--competitors=0` is rejected
|
|
||||||
- `--competitors-list` free-text CSV
|
|
||||||
- Normalize in `main()`: if `--competitors-list` is present, skip discovery and use the list. If `--competitors` is set and no list, trigger discovery with count = the flag value. Clamp count to 1..6 with a stderr warning at boundary.
|
|
||||||
- Thread the resulting entity list into the orchestrator added in Unit 3.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `--plan` argument at `scripts/last30days.py:187` — same skip-discovery-when-explicit shape.
|
|
||||||
- `--subreddits` / `--x-handle` at `scripts/last30days.py:180,189` — same override semantics.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: bare `--competitors` parses to count=3, empty list.
|
|
||||||
- Happy path: `--competitors=4` parses to count=4.
|
|
||||||
- Happy path: `--competitors-list="A,B,C"` parses to count=3, list=["A","B","C"], and is preferred over any discovery signal.
|
|
||||||
- Edge case: `--competitors=0` and `--competitors=-1` are rejected with a clear error.
|
|
||||||
- Edge case: `--competitors=99` clamps to 6 with a stderr warning.
|
|
||||||
- Edge case: `--competitors` combined with `--competitors-list` uses the list and logs that discovery was skipped.
|
|
||||||
- Edge case: `--competitors-list` value with whitespace ("A, B , C") normalizes correctly.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Running the binary with each flag variation produces the expected post-parse state without calling out to the network.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: `scripts/lib/competitors.py` discovery module**
|
|
||||||
|
|
||||||
**Goal:** Discover peer entities for a topic using web search + deterministic extraction, mirroring `resolve.auto_resolve()`.
|
|
||||||
|
|
||||||
**Requirements:** R6, R7
|
|
||||||
|
|
||||||
**Dependencies:** None (pure module; wired by Unit 3)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `scripts/lib/competitors.py`
|
|
||||||
- Test: `tests/test_competitors.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Public entry point `discover_competitors(topic: str, count: int, config: dict) -> list[str]`.
|
|
||||||
- Early return `[]` when `_has_backend(config)` is false (reuse the helper from `resolve.py`; factor if needed).
|
|
||||||
- Fan out 2-3 web searches in a `ThreadPoolExecutor`:
|
|
||||||
- `"{topic} competitors"`
|
|
||||||
- `"{topic} alternatives"`
|
|
||||||
- `"{topic} vs"` (captures "X vs Y" articles)
|
|
||||||
- Feed results into a deterministic `_extract_peer_entities(results, topic)` that:
|
|
||||||
- Mines titles and snippets for capitalized noun phrases other than the topic itself
|
|
||||||
- Scores by frequency across results
|
|
||||||
- Filters stopwords and the topic's own tokens
|
|
||||||
- Returns top `count` unique entities ordered by score
|
|
||||||
- Emit a single-line stderr log mirroring the `resolve._log` format.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/lib/resolve.py:179-258` for the function shape, executor usage, and empty-result fallback.
|
|
||||||
- `scripts/lib/resolve.py:98-140` for extractor style (small, deterministic, no external state).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: canned SERP fixtures for "OpenAI" return ["Anthropic", "xAI", "Google"] or close peers in the top 3.
|
|
||||||
- Happy path: canned SERP fixtures for "Kanye West" return rap peers (Drake, Kendrick) in the top 3.
|
|
||||||
- Edge case: empty SERP results return `[]` without raising.
|
|
||||||
- Edge case: extractor filters out the topic itself (case- and punctuation-insensitive).
|
|
||||||
- Edge case: near-duplicate entities ("OpenAI" vs "Open AI") dedupe to one slot.
|
|
||||||
- Error path: web search backend raises — the failure is logged and the function returns `[]`.
|
|
||||||
- Edge case: count=1 returns a single-element list; count=6 returns up to six entities.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Unit tests pass with fixtures committed under `tests/fixtures/competitors-*.json`.
|
|
||||||
- Manual run against a live backend for one topic confirms sensible output (recorded as a notes file, not a test assertion).
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Parallel fan-out orchestrator**
|
|
||||||
|
|
||||||
**Goal:** Run `pipeline.run()` once per entity (topic + discovered competitors) in parallel, collect `schema.Report` per entity, and hand them to the comparison renderer.
|
|
||||||
|
|
||||||
**Requirements:** R5, R7
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1, Unit 2
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py`
|
|
||||||
- Possibly create: `scripts/lib/fanout.py` if the orchestrator grows past ~60 lines
|
|
||||||
- Test: `tests/test_competitor_fanout.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- After arg parsing and before the existing `pipeline.run()` call, branch on `args.competitors`:
|
|
||||||
- If a list was provided or discovery returned entities, build `entities = [topic, *competitors]`.
|
|
||||||
- Spawn one `pipeline.run()` per entity via `ThreadPoolExecutor(max_workers=len(entities))`, passing the same `config`, `depth`, and all sub-run-relevant args (mock, plan, etc.). Respect `--plan` — if a plan is passed it applies to the main topic only; competitors use the internal planner fallback for v1.
|
|
||||||
- Collect `{entity: Report}` mapping. A per-entity failure logs a stderr warning and drops that entity from the comparison; the run continues as long as 2 entities succeed.
|
|
||||||
- If fewer than 2 entities survive, exit with a clear error.
|
|
||||||
- LAW 7-style stderr:
|
|
||||||
- If `args.competitors` is set, no list was passed, no web search backend is configured, emit a LAW 7 stderr message pointing to the `--competitors-list` override and exit non-zero. Reuse the tone from `planner.plan_query()` fallback (`scripts/lib/planner.py:125-135`).
|
|
||||||
|
|
||||||
**Execution note:** Start with a failing integration test that exercises the full main → orchestrator → mocked pipeline.run path; the orchestrator is where bugs hide.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/lib/resolve.py:225-239` for ThreadPoolExecutor + as_completed + per-future error handling.
|
|
||||||
- `scripts/lib/pipeline.py:310+` for how ThreadPoolExecutor is already used inside a single run (same idiom, outer layer).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: main + 2 competitors, all three `pipeline.run()` calls succeed (mocked), orchestrator returns 3 Reports.
|
|
||||||
- Happy path: discovery returns the competitor list; orchestrator fans out accordingly.
|
|
||||||
- Edge case: one of three competitor pipelines raises — the run continues with the surviving 2 and emits a warning.
|
|
||||||
- Edge case: all competitors fail but the main topic succeeds — orchestrator exits non-zero with a clear error rather than silently degrading to a single-entity render.
|
|
||||||
- Edge case: `--competitors` set, no backend, no list — orchestrator emits the LAW 7 stderr and exits non-zero before any pipeline call.
|
|
||||||
- Integration: wall-clock time for 3 mocked pipelines in parallel is close to the slowest single run, not the sum (timing assertion with generous margin).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- End-to-end test with mocked `pipeline.run()` and mocked competitors discovery produces 3 Reports and hands them to a stubbed renderer.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: Multi-report comparison renderer**
|
|
||||||
|
|
||||||
**Goal:** Compose N `schema.Report`s into a single comparison-mode output, reusing the existing 9-axis scaffold.
|
|
||||||
|
|
||||||
**Requirements:** R8
|
|
||||||
|
|
||||||
**Dependencies:** Unit 3
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/render.py`
|
|
||||||
- Test: `tests/test_render_comparison_multi.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `render_comparison_multi(reports: list[schema.Report], *, emit: str) -> str`.
|
|
||||||
- Build a synthetic comparison topic: `f"{entity_a} vs {entity_b} vs {entity_c}"`.
|
|
||||||
- Reuse `_render_comparison_scaffold()` for the table skeleton. Each entity column is populated from its own Report's top clusters and citations.
|
|
||||||
- For the narrative synthesis block, concatenate per-entity highlights, clearly labeled by entity, under a shared "Comparison" header.
|
|
||||||
- Preserve existing emit modes (`compact`, `md`, `json`, `context`). In `json` emit, return a `{"entities": [...], "reports": [...]}` shape; single-Report consumers remain unaffected because the single-report render path is untouched.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/lib/render.py:333-392` (`_parse_comparison_entities`, `_render_comparison_scaffold`) — the scaffold is the contract.
|
|
||||||
- `scripts/lib/render.py` single-report rendering — for per-entity narrative blocks.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: 3 Reports with distinct clusters render into a 3-column table and a "Comparison" section that mentions each entity at least once.
|
|
||||||
- Happy path: 2 Reports render as a 2-column table without breaking the scaffold.
|
|
||||||
- Edge case: a Report with an empty cluster list renders as "(no significant discussion this month)" in its column rather than crashing.
|
|
||||||
- Edge case: Reports with overlapping URLs (same article cited by two entities) dedupe citations at the footer but keep both column entries.
|
|
||||||
- Emit variants: `--emit=compact`, `--emit=md`, `--emit=json`, `--emit=context` each produce valid output with all entities represented.
|
|
||||||
- Integration: end-to-end snapshot test using fixture Reports, checked against a stored expected output (with a clear update path when the scaffold intentionally evolves).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Snapshot tests pass. Manual review of one real 3-way comparison confirms readability.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: Docs, SKILL.md mention, and sync**
|
|
||||||
|
|
||||||
**Goal:** Document the new flag so the hosting agent and human users both know it exists, and run the sync script.
|
|
||||||
|
|
||||||
**Requirements:** R1-R8 (surfaces them to users)
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-4
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md`
|
|
||||||
- Modify: `README.md` (brief flag reference)
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add a compact "Competitor mode" subsection under the existing comparison docs in `SKILL.md`. Document the flag, the default count, the override flag, and the LAW 7 fallback stderr.
|
|
||||||
- Keep `README.md` addition to a single example line.
|
|
||||||
- CHANGELOG entry mirrors the voice of recent entries (imperative, outcome-first).
|
|
||||||
- Sync via `scripts/sync.sh` per CLAUDE.md rules so `~/.claude/`, `~/.agents/`, `~/.codex/` pick up the new SKILL.md.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — documentation and sync only. Verification is by inspection and by running `sync.sh` and confirming target directories updated.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `sync.sh` completes without errors.
|
|
||||||
- `SKILL.md` rendered preview mentions `--competitors` in the comparison section.
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** `last30days.py main()` now orchestrates multiple `pipeline.run()` calls instead of one. No other callers of `pipeline.run()` are affected (it remains single-entity).
|
|
||||||
- **Error propagation:** Per-entity failures degrade gracefully as long as ≥2 entities survive; fewer survivors exits non-zero. Discovery failure with `--competitors` and no list is fatal.
|
|
||||||
- **State lifecycle risks:** Each sub-run uses its own `pipeline.run()` state; no shared mutable config. The `config` dict is read-only in `pipeline.run()` today — verify before committing to shared-reference passing, else deep-copy per sub-run.
|
|
||||||
- **API surface parity:** `--competitors` coexists with the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`. Both produce comparable output formats; the only difference is where the entity list came from.
|
|
||||||
- **Integration coverage:** The fan-out orchestrator crosses CLI → discovery → N pipelines → render; integration tests in Unit 3 and Unit 4 must exercise the full path end to end, not just unit-level.
|
|
||||||
- **Unchanged invariants:** `pipeline.run()` signature and single-entity semantics are unchanged. The single-entity render path in `render.py` is unchanged. No changes to `planner.plan_query()`. No changes to existing flags.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Competitor discovery returns garbage entities for niche topics. | `--competitors-list` override lets the user (or hosting agent) correct it. Unit tests with edge-case fixtures. Log discovery output to stderr under `--debug`. |
|
|
||||||
| Token cost scales linearly with N sub-runs. | Default count capped at 3, hard max 6, inherit `--quick` to let users throttle. Wall clock stays parallel. Emit a cost hint to stderr when N ≥ 4. |
|
|
||||||
| Merge conflicts against the single-entity render path during refactoring. | Keep the multi-report renderer strictly additive; do not modify the single-Report code path. |
|
|
||||||
| Config dict mutation inside sub-runs could leak state between entities. | Verify read-only usage before sharing references. If any sub-component mutates, deep-copy per sub-run before spawning threads. |
|
|
||||||
| A SERP extractor that works on Brave fixtures breaks on Exa/Serper result shapes. | Test fixtures for all three backends. Extractor operates on a normalized shape from `grounding.web_search()` (already the case), not raw provider output. |
|
|
||||||
| Hosting agent (Claude Code, Codex) unaware of the new flag when it could usefully pass `--competitors-list`. | SKILL.md updated in Unit 5 documents the flag in the same style as `--plan` and `--auto-resolve`. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first: per `CLAUDE.md`, experimental changes go to `mvanhorn/last30days-skill-private` on the `/last30days-beta` command. Land this on the private repo first, shake out on real topics for a day or two, then cherry-pick to public.
|
|
||||||
- After land-merge: run `scripts/sync.sh` to deploy SKILL.md + scripts to `~/.claude/`, `~/.agents/`, `~/.codex/`.
|
|
||||||
- Release notes entry in CHANGELOG.md follows the v3.0.9 voice — outcome-first, one paragraph.
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Related code: `scripts/lib/resolve.py:179` (`auto_resolve`), `scripts/lib/pipeline.py:162` (`pipeline.run`), `scripts/lib/planner.py:80` (`plan_query` LAW 7 fallback), `scripts/lib/render.py:333` (comparison scaffold)
|
|
||||||
- Related PRs: #305 (Step 0.55 category-peer subreddit expansion — the precedent for deterministic peer expansion, merged 2026-04-22)
|
|
||||||
- Related plan: `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md`
|
|
||||||
@@ -1,349 +0,0 @@
|
|||||||
---
|
|
||||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
|
||||||
type: fix
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
origin: docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md
|
|
||||||
---
|
|
||||||
|
|
||||||
# fix: per-entity resolution, default-2, and stale-path guard for --competitors
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Three test runs of v3.0.11 `--competitors` surfaced four real bugs plus one product tweak. This plan fixes all of them in a single follow-up:
|
|
||||||
|
|
||||||
1. Competitor sub-runs get no Step 0.55 resolution (no X handle, no subreddits, no GitHub repo). Drake / Kendrick / Travis ran with deterministic-fallback single-word queries while Kanye had the full targeting package. User called it "lazy" and was right.
|
|
||||||
2. Two of three test windows (Linear, Coinbase) never invoked the new flag at all. They loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (a Claude-Code-managed git clone pinned to origin/main, which predates PR #308) instead of `plugins/cache/last30days-skill/last30days/3.0.11/`, so `--help` showed no `--competitors` flag and the model fell back to the manual comparison path.
|
|
||||||
3. Each competitor sub-run emits a scary `[Planner] No --plan passed... deterministic fallback` stderr line because LAW 7 targets the hosting-model path, not internal fan-out sub-runs.
|
|
||||||
4. Default competitor count is 3 (→ 4-way comparison). User wants default 2 (→ 3-way: original + 2 peers). Flag keeps `--competitors=N` to customize.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
The 3 test runs (Kanye, Linear, Coinbase) showed a pattern:
|
|
||||||
|
|
||||||
| Window | Loaded SKILL.md from | Invoked --competitors? | Per-entity resolution? | Outcome |
|
|
||||||
|--------|----------------------|-----------------------|------------------------|---------|
|
|
||||||
| Kanye | cache/3.0.11/ (correct) | Yes | Only for main topic (Kanye) | Drake/Kendrick/Travis thin; Reddit 403 fallbacks |
|
|
||||||
| Linear | marketplaces/ (stale) | No — fell back to manual comparison | No | Thin run with noisy subreddits |
|
|
||||||
| Coinbase | marketplaces/ (stale) | No — fell back to manual comparison | Main only; keyword-search poisoned pool | Top subs: r/survivor, r/Airpodsmax (noise) |
|
|
||||||
|
|
||||||
Root causes:
|
|
||||||
- **Per-entity resolution gap:** `scripts/lib/fanout.py` calls `pipeline.run()` with topic + depth + web_backend + lookback_days only. It does not call `resolve.auto_resolve()` per entity, so sub-runs have no X handle, subreddit, or GitHub targeting. The original plan (`2026-04-22-002`) acknowledged this as a deliberate v1 simplification ("competitor sub-runs use planner defaults"). In practice this produces visibly asymmetric output and triggers downstream retrieval issues (403 fallbacks, keyword-search noise).
|
|
||||||
- **Stale-path loading:** Claude Code's skill loader alphabetizes `find` results with `marketplaces/` before `cache/`, and the model reads the first plausible SKILL.md it sees. SKILL.md line 823's `SKILL_ROOT` resolver is the correct path but only fires in engine-invocation blocks, not in the skill-load step.
|
|
||||||
- **LAW 7 in sub-runs:** LAW 7 exists because the *hosting reasoning model* is supposed to pass `--plan`. For competitor sub-runs, there is no hosting-model planning — it's an engine-internal fan-out. The warning is a false positive there.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. Default `--competitors` count is 2 peers (3-way comparison: original + 2).
|
|
||||||
- R2. Each competitor sub-run performs Step 0.55 resolution (X handle, subreddits, GitHub user/repos, news context) before its pipeline runs — not just the main topic.
|
|
||||||
- R3. Sub-runs do not emit the LAW 7 `No --plan passed` warning; they are internal fan-out, not hosting-model calls.
|
|
||||||
- R4. The rendered comparison output includes a visible "Resolved entities" block showing per-entity handles/subs/github for debug transparency (answers "did it resolve everyone?" without the user having to read stderr).
|
|
||||||
- R5. SKILL.md has a canonical-path self-check at the top: if the reader loaded it from anywhere other than `plugins/cache/last30days-skill/last30days/{VERSION}/`, re-read from the versioned path before proceeding.
|
|
||||||
- R6. Version bumps to 3.0.12; CHANGELOG entry; `scripts/sync.sh` deploys.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No new discovery strategy. The web-search + regex extraction in `scripts/lib/competitors.py` stays as-is.
|
|
||||||
- No new CLI flags beyond the behavior changes above. Specifically: no per-entity override flags like `--competitor-handles`. The hosting-model escape hatch remains `--competitors-list`.
|
|
||||||
- No changes to the explicit `A vs B` comparison path (topic-string parsing in `planner._comparison_entities`).
|
|
||||||
- No marketplace-clone auto-restore fix — that's Claude Code harness behavior. This plan only guards against the symptom on the skill side.
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Caching of per-entity resolution results: separate follow-up once hit rate justifies it.
|
|
||||||
- Fan-out rate-limiting tuning (currently `max_workers=len(entities)+1`, capped at 6): defer until we see real-world quota exhaustion.
|
|
||||||
- Pre-flight cost hint when N ≥ 4 (noted in `2026-04-22-002` risks): defer.
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py:205-219` — `--competitors` / `--competitors-list` argparse definition (const=3 today; changing to 2).
|
|
||||||
- `scripts/last30days.py:220-290` — `resolve_competitors_args()` validator; update `COMPETITORS_DEFAULT`.
|
|
||||||
- `scripts/last30days.py:438-520` — main() fan-out orchestration; currently passes only topic/depth to each `_competitor_runner`.
|
|
||||||
- `scripts/lib/fanout.py:40-95` — `run_competitor_fanout()` signature. The `competitor_runner` callable is where per-entity resolution needs to happen.
|
|
||||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the exact per-entity resolver to reuse. Already does X handle + subreddits + GitHub user/repos + news context in parallel via ThreadPoolExecutor.
|
|
||||||
- `scripts/lib/planner.py:80-135` — `plan_query()` emits the LAW 7 stderr. A `quiet: bool` keyword or `internal_subrun: bool` flag will suppress it.
|
|
||||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature. Needs a new keyword to propagate quiet-mode down to the planner.
|
|
||||||
- `scripts/lib/render.py:render_comparison_multi` — where the "Resolved entities" block is inserted.
|
|
||||||
- `SKILL.md` line 823 — canonical `SKILL_ROOT` resolver already exists but fires in engine bash, not at skill-load time.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md` acknowledged the per-entity-resolution gap as a v1 tradeoff. This plan closes that gap.
|
|
||||||
- Kanye run stderr: `[Planner] No --plan passed... deterministic fallback` × 3 (once per competitor sub-run). That's the LAW 7 noise R3 targets.
|
|
||||||
- Linear / Coinbase runs loaded `plugins/marketplaces/last30days-skill/CLAUDE.md` as the first hit. That's the stale-path issue R5 targets.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None. All patterns are in-repo.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Per-entity resolve happens inside fanout, not in SKILL.md.** The user-facing promise of `--competitors` is "one flag, engine does the work." Pushing resolution onto the hosting model creates another path-of-least-resistance trap (model skips it, output looks lazy). Auto-resolve inside each sub-run when a web backend is available makes the feature self-contained.
|
|
||||||
- **Stale-path guard is a SKILL.md self-check, not a code change.** We cannot stop Claude Code from auto-restoring the marketplace clone. But we can put a 3-line banner at the top of SKILL.md that forces any path-mismatched read to re-read from the versioned cache. Both the marketplace copy (once main catches up) and the cache copy carry the guard.
|
|
||||||
- **LAW 7 suppression is opt-in via `internal_subrun=True` keyword.** Do not remove the warning from the default path — it's load-bearing for the hosting-model contract. Add an explicit bypass for engine-internal fan-out only.
|
|
||||||
- **Default 2, hard max 6 unchanged.** "Original + 2" matches the Kanye/Drake/Kendrick mental model from the feature description. Still allow `--competitors=N` from 1 to 6.
|
|
||||||
- **Resolved block is inside the EVIDENCE envelope, not above it.** Keeps the rendered output structure stable for the synthesis contract (LAW 1–8). The block is context, not output.
|
|
||||||
- **Skip auto-resolve when `--mock` or no web backend.** Mirrors the existing `resolve.auto_resolve()` fast-fail and keeps the mock test path deterministic.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **Where does per-entity resolve live?** Inside `fanout.run_competitor_fanout`, not in `main()`. Each sub-run calls `auto_resolve()` just before `pipeline.run()`.
|
|
||||||
- **Should the hosting model still be able to override?** Yes — `--competitors-list` remains the escape hatch. When an explicit list is passed, the engine still does auto-resolve per entity; the user's list just skips discovery.
|
|
||||||
- **Should sub-runs run auto-resolve in parallel with each other?** Yes. The existing `ThreadPoolExecutor` in fanout already parallelizes sub-runs; auto-resolve happens inside each sub-run's thread, so resolve calls for different entities run concurrently.
|
|
||||||
- **Default count:** 2 peers (3-way). Confirmed.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Whether to expose a `--no-auto-resolve-competitors` flag for power users who want the fast, shallow behavior. Probably not needed v2; ship auto-resolve always-on and revisit if someone complains about cost.
|
|
||||||
- Whether to surface the per-entity resolution context back into the main topic's planner (cross-entity context sharing). Stays deferred.
|
|
||||||
- Whether the Resolved block should be collapsible or always inline. Start inline; revisit based on output length feedback.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: Default `--competitors` to 2 peers**
|
|
||||||
|
|
||||||
**Goal:** Change the bare `--competitors` default from 3 to 2 per user feedback. `--competitors=N` still overrides; range 1..6 unchanged.
|
|
||||||
|
|
||||||
**Requirements:** R1
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (`COMPETITORS_DEFAULT`, `--competitors` const, stderr messages if any reference 3)
|
|
||||||
- Modify: `SKILL.md` Competitor mode section ("discovered 2-6" wording, bare-flag default line)
|
|
||||||
- Modify: `README.md` auto-discovered example line (if it references count)
|
|
||||||
- Test: `tests/test_cli_competitors.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Change `COMPETITORS_DEFAULT = 3` → `2` in `scripts/last30days.py`.
|
|
||||||
- Change argparse `--competitors` `const=3` → `const=2`.
|
|
||||||
- Update any SKILL.md / README copy referencing "3 peers" to "2 peers" (default) or "2-6 peers" (range).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing default constants in `scripts/last30days.py` argparse block.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: bare `--competitors` yields count=2, enabled=True, empty explicit_list.
|
|
||||||
- Edge case: `--competitors=3` still works (explicit override).
|
|
||||||
- Edge case: existing `test_bare_flag_defaults_to_three` test is updated to `test_bare_flag_defaults_to_two` and asserts count=2.
|
|
||||||
- Edge case: `--competitors=5` with a `--competitors-list` of length 2 still logs the mismatch warning and uses the list.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `pytest tests/test_cli_competitors.py -v` passes with the updated default.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: Per-entity Step 0.55 resolution inside fanout**
|
|
||||||
|
|
||||||
**Goal:** Each competitor sub-run auto-resolves its own X handle, subreddits, GitHub user/repos, and news context via `resolve.auto_resolve()` before its `pipeline.run()` call — just like the main topic.
|
|
||||||
|
|
||||||
**Requirements:** R2
|
|
||||||
|
|
||||||
**Dependencies:** None (but Unit 3 should land together so sub-runs don't emit LAW 7 stderr while the resolution context is being passed)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/fanout.py`
|
|
||||||
- Modify: `scripts/last30days.py` (`_competitor_runner` closure builds the resolved args)
|
|
||||||
- Test: `tests/test_competitor_fanout.py`
|
|
||||||
- Test: `tests/test_competitors_resolve_integration.py` (new; covers the auto-resolve path)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- `_competitor_runner(entity)` in main() does:
|
|
||||||
1. Call `resolve.auto_resolve(entity, config)` when `not args.mock` and a web backend is configured (reuse `_has_backend`).
|
|
||||||
2. Extract resolved x_handle, subreddits, github_user, github_repos, context.
|
|
||||||
3. Pass them to `pipeline.run()` for that sub-run.
|
|
||||||
4. Inject resolved context into a per-entity config copy (so `_auto_resolve_context` does not leak across sub-runs — deep-copy the config or use a local dict).
|
|
||||||
5. Store the resolved block on the Report's `artifacts` so the renderer can surface it (Unit 4).
|
|
||||||
- When `args.mock` is True or no backend is available, skip auto-resolve (fall through to planner defaults, matching the existing `auto_resolve()` early-return contract).
|
|
||||||
- Update `fanout.run_competitor_fanout` docstring to note that auto-resolve happens inside the caller-provided runner.
|
|
||||||
|
|
||||||
**Execution note:** Start with a failing integration test that exercises two-entity fanout + auto-resolve via a mocked `resolve.auto_resolve` and asserts that `pipeline.run` receives the resolved x_handle/subreddits for each entity.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/last30days.py` main topic branch (`if args.auto_resolve and not external_plan`) already calls `resolve.auto_resolve` and propagates results — mirror the shape for competitors.
|
|
||||||
- Config isolation: `scripts/lib/pipeline.py:162-220` reads config as-is; use `dict(config)` to avoid cross-sub-run mutation of `_auto_resolve_context`.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: 3 entities, mocked `auto_resolve` returns distinct handles per entity; `pipeline.run` receives `x_handle=@drake` for Drake, `x_handle=@kendricklamar` for Kendrick, etc.
|
|
||||||
- Happy path: the main topic still uses the user-supplied `--x-handle` / `--subreddits` overrides (not overwritten by auto-resolve for the main). Competitors use their own auto-resolved values.
|
|
||||||
- Edge case: `--mock` skips auto-resolve entirely for all sub-runs (no `resolve.auto_resolve` calls).
|
|
||||||
- Edge case: `resolve.auto_resolve` returns empty dicts for one entity (low-signal topic) — the sub-run still executes with planner defaults; doesn't crash.
|
|
||||||
- Edge case: no web backend configured — auto-resolve returns empty for every entity, sub-runs fall through to planner defaults, no stack trace.
|
|
||||||
- Error path: `resolve.auto_resolve` raises — the sub-run logs a warning and continues with planner defaults (does not fail the whole comparison).
|
|
||||||
- Integration: config `_auto_resolve_context` from entity A does not leak into entity B's `pipeline.run`. Assert each sub-run gets its own context string.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- New integration test passes.
|
|
||||||
- End-to-end smoke (mock mode + explicit list): each sub-run's stderr shows `[AutoResolve]` lines per entity with distinct values.
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Suppress LAW 7 warning for engine-internal sub-runs**
|
|
||||||
|
|
||||||
**Goal:** The `[Planner] No --plan passed... deterministic fallback` warning does not fire during competitor sub-runs. LAW 7 is load-bearing for hosting-model contracts and must stay on the default path; this is an opt-in bypass for internal fan-out only.
|
|
||||||
|
|
||||||
**Requirements:** R3
|
|
||||||
|
|
||||||
**Dependencies:** Unit 2 (so the sub-run call site is already being modified)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/planner.py` (`plan_query` signature + conditional stderr)
|
|
||||||
- Modify: `scripts/lib/pipeline.py` (`run` signature + propagation)
|
|
||||||
- Modify: `scripts/last30days.py` or `scripts/lib/fanout.py` (pass `internal_subrun=True` for competitor runners)
|
|
||||||
- Test: `tests/test_planner_v3.py` (or new `tests/test_planner_quiet_mode.py`)
|
|
||||||
- Test: `tests/test_competitor_fanout.py` (assert sub-runs don't emit LAW 7 stderr)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add a keyword `internal_subrun: bool = False` to `planner.plan_query`. When True, skip the two `print(..., file=sys.stderr)` blocks that emit the LAW 7 banner and the `[Planner] No --plan passed` capability message.
|
|
||||||
- Add the same keyword to `pipeline.run()`; pass through to `plan_query`.
|
|
||||||
- In main()/fanout, set `internal_subrun=True` for every competitor sub-run's pipeline.run call. The main topic's pipeline.run keeps the default (LAW 7 stays on for the hosting-model path).
|
|
||||||
- Also suppress the LAW 7-triggered degraded-run warning block in the render layer for sub-reports when the envelope is going to be merged into a comparison output (or accept that the block is per-entity and surfaces once per entity).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing keyword-only parameters on `pipeline.run` (`mock`, `x_handle`, etc.).
|
|
||||||
- `planner.plan_query` signature is already keyword-only.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `plan_query(..., internal_subrun=True, provider=None, model=None)` returns the deterministic fallback plan WITHOUT writing the LAW 7 stderr block.
|
|
||||||
- Happy path: `plan_query(...)` with default `internal_subrun=False` still writes the LAW 7 warning (unchanged behavior).
|
|
||||||
- Integration: end-to-end competitor fanout; assert captured stderr contains zero occurrences of `No --plan passed` and zero of `YOU ARE the planner`.
|
|
||||||
- Integration: main topic is not part of competitor mode; if the user invokes bare `/last30days OpenAI` without `--plan`, LAW 7 stderr fires exactly once (regression test).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Running the Kanye-style smoke test shows zero `[Planner] No --plan passed` lines for Drake / Kendrick / Travis sub-runs.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: "Resolved entities" block in comparison output**
|
|
||||||
|
|
||||||
**Goal:** The rendered comparison output includes a visible block listing per-entity handles, subreddits, GitHub user, and resolved context. Answers "did it resolve everyone?" at a glance without reading stderr.
|
|
||||||
|
|
||||||
**Requirements:** R4
|
|
||||||
|
|
||||||
**Dependencies:** Unit 2 (needs resolved data on report artifacts)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/render.py` (`render_comparison_multi` and `render_comparison_multi_context`)
|
|
||||||
- Test: `tests/test_render_comparison_multi.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- When each entity's `Report.artifacts` contains a `resolved` dict (populated by Unit 2), `render_comparison_multi` emits a `## Resolved Entities` block early in the EVIDENCE envelope:
|
|
||||||
```
|
|
||||||
## Resolved Entities
|
|
||||||
- **Kanye West**: X @kanyewest | Subs r/Kanye, r/hiphopheads | GitHub: — | Context: BULLY released, UK ban…
|
|
||||||
- **Drake**: X @Drake | Subs r/DrakeTheType, r/hiphopheads | GitHub: — | Context: ICEMAN rollout…
|
|
||||||
- **Kendrick Lamar**: X @kendricklamar | Subs r/KendrickLamar | GitHub: — | Context: Grammy wins, dormant…
|
|
||||||
```
|
|
||||||
- Missing fields render as `—` not empty.
|
|
||||||
- When no entity has a `resolved` payload (mock mode, no web backend), omit the block entirely rather than emit an empty section.
|
|
||||||
- Context strings are truncated at 120 chars to keep the block scannable.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing `render_comparison_multi` envelope structure (lines ~395-480 in render.py).
|
|
||||||
- Existing per-entity evidence block format (`## {label}`) for consistency.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: 3 entities each with a `resolved` artifact → block lists all 3 with their fields.
|
|
||||||
- Happy path: 2 entities, one with full resolution, one with partial (x_handle only) → missing fields render as `—`.
|
|
||||||
- Edge case: no entity has a resolved artifact → block is omitted entirely.
|
|
||||||
- Edge case: context string > 120 chars → truncated with ellipsis.
|
|
||||||
- Integration: rendered output passes through the same EVIDENCE envelope comments and synthesis contract (LAW 1–8 unchanged).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Snapshot tests confirm the block appears in the right spot with the right formatting.
|
|
||||||
- End-to-end smoke shows a realistic 3-entity Resolved block in the rendered output.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: SKILL.md canonical-path self-check**
|
|
||||||
|
|
||||||
**Goal:** A top-of-file SKILL.md directive forces any reader (Claude Code, Codex, Hermes, Gemini) to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` before proceeding. If loaded from `marketplaces/` or any other path, re-read from the pinned versioned cache.
|
|
||||||
|
|
||||||
**Requirements:** R5
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md` (prepend a STEP 0 block before the existing STEP 0 / LAW list)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add a numbered first step at the top (before or bundled with existing "STEP 0: ToolSearch preload"):
|
|
||||||
```
|
|
||||||
## STEP 0: Canonical Path Self-Check (must run first)
|
|
||||||
|
|
||||||
Before reading anything else below, verify you loaded this SKILL.md from
|
|
||||||
the versioned cache, not the marketplace clone:
|
|
||||||
|
|
||||||
CANONICAL=$HOME/.claude/plugins/cache/last30days-skill/last30days/
|
|
||||||
CANONICAL_LATEST=$(ls -d "$CANONICAL"*/ 2>/dev/null | sort -V | tail -1)
|
|
||||||
|
|
||||||
If the SKILL.md you just read is not under $CANONICAL_LATEST, STOP. Re-read
|
|
||||||
$CANONICAL_LATEST/SKILL.md and restart from here. Marketplace clones
|
|
||||||
(`plugins/marketplaces/last30days-skill/`) are pinned to origin/main and
|
|
||||||
can be stale; the versioned cache is the ground truth.
|
|
||||||
```
|
|
||||||
- Reinforce in the existing LAW 7 block that `--help` output must be read from the same pinned `SKILL_ROOT` to avoid flag-list skew.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing STEP 0 ToolSearch preload (top of SKILL.md) for tone / imperative voice.
|
|
||||||
- Existing `SKILL_ROOT` resolver snippet (line ~823).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — SKILL.md is documentation; no unit test, verified by follow-up user invocation.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- In a fresh Claude Code window, `/last30days Test --competitors` loads SKILL.md, the model executes the STEP 0 self-check, and (if it had loaded from marketplaces/) switches to the cache path before running `--help` or the engine. Observable via the model's announced reasoning / task list.
|
|
||||||
|
|
||||||
- [ ] **Unit 6: Version bump, CHANGELOG, sync**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.12 and deploy to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** R6
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-5
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json` (version 3.0.11 → 3.0.12)
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG entry under `## [3.0.12]` dated 2026-04-22 covering the four fixes (Fixed: per-entity resolution; Fixed: LAW 7 sub-run noise; Changed: default count 3→2; Added: Resolved entities block; Added: canonical-path self-check in SKILL.md).
|
|
||||||
- `sync.sh` deploys to `~/.claude/plugins/cache/last30days-skill-private/...`, `~/.agents/`, `~/.codex/`, Hermes.
|
|
||||||
- Manual hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.12/` so the public `/last30days` slash command picks up the new version before PR merge (matches the 3.0.11 testing pattern).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging only. Verification is by inspection.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` returns `3.0.12`.
|
|
||||||
- `sync.sh` exits 0 with "Import check: OK" for each target.
|
|
||||||
- Hot-copied 3.0.12 directory contains the new files and `/last30days` picks up the new version (highest-version resolver).
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** Fanout sub-runs now call `resolve.auto_resolve` per entity. Each sub-run is independent; no shared mutable state with other sub-runs or with the main topic.
|
|
||||||
- **Error propagation:** `auto_resolve` failures inside a sub-run log a warning and degrade to planner defaults; do not propagate up to abort the comparison. Same contract as today for the main topic.
|
|
||||||
- **State lifecycle risks:** Config dict is mutated by `auto_resolve` (via `config["_auto_resolve_context"]`). Must deep-copy per sub-run or scope context to a local mapping — otherwise two sub-runs' context strings race.
|
|
||||||
- **API surface parity:** `pipeline.run` gains a keyword (`internal_subrun`); callers that don't pass it get the existing behavior. `planner.plan_query` gains the same. Backward compatible.
|
|
||||||
- **Integration coverage:** New integration test for the fanout + auto-resolve + render chain. Existing snapshot tests update to include the Resolved block.
|
|
||||||
- **Unchanged invariants:** Single-entity `/last30days` invocations (no `--competitors`) behave identically. Explicit `A vs B` comparison topics behave identically. LAW 7 still fires on the default hosting-model path. `render_compact` path is untouched.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Auto-resolving per competitor triples the WebSearch call volume (4 queries × 3 competitors = 12 extra web searches). | Fast-fail when no backend; user can pass `--competitors-list` to skip discovery but still get auto-resolve. Cost note in CHANGELOG. |
|
|
||||||
| Config mutation across sub-runs via `_auto_resolve_context`. | Unit 2 deep-copies config per sub-run before each `auto_resolve` + `pipeline.run` call. Integration test asserts no cross-entity leak. |
|
|
||||||
| LAW 7 suppression leaks onto the hosting-model path via a wrong default. | Default `internal_subrun=False`. Only fanout's competitor sub-runs set True. Unit test asserts bare-topic invocation still emits LAW 7. |
|
|
||||||
| SKILL.md STEP 0 banner gets ignored by the model (same failure mode as line 823 today). | Put it in the guaranteed-read top band (before LAW 1, above all other content), imperative voice, concrete `STOP` verb. Still not bulletproof but strictly better than current. |
|
|
||||||
| Default count change breaks assumptions in downstream tools or existing user muscle memory. | Changelog calls it out as Changed; `--competitors=3` still works for users who want the old default. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first: merge behind `/last30days-beta` via the private repo before cherry-picking to public. Follows the same process as 3.0.11.
|
|
||||||
- Version 3.0.12 is a fix release; no marketing post required.
|
|
||||||
- After merge, add a line to the PR description pointing at this plan.
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Origin plan: `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
|
||||||
- Related PR: #308 (v3.0.11 shipping --competitors)
|
|
||||||
- Test windows that surfaced the bugs: Kanye, Linear, Coinbase (2026-04-22 session)
|
|
||||||
- Related code: `scripts/lib/fanout.py`, `scripts/lib/resolve.py` (`auto_resolve`), `scripts/lib/planner.py` (`plan_query`), `scripts/lib/render.py` (`render_comparison_multi`)
|
|
||||||
@@ -1,394 +0,0 @@
|
|||||||
---
|
|
||||||
title: "fix: --competitors runs a full last30days per entity with hosting-model pre-resolve"
|
|
||||||
type: fix
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
origin: docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md
|
|
||||||
---
|
|
||||||
|
|
||||||
# fix: --competitors runs a full last30days per entity with hosting-model pre-resolve
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
User intent confirmed 2026-04-22: `--competitors` should run a full single-entity `last30days` pipeline for the main topic AND for each discovered peer — three independent full-depth passes, each with its own Step 0.55 resolution, own X handle primary weight, own subreddit targeting, own GitHub repo scoping. Then merge them into the comparison output.
|
|
||||||
|
|
||||||
3.0.12 already built the N-parallel-pipelines orchestration (`scripts/lib/fanout.py`). What it got wrong: it tried to do per-entity Step 0.55 engine-side via `resolve.auto_resolve()`, which requires a web search backend key (BRAVE/EXA/SERPER/PARALLEL/OPENROUTER). Matt runs from Claude Code, which has its own WebSearch tool. The engine has none of those keys, so per-entity auto_resolve silently no-ops and all peer sub-runs fall through to deterministic single-word planner queries.
|
|
||||||
|
|
||||||
Four 2026-04-22 test runs (Warriors, Seattle, Arizona Wildcats, Kanye West) confirmed this via engine receipts:
|
|
||||||
|
|
||||||
- Compact Resolved Entities block shows peers as `X - | Subs - | GitHub - | Context: -`.
|
|
||||||
- Sub-run planner lines show `source=deterministic, subqueries=1` — the "I gave up and keyword-searched" shape.
|
|
||||||
- Engine footer keeps nudging `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY`, which is wrong advice for a Claude Code user who already has WebSearch.
|
|
||||||
- Kanye run leaked main topic's `--subreddits` into Drake's and Kendrick's sub-runs (regression bug).
|
|
||||||
|
|
||||||
The fix is to flip the resolution responsibility: the hosting model (Claude Code, Codex, Hermes, Gemini) does Step 0.55 via its own WebSearch tool for every entity, then passes the resolved targeting to the engine via a new `--competitors-plan` JSON flag. Engine fan-out remains — each peer still runs a full `pipeline.run()`. The difference is the peers now arrive with full targeting, equivalent to the main topic, so retrieval is apples-to-apples.
|
|
||||||
|
|
||||||
Why not just reuse vs-mode? vs-mode is a SINGLE `pipeline.run()` with a comparison-optimized plan. It pre-resolves Step 0.55 per entity but merges everything into one retrieval pool with lower-weight `--x-related` for peers, merged subreddits, and cross-entity keyword noise. That is not "three full passes." The user explicitly wants three full passes.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
3.0.12's architecture was correct; its data dependency was wrong.
|
|
||||||
|
|
||||||
| Capability | 3.0.12 path | Target path (this plan) |
|
|
||||||
|---|---|---|
|
|
||||||
| Fan out to N parallel pipelines | Yes (`fanout.run_competitor_fanout`) | Same — keep |
|
|
||||||
| Per-entity Step 0.55 resolution | Engine-internal `resolve.auto_resolve()` — needs BRAVE/EXA/SERPER/PARALLEL key | Hosting model does it via its own WebSearch, passes to engine |
|
|
||||||
| Per-entity targeting threaded into `pipeline.run()` | Main topic only via outer flags; peers via auto_resolve (failing) or nothing | Main topic via outer flags; peers via `--competitors-plan` JSON |
|
|
||||||
| Footer nudge | Unconditional BRAVE/SERPER | Suppressed when `--plan` or `--competitors-plan` present |
|
|
||||||
| Resolved Entities block in raw save file | Stdout only | Also in `--save-dir` raw file |
|
|
||||||
| Override-leak from main into peers | Present (Kanye receipt) | Fixed via explicit per-entity kwargs scrub |
|
|
||||||
| Polymarket noise on ambiguous topics | Present (Warriors, Arizona receipts) | `--polymarket-keywords` + auto-skip for single-token-ambiguous |
|
|
||||||
|
|
||||||
The key architectural change is who owns per-entity resolution. The engine stops trying to do it itself; the hosting model does it upstream (it already has WebSearch) and passes results in.
|
|
||||||
|
|
||||||
This is the same pattern `--plan` already uses for the main topic: hosting model generates the plan via its own reasoning, passes it in, engine accepts. We apply the pattern to peers.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. New `--competitors-plan` JSON flag accepting per-entity targeting: `x_handle`, `x_related`, `subreddits`, `github_user`, `github_repos`, `context`. Implies `--competitors`. Per-entity values thread into that entity's `pipeline.run()`. Bypasses engine-internal `auto_resolve` for covered entities.
|
|
||||||
- R2. SKILL.md "Competitor mode" rewritten to make the hosting-model path canonical: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers) via WebSearch, (c) assemble `--competitors-plan` JSON, (d) invoke engine. Engine-internal auto_resolve remains as headless fallback.
|
|
||||||
- R3. The LAW 7-style stderr emitted when `--competitors` has no list, no plan, no backend is reframed: leads with "hosting reasoning model, use your WebSearch to run Step 0.55 per entity and pass `--competitors-plan`." Does not lead with BRAVE_API_KEY.
|
|
||||||
- R4. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` OR `--competitors-plan` was passed. Signal: hosting model is driving and already has WebSearch.
|
|
||||||
- R5. Override-leak fix: competitor sub-runs do not inherit main topic's `--subreddits`, `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`. Sub-runs use only their own per-entity targeting (from `--competitors-plan` if provided, else engine-internal auto_resolve if backend, else planner defaults).
|
|
||||||
- R6. The `## Resolved Entities` block is also appended to the saved raw file when `--save-dir` is in use. Each entity's effective targeting (whatever was actually passed to its `pipeline.run()`) is visible on audit.
|
|
||||||
- R6b. When `--save-dir` is in use with a comparison run, each entity's sub-run ALSO saves its own standalone raw file — same format as a single-entity run. `/last30days Kanye West --competitors` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md` (one per entity) plus the merged comparison file. Matches the historical vs-mode behavior when it ran as N passes.
|
|
||||||
- R7. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
|
||||||
- R8. Default `--competitors` count remains 2 (3-way: main + 2 peers). Unchanged from 3.0.12.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No changes to `scripts/lib/fanout.py` architecture. N parallel pipelines stays. Only the data each sub-run receives changes.
|
|
||||||
- No changes to the vs-mode (topic contains "vs" / "versus") behavior. That path is independent.
|
|
||||||
- No new emit modes. Comparison output format unchanged.
|
|
||||||
- No deprecation of `--competitors-list`. Stays as the minimum escape hatch for hosting models that skip per-entity Step 0.55 (names-only).
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Cache layer for hosting-model competitor resolution: separate plan once cost evidence exists.
|
|
||||||
- Cross-source disambiguation beyond Polymarket: separate plan.
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py` — `--competitors` / `--competitors-list` argparse block, `resolve_competitors_args` validator, `_main_runner` closure, `_competitor_runner` closure, the `[Competitors] --competitors requires...` stderr block. Primary file for this plan.
|
|
||||||
- `scripts/lib/fanout.py` — `run_competitor_fanout` orchestrator. Signature unchanged; `_competitor_runner` closure now builds kwargs from `--competitors-plan`.
|
|
||||||
- `scripts/lib/pipeline.py` — `pipeline.run()` signature; no changes required (all per-entity flags already exist as kwargs).
|
|
||||||
- `scripts/lib/planner.py` — existing `--plan` parsing and validation, pattern to mirror for `--competitors-plan`.
|
|
||||||
- `scripts/lib/render.py` `_render_resolved_entities_block` (added in 3.0.12) — already reads `report.artifacts["resolved"]`; no change needed.
|
|
||||||
- `scripts/last30days.py` `save_output` / `render.render_full` — the save path. Needs to include the Resolved Entities block for comparison runs.
|
|
||||||
- `scripts/lib/quality_nudge.py` — where the BRAVE/SERPER footer nudge is emitted. Needs a context-aware suppression check.
|
|
||||||
- `scripts/lib/polymarket.py` — source adapter. Entry point for `--polymarket-keywords` filter and single-token-ambiguous auto-skip.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- 3.0.11 plan (`2026-04-22-002`): built the initial fanout, deferred per-entity resolve as "v1 simplification."
|
|
||||||
- 3.0.12 plan (`2026-04-22-003`): tried to close the gap via engine-internal `auto_resolve`. Works only with backend keys. Fails silently without.
|
|
||||||
- 2026-04-22 test session receipts: confirmed all four fixes in this plan are real, reproducible bugs.
|
|
||||||
- User's architectural steer 2026-04-22: "runs a full last30days on all 3 topics" — this plan encodes that explicitly as N full `pipeline.run()` calls with pre-resolved targeting per entity.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None. All patterns in-repo.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **`--competitors-plan` is a single JSON flag, not a fan of separate flags.** Mirrors `--plan`. Stable schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Accept inline JSON or a file path (matches `--plan`).
|
|
||||||
- **Hosting-model-driven resolution is the documented default.** Engine-internal `auto_resolve` is the headless / cron fallback. SKILL.md routes hosting models to the JSON-flag path; engine keeps auto_resolve alive for BRAVE/EXA/SERPER users running CI.
|
|
||||||
- **Override-leak fix is call-site scrubbing, not a signature change.** `_competitor_runner` builds an explicit kwargs dict per entity from `_subrun_kwargs(entity, plan_entry)`. No closure-default fallthrough from main scope. The 3.0.12 `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag.
|
|
||||||
- **Footer nudge becomes context-aware.** Suppressed when `--plan` or `--competitors-plan` present. Not suppressed for bare `--competitors-list` or bare invocations. Headless cron without keys still sees the nudge.
|
|
||||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known list of single-token-ambiguous names (states, common nouns). Stderr notes the skip so it is observable and overridable.
|
|
||||||
- **Per-entity sub-runs get the full `pipeline.run()` pass.** Same depth, same sources, same API cost per entity as a single-topic run. This is the explicit user intent — three full passes, not one merged pass.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **JSON or multi-flag?** JSON. Matches `--plan`.
|
|
||||||
- **Default count?** 2 peers (3-way comparison). Unchanged from 3.0.12.
|
|
||||||
- **Does engine-internal auto_resolve stay alive?** Yes, for entities not covered by `--competitors-plan` when a backend is configured. Headless/cron users with keys keep the current 3.0.12 behavior.
|
|
||||||
- **vs-mode or fanout?** Fanout. User's explicit ask: three full passes, not one merged pass. vs-mode merges into one pipeline with lower peer weighting, which is not what the user wants.
|
|
||||||
- **Does the save file need per-entity clusters?** Start with the Resolved block appended. Per-entity cluster sections can follow in a separate task; they are nice-to-have, not blocking.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Exact trace of override-leak source. Candidates: closure capture of `subreddits` in `_competitor_runner`, shared `_auto_resolve_context` leak, Reddit adapter inheriting global config. Test-first; trace at implementation time.
|
|
||||||
- Heuristic for "single-token-ambiguous topic" auto-skip. Start with a short hard-coded list (US state names, US city names, common nouns like "Warriors", "Suns", "Jets"); revisit after dogfood.
|
|
||||||
- Whether per-entity coverage warnings fire when `--competitors-plan` under-resolves an entity (e.g., only `x_handle`, no subreddits). Start with stderr logging; revisit UX.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: `--competitors-plan` JSON flag + per-entity kwargs threading**
|
|
||||||
|
|
||||||
**Goal:** New CLI flag accepting per-entity targeting JSON. Each covered entity's `pipeline.run()` receives its own `x_handle` / `x_related` / `subreddits` / `github_user` / `github_repos` / `context`. Skips engine-internal `auto_resolve` for covered entities.
|
|
||||||
|
|
||||||
**Requirements:** R1, R5 (primary leak fix site)
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner`)
|
|
||||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; verify)
|
|
||||||
- Test: `tests/test_cli_competitors.py` (extend)
|
|
||||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON OR a file path (mirror `--plan`).
|
|
||||||
- Validation: parse JSON; must be a dict; each value must be a dict; unknown fields log warnings; malformed input exits 2.
|
|
||||||
- Schema per entity: optional fields `x_handle` (str), `x_related` (list), `subreddits` (list), `github_user` (str), `github_repos` (list), `context` (str).
|
|
||||||
- Case-insensitive matching against `--competitors-list` / discovered entities.
|
|
||||||
- Build `_subrun_kwargs(entity, plan_entry)` helper. Returns a complete, explicit kwargs dict for `pipeline.run()` with no closure-default fallthrough from main scope. This helper is the single source of truth for per-entity call args. It also fixes the override-leak (R5) by scrubbing all per-entity flags to None unless the plan (or auto_resolve) sets them.
|
|
||||||
- `_competitor_runner(entity)`:
|
|
||||||
1. Look up `plan_entry` from `--competitors-plan` (if any).
|
|
||||||
2. If plan covers entity fully, build kwargs from it; skip `auto_resolve`.
|
|
||||||
3. If plan partially covers or is absent, fall back to `auto_resolve` (3.0.12 behavior) when a backend is configured. Plan values win over auto_resolve values on conflict.
|
|
||||||
4. If neither plan nor backend, fall through to `pipeline.run()` with per-entity kwargs all None — engine uses planner defaults for that entity only (no leak).
|
|
||||||
- Deep-copy config per sub-run (already done in 3.0.12); merge per-entity `context` into `entity_config["_auto_resolve_context"]` only.
|
|
||||||
|
|
||||||
**Execution note:** Test-first for the override-leak regression (pass `--subreddits=A,B` on main + a peer, assert peer's `pipeline.run(subreddits=...)` is None or peer-specific).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `--plan` parsing at `scripts/last30days.py` (inline JSON or file path).
|
|
||||||
- 3.0.12's `_competitor_runner` closure for scope; extract the kwargs-build into `_subrun_kwargs` helper.
|
|
||||||
- `entity_config = dict(config)` deep-copy pattern from 3.0.12.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors-plan '{"Drake": {"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's `pipeline.run` receives `x_handle="Drake"` and `subreddits=["Drizzy"]`; no `auto_resolve` call for Drake.
|
|
||||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered entities skip auto_resolve; third falls back to auto_resolve.
|
|
||||||
- Happy path: plan file path accepted like `--plan` file path.
|
|
||||||
- Happy path: case-insensitive entity match (`Drake` in plan, `drake` in list).
|
|
||||||
- Edge case: unknown fields in plan entry → logged, ignored, run continues.
|
|
||||||
- Edge case: plan entry for entity not in list → ignored with warning.
|
|
||||||
- Error path: malformed JSON → exit 2.
|
|
||||||
- Error path: top-level JSON is list not dict → exit 2.
|
|
||||||
- Regression (leak fix): main `--subreddits=A,B` + `--competitors-list "Drake"` + no plan → Drake's `pipeline.run` receives `subreddits=None` (no leak).
|
|
||||||
- Regression (leak fix): same for `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`.
|
|
||||||
- Regression (leak fix): main `--x-handle=kanyewest` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanyewest"`.
|
|
||||||
- Integration: full main + 2 peers run via `--competitors-plan`; assert each sub-run's effective kwargs match expected per-entity values.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- All new and regression tests pass.
|
|
||||||
- Smoke run (mock mode + `--competitors-plan`): stderr shows `[Competitors] Drake: x=@Drake subs=Drizzy` line per entity; no `[AutoResolve]` calls for plan-covered entities; no leak of main topic's flags.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: Reframe LAW 7-style stderr for hosting-model context**
|
|
||||||
|
|
||||||
**Goal:** When `--competitors` has no `--competitors-list`, no `--competitors-plan`, and no backend, stderr tells the hosting reasoning model to use its WebSearch tool for Step 0.55 per entity and pass `--competitors-plan`. Stops leading with BRAVE_API_KEY.
|
|
||||||
|
|
||||||
**Requirements:** R3
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1 (flag must exist)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (the existing `[Competitors] --competitors requires...` block)
|
|
||||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Rewrite stderr in this order:
|
|
||||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent runtime with a WebSearch tool), YOU should: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers), (c) assemble a `--competitors-plan` JSON, (d) re-invoke. Skip this step and quality degrades — peer entities will run with planner defaults."
|
|
||||||
2. "If you are running headless (cron, CI, no hosting model), set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
|
||||||
3. "Minimum escape hatch: `--competitors-list "A,B,C"` skips discovery but does not pre-resolve peers. Use only for quick tests."
|
|
||||||
- Exits non-zero as today.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing LAW 7 stderr in `planner.plan_query` for tone.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: stderr leads with "If you are the hosting reasoning model" and names `--competitors-plan` before any backend key.
|
|
||||||
- Happy path: stderr explicitly names `--competitors-plan` as the preferred override.
|
|
||||||
- Happy path: stderr does NOT say "requires either a configured web search backend OR an explicit --competitors-list" (the current 3.0.12 wording).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Test asserts ordering and required phrases.
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Suppress BRAVE/SERPER footer nudge when hosting-model-driven**
|
|
||||||
|
|
||||||
**Goal:** The `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY` footer is suppressed when `--plan` or `--competitors-plan` was passed (signal: hosting model is driving and already has WebSearch).
|
|
||||||
|
|
||||||
**Requirements:** R4
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever nudge is emitted; verify during implementation)
|
|
||||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Locate the nudge emission point.
|
|
||||||
- Add a suppression check: if `--plan` OR `--competitors-plan` was passed, skip the nudge. Otherwise, current behavior.
|
|
||||||
- Don't suppress the nudge for bare `--competitors-list` alone — that path isn't necessarily hosting-model-driven.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--plan` passed, no backend → nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-plan` passed, no backend → nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-list` only, no backend → nudge fires (current behavior).
|
|
||||||
- Happy path: no `--competitors`, no `--plan`, no backend → nudge fires (current behavior unchanged).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- All four scenarios produce expected nudge presence/absence.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: Per-entity save files + Resolved block in each**
|
|
||||||
|
|
||||||
**Goal:** When `--save-dir` is in use with a comparison run, each entity's sub-run saves its own standalone raw file (same format as a single-entity run), and each file includes the `## Resolved Entities` block so audits can see what targeting that entity received. Matches the historical vs-mode behavior when it was N passes.
|
|
||||||
|
|
||||||
**Requirements:** R6, R6b
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (`save_output`, the save loop after fanout completes)
|
|
||||||
- Possibly modify: `scripts/lib/render.py` (`render_full` branch to include Resolved block when artifact is present)
|
|
||||||
- Test: `tests/test_save_raw_competitor_files.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]`. For each `(entity, entity_report)` tuple, call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)` — same path a single-entity run takes.
|
|
||||||
- Each saved file uses its entity's slug as the filename (`drake-raw.md`, `kendrick-lamar-raw.md`). Main topic keeps the existing `kanye-west-raw.md` filename.
|
|
||||||
- Each file includes its own `## Resolved Entities` block (single-entity variant: one row for that entity only). This makes each sub-run's file self-describing — you can see what targeting was used without opening the comparison file.
|
|
||||||
- The merged comparison output (stdout) still includes the 3-row Resolved Entities block.
|
|
||||||
- Optional: also save a comparison summary file (e.g., `kanye-west-comparison-raw.md`) holding the merged multi-entity render. Start with per-entity files only; comparison summary is a follow-up if stdout-plus-individual-files is insufficient.
|
|
||||||
- Single-entity runs unchanged (no additional files, no block change).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing `save_output` invocation for single-entity runs (line 501 of current `scripts/last30days.py`).
|
|
||||||
- Existing slug generation (`slugify(topic)`) for filename consistency.
|
|
||||||
- `_render_resolved_entities_block` from 3.0.12 for the single-entity variant.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors-list "Drake,Kendrick Lamar"` + `--save-dir=/tmp/x` → `/tmp/x/kanye-west-raw.md`, `/tmp/x/drake-raw.md`, `/tmp/x/kendrick-lamar-raw.md` all exist.
|
|
||||||
- Happy path: each peer file's first sections include that entity's Resolved Entities block with its own row only.
|
|
||||||
- Happy path: single-entity run with `--save-dir` → one file, unchanged from today's behavior.
|
|
||||||
- Edge case: entity slug collides with existing file → overwrite (matches single-entity behavior).
|
|
||||||
- Edge case: `--save-suffix=v3` → all 3 files get the suffix (`kanye-west-raw-v3.md`, `drake-raw-v3.md`, `kendrick-lamar-raw-v3.md`).
|
|
||||||
- Edge case: comparison run with one peer whose sub-run failed → that entity's file is NOT saved; others are.
|
|
||||||
- Integration: stderr after save shows three `[last30days] Saved output to <path>` lines, one per entity.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- After `/last30days Kanye West --competitors-list "Drake,Kendrick Lamar" --save-dir=/tmp/x`: `ls /tmp/x/*-raw.md` shows 3 files. Each contains its entity's Resolved block.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: SKILL.md "Competitor mode" rewrite — hosting-model Step 0.55 canonical**
|
|
||||||
|
|
||||||
**Goal:** SKILL.md documents the hosting-model-driven path as canonical: discover N peers via WebSearch, run Step 0.55 per entity, assemble `--competitors-plan`, invoke engine. Engine-internal `auto_resolve` is labeled the headless fallback.
|
|
||||||
|
|
||||||
**Requirements:** R2
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1 (flag must exist before documented)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md` (Competitor mode subsection)
|
|
||||||
- Modify: `README.md` (one-line example update)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Replace the 3.0.12 Competitor mode subsection with a clear flow:
|
|
||||||
1. User invokes with `--competitors` or `--competitors=N`.
|
|
||||||
2. Hosting model runs WebSearch for "[topic] competitors" / "[topic] alternatives" → picks top N peers.
|
|
||||||
3. Hosting model runs Step 0.55 for main + each peer (x_handle, subreddits, github_user, github_repos, context) — same protocol as vs-mode per SKILL.md §679.
|
|
||||||
4. Hosting model assembles a `--competitors-plan` JSON object.
|
|
||||||
5. Hosting model invokes the engine with `--competitors-list "A,B,C" --competitors-plan '{...}'`.
|
|
||||||
6. Engine fans out N full pipelines (main + peers), each with its own full Step 0.55-grade targeting. Each entity also saves its own `*-raw.md` file when `--save-dir` is set (three full passes → three save files, matching the historical vs-mode behavior). Comparison output merges them for display.
|
|
||||||
- Concrete JSON example in SKILL.md showing the schema.
|
|
||||||
- Failure-mode warning: a `## Resolved Entities` block with dashes for any entity means hosting model skipped Step 0.55 for that one. Re-run with corrected plan.
|
|
||||||
- "Headless fallback" sub-subsection: when BRAVE/EXA/SERPER/PARALLEL/OPENROUTER is set, engine's internal `auto_resolve` handles peers and `--competitors-plan` is optional.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- SKILL.md "Step 0.55" section for per-entity resolve protocol.
|
|
||||||
- SKILL.md "If QUERY_TYPE = COMPARISON" section for the same-protocol-as-vs-mode reference.
|
|
||||||
- Tone of existing 3.0.12 Competitor mode prose.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — documentation. Verification is a fresh Claude Code window dogfood run.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `/last30days Kanye West --competitors` in a new window: hosting model does Step 0.55 for Kanye + 2 discovered peers; passes `--competitors-plan`; rendered Resolved block shows non-empty fields for all 3; top voices include at least one peer-specific handle.
|
|
||||||
|
|
||||||
- [ ] **Unit 6: Polymarket disambiguation guard**
|
|
||||||
|
|
||||||
**Goal:** Support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
|
||||||
|
|
||||||
**Requirements:** R7
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` argparse (`--polymarket-keywords`)
|
|
||||||
- Modify: `scripts/lib/polymarket.py`
|
|
||||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--polymarket-keywords "kw1,kw2"` flag. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
|
||||||
- Auto-skip rule: if topic is one token AND token matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords` provided, skip Polymarket with a stderr note.
|
|
||||||
- SKILL.md Step 0.55 protocol gets a small addition: for ambiguous topics, hosting model passes `--polymarket-keywords` with topic-specific qualifiers.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing Polymarket adapter match logic.
|
|
||||||
- Single-token detection heuristic.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr notes the skip.
|
|
||||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs; matches filtered.
|
|
||||||
- Happy path: topic "OpenAI" (no ambiguity) → Polymarket runs as before.
|
|
||||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
|
||||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Warriors smoke run → Polymarket footer absent OR filtered to nba/gsw markets.
|
|
||||||
|
|
||||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.13 to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** Closes R1-R7
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-6
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json`
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG entry groups the fixes: Added `--competitors-plan` JSON flag for per-entity hosting-model pre-resolve. Fixed override-leak from main into peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when `--plan` / `--competitors-plan` is present. Added: Resolved Entities block persists to saved raw file. Added: `--polymarket-keywords` + auto-skip for ambiguous single-token topics.
|
|
||||||
- Beta channel first per CLAUDE.md.
|
|
||||||
- Hot-copy so public `/last30days` picks up 3.0.13 immediately.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` returns 3.0.13.
|
|
||||||
- `sync.sh` exits 0.
|
|
||||||
- Hot-copy contains the new files with competitors.py, fanout.py, the updated SKILL.md, and plugin.json 3.0.13.
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** `_competitor_runner` becomes the single source of truth for sub-run kwargs via `_subrun_kwargs(entity, plan_entry)`. Every per-entity flag flows through one helper. No closure-default leaks.
|
|
||||||
- **Error propagation:** `--competitors-plan` JSON parse errors exit 2 with stderr (same as `--plan`). Per-entity plan entries with malformed values log warnings and fall back; don't abort the whole run.
|
|
||||||
- **State lifecycle risks:** `entity_config = dict(config)` already deep-copies for `_auto_resolve_context`; extend the isolation discipline to every per-entity flag. Verified in Unit 1 regression tests.
|
|
||||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors` and `--competitors-list` unchanged. `--plan` unchanged. `--polymarket-keywords` additive.
|
|
||||||
- **Integration coverage:** New regression tests for override-leak. New integration test for plan-driven sub-run threading. New nudge-suppression test. New Polymarket disambiguation test.
|
|
||||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. `planner.plan_query` LAW 7 behavior for the default path unchanged. Single-entity render path unchanged. vs-mode behavior unchanged.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Hosting model takes the lazy path and uses `--competitors-list` names-only. | Unit 2 stderr explicitly steers to `--competitors-plan` with Step 0.55 protocol named. Unit 5 SKILL.md docs. Resolved Entities dashes in output make the gap visible. |
|
|
||||||
| JSON gets verbose for the hosting model to construct repeatedly. | Schema is small (≤6 fields per entity). Hosting model already runs Step 0.55 for main topic in every comparison run; peers use the same protocol. One JSON block replaces N CLI flags. |
|
|
||||||
| Override-leak source is deeper than `_competitor_runner` closure. | Test-first per Unit 1. Receipts from 2026-04-22 Kanye run are reproducible. Trace methodically from call site. |
|
|
||||||
| Plan-covered entity bypasses auto_resolve but plan data is incomplete (e.g., no subreddits). | Hosting model's own SKILL.md contract says Step 0.55 must cover all fields. Stderr logs per-entity coverage so under-resolved entities are visible. Next-run correction, not engine-side rescue. |
|
|
||||||
| Polymarket auto-skip false-positives on legitimate ambiguous topics with real markets. | Conservative match (single-token + known list). `--polymarket-keywords` override is explicit and unambiguous. Stderr notes the skip. |
|
|
||||||
| Footer nudge suppression hides the message from headless users who genuinely need it. | Suppression only fires when `--plan` or `--competitors-plan` is present. Cron / CI runs that pass neither still see the nudge. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first per CLAUDE.md (private repo `/last30days-beta`).
|
|
||||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
|
||||||
- CHANGELOG voice should call this out as the feedback-driven follow-up to 3.0.12. Reader should see "we tried engine-internal resolve in 3.0.12; it needs backend keys we don't have; we moved resolution to the hosting model in 3.0.13."
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Origin plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
|
||||||
- Earlier plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
|
||||||
- 2026-04-22 test session receipts: Warriors, Seattle, Arizona Wildcats, Kanye West
|
|
||||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol
|
|
||||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
|
||||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
|
||||||
@@ -1,451 +0,0 @@
|
|||||||
---
|
|
||||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
|
||||||
type: feat
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
origin: docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded
|
|
||||||
---
|
|
||||||
|
|
||||||
# feat: vs mode runs N full passes and --competitors is vs with auto-discovery
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Architectural unification driven by user correction 2026-04-22: vs mode and `--competitors` are the same thing. A user typing `/last30days OpenAI vs Anthropic vs xAI` should get a full single-entity last30days pass for each of the three entities — three full pipelines, three saved `*-raw.md` files, merged into one comparison output. A user typing `/last30days OpenAI --competitors` should get the same output after the hosting model auto-picks 2 peers; i.e., `--competitors` is a thin shortcut that expands "topic + `--competitors`" into "topic vs peer1 vs peer2" and then runs the unified vs pipeline.
|
|
||||||
|
|
||||||
Current state diverges from this:
|
|
||||||
|
|
||||||
- **vs mode today**: one `pipeline.run()` with a comparison-optimized plan that merges all entities' targeting into a single retrieval pool. Lower-weight `--x-related` for peers, merged subreddits, cross-entity keyword noise. One saved file.
|
|
||||||
- **`--competitors` today (3.0.12)**: N parallel `pipeline.run()` calls via `scripts/lib/fanout.py`, but per-entity Step 0.55 depends on an engine-side web backend key Matt doesn't have. Silently degrades to planner defaults for peers. One saved file (main topic only). Override-leak from main into peers.
|
|
||||||
|
|
||||||
After this plan:
|
|
||||||
|
|
||||||
- **vs mode**: N parallel `pipeline.run()` calls, one per entity, each with its own full Step 0.55-grade targeting, each saving its own `*-raw.md`. Merged into one comparison output.
|
|
||||||
- **`--competitors`**: SKILL.md shortcut. Hosting model discovers N peers, builds `"topic vs peer1 vs peer2"`, and invokes the same vs pipeline. No separate orchestration path.
|
|
||||||
- **Same fanout machinery (`scripts/lib/fanout.py`)** serves both. One fix, both behaviors improve.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
The product insight from 2026-04-22 test runs is simple: the user wants three full last30days reports plus a comparison merge. Not one comparison pass with N-way targeting merged into a single retrieval pool. Not one save file. Not "main gets Step 0.55, peers get planner defaults." Three full passes. Three save files. Merged output.
|
|
||||||
|
|
||||||
The historical vs mode did that (it ran as 3 passes, saving 3 files). SKILL.md §551 currently says:
|
|
||||||
|
|
||||||
> "When the user asks 'X vs Y', run ONE research pass with a comparison-optimized plan that covers both entities AND their rivalry. This replaces the old 3-pass approach (which took 13+ minutes and produced tangential content)."
|
|
||||||
|
|
||||||
That change was a latency optimization that removed the user-visible behavior the user wants. The fix is to revert the architectural direction: N passes per entity, in parallel rather than serial (parallelism lowers wall-clock to ~1× a single pass, not N×), with per-entity save files.
|
|
||||||
|
|
||||||
The 3.0.11 `--competitors` flag already introduced parallel N-pass machinery (`fanout.run_competitor_fanout`). The 3.0.12 follow-up tried to wire per-entity Step 0.55 into it but failed when no web backend was configured. The elegant move: stop maintaining two architectures. vs-mode and `--competitors` both use `fanout.py`. `--competitors` becomes a SKILL.md-level shortcut that discovers 2 peers and hands off to vs-mode.
|
|
||||||
|
|
||||||
Four 2026-04-22 test receipts (Warriors, Seattle, Arizona Wildcats, Kanye West) all confirmed the user's pain points:
|
|
||||||
|
|
||||||
- Peers thin because they ran without per-entity handle/sub targeting.
|
|
||||||
- Only one `*-raw.md` per run — no per-entity audit.
|
|
||||||
- Kanye peers leaked main topic's `--subreddits`.
|
|
||||||
- Engine footer nudging `BRAVE_API_KEY` to Claude Code users who already have WebSearch.
|
|
||||||
- Polymarket noise on ambiguous topics (Warriors → Glasgow rugby; Arizona → Diamondbacks).
|
|
||||||
|
|
||||||
This plan closes all of them by unifying the architecture and making hosting-model-driven Step 0.55 per entity the canonical path.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. vs mode (any topic containing ` vs ` / ` versus `) runs N full `pipeline.run()` calls in parallel, one per entity. Each sub-run uses its entity's own Step 0.55 targeting (from the hosting model's pre-resolution, passed via a new `--competitors-plan` JSON).
|
|
||||||
- R2. `--competitors` (and `--competitors=N`) becomes a SKILL.md-level shortcut: the hosting model (a) discovers N peers via WebSearch, (b) runs Step 0.55 per entity (main + peers), (c) rewrites the topic to `"main vs peer1 vs peer2"`, (d) invokes the engine with `--competitors-plan` containing each entity's targeting.
|
|
||||||
- R3. New `--competitors-plan` JSON flag. Schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Implies vs mode when present with a single-entity topic. Applies per-entity targeting to each sub-run. Accepts inline JSON or a file path (matches `--plan`).
|
|
||||||
- R4. Each entity's sub-run saves its own `*-raw.md` file when `--save-dir` is in use. Example: `/last30days "Kanye West vs Drake vs Kendrick Lamar" --save-dir=~/Documents/Last30Days` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`. Same filenames a single-entity run of each topic would produce. Matches historical vs-mode behavior.
|
|
||||||
- R5. Each per-entity saved file includes its own single-row `## Resolved Entities` block so the audit survives. The merged comparison stdout still shows the full 3-row block.
|
|
||||||
- R6. Override-leak fix: no main-topic flags (`--subreddits`, `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`) leak into peer sub-runs. Every per-entity kwarg is scrubbed at the sub-run call site.
|
|
||||||
- R7. LAW 7-style stderr for `--competitors` invocations with no list, no plan, no backend is reframed for hosting-model context: leads with "use your WebSearch to discover peers, resolve Step 0.55 per entity, re-invoke with `topic vs peer1 vs peer2 --competitors-plan '...'`." Does not lead with BRAVE_API_KEY.
|
|
||||||
- R8. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` or `--competitors-plan` was passed.
|
|
||||||
- R9. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
|
||||||
- R10. Default `--competitors` count stays 2 peers (3-way comparison). Unchanged from 3.0.12.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No changes to single-entity `pipeline.run()` semantics. Each sub-run in vs mode behaves identically to a bare `/last30days {entity}` invocation.
|
|
||||||
- No changes to the planner's comparison-intent logic for single-entity-containing topics. The `_should_force_deterministic_plan` shortcut for vs-topics routes to fanout, not to its current single-pipeline path.
|
|
||||||
- No new emit modes. Comparison output format unchanged.
|
|
||||||
- No removal of `--competitors-list`. Stays as a minimum escape hatch (names-only, no per-entity targeting) for scripted headless use.
|
|
||||||
- No removal of engine-internal `resolve.auto_resolve()` in fanout. Remains as headless / cron fallback for users with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys. The dominant Claude Code path bypasses it via `--competitors-plan`.
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Explicit "head-to-head" rivalry pass in vs-mode (a supplemental subquery like `"A vs B"` that catches rivalry articles missing from pure entity-scoped passes). Start with N independent passes; add a head-to-head supplemental pass if the rivalry-content gap shows up in dogfood.
|
|
||||||
- Cache layer for hosting-model pre-resolution.
|
|
||||||
- Cross-source disambiguation (not just Polymarket).
|
|
||||||
- Latency knob for users who want the old one-pass vs behavior (probably not needed; parallel N-pass is ~1× wall clock).
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py` — main(), `_main_runner`, `_competitor_runner`, the competitor enable/discovery branch. Primary file.
|
|
||||||
- `scripts/lib/fanout.py` — existing orchestrator (3.0.11). Reused as-is; `competitor_runner` closure is where per-entity kwargs apply.
|
|
||||||
- `scripts/lib/planner.py` — `_should_force_deterministic_plan` detects vs-topics via regex. Current path synthesizes ONE comparison plan; new path routes to fanout.
|
|
||||||
- `scripts/lib/render.py` — `render_comparison_multi` (3.0.12) + `_render_resolved_entities_block`. Both reused. `render_full` needs a per-entity variant when saving sub-run files.
|
|
||||||
- `scripts/last30days.py` `save_output` — where raw files are written. Needs to iterate per entity when competitor_reports artifact present.
|
|
||||||
- `scripts/lib/quality_nudge.py` — BRAVE/SERPER nudge emission.
|
|
||||||
- `scripts/lib/polymarket.py` — source adapter for `--polymarket-keywords` and ambiguous-topic auto-skip.
|
|
||||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol — the hosting-model contract that drives per-entity pre-resolution for both vs mode and `--competitors`.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- 3.0.11 plan (`2026-04-22-002`): built fanout.
|
|
||||||
- 3.0.12 plan (`2026-04-22-003`): tried engine-internal per-entity auto_resolve; failed without backend keys.
|
|
||||||
- 3.0.13 plan draft (`2026-04-22-004-...superseded`): proposed `--competitors-plan` JSON + vs-mode-shortcut path but kept them separate. User's 2026-04-22 correction unifies them.
|
|
||||||
- 2026-04-22 test receipts: Warriors, Seattle, Arizona Wildcats, Kanye West runs all reproduced the per-entity resolve gap.
|
|
||||||
- User's architectural steer: "vs mode should work that way too" + "--competitors is just vs mode with auto-discovery." This plan encodes that.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None. All patterns in-repo.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Unify vs-mode and --competitors on one orchestrator.** `fanout.run_competitor_fanout` serves both. vs-mode is "topic contains ' vs '" detection → fanout. `--competitors` is "SKILL.md shortcut → hosting model rewrites topic to vs form → fanout." One code path.
|
|
||||||
- **Per-entity targeting via `--competitors-plan` JSON.** Schema `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Mirrors `--plan`. Applies to both vs-mode and `--competitors` paths. Hosting model passes it after running Step 0.55 per entity.
|
|
||||||
- **N save files, one per entity.** Each sub-run writes a `{entity-slug}-raw.md` file when `--save-dir` is set. Matches historical vs-mode behavior. Single-entity runs unchanged.
|
|
||||||
- **Revert the "one pass for latency" optimization that removed per-entity passes.** Parallel execution via `ThreadPoolExecutor` means wall-clock is ~max(per-entity-latency), not sum. The old latency concern (13+ minutes for 3 serial passes) does not apply to a parallel fan-out.
|
|
||||||
- **Override-leak fix at the call site.** `_subrun_kwargs(entity, plan_entry)` helper returns fully explicit per-entity kwargs; no closure-default fallthrough from main scope.
|
|
||||||
- **LAW 7 stderr reframed, not just updated.** Current message treats BRAVE_API_KEY as the solution. New message treats hosting-model Step 0.55 as the solution, with backend keys listed only as the headless fallback.
|
|
||||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known-ambiguous single-token list.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **vs mode N passes or single-pass?** N passes. User's architectural correction.
|
|
||||||
- **Should --competitors still be an engine flag at all?** Yes, kept for headless / cron contexts with backend keys. Dominant Claude Code path is SKILL.md shortcut → vs-mode fanout. Engine flag stays as compatibility surface.
|
|
||||||
- **`--competitors-plan` JSON or multi-flag?** JSON. Matches `--plan`.
|
|
||||||
- **Default count?** 2 peers → 3-way comparison. Unchanged.
|
|
||||||
- **Saved-file naming?** `{entity-slug}-raw.md` per entity, same as single-entity runs would produce.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Exact trace of override-leak path (closure capture vs shared config vs Reddit adapter fallback). Test-first per Unit 2; patch at the right layer.
|
|
||||||
- Heuristic for single-token-ambiguous Polymarket auto-skip. Start with a short hard-coded list; iterate.
|
|
||||||
- Whether to include a head-to-head rivalry supplemental pass in vs-mode. Ship N-independent passes first; revisit after dogfood if rivalry content is missing.
|
|
||||||
- Exact filename convention when the comparison merged output is saved (if saved at all). Not blocking — per-entity files are the primary save artifact.
|
|
||||||
|
|
||||||
## High-Level Technical Design
|
|
||||||
|
|
||||||
> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.*
|
|
||||||
|
|
||||||
```
|
|
||||||
User invokes:
|
|
||||||
/last30days "OpenAI vs Anthropic vs xAI"
|
|
||||||
OR
|
|
||||||
/last30days OpenAI --competitors (hosting model rewrites to vs form)
|
|
||||||
OR
|
|
||||||
/last30days OpenAI --competitors-list "Anthropic,xAI"
|
|
||||||
OR
|
|
||||||
/last30days "OpenAI vs Anthropic vs xAI" --competitors-plan '{...per-entity...}'
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
scripts/last30days.py main():
|
|
||||||
- Detect: topic has " vs " OR --competitors enabled
|
|
||||||
- If --competitors and no list/plan: emit LAW 7-style stderr with hosting-model instruction
|
|
||||||
- If --competitors with list or discovery: rewrite topic to vs form, continue
|
|
||||||
- Parse --competitors-plan JSON, map to entities
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
fanout.run_competitor_fanout (shared path):
|
|
||||||
- For each entity (main + peers):
|
|
||||||
- entity_config = dict(config) [deep copy to prevent leak]
|
|
||||||
- kwargs = _subrun_kwargs(entity, plan_entry) [explicit; no main-topic leak]
|
|
||||||
- If plan_entry missing a field AND backend available: auto_resolve() fill
|
|
||||||
- pipeline.run(topic=entity, **kwargs, internal_subrun=True)
|
|
||||||
- Parallel ThreadPoolExecutor
|
|
||||||
- Collect per-entity Reports
|
|
||||||
- Attach resolved targeting to each Report.artifacts["resolved"]
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
scripts/last30days.py after fanout:
|
|
||||||
- If --save-dir: save each entity's Report as {entity-slug}-raw.md
|
|
||||||
Each file includes its own single-row Resolved Entities block
|
|
||||||
- emit_comparison_output → render_comparison_multi (merged stdout)
|
|
||||||
Includes full N-row Resolved Entities block
|
|
||||||
```
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: vs-topic detection routes to fanout (not single-pipeline)**
|
|
||||||
|
|
||||||
**Goal:** A topic containing ` vs ` / ` versus ` triggers `fanout.run_competitor_fanout` with the parsed entities. Each entity runs a full `pipeline.run()`. Replace the current single-pipeline-with-comparison-plan behavior.
|
|
||||||
|
|
||||||
**Requirements:** R1
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (main() — detect vs-topic, route to fanout)
|
|
||||||
- Modify: `scripts/lib/planner.py` (remove / bypass the `_should_force_deterministic_plan` special case for vs topics; vs topics no longer go through `plan_query` as a single comparison plan)
|
|
||||||
- Test: `tests/test_vs_mode_fanout.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Parse the incoming topic: if it contains ` vs ` or ` versus ` (case-insensitive), split into entities (reuse `planner._comparison_entities`-style logic or move that utility into main()).
|
|
||||||
- When vs-entities are detected, route to the same fanout branch `--competitors` uses today. The entity list comes from the topic string; no discovery step needed.
|
|
||||||
- Each entity runs `pipeline.run()` with its own plan (either from `--competitors-plan[entity]` or from the engine's per-entity fallback path).
|
|
||||||
- For back-compat, if the user passes both a vs-topic AND `--plan`, honor `--plan` for the main (first) entity and use per-entity defaults for peers unless `--competitors-plan` is also provided.
|
|
||||||
|
|
||||||
**Execution note:** Start with an integration test that runs `"A vs B"` via mock mode and asserts fanout was called with two entities + two pipeline.run calls.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- 3.0.11 fanout wiring in `scripts/last30days.py`'s `--competitors` branch.
|
|
||||||
- `planner._comparison_entities` for the split logic.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: topic `"A vs B"` → two pipeline.run calls, two Reports returned, merged render.
|
|
||||||
- Happy path: topic `"A vs B vs C"` → three pipeline.run calls.
|
|
||||||
- Happy path: topic `"A versus B"` → matches the same regex, two pipelines.
|
|
||||||
- Edge case: topic `"OpenAI vs"` (trailing empty entity) → treated as single-entity `"OpenAI"`, not vs mode.
|
|
||||||
- Edge case: topic contains "vs." (dot, no trailing space) → existing regex tolerates it; verify.
|
|
||||||
- Edge case: topic `"A vs B"` plus `--plan` → plan applies to first entity only, peers use per-entity defaults.
|
|
||||||
- Integration: full vs-mode run end-to-end in mock mode; verify rendered output, stderr has one `[Competitors] Comparing: A vs B vs ...` line.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Test assertions pass.
|
|
||||||
- Mock-mode smoke of `/last30days "OpenAI vs Anthropic"` shows fanout invocation, per-entity Reports, merged comparison output.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: `--competitors-plan` JSON flag + `_subrun_kwargs` helper + override-leak fix**
|
|
||||||
|
|
||||||
**Goal:** New JSON flag threads per-entity targeting into each sub-run's `pipeline.run()`. A `_subrun_kwargs(entity, plan_entry)` helper is the single source of truth for per-entity kwargs, eliminating override-leak.
|
|
||||||
|
|
||||||
**Requirements:** R3, R6
|
|
||||||
|
|
||||||
**Dependencies:** None (can land alongside or before Unit 1)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner` + `_subrun_kwargs` helper)
|
|
||||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; the competitor_runner contract is unchanged)
|
|
||||||
- Test: `tests/test_cli_competitors.py` (extend)
|
|
||||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
|
||||||
- Test: `tests/test_competitor_subrun_isolation.py` (new, regression)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON or file path (mirror `--plan`).
|
|
||||||
- Validation: top-level dict; each value is a dict; unknown fields log warnings; malformed input exits 2. Case-insensitive entity matching.
|
|
||||||
- Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`.
|
|
||||||
- Build `_subrun_kwargs(entity, plan_entry)` — returns an explicit dict with every per-entity flag. No closure-default fallthrough. This is the leak fix.
|
|
||||||
- `_competitor_runner(entity)`:
|
|
||||||
1. Get `plan_entry` from `--competitors-plan` if present.
|
|
||||||
2. Build base kwargs with `_subrun_kwargs(entity, plan_entry)`.
|
|
||||||
3. Fill missing fields via `resolve.auto_resolve(entity, entity_config)` only if backend is configured (3.0.12 fallback path).
|
|
||||||
4. Call `pipeline.run(topic=entity, internal_subrun=True, **kwargs)`.
|
|
||||||
5. Attach `resolved` dict to `report.artifacts`.
|
|
||||||
- Verify no per-entity flag from main() leaks via closure. The helper is the only source of per-entity values.
|
|
||||||
|
|
||||||
**Execution note:** Test-first for the override-leak regression. Use the Kanye 2026-04-22 receipt as the failing test input (main `--subreddits=Kanye,hiphopheads` + `--competitors-list "Drake"` → assert Drake's pipeline.run receives `subreddits=None`).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `--plan` parsing block in `scripts/last30days.py`.
|
|
||||||
- 3.0.12's `entity_config = dict(config)` deep-copy pattern.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors-plan '{"Drake":{"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's pipeline.run receives `x_handle="Drake"`, `subreddits=["Drizzy"]`. No auto_resolve call for Drake.
|
|
||||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered skip auto_resolve; third falls back.
|
|
||||||
- Happy path: plan file path accepted like `--plan`.
|
|
||||||
- Happy path: case-insensitive entity match.
|
|
||||||
- Edge case: unknown fields → warn, ignore.
|
|
||||||
- Edge case: plan entry for entity not in list → warn, ignore.
|
|
||||||
- Error path: malformed JSON → exit 2.
|
|
||||||
- Error path: top-level JSON is list → exit 2.
|
|
||||||
- Regression (leak): main `--subreddits=A,B` + `--competitors-list "X"` + no plan → X's pipeline.run gets `subreddits=None`.
|
|
||||||
- Regression (leak): same for `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`.
|
|
||||||
- Regression (leak): main `--x-handle=kanye` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanye"`.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- All regression tests pass.
|
|
||||||
- Smoke run (mock mode + plan): stderr shows per-entity `[Competitors] {entity}: x=... subs=...` line; no leak from main topic's flags.
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Per-entity save files**
|
|
||||||
|
|
||||||
**Goal:** When `--save-dir` is set in a vs-mode or `--competitors` run, each entity's sub-run saves its own `{entity-slug}-raw.md` file — same format as a single-entity run would produce.
|
|
||||||
|
|
||||||
**Requirements:** R4, R5
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1, Unit 2
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (`save_output` iteration after fanout)
|
|
||||||
- Modify: `scripts/lib/render.py` (`render_full` includes single-row Resolved Entities block when that entity's `artifacts["resolved"]` is present)
|
|
||||||
- Test: `tests/test_save_raw_per_entity.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]` (or equivalent). For each `(entity, entity_report)`:
|
|
||||||
- Call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)`.
|
|
||||||
- Uses entity's `slugify(entity)` for the filename. Same pattern a single-entity run uses.
|
|
||||||
- Each saved file invokes `render_full` (or the save-variant). `render_full` now checks for `report.artifacts["resolved"]` and prepends a single-row Resolved Entities block.
|
|
||||||
- Stderr logs one `[last30days] Saved output to <path>` line per entity.
|
|
||||||
- Single-entity runs unchanged (no extra files, render_full unchanged for them).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing `save_output` invocation in main() for single-entity runs.
|
|
||||||
- `slugify(topic)` for filename.
|
|
||||||
- 3.0.12's `_render_resolved_entities_block` (reused, single-row mode).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `/last30days "A vs B vs C" --save-dir=/tmp/x` → `/tmp/x/a-raw.md`, `/tmp/x/b-raw.md`, `/tmp/x/c-raw.md` exist.
|
|
||||||
- Happy path: `--competitors-list "Drake,Kendrick" --save-dir=/tmp/x` on topic Kanye → three files: `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`.
|
|
||||||
- Happy path: each file includes a single-row Resolved Entities block for its entity.
|
|
||||||
- Happy path: single-entity run with `--save-dir` → one file, no Resolved block (unchanged).
|
|
||||||
- Edge case: `--save-suffix=v3` → all N files get the suffix.
|
|
||||||
- Edge case: one entity sub-run failed → its file is NOT saved; the others are.
|
|
||||||
- Integration: `ls {save-dir}/*-raw.md` returns N files after a vs-mode run.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Test assertions pass.
|
|
||||||
- Manual vs-mode smoke saves N files.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: LAW 7-style stderr reframe + footer-nudge suppression**
|
|
||||||
|
|
||||||
**Goal:** The `--competitors`-with-no-backend stderr tells the hosting model to do Step 0.55 per entity and pass `--competitors-plan`. The BRAVE/SERPER footer nudge is suppressed when `--plan` or `--competitors-plan` is present.
|
|
||||||
|
|
||||||
**Requirements:** R7, R8
|
|
||||||
|
|
||||||
**Dependencies:** Unit 2 (flag must exist)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (the `[Competitors] --competitors requires...` stderr block)
|
|
||||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever footer nudge emits; verify during implementation)
|
|
||||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
|
||||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Rewrite stderr in this order:
|
|
||||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent with WebSearch), the recommended path: (a) discover N peers via WebSearch, (b) run Step 0.55 for main + each peer, (c) re-invoke as `/last30days 'topic vs peer1 vs peer2' --competitors-plan '{...}'`. See SKILL.md 'Competitor mode'."
|
|
||||||
2. "Headless / cron path: set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
|
||||||
3. "Minimum escape hatch: `--competitors-list 'A,B,C'` skips discovery but does not pre-resolve peers."
|
|
||||||
- Suppress footer nudge when `external_plan` OR `competitors_plan` was passed.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors` with no backend, no list, no plan → stderr leads with "If you are the hosting reasoning model" and references `--competitors-plan` before naming API keys.
|
|
||||||
- Happy path: `--plan` passed → footer nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-plan` passed → footer nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-list` only (no plan, no backend) → footer nudge still fires (hosting model didn't fully engage).
|
|
||||||
- Happy path: no `--competitors`, no `--plan` → footer nudge unchanged.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Tests pass.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: Polymarket disambiguation guard**
|
|
||||||
|
|
||||||
**Goal:** `--polymarket-keywords "kw1,kw2"` filters market matches; auto-skip Polymarket on single-token-ambiguous topics without override.
|
|
||||||
|
|
||||||
**Requirements:** R9
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (argparse)
|
|
||||||
- Modify: `scripts/lib/polymarket.py`
|
|
||||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--polymarket-keywords "kw1,kw2"`. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
|
||||||
- Auto-skip: if topic is one token AND matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords`, skip Polymarket with stderr note.
|
|
||||||
- SKILL.md update (small): mention `--polymarket-keywords` in Step 0.55 instructions for ambiguous topics.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr note.
|
|
||||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs, filtered.
|
|
||||||
- Happy path: topic "OpenAI" → Polymarket runs as before.
|
|
||||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
|
||||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Warriors smoke → Polymarket footer absent or filtered.
|
|
||||||
|
|
||||||
- [ ] **Unit 6: SKILL.md rewrite — vs mode is the canonical path, `--competitors` is a shortcut**
|
|
||||||
|
|
||||||
**Goal:** SKILL.md documents the unified architecture. vs mode runs N full passes. `--competitors` is a SKILL.md-level shortcut that discovers 2 peers and invokes vs mode with `--competitors-plan`.
|
|
||||||
|
|
||||||
**Requirements:** R1, R2, R10 (surfaces them)
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-4
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md` (§551 "If QUERY_TYPE = COMPARISON" rewrite; Competitor mode subsection rewrite)
|
|
||||||
- Modify: `README.md` (one-line example)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Rewrite §551 to describe the N-pass architecture: "When the user asks 'X vs Y' (or 'X vs Y vs Z'), run Step 0.55 per entity, then invoke the engine. The engine fans out N full pipelines in parallel. Each entity gets its own single-entity-grade coverage. Wall clock is close to a single run."
|
|
||||||
- Remove the "ONE research pass with a comparison-optimized plan that replaces the old 3-pass approach" language.
|
|
||||||
- Add a `--competitors-plan` JSON example.
|
|
||||||
- Rewrite the Competitor mode subsection: "`--competitors` is a shortcut. The hosting model: (1) runs WebSearch to discover N=2 peers, (2) runs Step 0.55 for main + each peer, (3) rewrites topic to `'main vs peer1 vs peer2'`, (4) invokes engine with `--competitors-plan '{...}'`. Engine flag `--competitors` and `--competitors-list` remain for headless fallback."
|
|
||||||
- Cross-reference §679 (per-entity Step 0.55 protocol).
|
|
||||||
- Warning: a thin `## Resolved Entities` block (dashes for any entity) means the hosting model skipped Step 0.55 for that one.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing §679 per-entity Step 0.55 protocol for tone.
|
|
||||||
- 3.0.12 Competitor mode prose for terseness.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — documentation. Verification is dogfood.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `/last30days "OpenAI vs Anthropic vs xAI"` in a fresh Claude Code window produces 3 save files with populated Resolved blocks and non-dash per-entity targeting.
|
|
||||||
- `/last30days OpenAI --competitors` produces same after discovery step.
|
|
||||||
|
|
||||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.13 to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** Closes R1-R10
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-6
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json`
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG: group the changes. "Changed: vs mode now runs N full passes in parallel, one per entity — reverting the one-pass optimization to restore per-entity depth. Added: --competitors-plan JSON for per-entity Step 0.55 targeting (applies to vs mode and --competitors). Changed: --competitors is now a SKILL.md shortcut for vs-with-discovery. Added: per-entity *-raw.md save files. Fixed: override-leak from main to peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when --plan / --competitors-plan present. Added: --polymarket-keywords + auto-skip for ambiguous topics."
|
|
||||||
- Beta channel first per CLAUDE.md.
|
|
||||||
- Hot-copy so public `/last30days` picks up 3.0.13.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` → 3.0.13.
|
|
||||||
- `sync.sh` exits 0.
|
|
||||||
- Hot-copy contains the new files.
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** vs-mode and `--competitors` share one orchestrator (`fanout.run_competitor_fanout`). `_subrun_kwargs` is the single source of per-entity kwargs. Save loop iterates per entity.
|
|
||||||
- **Error propagation:** Per-entity sub-run failure → logged, dropped, continue (3.0.11 behavior unchanged). `--competitors-plan` JSON parse errors exit 2 (same shape as `--plan`).
|
|
||||||
- **State lifecycle risks:** `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag (Unit 2 fix). No cross-entity context leak.
|
|
||||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors`, `--competitors-list`, `--plan` unchanged. `--polymarket-keywords` additive. vs-mode keeps its topic-string surface.
|
|
||||||
- **Integration coverage:** New vs-mode-fanout integration test. New override-leak regression test. New plan-threading test. New nudge-suppression test. New per-entity-save test. New Polymarket disambiguation test.
|
|
||||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. Single-entity render path unchanged. LAW 7 on the default path unchanged (still fires when a single-entity run lacks `--plan`).
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| vs-mode N-pass latency feels slower for users who remember the one-pass shortcut. | Parallel execution keeps wall-clock ~= max(per-entity-latency), not sum. `--quick` on a vs-topic still applies to each sub-run. CHANGELOG calls out the revert + parallelism. |
|
|
||||||
| API cost scales linearly with N (per source). | Default count 2 caps it. Hard max 6 on `--competitors`. vs-mode users opted into N entities explicitly. |
|
|
||||||
| Rivalry content ("A vs B" articles) missed in N-independent passes. | Deferred to separate task (head-to-head supplemental pass). Start shipping and observe whether this is actually a gap. |
|
|
||||||
| Hosting model skips `--competitors-plan` and uses `--competitors-list` only. | Unit 4 stderr reframe steers explicitly. SKILL.md Unit 6 makes the plan-path canonical. Thin Resolved block in output makes skipped-Step-0.55 visible. |
|
|
||||||
| Override-leak fix misses a subtle closure path. | Unit 2 is test-first with the Kanye receipt as the failing input. Regression test asserts every per-entity flag is None unless plan provides it. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first per CLAUDE.md.
|
|
||||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
|
||||||
- CHANGELOG explicitly frames the vs-mode change as an architectural revert-with-parallelism, not a regression to the old serial N-pass.
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Superseded plan: `docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded`
|
|
||||||
- Previous plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
|
||||||
- Initial plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
|
||||||
- 2026-04-22 test session receipts (Warriors, Seattle, Arizona Wildcats, Kanye West)
|
|
||||||
- SKILL.md §551 + §679 — the per-entity Step 0.55 protocol the hosting model uses for both paths
|
|
||||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/planner.py` vs-topic special-case, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
|
||||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
---
|
|
||||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
|
||||||
type: fix
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
---
|
|
||||||
|
|
||||||
# fix: comparison title says (/Last30Days) instead of (Last 30 Days)
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
User feedback 2026-04-22 on the 3.0.13 release runs (Kanye vs Drake, Mercer Island, Figma): the comparison title currently reads `# Kanye West vs Drake: What the Community Says (Last 30 Days)`. It should read `# Kanye West vs Drake: What the Community Says (/Last30Days)` — attributing the output to the slash command rather than describing the date range generically.
|
|
||||||
|
|
||||||
Single-line change in SKILL.md, three occurrences. No code change.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. Comparison title pattern in SKILL.md changes from `(Last 30 Days)` to `(/Last30Days)` so synthesis outputs read `... What the Community Says (/Last30Days)`.
|
|
||||||
- R2. Both the rule statement (line 113) and the COMPARISON-exception statement (line 131) and the synthesis template example (line 1208) all use the new suffix.
|
|
||||||
- R3. Version bumps to 3.0.14, CHANGELOG entry, sync, hot-copy. Public cache picks up the new title pattern.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No changes to the single-entity output title (no `(/Last30Days)` suffix there — only comparison topics carry it).
|
|
||||||
- No changes to engine code. Pure SKILL.md content.
|
|
||||||
- No changes to anything else surfaced in the test runs.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Replace all three occurrences of the suffix string in one pass.** They are identical strings; changing one without the others would cause synthesis-time confusion when the model reaches a different reference.
|
|
||||||
- **Ship as 3.0.14, not 3.0.13.x.** Patch-level bump matches the small scope and keeps the release log clean.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: Replace `(Last 30 Days)` → `(/Last30Days)` in SKILL.md**
|
|
||||||
|
|
||||||
**Goal:** All three SKILL.md references to the comparison title use the new suffix.
|
|
||||||
|
|
||||||
**Requirements:** R1, R2
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- `replace_all` swap of `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Three occurrences, no other strings overlap.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — pure documentation. Verification by inspection + dogfood run.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep -c "What the Community Says (/Last30Days)" SKILL.md` returns 3.
|
|
||||||
- `grep -c "What the Community Says (Last 30 Days)" SKILL.md` returns 0.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: Version 3.0.14 + CHANGELOG + sync + hot-copy**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.14 to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** R3
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json`
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.14/`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG: "Changed: comparison-mode title attribution — `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range."
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` → 3.0.14.
|
|
||||||
- Hot-copy contains the updated SKILL.md.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Hosting model has the old title pattern memorized from a prior run and re-emits `(Last 30 Days)`. | SKILL.md is read top-to-bottom each invocation. STEP 0 canonical-path self-check (3.0.12) ensures the model loads the new SKILL.md, not the marketplace stale copy. |
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- 2026-04-22 dogfood runs (Kanye West vs Drake, Mercer Island --competitors, Figma --competitors)
|
|
||||||
- Related code: `SKILL.md` lines 113, 131, 1208
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
# Mirror of skills/last30days/scripts/, populated by scripts/sync-engine.sh.
|
|
||||||
# Source of truth lives in the Python skill; never commit the mirror.
|
|
||||||
# Lives inside internal/engine/ because //go:embed cannot reach outside
|
|
||||||
# its own package directory.
|
|
||||||
internal/engine/vendored/*
|
|
||||||
!internal/engine/vendored/.gitkeep
|
|
||||||
|
|
||||||
# Local build output: cross-compiled binaries and packaged .mcpb files.
|
|
||||||
build/
|
|
||||||
# Anchor to the mcp/ root so the cmd/last30days-pp-mcp/ package directory
|
|
||||||
# is not also excluded (subdirs with the same name would otherwise match).
|
|
||||||
/last30days-pp-mcp
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
# last30days-pp-mcp
|
|
||||||
|
|
||||||
Go MCP server that wraps the last30days Python engine for Claude Desktop. Packaged as a `.mcpb` bundle (drag-drop install into Claude Desktop).
|
|
||||||
|
|
||||||
The MCP server exposes a single `research` tool that mirrors the `/last30days <topic>` slash command available in Claude Code. At runtime the binary extracts the vendored Python engine into a per-user cache and shells out to `python3` to produce the synthesis input Claude renders.
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
- `cmd/last30days-pp-mcp/` - server entry point
|
|
||||||
- `internal/engine/` - `embed.FS` of the Python engine + cache extractor + subprocess wrapper
|
|
||||||
- `internal/tools/` - MCP tool handlers (currently `research`)
|
|
||||||
- `internal/engine/vendored/` - mirror of `skills/last30days/scripts/`, generated by `scripts/sync-engine.sh` (gitignored). Lives inside the engine package because `//go:embed` cannot reach files outside its own package directory.
|
|
||||||
- `manifest.json` - MCPB v0.3 manifest consumed by Claude Desktop and `printing-press bundle`
|
|
||||||
|
|
||||||
## Local build
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Mirror the Python engine into vendored/.
|
|
||||||
bash scripts/sync-engine.sh
|
|
||||||
|
|
||||||
# Build for the current host.
|
|
||||||
go build -ldflags "-X main.Version=dev" -o build/last30days-pp-mcp ./cmd/last30days-pp-mcp
|
|
||||||
|
|
||||||
# Package as a .mcpb (requires the printing-press binary on PATH).
|
|
||||||
printing-press bundle . --skip-build --binary build/last30days-pp-mcp
|
|
||||||
```
|
|
||||||
|
|
||||||
The output `.mcpb` lands at `build/last30days-pp-mcp-<os>-<arch>.mcpb`. Drag it into Claude Desktop's Extensions panel to install.
|
|
||||||
|
|
||||||
## Runtime requirements
|
|
||||||
|
|
||||||
End users need Python 3.12+ on PATH. The bundle ships the engine source but relies on the host interpreter.
|
|
||||||
|
|
||||||
## Versioning
|
|
||||||
|
|
||||||
The MCPB `manifest.json` version is hand-bumped in the same PR that ships engine changes worth releasing. Release CI stamps the Go binary's `main.Version` from the tag.
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
// Package main is the entry point for the last30days MCP server bundled
|
|
||||||
// as a .mcpb for Claude Desktop. The server registers a single research
|
|
||||||
// tool (see internal/tools) and serves it over stdio. See mcp/README.md
|
|
||||||
// for build and packaging instructions.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/mark3labs/mcp-go/server"
|
|
||||||
|
|
||||||
"github.com/mvanhorn/last30days-skill/mcp/internal/tools"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Version is stamped at build time via -ldflags "-X main.Version=<tag>".
|
|
||||||
// It namespaces the per-user cache directory in internal/engine so multiple
|
|
||||||
// installed versions can coexist without clobbering each other.
|
|
||||||
var Version = "dev"
|
|
||||||
|
|
||||||
const (
|
|
||||||
serverName = "last30days"
|
|
||||||
serverVersion = "1"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
s := server.NewMCPServer(
|
|
||||||
serverName,
|
|
||||||
serverVersion,
|
|
||||||
server.WithToolCapabilities(false),
|
|
||||||
)
|
|
||||||
|
|
||||||
tools.Register(s, tools.Config{Version: Version})
|
|
||||||
|
|
||||||
if err := server.ServeStdio(s); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "last30days-pp-mcp: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
module github.com/mvanhorn/last30days-skill/mcp
|
|
||||||
|
|
||||||
go 1.25.5
|
|
||||||
|
|
||||||
require github.com/mark3labs/mcp-go v0.54.0
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/google/jsonschema-go v0.4.2 // indirect
|
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
|
||||||
github.com/spf13/cast v1.7.1 // indirect
|
|
||||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
|
||||||
golang.org/x/text v0.14.0 // indirect
|
|
||||||
)
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
|
||||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
|
||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
|
||||||
github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
|
|
||||||
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
|
||||||
github.com/mark3labs/mcp-go v0.54.0 h1:PZhQvd+5xrT43cUoiaKn/hDcvLUhcLc1twSEKYPTcTA=
|
|
||||||
github.com/mark3labs/mcp-go v0.54.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
||||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
|
||||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
|
||||||
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
|
|
||||||
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
|
||||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
|
||||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
|
||||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
// Package engine wraps the vendored Python last30days engine. The engine
|
|
||||||
// is embedded at build time via //go:embed and extracted into a per-user
|
|
||||||
// cache directory on first use, then invoked through python3 in a
|
|
||||||
// subprocess. Consumers should call EnsureUserCache to materialize the
|
|
||||||
// engine and Run to execute it.
|
|
||||||
package engine
|
|
||||||
|
|
||||||
import (
|
|
||||||
"embed"
|
|
||||||
"io/fs"
|
|
||||||
)
|
|
||||||
|
|
||||||
// EngineSourceDir is the embed root inside the binary. scripts/sync-engine.sh
|
|
||||||
// mirrors skills/last30days/scripts/ into this directory before each build.
|
|
||||||
// The all: prefix preserves files starting with "." or "_" so the .gitkeep
|
|
||||||
// anchor file survives - without it the embed would error before sync runs.
|
|
||||||
//
|
|
||||||
//go:embed all:vendored
|
|
||||||
var vendored embed.FS
|
|
||||||
|
|
||||||
// EngineFS returns the embedded engine as a filesystem rooted at the
|
|
||||||
// vendored/ directory contents (so callers see "last30days.py" at the
|
|
||||||
// root, not "vendored/last30days.py").
|
|
||||||
func EngineFS() (fs.FS, error) {
|
|
||||||
return fs.Sub(vendored, "vendored")
|
|
||||||
}
|
|
||||||
@@ -1,168 +0,0 @@
|
|||||||
package engine
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SentinelFilename names the file Ensure writes inside the cache directory
|
|
||||||
// after a successful extraction. Its contents are compared to the requested
|
|
||||||
// version; a match short-circuits re-extraction on subsequent calls.
|
|
||||||
const SentinelFilename = ".version"
|
|
||||||
|
|
||||||
// cacheSubdir namespaces our cache under the OS user cache directory so
|
|
||||||
// multiple printing-press-style bundles can coexist.
|
|
||||||
const cacheSubdir = "last30days-pp-mcp"
|
|
||||||
|
|
||||||
// CacheEnvOverride lets users redirect the cache directory when the default
|
|
||||||
// OS cache location is read-only (locked-down corp images, ephemeral CI
|
|
||||||
// containers). Pointed at by extract errors via the documented escape hatch.
|
|
||||||
const CacheEnvOverride = "LAST30DAYS_CACHE_DIR"
|
|
||||||
|
|
||||||
// Ensure extracts src into baseDir/last30days-pp-mcp/<version> and returns
|
|
||||||
// the cache path. If the sentinel file already records the same version the
|
|
||||||
// directory is reused without rewriting. version must be non-empty so the
|
|
||||||
// cache layout always namespaces by version.
|
|
||||||
//
|
|
||||||
// Extraction writes to a sibling .tmp directory and renames it on success
|
|
||||||
// so a partial extraction can never be mistaken for a complete one. Concurrent
|
|
||||||
// callers within the same process serialize behind a per-cache-dir sync.Once
|
|
||||||
// so the rename happens exactly once.
|
|
||||||
func Ensure(src fs.FS, baseDir, version string) (string, error) {
|
|
||||||
if version == "" {
|
|
||||||
return "", errors.New("engine: version is required")
|
|
||||||
}
|
|
||||||
cacheDir := filepath.Join(baseDir, cacheSubdir, version)
|
|
||||||
|
|
||||||
once := getOnce(cacheDir)
|
|
||||||
var extractErr error
|
|
||||||
once.Do(func() {
|
|
||||||
extractErr = ensureLocked(src, cacheDir, version)
|
|
||||||
})
|
|
||||||
if extractErr != nil {
|
|
||||||
// Reset the sync.Once so a follow-up call can retry rather than
|
|
||||||
// permanently caching the error. Retry is the right default when
|
|
||||||
// the failure is transient (e.g., disk full, parent dir restored).
|
|
||||||
resetOnce(cacheDir)
|
|
||||||
return "", extractErr
|
|
||||||
}
|
|
||||||
return cacheDir, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnsureUserCache wraps Ensure with the OS user cache dir (or the
|
|
||||||
// LAST30DAYS_CACHE_DIR override) as base. Production callers use this; tests
|
|
||||||
// use Ensure with an explicit temp dir.
|
|
||||||
func EnsureUserCache(src fs.FS, version string) (string, error) {
|
|
||||||
if override := os.Getenv(CacheEnvOverride); override != "" {
|
|
||||||
return Ensure(src, override, version)
|
|
||||||
}
|
|
||||||
base, err := os.UserCacheDir()
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("engine: resolve user cache dir (set %s to override): %w", CacheEnvOverride, err)
|
|
||||||
}
|
|
||||||
return Ensure(src, base, version)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ensureLocked(src fs.FS, cacheDir, version string) error {
|
|
||||||
if sentinelMatches(cacheDir, version) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
tmpDir := cacheDir + ".tmp"
|
|
||||||
if err := os.RemoveAll(tmpDir); err != nil {
|
|
||||||
return fmt.Errorf("engine: clean tmp cache: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(tmpDir, 0o755); err != nil {
|
|
||||||
return fmt.Errorf("engine: create tmp cache (%s, set %s to override): %w", tmpDir, CacheEnvOverride, err)
|
|
||||||
}
|
|
||||||
if err := extractAll(src, tmpDir); err != nil {
|
|
||||||
_ = os.RemoveAll(tmpDir)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
sentinel := filepath.Join(tmpDir, SentinelFilename)
|
|
||||||
if err := os.WriteFile(sentinel, []byte(version), 0o644); err != nil {
|
|
||||||
_ = os.RemoveAll(tmpDir)
|
|
||||||
return fmt.Errorf("engine: write sentinel: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.RemoveAll(cacheDir); err != nil {
|
|
||||||
_ = os.RemoveAll(tmpDir)
|
|
||||||
return fmt.Errorf("engine: clean old cache: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.Rename(tmpDir, cacheDir); err != nil {
|
|
||||||
_ = os.RemoveAll(tmpDir)
|
|
||||||
return fmt.Errorf("engine: promote tmp cache: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func sentinelMatches(cacheDir, version string) bool {
|
|
||||||
data, err := os.ReadFile(filepath.Join(cacheDir, SentinelFilename))
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return string(data) == version
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractAll(src fs.FS, dst string) error {
|
|
||||||
return fs.WalkDir(src, ".", func(path string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if path == "." {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
target := filepath.Join(dst, path)
|
|
||||||
if d.IsDir() {
|
|
||||||
return os.MkdirAll(target, 0o755)
|
|
||||||
}
|
|
||||||
return copyEmbeddedFile(src, path, target)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyEmbeddedFile(src fs.FS, srcPath, dst string) error {
|
|
||||||
in, err := src.Open(srcPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("engine: open %s: %w", srcPath, err)
|
|
||||||
}
|
|
||||||
defer func() { _ = in.Close() }()
|
|
||||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
|
||||||
return fmt.Errorf("engine: ensure parent of %s: %w", dst, err)
|
|
||||||
}
|
|
||||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("engine: create %s: %w", dst, err)
|
|
||||||
}
|
|
||||||
defer func() { _ = out.Close() }()
|
|
||||||
if _, err := io.Copy(out, in); err != nil {
|
|
||||||
return fmt.Errorf("engine: write %s: %w", dst, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// onceRegistry serializes first-call extraction per cache directory so the
|
|
||||||
// rename in ensureLocked happens exactly once across goroutines.
|
|
||||||
var (
|
|
||||||
onceMu sync.Mutex
|
|
||||||
onceRegistry = map[string]*sync.Once{}
|
|
||||||
)
|
|
||||||
|
|
||||||
func getOnce(cacheDir string) *sync.Once {
|
|
||||||
onceMu.Lock()
|
|
||||||
defer onceMu.Unlock()
|
|
||||||
if o, ok := onceRegistry[cacheDir]; ok {
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
o := &sync.Once{}
|
|
||||||
onceRegistry[cacheDir] = o
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
func resetOnce(cacheDir string) {
|
|
||||||
onceMu.Lock()
|
|
||||||
defer onceMu.Unlock()
|
|
||||||
delete(onceRegistry, cacheDir)
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
package engine
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"testing/fstest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func newTestFS() fstest.MapFS {
|
|
||||||
return fstest.MapFS{
|
|
||||||
"last30days.py": &fstest.MapFile{Data: []byte("# last30days entry\n"), Mode: 0o644},
|
|
||||||
"lib/__init__.py": &fstest.MapFile{Data: []byte(""), Mode: 0o644},
|
|
||||||
"lib/env.py": &fstest.MapFile{Data: []byte("# env helpers\n"), Mode: 0o644},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnsureExtractsEngine(t *testing.T) {
|
|
||||||
src := newTestFS()
|
|
||||||
base := t.TempDir()
|
|
||||||
|
|
||||||
cacheDir, err := Ensure(src, base, "v1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Ensure: %v", err)
|
|
||||||
}
|
|
||||||
if cacheDir != filepath.Join(base, cacheSubdir, "v1") {
|
|
||||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, filepath.Join(base, cacheSubdir, "v1"))
|
|
||||||
}
|
|
||||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
|
||||||
mustReadFile(t, filepath.Join(cacheDir, "lib/env.py"), "# env helpers\n")
|
|
||||||
mustReadFile(t, filepath.Join(cacheDir, SentinelFilename), "v1")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnsureSkipsWhenSentinelMatches(t *testing.T) {
|
|
||||||
src := newTestFS()
|
|
||||||
base := t.TempDir()
|
|
||||||
|
|
||||||
cacheDir, err := Ensure(src, base, "v1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first Ensure: %v", err)
|
|
||||||
}
|
|
||||||
target := filepath.Join(cacheDir, "last30days.py")
|
|
||||||
info1, err := os.Stat(target)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stat: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reset the sync.Once so a second call would re-extract if not for the
|
|
||||||
// sentinel short-circuit. Without the reset, sync.Once would skip the
|
|
||||||
// extraction regardless of sentinel state.
|
|
||||||
resetOnce(cacheDir)
|
|
||||||
|
|
||||||
if _, err := Ensure(src, base, "v1"); err != nil {
|
|
||||||
t.Fatalf("second Ensure: %v", err)
|
|
||||||
}
|
|
||||||
info2, err := os.Stat(target)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stat second: %v", err)
|
|
||||||
}
|
|
||||||
if !info2.ModTime().Equal(info1.ModTime()) {
|
|
||||||
t.Fatalf("expected file untouched on sentinel match; got mtime %v -> %v", info1.ModTime(), info2.ModTime())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnsureReExtractsOnVersionChange(t *testing.T) {
|
|
||||||
v1 := fstest.MapFS{
|
|
||||||
"last30days.py": &fstest.MapFile{Data: []byte("v1\n"), Mode: 0o644},
|
|
||||||
}
|
|
||||||
v2 := fstest.MapFS{
|
|
||||||
"last30days.py": &fstest.MapFile{Data: []byte("v2\n"), Mode: 0o644},
|
|
||||||
}
|
|
||||||
base := t.TempDir()
|
|
||||||
|
|
||||||
cache1, err := Ensure(v1, base, "v1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Ensure v1: %v", err)
|
|
||||||
}
|
|
||||||
cache2, err := Ensure(v2, base, "v2")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Ensure v2: %v", err)
|
|
||||||
}
|
|
||||||
if cache1 == cache2 {
|
|
||||||
t.Fatalf("expected distinct cache dirs per version, got %q == %q", cache1, cache2)
|
|
||||||
}
|
|
||||||
mustReadFile(t, filepath.Join(cache1, "last30days.py"), "v1\n")
|
|
||||||
mustReadFile(t, filepath.Join(cache2, "last30days.py"), "v2\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnsureConcurrentFirstCall(t *testing.T) {
|
|
||||||
src := newTestFS()
|
|
||||||
base := t.TempDir()
|
|
||||||
|
|
||||||
const goroutines = 10
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
wg.Add(goroutines)
|
|
||||||
results := make([]string, goroutines)
|
|
||||||
errs := make([]error, goroutines)
|
|
||||||
for i := 0; i < goroutines; i++ {
|
|
||||||
i := i
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
results[i], errs[i] = Ensure(src, base, "v1")
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
for i, err := range errs {
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("goroutine %d: %v", i, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 1; i < goroutines; i++ {
|
|
||||||
if results[i] != results[0] {
|
|
||||||
t.Fatalf("goroutine 0 saw %q, goroutine %d saw %q", results[0], i, results[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mustReadFile(t, filepath.Join(results[0], "last30days.py"), "# last30days entry\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnsureRejectsEmptyVersion(t *testing.T) {
|
|
||||||
if _, err := Ensure(newTestFS(), t.TempDir(), ""); err == nil {
|
|
||||||
t.Fatal("expected error for empty version")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnsureReturnsErrorWhenCacheUnwritable(t *testing.T) {
|
|
||||||
// Place the cache root at a path that cannot exist (a regular file).
|
|
||||||
// MkdirAll will refuse and Ensure must surface a wrapped error.
|
|
||||||
base := t.TempDir()
|
|
||||||
blocker := filepath.Join(base, "blocker")
|
|
||||||
if err := os.WriteFile(blocker, []byte("not a dir"), 0o644); err != nil {
|
|
||||||
t.Fatalf("setup: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := Ensure(newTestFS(), blocker, "v1")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error when cache parent is not a directory")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnsureUserCacheHonorsOverride(t *testing.T) {
|
|
||||||
override := t.TempDir()
|
|
||||||
t.Setenv(CacheEnvOverride, override)
|
|
||||||
|
|
||||||
src := newTestFS()
|
|
||||||
cacheDir, err := EnsureUserCache(src, "v1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("EnsureUserCache: %v", err)
|
|
||||||
}
|
|
||||||
want := filepath.Join(override, cacheSubdir, "v1")
|
|
||||||
if cacheDir != want {
|
|
||||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, want)
|
|
||||||
}
|
|
||||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustReadFile(t *testing.T, path, want string) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read %s: %v", path, err)
|
|
||||||
}
|
|
||||||
if string(data) != want {
|
|
||||||
t.Fatalf("%s: got %q, want %q", path, string(data), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,163 +0,0 @@
|
|||||||
package engine
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DefaultPythonBinary is the interpreter we look up unless RunOptions
|
|
||||||
// overrides it. Windows installs may expose only "python"; we surface a
|
|
||||||
// clear error in that case rather than silently picking the wrong binary.
|
|
||||||
const DefaultPythonBinary = "python3"
|
|
||||||
|
|
||||||
// MinPythonVersion mirrors the engine's MIN_PYTHON constant in
|
|
||||||
// last30days.py. Surfaced in errors so users know what they're missing.
|
|
||||||
const MinPythonVersion = "3.12"
|
|
||||||
|
|
||||||
// PythonInstallURL is included in the missing-interpreter error so users
|
|
||||||
// have a direct route from the failure to a fix.
|
|
||||||
const PythonInstallURL = "https://www.python.org/downloads/"
|
|
||||||
|
|
||||||
// DefaultTimeout caps a single research subprocess. The engine's deep mode
|
|
||||||
// can run several minutes; five minutes is a safe upper bound that still
|
|
||||||
// fails fast when something hangs.
|
|
||||||
const DefaultTimeout = 5 * time.Minute
|
|
||||||
|
|
||||||
// TimeoutEnvOverride lets operators override DefaultTimeout per install
|
|
||||||
// (seconds, integer). Honored by Run when RunOptions.Timeout is zero.
|
|
||||||
const TimeoutEnvOverride = "LAST30DAYS_MCP_TIMEOUT"
|
|
||||||
|
|
||||||
// RunOptions configures one invocation of the embedded Python engine.
|
|
||||||
// PythonPath is exposed so tests can substitute a stub interpreter without
|
|
||||||
// manipulating the process PATH.
|
|
||||||
type RunOptions struct {
|
|
||||||
PythonPath string // resolved python3 binary; empty means look up DefaultPythonBinary on PATH
|
|
||||||
CacheDir string // engine.Ensure result; lib/ here is added to PYTHONPATH
|
|
||||||
Args []string // arguments after last30days.py (topic, --emit=..., etc.)
|
|
||||||
ExtraEnv []string // appended to os.Environ() for the child process
|
|
||||||
Timeout time.Duration // zero means DefaultTimeout or TimeoutEnvOverride
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunResult captures the engine's full output. Stdout is what we surface to
|
|
||||||
// the agent; Stderr is included in error messages so users can diagnose
|
|
||||||
// engine failures without leaving Claude Desktop.
|
|
||||||
type RunResult struct {
|
|
||||||
Stdout []byte
|
|
||||||
Stderr []byte
|
|
||||||
ExitCode int
|
|
||||||
TimedOut bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run shells out to python3 with last30days.py inside cacheDir. The child
|
|
||||||
// receives the parent environment (so MCPB user_config env-injection
|
|
||||||
// reaches the engine) plus ExtraEnv and a PYTHONPATH that points at the
|
|
||||||
// cache so the engine's `from lib import ...` statements resolve.
|
|
||||||
//
|
|
||||||
// A missing interpreter, a non-zero exit, and a timeout each surface as
|
|
||||||
// distinct errors so the tool handler can map them to user-facing
|
|
||||||
// messages without re-parsing stderr.
|
|
||||||
func Run(ctx context.Context, opts RunOptions) (*RunResult, error) {
|
|
||||||
if opts.CacheDir == "" {
|
|
||||||
return nil, errors.New("engine: CacheDir is required")
|
|
||||||
}
|
|
||||||
pythonPath, err := resolvePython(opts.PythonPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
scriptPath := filepath.Join(opts.CacheDir, "last30days.py")
|
|
||||||
if _, err := os.Stat(scriptPath); err != nil {
|
|
||||||
return nil, fmt.Errorf("engine: last30days.py not found in cache %s: %w", opts.CacheDir, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout := resolveTimeout(opts.Timeout)
|
|
||||||
subCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
args := append([]string{scriptPath}, opts.Args...)
|
|
||||||
cmd := exec.CommandContext(subCtx, pythonPath, args...)
|
|
||||||
cmd.Env = buildEnv(opts.CacheDir, opts.ExtraEnv)
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
cmd.Stdout = &stdout
|
|
||||||
cmd.Stderr = &stderr
|
|
||||||
|
|
||||||
err = cmd.Run()
|
|
||||||
res := &RunResult{
|
|
||||||
Stdout: stdout.Bytes(),
|
|
||||||
Stderr: stderr.Bytes(),
|
|
||||||
ExitCode: 0,
|
|
||||||
TimedOut: errors.Is(subCtx.Err(), context.DeadlineExceeded),
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var exitErr *exec.ExitError
|
|
||||||
if errors.As(err, &exitErr) {
|
|
||||||
res.ExitCode = exitErr.ExitCode()
|
|
||||||
if res.TimedOut {
|
|
||||||
return res, fmt.Errorf("engine: subprocess exceeded %s timeout", timeout)
|
|
||||||
}
|
|
||||||
return res, fmt.Errorf("engine: subprocess exited with code %d", res.ExitCode)
|
|
||||||
}
|
|
||||||
return res, fmt.Errorf("engine: subprocess failed to start: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolvePython returns an absolute path to the interpreter or an error
|
|
||||||
// naming the install URL. If the caller supplied a path we trust it - tests
|
|
||||||
// rely on this to inject a stub. Otherwise we look up python3 on PATH.
|
|
||||||
func resolvePython(override string) (string, error) {
|
|
||||||
if override != "" {
|
|
||||||
return override, nil
|
|
||||||
}
|
|
||||||
path, err := exec.LookPath(DefaultPythonBinary)
|
|
||||||
if err == nil {
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
return "", fmt.Errorf(
|
|
||||||
"engine: %s not found on PATH (need Python %s+, install from %s; current GOOS=%s)",
|
|
||||||
DefaultPythonBinary, MinPythonVersion, PythonInstallURL, runtime.GOOS,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveTimeout(explicit time.Duration) time.Duration {
|
|
||||||
if explicit > 0 {
|
|
||||||
return explicit
|
|
||||||
}
|
|
||||||
if raw := os.Getenv(TimeoutEnvOverride); raw != "" {
|
|
||||||
if d, err := time.ParseDuration(raw); err == nil && d > 0 {
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return DefaultTimeout
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildEnv stitches PYTHONPATH onto os.Environ + ExtraEnv. Any pre-existing
|
|
||||||
// PYTHONPATH in the parent environment is dropped before appending the
|
|
||||||
// cache dir; otherwise the child sees two PYTHONPATH= entries and POSIX
|
|
||||||
// getenv returns the first one, so the user's value wins and the engine's
|
|
||||||
// `from lib import ...` fails with ModuleNotFoundError. The engine is
|
|
||||||
// self-contained and does not need the user's Python module search path.
|
|
||||||
func buildEnv(cacheDir string, extra []string) []string {
|
|
||||||
const pyKey = "PYTHONPATH="
|
|
||||||
parent := os.Environ()
|
|
||||||
base := make([]string, 0, len(parent)+1+len(extra))
|
|
||||||
for _, kv := range parent {
|
|
||||||
if strings.HasPrefix(kv, pyKey) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
base = append(base, kv)
|
|
||||||
}
|
|
||||||
base = append(base, pyKey+cacheDir)
|
|
||||||
base = append(base, extra...)
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
@@ -1,281 +0,0 @@
|
|||||||
package engine
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// makeStubPython writes a shell script that simulates python3 and returns
|
|
||||||
// its absolute path. The script honors a small env-driven protocol so each
|
|
||||||
// test can shape its output:
|
|
||||||
//
|
|
||||||
// STUB_STDOUT - text printed to stdout
|
|
||||||
// STUB_STDERR - text printed to stderr
|
|
||||||
// STUB_EXIT_CODE - integer exit code (default 0)
|
|
||||||
// STUB_SLEEP_SECS - sleep before exiting (for timeout tests)
|
|
||||||
// STUB_ECHO_ENV - name of an env var; the stub prints "<NAME>=<VALUE>"
|
|
||||||
// STUB_ECHO_ARG - integer index; the stub prints "ARG<i>=<args[i]>"
|
|
||||||
//
|
|
||||||
// The stub ignores its first argument (the script path), matching how a
|
|
||||||
// real python3 invocation treats `python3 last30days.py ...`.
|
|
||||||
func makeStubPython(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("stub-python tests rely on POSIX shell")
|
|
||||||
}
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "python3-stub.sh")
|
|
||||||
script := `#!/usr/bin/env bash
|
|
||||||
if [ -n "${STUB_SLEEP_SECS:-}" ]; then sleep "$STUB_SLEEP_SECS"; fi
|
|
||||||
if [ -n "${STUB_STDOUT:-}" ]; then printf "%s" "$STUB_STDOUT"; fi
|
|
||||||
if [ -n "${STUB_STDERR:-}" ]; then printf "%s" "$STUB_STDERR" >&2; fi
|
|
||||||
if [ -n "${STUB_ECHO_ENV:-}" ]; then echo "${STUB_ECHO_ENV}=${!STUB_ECHO_ENV:-<unset>}"; fi
|
|
||||||
if [ -n "${STUB_ECHO_ARG:-}" ]; then echo "ARG${STUB_ECHO_ARG}=${!STUB_ECHO_ARG:-<unset>}"; fi
|
|
||||||
exit "${STUB_EXIT_CODE:-0}"
|
|
||||||
`
|
|
||||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
|
||||||
t.Fatalf("write stub: %v", err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
// stageCache materializes a fake CacheDir with a no-op last30days.py so
|
|
||||||
// the existence check in Run passes. The stub python3 ignores the script
|
|
||||||
// contents, so the file just has to exist.
|
|
||||||
func stageCache(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "last30days.py"), []byte("# stub\n"), 0o644); err != nil {
|
|
||||||
t.Fatalf("stage cache: %v", err)
|
|
||||||
}
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunHappyPath(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
cache := stageCache(t)
|
|
||||||
t.Setenv("STUB_STDOUT", "synthesis output\n")
|
|
||||||
|
|
||||||
res, err := Run(context.Background(), RunOptions{
|
|
||||||
PythonPath: stub,
|
|
||||||
CacheDir: cache,
|
|
||||||
Args: []string{"my topic", "--emit=compact"},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run: %v", err)
|
|
||||||
}
|
|
||||||
if string(res.Stdout) != "synthesis output\n" {
|
|
||||||
t.Fatalf("stdout = %q, want %q", res.Stdout, "synthesis output\n")
|
|
||||||
}
|
|
||||||
if res.ExitCode != 0 {
|
|
||||||
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
|
||||||
}
|
|
||||||
if res.TimedOut {
|
|
||||||
t.Fatal("TimedOut = true, want false")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunForwardsEnv(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
cache := stageCache(t)
|
|
||||||
t.Setenv("OPENAI_API_KEY", "sk-test-value")
|
|
||||||
t.Setenv("STUB_ECHO_ENV", "OPENAI_API_KEY")
|
|
||||||
|
|
||||||
res, err := Run(context.Background(), RunOptions{
|
|
||||||
PythonPath: stub,
|
|
||||||
CacheDir: cache,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run: %v", err)
|
|
||||||
}
|
|
||||||
if got := strings.TrimSpace(string(res.Stdout)); got != "OPENAI_API_KEY=sk-test-value" {
|
|
||||||
t.Fatalf("stdout = %q, want OPENAI_API_KEY=sk-test-value", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunSetsPythonPath(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
cache := stageCache(t)
|
|
||||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
|
||||||
|
|
||||||
res, err := Run(context.Background(), RunOptions{
|
|
||||||
PythonPath: stub,
|
|
||||||
CacheDir: cache,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run: %v", err)
|
|
||||||
}
|
|
||||||
want := "PYTHONPATH=" + cache
|
|
||||||
if got := strings.TrimSpace(string(res.Stdout)); got != want {
|
|
||||||
t.Fatalf("stdout = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunDropsPreExistingPythonPath guards the buildEnv dedup: when the
|
|
||||||
// parent already sets PYTHONPATH (common on dev machines and CI runners
|
|
||||||
// that touch Python), the child must NOT see two PYTHONPATH= entries.
|
|
||||||
// POSIX getenv returns the first match, so a duplicate from os.Environ
|
|
||||||
// would shadow our cache-dir entry and break `from lib import ...`.
|
|
||||||
func TestRunDropsPreExistingPythonPath(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
cache := stageCache(t)
|
|
||||||
t.Setenv("PYTHONPATH", "/users-stale-pythonpath")
|
|
||||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
|
||||||
|
|
||||||
res, err := Run(context.Background(), RunOptions{
|
|
||||||
PythonPath: stub,
|
|
||||||
CacheDir: cache,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run: %v", err)
|
|
||||||
}
|
|
||||||
got := strings.TrimSpace(string(res.Stdout))
|
|
||||||
want := "PYTHONPATH=" + cache
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("stdout = %q, want %q (stale parent value leaked through)", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildEnvDropsAllPreExistingPythonPath(t *testing.T) {
|
|
||||||
// Direct unit test on buildEnv to catch the case where the parent has
|
|
||||||
// PYTHONPATH set: the returned slice must contain exactly one
|
|
||||||
// PYTHONPATH= entry, and it must be ours.
|
|
||||||
t.Setenv("PYTHONPATH", "/parent/one")
|
|
||||||
cache := "/cache/dir"
|
|
||||||
out := buildEnv(cache, []string{"EXTRA=1"})
|
|
||||||
|
|
||||||
var pythonPaths []string
|
|
||||||
for _, kv := range out {
|
|
||||||
if strings.HasPrefix(kv, "PYTHONPATH=") {
|
|
||||||
pythonPaths = append(pythonPaths, kv)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(pythonPaths) != 1 {
|
|
||||||
t.Fatalf("got %d PYTHONPATH entries, want 1: %v", len(pythonPaths), pythonPaths)
|
|
||||||
}
|
|
||||||
if pythonPaths[0] != "PYTHONPATH="+cache {
|
|
||||||
t.Fatalf("PYTHONPATH = %q, want %q", pythonPaths[0], "PYTHONPATH="+cache)
|
|
||||||
}
|
|
||||||
// Confirm ExtraEnv still rides along.
|
|
||||||
found := false
|
|
||||||
for _, kv := range out {
|
|
||||||
if kv == "EXTRA=1" {
|
|
||||||
found = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("EXTRA=1 missing from buildEnv output")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunSurfacesExitCode(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
cache := stageCache(t)
|
|
||||||
t.Setenv("STUB_STDERR", "engine boom\n")
|
|
||||||
t.Setenv("STUB_EXIT_CODE", "2")
|
|
||||||
|
|
||||||
res, err := Run(context.Background(), RunOptions{
|
|
||||||
PythonPath: stub,
|
|
||||||
CacheDir: cache,
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for non-zero exit")
|
|
||||||
}
|
|
||||||
if res == nil {
|
|
||||||
t.Fatal("res is nil; want populated result alongside error")
|
|
||||||
}
|
|
||||||
if res.ExitCode != 2 {
|
|
||||||
t.Fatalf("ExitCode = %d, want 2", res.ExitCode)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(res.Stderr), "engine boom") {
|
|
||||||
t.Fatalf("stderr did not surface engine output: %q", res.Stderr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunTimesOut(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
cache := stageCache(t)
|
|
||||||
t.Setenv("STUB_SLEEP_SECS", "3")
|
|
||||||
|
|
||||||
res, err := Run(context.Background(), RunOptions{
|
|
||||||
PythonPath: stub,
|
|
||||||
CacheDir: cache,
|
|
||||||
Timeout: 200 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected timeout error")
|
|
||||||
}
|
|
||||||
if !res.TimedOut {
|
|
||||||
t.Fatal("TimedOut = false, want true")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "timeout") {
|
|
||||||
t.Fatalf("error %q lacks 'timeout' marker", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunMissingPython(t *testing.T) {
|
|
||||||
cache := stageCache(t)
|
|
||||||
// Empty PATH guarantees the lookup fails. PythonPath stays unset so Run
|
|
||||||
// falls through to exec.LookPath.
|
|
||||||
t.Setenv("PATH", "")
|
|
||||||
|
|
||||||
_, err := Run(context.Background(), RunOptions{CacheDir: cache})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected lookup failure with empty PATH")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), DefaultPythonBinary) {
|
|
||||||
t.Fatalf("error %q does not mention %s", err, DefaultPythonBinary)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), PythonInstallURL) {
|
|
||||||
t.Fatalf("error %q does not include install URL", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunMissingScript(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
// CacheDir exists but contains no last30days.py.
|
|
||||||
cache := t.TempDir()
|
|
||||||
|
|
||||||
_, err := Run(context.Background(), RunOptions{
|
|
||||||
PythonPath: stub,
|
|
||||||
CacheDir: cache,
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error when last30days.py missing")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "last30days.py") {
|
|
||||||
t.Fatalf("error %q does not name missing script", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRejectsEmptyCacheDir(t *testing.T) {
|
|
||||||
stub := makeStubPython(t)
|
|
||||||
_, err := Run(context.Background(), RunOptions{PythonPath: stub})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for empty CacheDir")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, err) || !strings.Contains(err.Error(), "CacheDir") {
|
|
||||||
t.Fatalf("error %q does not name CacheDir", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveTimeoutHonorsEnv(t *testing.T) {
|
|
||||||
t.Setenv(TimeoutEnvOverride, "750ms")
|
|
||||||
if got := resolveTimeout(0); got != 750*time.Millisecond {
|
|
||||||
t.Fatalf("resolveTimeout = %v, want 750ms", got)
|
|
||||||
}
|
|
||||||
t.Setenv(TimeoutEnvOverride, "garbage")
|
|
||||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
|
||||||
t.Fatalf("garbage value: got %v, want default %v", got, DefaultTimeout)
|
|
||||||
}
|
|
||||||
if got := resolveTimeout(time.Minute); got != time.Minute {
|
|
||||||
t.Fatalf("explicit value not honored: got %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
Populated at build time by scripts/sync-engine.sh.
|
|
||||||
Source of truth: skills/last30days/scripts/.
|
|
||||||
@@ -1,189 +0,0 @@
|
|||||||
// Package manifest holds tests for mcp/manifest.json. It contains no
|
|
||||||
// production code - the manifest itself is the artifact, and these tests
|
|
||||||
// guard structural invariants the bundling pipeline depends on.
|
|
||||||
package manifest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// envBinding is a minimal subset of the MCPB v0.3 manifest just covering
|
|
||||||
// the fields these tests assert on. We deliberately do not depend on the
|
|
||||||
// printing-press internal/pipeline types (that's an internal/ package and
|
|
||||||
// not importable across modules) - the structural invariants below are
|
|
||||||
// what actually matter for Claude Desktop install correctness.
|
|
||||||
type manifestShape struct {
|
|
||||||
ManifestVersion string `json:"manifest_version"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
Version string `json:"version"`
|
|
||||||
Server struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
EntryPoint string `json:"entry_point"`
|
|
||||||
MCPConfig struct {
|
|
||||||
Command string `json:"command"`
|
|
||||||
Env map[string]string `json:"env"`
|
|
||||||
} `json:"mcp_config"`
|
|
||||||
} `json:"server"`
|
|
||||||
UserConfig map[string]struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
Sensitive bool `json:"sensitive"`
|
|
||||||
Required bool `json:"required"`
|
|
||||||
} `json:"user_config"`
|
|
||||||
Compatibility struct {
|
|
||||||
ClaudeDesktop string `json:"claude_desktop"`
|
|
||||||
Platforms []string `json:"platforms"`
|
|
||||||
} `json:"compatibility"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadManifest reads mcp/manifest.json relative to this test file so the
|
|
||||||
// test passes regardless of where `go test` is invoked from.
|
|
||||||
func loadManifest(t *testing.T) manifestShape {
|
|
||||||
t.Helper()
|
|
||||||
_, thisFile, _, ok := runtime.Caller(0)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("runtime.Caller failed")
|
|
||||||
}
|
|
||||||
// manifest_test.go is at mcp/internal/manifest/; manifest.json at mcp/.
|
|
||||||
manifestPath := filepath.Join(filepath.Dir(thisFile), "..", "..", "manifest.json")
|
|
||||||
data, err := os.ReadFile(manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read manifest: %v", err)
|
|
||||||
}
|
|
||||||
var m manifestShape
|
|
||||||
if err := json.Unmarshal(data, &m); err != nil {
|
|
||||||
t.Fatalf("parse manifest: %v", err)
|
|
||||||
}
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestRequiredFields(t *testing.T) {
|
|
||||||
m := loadManifest(t)
|
|
||||||
if m.ManifestVersion != "0.3" {
|
|
||||||
t.Errorf("manifest_version = %q, want 0.3", m.ManifestVersion)
|
|
||||||
}
|
|
||||||
if m.Name != "last30days-pp-mcp" {
|
|
||||||
t.Errorf("name = %q, want last30days-pp-mcp", m.Name)
|
|
||||||
}
|
|
||||||
if m.Version == "" {
|
|
||||||
t.Error("version is empty")
|
|
||||||
}
|
|
||||||
if m.Server.Type != "binary" {
|
|
||||||
t.Errorf("server.type = %q, want binary", m.Server.Type)
|
|
||||||
}
|
|
||||||
if m.Server.EntryPoint != "bin/last30days-pp-mcp" {
|
|
||||||
t.Errorf("server.entry_point = %q, want bin/last30days-pp-mcp", m.Server.EntryPoint)
|
|
||||||
}
|
|
||||||
if m.Compatibility.ClaudeDesktop == "" {
|
|
||||||
t.Error("compatibility.claude_desktop is empty")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEnvAndUserConfigCrossReference is the key invariant: every
|
|
||||||
// ${user_config.<key>} substitution in server.mcp_config.env must point
|
|
||||||
// at a real user_config entry, and every declared user_config must be
|
|
||||||
// wired to an env var. A typo on either side silently disables a credential
|
|
||||||
// at install time without the binary or Claude Desktop noticing.
|
|
||||||
func TestEnvAndUserConfigCrossReference(t *testing.T) {
|
|
||||||
m := loadManifest(t)
|
|
||||||
|
|
||||||
if len(m.Server.MCPConfig.Env) == 0 {
|
|
||||||
t.Fatal("server.mcp_config.env is empty; expected user_config substitutions")
|
|
||||||
}
|
|
||||||
if len(m.UserConfig) == 0 {
|
|
||||||
t.Fatal("user_config is empty; expected per-key declarations")
|
|
||||||
}
|
|
||||||
|
|
||||||
for envName, value := range m.Server.MCPConfig.Env {
|
|
||||||
key, ok := parseUserConfigRef(value)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("env[%s] = %q is not a ${user_config.<key>} reference", envName, value)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, declared := m.UserConfig[key]; !declared {
|
|
||||||
t.Errorf("env[%s] references user_config[%q], which is not declared", envName, key)
|
|
||||||
}
|
|
||||||
// The user_config key must be the lowercased env var so Claude
|
|
||||||
// Desktop's substitution rule matches PP's emitted shape.
|
|
||||||
if got := strings.ToLower(envName); key != got {
|
|
||||||
t.Errorf("env[%s] -> user_config[%q]; convention requires user_config[%q]", envName, key, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
envValues := make(map[string]bool, len(m.Server.MCPConfig.Env))
|
|
||||||
for _, value := range m.Server.MCPConfig.Env {
|
|
||||||
if key, ok := parseUserConfigRef(value); ok {
|
|
||||||
envValues[key] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for key := range m.UserConfig {
|
|
||||||
if !envValues[key] {
|
|
||||||
t.Errorf("user_config[%q] is declared but never substituted into env", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUserConfigShape(t *testing.T) {
|
|
||||||
m := loadManifest(t)
|
|
||||||
for key, slot := range m.UserConfig {
|
|
||||||
if slot.Type != "string" {
|
|
||||||
t.Errorf("user_config[%q].type = %q, want string", key, slot.Type)
|
|
||||||
}
|
|
||||||
if slot.Title == "" {
|
|
||||||
t.Errorf("user_config[%q].title is empty", key)
|
|
||||||
}
|
|
||||||
if slot.Description == "" {
|
|
||||||
t.Errorf("user_config[%q].description is empty", key)
|
|
||||||
}
|
|
||||||
if !slot.Sensitive {
|
|
||||||
// API keys must be flagged sensitive so Claude Desktop masks
|
|
||||||
// the input and prefers OS-keychain storage.
|
|
||||||
t.Errorf("user_config[%q].sensitive = false; want true for API credentials", key)
|
|
||||||
}
|
|
||||||
if slot.Required {
|
|
||||||
// The engine degrades to web-only mode without keys, so no
|
|
||||||
// key is install-blocking.
|
|
||||||
t.Errorf("user_config[%q].required = true; engine degrades without keys, so all keys are optional", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPlatformsMatchShippingMatrix(t *testing.T) {
|
|
||||||
// compatibility.platforms must list exactly what the release CI
|
|
||||||
// actually packages. Listing a platform we don't ship would let
|
|
||||||
// Claude Desktop start an install that has no matching binary inside
|
|
||||||
// the bundle, producing a silent failure. The CI matrix in
|
|
||||||
// .github/workflows/release.yml currently covers darwin (arm64 +
|
|
||||||
// amd64) and linux/amd64; Windows is deferred.
|
|
||||||
m := loadManifest(t)
|
|
||||||
required := map[string]bool{"darwin": false, "linux": false}
|
|
||||||
forbidden := map[string]bool{"win32": true}
|
|
||||||
for _, p := range m.Compatibility.Platforms {
|
|
||||||
if _, ok := required[p]; ok {
|
|
||||||
required[p] = true
|
|
||||||
}
|
|
||||||
if forbidden[p] {
|
|
||||||
t.Errorf("compatibility.platforms contains %q but the release matrix does not ship that platform; add it to the matrix or remove from the manifest", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for p, found := range required {
|
|
||||||
if !found {
|
|
||||||
t.Errorf("compatibility.platforms missing %q", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseUserConfigRef(value string) (string, bool) {
|
|
||||||
const prefix = "${user_config."
|
|
||||||
const suffix = "}"
|
|
||||||
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
return value[len(prefix) : len(value)-len(suffix)], true
|
|
||||||
}
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
// Package tools owns the MCP tool surface for last30days. Today there is
|
|
||||||
// exactly one tool, research, mirroring the /last30days <topic> slash
|
|
||||||
// command available in Claude Code. Adding new tools means another file
|
|
||||||
// here plus an additional s.AddTool call in Register.
|
|
||||||
package tools
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
|
||||||
"github.com/mark3labs/mcp-go/server"
|
|
||||||
|
|
||||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Config carries the version string used to namespace the per-user cache.
|
|
||||||
// main passes its ldflags-stamped Version here.
|
|
||||||
type Config struct {
|
|
||||||
Version string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Register adds every tool this server exposes to s. The caller supplies a
|
|
||||||
// Config so test harnesses can pin a version without touching globals.
|
|
||||||
func Register(s *server.MCPServer, cfg Config) {
|
|
||||||
s.AddTool(
|
|
||||||
mcplib.NewTool("research",
|
|
||||||
mcplib.WithDescription(
|
|
||||||
"Research what people are actually saying about any topic in the last 30 days. "+
|
|
||||||
"Aggregates Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web, "+
|
|
||||||
"scored by upvotes, likes, transcripts, and real-money prediction-market odds. "+
|
|
||||||
"Returns the engine's compact output for the model to synthesize.",
|
|
||||||
),
|
|
||||||
mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")),
|
|
||||||
mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")),
|
|
||||||
mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")),
|
|
||||||
mcplib.WithReadOnlyHintAnnotation(true),
|
|
||||||
mcplib.WithDestructiveHintAnnotation(false),
|
|
||||||
mcplib.WithOpenWorldHintAnnotation(true),
|
|
||||||
),
|
|
||||||
makeResearchHandler(cfg),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
|
||||||
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
|
|
||||||
args := req.GetArguments()
|
|
||||||
topic, err := requireString(args, "topic")
|
|
||||||
if err != nil {
|
|
||||||
return mcplib.NewToolResultError(err.Error()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
emit, err := emitArgument(args)
|
|
||||||
if err != nil {
|
|
||||||
return mcplib.NewToolResultError(err.Error()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
save, err := boolArgument(args, "save")
|
|
||||||
if err != nil {
|
|
||||||
return mcplib.NewToolResultError(err.Error()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
src, err := engine.EngineFS()
|
|
||||||
if err != nil {
|
|
||||||
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
|
|
||||||
}
|
|
||||||
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
|
|
||||||
if err != nil {
|
|
||||||
return mcplib.NewToolResultError(fmt.Sprintf(
|
|
||||||
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
|
|
||||||
err, engine.CacheEnvOverride,
|
|
||||||
)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
runArgs := []string{topic, "--emit=" + emit}
|
|
||||||
if save {
|
|
||||||
runArgs = append(runArgs, "--save")
|
|
||||||
}
|
|
||||||
|
|
||||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
|
||||||
CacheDir: cacheDir,
|
|
||||||
Args: runArgs,
|
|
||||||
})
|
|
||||||
if runErr != nil {
|
|
||||||
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
|
|
||||||
}
|
|
||||||
return mcplib.NewToolResultText(string(res.Stdout)), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireString(args map[string]any, name string) (string, error) {
|
|
||||||
raw, ok := args[name]
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("%s is required", name)
|
|
||||||
}
|
|
||||||
value, ok := raw.(string)
|
|
||||||
if !ok || strings.TrimSpace(value) == "" {
|
|
||||||
return "", fmt.Errorf("%s must be a non-empty string", name)
|
|
||||||
}
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func emitArgument(args map[string]any) (string, error) {
|
|
||||||
raw, ok := args["emit"]
|
|
||||||
if !ok {
|
|
||||||
return "compact", nil
|
|
||||||
}
|
|
||||||
value, ok := raw.(string)
|
|
||||||
if !ok {
|
|
||||||
return "", errors.New("emit must be a string")
|
|
||||||
}
|
|
||||||
switch value {
|
|
||||||
case "":
|
|
||||||
return "compact", nil
|
|
||||||
case "compact", "html":
|
|
||||||
return value, nil
|
|
||||||
default:
|
|
||||||
return "", fmt.Errorf("emit must be 'compact' or 'html', got %q", value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func boolArgument(args map[string]any, name string) (bool, error) {
|
|
||||||
raw, ok := args[name]
|
|
||||||
if !ok {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
value, ok := raw.(bool)
|
|
||||||
if !ok {
|
|
||||||
return false, fmt.Errorf("%s must be a boolean", name)
|
|
||||||
}
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// formatRunError flattens engine.Run's distinct error shapes into a single
|
|
||||||
// user-facing message that includes the relevant stderr context.
|
|
||||||
func formatRunError(runErr error, res *engine.RunResult) string {
|
|
||||||
var msg strings.Builder
|
|
||||||
msg.WriteString(runErr.Error())
|
|
||||||
if res != nil && len(res.Stderr) > 0 {
|
|
||||||
msg.WriteString("\nengine stderr:\n")
|
|
||||||
msg.Write(res.Stderr)
|
|
||||||
}
|
|
||||||
return msg.String()
|
|
||||||
}
|
|
||||||
@@ -1,145 +0,0 @@
|
|||||||
package tools
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
|
||||||
|
|
||||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
|
||||||
)
|
|
||||||
|
|
||||||
func newCallToolRequest(args map[string]any) mcplib.CallToolRequest {
|
|
||||||
var req mcplib.CallToolRequest
|
|
||||||
req.Params.Arguments = args
|
|
||||||
return req
|
|
||||||
}
|
|
||||||
|
|
||||||
// resultText pulls text content out of a tool result so tests can assert on
|
|
||||||
// the body Claude will see. Returns empty string when the result is nil or
|
|
||||||
// has no text content.
|
|
||||||
func resultText(res *mcplib.CallToolResult) string {
|
|
||||||
if res == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
var out strings.Builder
|
|
||||||
for _, item := range res.Content {
|
|
||||||
if tc, ok := item.(mcplib.TextContent); ok {
|
|
||||||
out.WriteString(tc.Text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRequireStringRejectsMissingAndBlank(t *testing.T) {
|
|
||||||
if _, err := requireString(map[string]any{}, "topic"); err == nil {
|
|
||||||
t.Fatal("expected error for missing topic")
|
|
||||||
}
|
|
||||||
if _, err := requireString(map[string]any{"topic": ""}, "topic"); err == nil {
|
|
||||||
t.Fatal("expected error for empty topic")
|
|
||||||
}
|
|
||||||
if _, err := requireString(map[string]any{"topic": " "}, "topic"); err == nil {
|
|
||||||
t.Fatal("expected error for whitespace-only topic")
|
|
||||||
}
|
|
||||||
if _, err := requireString(map[string]any{"topic": 42}, "topic"); err == nil {
|
|
||||||
t.Fatal("expected error for non-string topic")
|
|
||||||
}
|
|
||||||
v, err := requireString(map[string]any{"topic": "OpenAI"}, "topic")
|
|
||||||
if err != nil || v != "OpenAI" {
|
|
||||||
t.Fatalf("requireString ok = %q, %v", v, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmitArgumentDefaultsAndValidates(t *testing.T) {
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
args map[string]any
|
|
||||||
want string
|
|
||||||
wantErr bool
|
|
||||||
}{
|
|
||||||
{"missing defaults to compact", map[string]any{}, "compact", false},
|
|
||||||
{"empty string defaults to compact", map[string]any{"emit": ""}, "compact", false},
|
|
||||||
{"compact passes through", map[string]any{"emit": "compact"}, "compact", false},
|
|
||||||
{"html passes through", map[string]any{"emit": "html"}, "html", false},
|
|
||||||
{"invalid value rejected", map[string]any{"emit": "json"}, "", true},
|
|
||||||
{"non-string rejected", map[string]any{"emit": 7}, "", true},
|
|
||||||
}
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
got, err := emitArgument(tc.args)
|
|
||||||
if (err != nil) != tc.wantErr {
|
|
||||||
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
|
|
||||||
}
|
|
||||||
if got != tc.want {
|
|
||||||
t.Fatalf("got %q, want %q", got, tc.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBoolArgument(t *testing.T) {
|
|
||||||
v, err := boolArgument(map[string]any{}, "save")
|
|
||||||
if err != nil || v {
|
|
||||||
t.Fatalf("missing: %v, %v", v, err)
|
|
||||||
}
|
|
||||||
v, err = boolArgument(map[string]any{"save": true}, "save")
|
|
||||||
if err != nil || !v {
|
|
||||||
t.Fatalf("true: %v, %v", v, err)
|
|
||||||
}
|
|
||||||
v, err = boolArgument(map[string]any{"save": false}, "save")
|
|
||||||
if err != nil || v {
|
|
||||||
t.Fatalf("false: %v, %v", v, err)
|
|
||||||
}
|
|
||||||
if _, err := boolArgument(map[string]any{"save": "true"}, "save"); err == nil {
|
|
||||||
t.Fatal("expected error for string value")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) {
|
|
||||||
// Validation failures are returned as MCP tool errors (not Go errors)
|
|
||||||
// so Claude sees a structured failure with a readable message rather
|
|
||||||
// than a transport-level fault.
|
|
||||||
handler := makeResearchHandler(Config{Version: "test"})
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
args map[string]any
|
|
||||||
wantSub string
|
|
||||||
}{
|
|
||||||
{"missing topic", map[string]any{}, "topic is required"},
|
|
||||||
{"blank topic", map[string]any{"topic": " "}, "non-empty string"},
|
|
||||||
{"invalid emit", map[string]any{"topic": "OpenAI", "emit": "json"}, "must be 'compact' or 'html'"},
|
|
||||||
{"non-bool save", map[string]any{"topic": "OpenAI", "save": "yes"}, "save must be a boolean"},
|
|
||||||
}
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
res, err := handler(context.Background(), newCallToolRequest(tc.args))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("handler should not return Go error for validation; got %v", err)
|
|
||||||
}
|
|
||||||
if res == nil || !res.IsError {
|
|
||||||
t.Fatalf("expected IsError result, got %+v", res)
|
|
||||||
}
|
|
||||||
if !strings.Contains(resultText(res), tc.wantSub) {
|
|
||||||
t.Fatalf("result text %q missing substring %q", resultText(res), tc.wantSub)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatRunErrorIncludesStderr(t *testing.T) {
|
|
||||||
res := &engine.RunResult{Stderr: []byte("engine exploded\n")}
|
|
||||||
msg := formatRunError(errors.New("boom"), res)
|
|
||||||
if !strings.Contains(msg, "boom") || !strings.Contains(msg, "engine exploded") {
|
|
||||||
t.Fatalf("formatRunError missed pieces: %q", msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatRunErrorHandlesNilResult(t *testing.T) {
|
|
||||||
msg := formatRunError(errors.New("boom"), nil)
|
|
||||||
if msg != "boom" {
|
|
||||||
t.Fatalf("nil result: got %q, want %q", msg, "boom")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
{
|
|
||||||
"manifest_version": "0.3",
|
|
||||||
"name": "last30days-pp-mcp",
|
|
||||||
"display_name": "Last30Days",
|
|
||||||
"version": "3.0.0",
|
|
||||||
"description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.",
|
|
||||||
"author": {
|
|
||||||
"name": "Matt Van Horn",
|
|
||||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
|
||||||
},
|
|
||||||
"repository": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
|
||||||
},
|
|
||||||
"license": "MIT",
|
|
||||||
"keywords": [
|
|
||||||
"research",
|
|
||||||
"reddit",
|
|
||||||
"twitter",
|
|
||||||
"x",
|
|
||||||
"youtube",
|
|
||||||
"hacker-news",
|
|
||||||
"polymarket",
|
|
||||||
"github",
|
|
||||||
"search",
|
|
||||||
"synthesis"
|
|
||||||
],
|
|
||||||
"server": {
|
|
||||||
"type": "binary",
|
|
||||||
"entry_point": "bin/last30days-pp-mcp",
|
|
||||||
"mcp_config": {
|
|
||||||
"command": "${__dirname}/bin/last30days-pp-mcp",
|
|
||||||
"args": [],
|
|
||||||
"env": {
|
|
||||||
"OPENAI_API_KEY": "${user_config.openai_api_key}",
|
|
||||||
"XAI_API_KEY": "${user_config.xai_api_key}",
|
|
||||||
"BRAVE_API_KEY": "${user_config.brave_api_key}",
|
|
||||||
"EXA_API_KEY": "${user_config.exa_api_key}",
|
|
||||||
"SERPER_API_KEY": "${user_config.serper_api_key}",
|
|
||||||
"GOOGLE_API_KEY": "${user_config.google_api_key}",
|
|
||||||
"GEMINI_API_KEY": "${user_config.gemini_api_key}",
|
|
||||||
"GOOGLE_GENAI_API_KEY": "${user_config.google_genai_api_key}",
|
|
||||||
"APIFY_API_TOKEN": "${user_config.apify_api_token}",
|
|
||||||
"BSKY_APP_PASSWORD": "${user_config.bsky_app_password}",
|
|
||||||
"PARALLEL_API_KEY": "${user_config.parallel_api_key}",
|
|
||||||
"SCRAPECREATORS_API_KEY": "${user_config.scrapecreators_api_key}",
|
|
||||||
"OPENROUTER_API_KEY": "${user_config.openrouter_api_key}"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"user_config": {
|
|
||||||
"openai_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "OPENAI_API_KEY",
|
|
||||||
"description": "OpenAI API key. Powers Reddit research via OpenAI's web_search tool. Get one at https://platform.openai.com/api-keys.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"xai_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "XAI_API_KEY",
|
|
||||||
"description": "xAI API key. Powers X / Twitter research via xAI's x_search tool. Get one at https://console.x.ai/.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"brave_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "BRAVE_API_KEY",
|
|
||||||
"description": "Brave Search API key. Used for grounded web search results. Get one at https://brave.com/search/api/.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"exa_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "EXA_API_KEY",
|
|
||||||
"description": "Exa search API key. Alternative web search backend with semantic ranking. Get one at https://exa.ai/.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"serper_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "SERPER_API_KEY",
|
|
||||||
"description": "Serper API key. Google search via API. Get one at https://serper.dev/.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"google_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "GOOGLE_API_KEY",
|
|
||||||
"description": "Google API key for YouTube transcript fetching and other Google services. Get one at https://console.cloud.google.com/apis/credentials.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"gemini_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "GEMINI_API_KEY",
|
|
||||||
"description": "Gemini API key. Used for synthesis fallback when other LLM providers are unavailable. Get one at https://aistudio.google.com/apikey.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"google_genai_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "GOOGLE_GENAI_API_KEY",
|
|
||||||
"description": "Alternative Google generative-AI API key. Same source as GEMINI_API_KEY; set whichever name your tooling expects.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"apify_api_token": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "APIFY_API_TOKEN",
|
|
||||||
"description": "Apify API token. Powers TikTok and Instagram Reels search via Apify actors. Get one at https://console.apify.com/account/integrations.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"bsky_app_password": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "BSKY_APP_PASSWORD",
|
|
||||||
"description": "Bluesky app password (not your main password). Powers AT Protocol post search. Create at https://bsky.app/settings/app-passwords.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"parallel_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "PARALLEL_API_KEY",
|
|
||||||
"description": "Parallel AI key. Powers parallel research runs across sources. Get one at https://parallel.ai/.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"scrapecreators_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "SCRAPECREATORS_API_KEY",
|
|
||||||
"description": "ScrapeCreators API key. Powers creator-focused social search across TikTok, Instagram, and YouTube. Get one at https://scrapecreators.com/.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
},
|
|
||||||
"openrouter_api_key": {
|
|
||||||
"type": "string",
|
|
||||||
"title": "OPENROUTER_API_KEY",
|
|
||||||
"description": "OpenRouter API key. Alternative LLM provider gateway for synthesis. Get one at https://openrouter.ai/keys.",
|
|
||||||
"sensitive": true,
|
|
||||||
"required": false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"compatibility": {
|
|
||||||
"claude_desktop": ">=1.0.0",
|
|
||||||
"platforms": [
|
|
||||||
"darwin",
|
|
||||||
"linux"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Mirrors skills/last30days/scripts/{last30days.py,lib/} into mcp/vendored/
|
|
||||||
# so the Go binary's embed.FS captures the engine at build time.
|
|
||||||
#
|
|
||||||
# Source of truth: skills/last30days/scripts/. Never edit mcp/vendored/ directly.
|
|
||||||
# Run before `go build` locally and in CI before `printing-press bundle`.
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
MCP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
||||||
REPO_ROOT="$(cd "${MCP_DIR}/.." && pwd)"
|
|
||||||
ENGINE_SRC="${REPO_ROOT}/skills/last30days/scripts"
|
|
||||||
# Embed path must live inside the consuming package (Go //go:embed cannot
|
|
||||||
# reach outside its own directory tree), so vendored/ sits under engine/.
|
|
||||||
VENDORED="${MCP_DIR}/internal/engine/vendored"
|
|
||||||
|
|
||||||
if [ ! -f "${ENGINE_SRC}/last30days.py" ]; then
|
|
||||||
echo "sync-engine: ${ENGINE_SRC}/last30days.py not found" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
mkdir -p "${VENDORED}"
|
|
||||||
# Clear stale content while keeping the .gitkeep that anchors the embed path.
|
|
||||||
find "${VENDORED}" -mindepth 1 -not -name ".gitkeep" -delete
|
|
||||||
|
|
||||||
# Copy the entry script and the lib/ tree (modules + lib/vendor/).
|
|
||||||
cp "${ENGINE_SRC}/last30days.py" "${VENDORED}/last30days.py"
|
|
||||||
cp -R "${ENGINE_SRC}/lib" "${VENDORED}/lib"
|
|
||||||
|
|
||||||
# Strip caches so the embed.FS stays deterministic.
|
|
||||||
find "${VENDORED}" -type d -name "__pycache__" -prune -exec rm -rf {} +
|
|
||||||
find "${VENDORED}" -type f -name "*.pyc" -delete
|
|
||||||
|
|
||||||
echo "sync-engine: vendored engine at ${VENDORED}"
|
|
||||||
|
Before Width: | Height: | Size: 2.4 MiB |
@@ -1,10 +1,12 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "last30days-skill"
|
name = "last30days-skill"
|
||||||
version = "3.2.3"
|
version = "3.0.0"
|
||||||
description = "Multi-source last-30-days research skill"
|
description = "Multi-source last-30-days research skill"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
dependencies = []
|
dependencies = [
|
||||||
|
"requests>=2.32,<3",
|
||||||
|
]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
@@ -22,9 +24,9 @@ addopts = [
|
|||||||
|
|
||||||
[tool.coverage.run]
|
[tool.coverage.run]
|
||||||
branch = true
|
branch = true
|
||||||
source = ["skills/last30days/scripts", "tests"]
|
source = ["scripts", "tests"]
|
||||||
omit = [
|
omit = [
|
||||||
"skills/last30days/scripts/lib/vendor/*",
|
"scripts/lib/vendor/*",
|
||||||
"dist/*",
|
"dist/*",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -32,6 +34,7 @@ omit = [
|
|||||||
skip_empty = true
|
skip_empty = true
|
||||||
show_missing = true
|
show_missing = true
|
||||||
omit = [
|
omit = [
|
||||||
"skills/last30days/scripts/lib/vendor/*",
|
"scripts/lib/vendor/*",
|
||||||
"dist/*",
|
"dist/*",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ OpenClaw:
|
|||||||
clawhub install last30days-official
|
clawhub install last30days-official
|
||||||
```
|
```
|
||||||
|
|
||||||
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`.
|
OpenAI Codex CLI: run `codex` from a checkout of this repo and v3's skill at `.agents/skills/last30days/SKILL.md` will be discovered automatically. Or copy `SKILL.md` to `~/.agents/skills/last30days/SKILL.md` for a global install.
|
||||||
|
|
||||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,13 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
||||||
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root)
|
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||||
#
|
#
|
||||||
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
||||||
# directory containing SKILL.md and the scripts/ runtime from skills/last30days.
|
# directory containing SKILL.md and the scripts/ runtime. See
|
||||||
# See
|
|
||||||
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
cd "$REPO_ROOT"
|
cd "$REPO_ROOT"
|
||||||
|
|
||||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||||
@@ -18,7 +17,14 @@ fi
|
|||||||
|
|
||||||
mkdir -p dist
|
mkdir -p dist
|
||||||
OUT="dist/last30days.skill"
|
OUT="dist/last30days.skill"
|
||||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days
|
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD
|
||||||
|
|
||||||
|
# claude.ai's .skill bundle only needs the root SKILL.md + scripts/ runtime.
|
||||||
|
# Claude Code needs skills/ and .claude-plugin/ in the git archive
|
||||||
|
# (that's why they're NOT in .gitattributes export-ignore), but the .skill
|
||||||
|
# bundle must strip them to keep a single canonical SKILL.md and stay under
|
||||||
|
# the 200-file cap.
|
||||||
|
zip -d "$OUT" "last30days/skills/*" "last30days/.claude-plugin/*" > /dev/null 2>&1 || true
|
||||||
|
|
||||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||||
SIZE=$(du -h "$OUT" | cut -f1)
|
SIZE=$(du -h "$OUT" | cut -f1)
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# A/B test runner: public release vs private beta
|
# A/B test runner: public release vs private beta
|
||||||
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West"
|
# Usage: bash scripts/compare.sh "Kanye West"
|
||||||
#
|
#
|
||||||
# Runs /last30days (public release) and /last30days-beta (private beta)
|
# Runs /last30days (public release) and /last30days-beta (private beta)
|
||||||
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
||||||
@@ -9,14 +9,13 @@
|
|||||||
set -e
|
set -e
|
||||||
|
|
||||||
if [ $# -eq 0 ]; then
|
if [ $# -eq 0 ]; then
|
||||||
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>"
|
echo "Usage: bash scripts/compare.sh <topic>"
|
||||||
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose"
|
echo " Example: bash scripts/compare.sh Kevin Rose"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
TOPIC="$*"
|
TOPIC="$*"
|
||||||
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//')
|
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//')
|
||||||
LAST30DAYS_MEMORY_DIR="${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}"
|
DIR="$HOME/Documents/Last30Days"
|
||||||
DIR="$LAST30DAYS_MEMORY_DIR"
|
|
||||||
DATE=$(date +%Y-%m-%d)
|
DATE=$(date +%Y-%m-%d)
|
||||||
|
|
||||||
echo "=============================================="
|
echo "=============================================="
|
||||||
@@ -22,8 +22,7 @@ from lib import env as envlib
|
|||||||
from lib import schema
|
from lib import schema
|
||||||
|
|
||||||
|
|
||||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
||||||
|
|
||||||
|
|
||||||
@@ -308,10 +307,7 @@ def create_eval_env() -> dict[str, str]:
|
|||||||
|
|
||||||
|
|
||||||
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
||||||
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py"
|
cmd = [sys.executable, "scripts/last30days.py", topic, "--emit=json"]
|
||||||
if not engine.exists():
|
|
||||||
engine = repo_dir / "scripts" / "last30days.py"
|
|
||||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
|
||||||
if search:
|
if search:
|
||||||
cmd.extend(["--search", search])
|
cmd.extend(["--search", search])
|
||||||
if quick:
|
if quick:
|
||||||
@@ -0,0 +1,433 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# ruff: noqa: E402
|
||||||
|
"""last30days v3.0.0 CLI."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import atexit
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
MIN_PYTHON = (3, 12)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_supported_python(version_info: tuple[int, int, int] | object | None = None) -> None:
|
||||||
|
if version_info is None:
|
||||||
|
version_info = sys.version_info
|
||||||
|
major, minor, micro = tuple(version_info[:3])
|
||||||
|
if (major, minor) >= MIN_PYTHON:
|
||||||
|
return
|
||||||
|
sys.stderr.write(
|
||||||
|
"last30days v3 requires Python 3.12+.\n"
|
||||||
|
f"Detected Python {major}.{minor}.{micro}.\n"
|
||||||
|
"Install and use python3.12 or python3.13, then rerun this command.\n"
|
||||||
|
)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
|
||||||
|
ensure_supported_python()
|
||||||
|
|
||||||
|
if os.name == "nt":
|
||||||
|
for stream in (sys.stdout, sys.stderr):
|
||||||
|
if hasattr(stream, "reconfigure"):
|
||||||
|
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from lib import env, pipeline, render, schema, ui
|
||||||
|
|
||||||
|
_child_pids: set[int] = set()
|
||||||
|
_child_pids_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def register_child_pid(pid: int) -> None:
|
||||||
|
with _child_pids_lock:
|
||||||
|
_child_pids.add(pid)
|
||||||
|
|
||||||
|
|
||||||
|
def unregister_child_pid(pid: int) -> None:
|
||||||
|
with _child_pids_lock:
|
||||||
|
_child_pids.discard(pid)
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_children() -> None:
|
||||||
|
with _child_pids_lock:
|
||||||
|
pids = list(_child_pids)
|
||||||
|
for pid in pids:
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||||
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
|
atexit.register(_cleanup_children)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_search_flag(raw: str) -> list[str]:
|
||||||
|
sources = []
|
||||||
|
for source in raw.split(","):
|
||||||
|
source = source.strip().lower()
|
||||||
|
if not source:
|
||||||
|
continue
|
||||||
|
normalized = pipeline.SEARCH_ALIAS.get(source, source)
|
||||||
|
if normalized not in pipeline.MOCK_AVAILABLE_SOURCES:
|
||||||
|
raise SystemExit(f"Unknown search source: {source}")
|
||||||
|
if normalized not in sources:
|
||||||
|
sources.append(normalized)
|
||||||
|
if not sources:
|
||||||
|
raise SystemExit("--search requires at least one source.")
|
||||||
|
return sources
|
||||||
|
|
||||||
|
|
||||||
|
def slugify(value: str) -> str:
|
||||||
|
slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
|
||||||
|
return slug or "last30days"
|
||||||
|
|
||||||
|
|
||||||
|
def save_output(report: schema.Report, emit: str, save_dir: str, suffix: str = "") -> Path:
|
||||||
|
from datetime import datetime
|
||||||
|
path = Path(save_dir).expanduser().resolve()
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
slug = slugify(report.topic)
|
||||||
|
extension = "json" if emit == "json" else "md"
|
||||||
|
suffix_part = f"-{suffix}" if suffix else ""
|
||||||
|
out_path = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||||
|
if out_path.exists():
|
||||||
|
out_path = path / f"{slug}-raw{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||||
|
# Always save the FULL dump to disk (all items, all sources, transcripts).
|
||||||
|
# Claude sees compact clusters via --emit=compact on stdout.
|
||||||
|
# The saved file is the complete debug artifact.
|
||||||
|
if emit == "json":
|
||||||
|
content = emit_output(report, emit)
|
||||||
|
else:
|
||||||
|
content = render.render_full(report)
|
||||||
|
out_path.write_text(content, encoding="utf-8")
|
||||||
|
return out_path
|
||||||
|
|
||||||
|
|
||||||
|
def emit_output(report: schema.Report, emit: str, fun_level: str = "medium", save_path: str | None = None) -> str:
|
||||||
|
if emit == "json":
|
||||||
|
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
|
||||||
|
if emit in {"compact", "md"}:
|
||||||
|
return render.render_compact(report, fun_level=fun_level, save_path=save_path)
|
||||||
|
if emit == "context":
|
||||||
|
return render.render_context(report)
|
||||||
|
raise SystemExit(f"Unsupported emit mode: {emit}")
|
||||||
|
|
||||||
|
|
||||||
|
def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str) -> str:
|
||||||
|
"""Compute the user-friendly save path string that will be shown in the footer.
|
||||||
|
|
||||||
|
Uses ~ for the home directory so the footer reads "~/Documents/Last30Days/slug-raw.md"
|
||||||
|
instead of an absolute machine-local path.
|
||||||
|
"""
|
||||||
|
from pathlib import Path as _Path
|
||||||
|
path = _Path(save_dir).expanduser().resolve()
|
||||||
|
slug = slugify(topic)
|
||||||
|
extension = "json" if emit == "json" else "md"
|
||||||
|
suffix_part = f"-{suffix}" if suffix else ""
|
||||||
|
raw = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||||
|
try:
|
||||||
|
home = _Path.home().resolve()
|
||||||
|
relative = raw.relative_to(home)
|
||||||
|
return f"~/{relative}"
|
||||||
|
except ValueError:
|
||||||
|
return str(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def persist_report(report: schema.Report) -> dict[str, int]:
|
||||||
|
import store
|
||||||
|
|
||||||
|
store.init_db()
|
||||||
|
topic_row = store.add_topic(report.topic)
|
||||||
|
topic_id = topic_row["id"]
|
||||||
|
source_mode = ",".join(sorted(report.items_by_source)) or "v3"
|
||||||
|
run_id = store.record_run(topic_id, source_mode=source_mode, status="running")
|
||||||
|
try:
|
||||||
|
findings = store.findings_from_report(report)
|
||||||
|
counts = store.store_findings(run_id, topic_id, findings)
|
||||||
|
store.update_run(
|
||||||
|
run_id,
|
||||||
|
status="completed",
|
||||||
|
findings_new=counts["new"],
|
||||||
|
findings_updated=counts["updated"],
|
||||||
|
)
|
||||||
|
return counts
|
||||||
|
except Exception as exc:
|
||||||
|
store.update_run(run_id, status="failed", error_message=str(exc)[:500])
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
|
||||||
|
parser.add_argument("topic", nargs="*", help="Research topic")
|
||||||
|
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md"])
|
||||||
|
parser.add_argument("--search", help="Comma-separated source list")
|
||||||
|
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
|
||||||
|
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
|
||||||
|
parser.add_argument("--debug", action="store_true", help="Enable HTTP debug logging")
|
||||||
|
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
||||||
|
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
||||||
|
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
||||||
|
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
|
||||||
|
parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
|
||||||
|
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
|
||||||
|
parser.add_argument("--web-backend", default="auto",
|
||||||
|
choices=["auto", "brave", "exa", "serper", "parallel", "none"],
|
||||||
|
help="Web search backend (default: auto, tries Brave then Exa then Serper then Parallel)")
|
||||||
|
parser.add_argument("--deep-research", action="store_true",
|
||||||
|
help="Use Perplexity Deep Research (~$0.90/query) for in-depth analysis. Requires OPENROUTER_API_KEY.")
|
||||||
|
parser.add_argument("--plan", help="JSON query plan (skips internal LLM planner). Can be a JSON string or a file path.")
|
||||||
|
parser.add_argument("--save-suffix", help="Suffix for saved output filename (e.g., 'gemini' → kanye-west-raw-gemini.md)")
|
||||||
|
parser.add_argument("--subreddits", help="Comma-separated subreddit names to search (e.g., SaaS,Entrepreneur)")
|
||||||
|
parser.add_argument("--tiktok-hashtags", help="Comma-separated TikTok hashtags without # (e.g., tella,screenrecording)")
|
||||||
|
parser.add_argument("--tiktok-creators", help="Comma-separated TikTok creator handles (e.g., TellaHQ,taborplace)")
|
||||||
|
parser.add_argument("--ig-creators", help="Comma-separated Instagram creator handles (e.g., tella.tv,laborstories)")
|
||||||
|
parser.add_argument(
|
||||||
|
"--days",
|
||||||
|
"--lookback-days",
|
||||||
|
dest="lookback_days",
|
||||||
|
type=int,
|
||||||
|
default=30,
|
||||||
|
help="Number of days to look back for research (default: 30, watchlist uses 90)",
|
||||||
|
)
|
||||||
|
parser.add_argument("--auto-resolve", action="store_true",
|
||||||
|
help="Use web search to discover subreddits/handles before planning (for platforms without WebSearch)")
|
||||||
|
parser.add_argument("--github-user", help="GitHub username for person-mode search (e.g., steipete)")
|
||||||
|
parser.add_argument("--github-repo", help="Comma-separated owner/repo for project-mode search (e.g., openclaw/openclaw,paperclipai/paperclip)")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def _missing_sources_for_promo(diag: dict[str, object]) -> str | None:
|
||||||
|
available = set(diag.get("available_sources") or [])
|
||||||
|
missing = []
|
||||||
|
if "reddit" not in available:
|
||||||
|
missing.append("reddit")
|
||||||
|
if "x" not in available:
|
||||||
|
missing.append("x")
|
||||||
|
if "grounding" not in available:
|
||||||
|
missing.append("web")
|
||||||
|
if not missing:
|
||||||
|
return None
|
||||||
|
if "reddit" in missing and "x" in missing:
|
||||||
|
return "both"
|
||||||
|
return missing[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _show_runtime_ui(report: schema.Report, progress: ui.ProgressDisplay, diag: dict[str, object]) -> None:
|
||||||
|
counts = {source: len(items) for source, items in report.items_by_source.items()}
|
||||||
|
display_sources = list(
|
||||||
|
dict.fromkeys(
|
||||||
|
[
|
||||||
|
*report.query_plan.source_weights.keys(),
|
||||||
|
*report.items_by_source.keys(),
|
||||||
|
*report.errors_by_source.keys(),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
progress.end_processing()
|
||||||
|
progress.show_complete(
|
||||||
|
source_counts=counts,
|
||||||
|
display_sources=display_sources,
|
||||||
|
)
|
||||||
|
promo = _missing_sources_for_promo(diag)
|
||||||
|
if promo:
|
||||||
|
progress.show_promo(promo, diag=diag)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = build_parser()
|
||||||
|
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
||||||
|
# --openclaw) pass through without argparse hard-exiting.
|
||||||
|
args, extra_argv = parser.parse_known_args()
|
||||||
|
if args.debug:
|
||||||
|
os.environ["LAST30DAYS_DEBUG"] = "1"
|
||||||
|
|
||||||
|
config = env.get_config()
|
||||||
|
|
||||||
|
# Handle setup subcommand
|
||||||
|
topic = " ".join(args.topic).strip()
|
||||||
|
if topic.lower() == "setup":
|
||||||
|
from lib import setup_wizard
|
||||||
|
if "--openclaw" in extra_argv:
|
||||||
|
results = setup_wizard.run_openclaw_setup(config)
|
||||||
|
print(json.dumps(results))
|
||||||
|
return 0
|
||||||
|
if "--github" in extra_argv:
|
||||||
|
results = setup_wizard.run_github_auth()
|
||||||
|
print(json.dumps(results))
|
||||||
|
return 0
|
||||||
|
if "--device-auth" in extra_argv:
|
||||||
|
results = setup_wizard.run_full_device_auth()
|
||||||
|
print(json.dumps(results))
|
||||||
|
return 0
|
||||||
|
sys.stderr.write("Running auto-setup...\n")
|
||||||
|
results = setup_wizard.run_auto_setup(config)
|
||||||
|
from_browser = "auto"
|
||||||
|
if results.get("cookies_found"):
|
||||||
|
first_browser = next(iter(results["cookies_found"].values()))
|
||||||
|
from_browser = first_browser
|
||||||
|
setup_wizard.write_setup_config(env.CONFIG_FILE, from_browser=from_browser)
|
||||||
|
results["env_written"] = True
|
||||||
|
sys.stderr.write(setup_wizard.get_setup_status_text(results) + "\n")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
requested_sources = parse_search_flag(args.search) if args.search else None
|
||||||
|
diag = pipeline.diagnose(config, requested_sources)
|
||||||
|
|
||||||
|
if args.diagnose:
|
||||||
|
print(json.dumps(diag, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if not topic:
|
||||||
|
parser.print_usage(sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
|
||||||
|
from lib import preflight
|
||||||
|
refuse_msg = preflight.check_class_1_trap(topic)
|
||||||
|
if refuse_msg:
|
||||||
|
sys.stderr.write(refuse_msg)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
progress = ui.ProgressDisplay(topic, show_banner=True)
|
||||||
|
progress.start_processing()
|
||||||
|
|
||||||
|
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
||||||
|
try:
|
||||||
|
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
||||||
|
subreddits = [s.strip().lstrip("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||||
|
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
||||||
|
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
||||||
|
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
||||||
|
# Parse external plan if provided via --plan flag
|
||||||
|
external_plan = None
|
||||||
|
if args.plan:
|
||||||
|
import json as _json
|
||||||
|
plan_str = args.plan
|
||||||
|
if os.path.isfile(plan_str):
|
||||||
|
plan_str = open(plan_str).read()
|
||||||
|
try:
|
||||||
|
external_plan = _json.loads(plan_str)
|
||||||
|
except _json.JSONDecodeError as exc:
|
||||||
|
sys.stderr.write(f"[Planner] Invalid --plan JSON: {exc}\n")
|
||||||
|
|
||||||
|
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
||||||
|
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
||||||
|
# without WebSearch (OpenClaw, Codex, raw CLI).
|
||||||
|
if args.auto_resolve and not external_plan:
|
||||||
|
from lib import resolve
|
||||||
|
resolution = resolve.auto_resolve(topic, config)
|
||||||
|
if resolution.get("subreddits") and not subreddits:
|
||||||
|
subreddits = resolution["subreddits"]
|
||||||
|
sys.stderr.write(f"[AutoResolve] Subreddits: {', '.join(subreddits)}\n")
|
||||||
|
if resolution.get("x_handle") and not args.x_handle:
|
||||||
|
args.x_handle = resolution["x_handle"]
|
||||||
|
sys.stderr.write(f"[AutoResolve] X handle: @{args.x_handle}\n")
|
||||||
|
if resolution.get("github_user") and not args.github_user:
|
||||||
|
args.github_user = resolution["github_user"]
|
||||||
|
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
||||||
|
if resolution.get("github_repos") and not args.github_repo:
|
||||||
|
args.github_repo = ",".join(resolution["github_repos"])
|
||||||
|
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
||||||
|
if resolution.get("context"):
|
||||||
|
# Inject context into external_plan metadata for the planner to use
|
||||||
|
if not external_plan:
|
||||||
|
external_plan = None # planner will use its own, but with context
|
||||||
|
# Store context for the planner prompt injection
|
||||||
|
config["_auto_resolve_context"] = resolution["context"]
|
||||||
|
sys.stderr.write(f"[AutoResolve] Context: {resolution['context'][:80]}...\n")
|
||||||
|
|
||||||
|
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
||||||
|
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
||||||
|
|
||||||
|
# --deep-research: auto-enable perplexity source and set deep flag
|
||||||
|
if args.deep_research:
|
||||||
|
if not config.get("OPENROUTER_API_KEY"):
|
||||||
|
print("Error: --deep-research requires OPENROUTER_API_KEY", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
config["_deep_research"] = True
|
||||||
|
# Auto-enable perplexity in INCLUDE_SOURCES
|
||||||
|
include = config.get("INCLUDE_SOURCES") or ""
|
||||||
|
if "perplexity" not in include.lower():
|
||||||
|
config["INCLUDE_SOURCES"] = f"{include},perplexity" if include else "perplexity"
|
||||||
|
|
||||||
|
report = pipeline.run(
|
||||||
|
topic=topic,
|
||||||
|
config=config,
|
||||||
|
depth=depth,
|
||||||
|
requested_sources=requested_sources,
|
||||||
|
mock=args.mock,
|
||||||
|
x_handle=args.x_handle,
|
||||||
|
x_related=x_related,
|
||||||
|
web_backend=args.web_backend,
|
||||||
|
external_plan=external_plan,
|
||||||
|
subreddits=subreddits,
|
||||||
|
tiktok_hashtags=tiktok_hashtags,
|
||||||
|
tiktok_creators=tiktok_creators,
|
||||||
|
ig_creators=ig_creators,
|
||||||
|
lookback_days=args.lookback_days,
|
||||||
|
github_user=github_user,
|
||||||
|
github_repos=github_repos,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
progress.end_processing()
|
||||||
|
progress.show_error(str(exc))
|
||||||
|
raise
|
||||||
|
_show_runtime_ui(report, progress, diag)
|
||||||
|
if args.store:
|
||||||
|
counts = persist_report(report)
|
||||||
|
sys.stderr.write(
|
||||||
|
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
||||||
|
)
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
# Show quality nudge if applicable
|
||||||
|
try:
|
||||||
|
from lib import quality_nudge
|
||||||
|
quality = quality_nudge.compute_quality_score(config, {})
|
||||||
|
if quality.get("nudge_text"):
|
||||||
|
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
fun_level = config.get("FUN_LEVEL", "medium").lower()
|
||||||
|
footer_save_path = None
|
||||||
|
if args.save_dir:
|
||||||
|
footer_save_path = compute_save_path_display(
|
||||||
|
args.save_dir, report.topic, args.save_suffix or "", args.emit
|
||||||
|
)
|
||||||
|
|
||||||
|
# Signal to render_compact whether pre-research flags were supplied.
|
||||||
|
# Used to emit a Pre-Research Status warning when the model skipped
|
||||||
|
# Step 0.5 / 0.55 and invoked the engine bare on an eligible topic.
|
||||||
|
pre_research_flags_present = bool(
|
||||||
|
args.x_handle
|
||||||
|
or args.github_user
|
||||||
|
or args.subreddits
|
||||||
|
or args.plan
|
||||||
|
or args.auto_resolve
|
||||||
|
or args.tiktok_creators
|
||||||
|
or args.ig_creators
|
||||||
|
)
|
||||||
|
report.artifacts["pre_research_flags_present"] = pre_research_flags_present
|
||||||
|
|
||||||
|
rendered = emit_output(report, args.emit, fun_level=fun_level, save_path=footer_save_path)
|
||||||
|
if args.save_dir:
|
||||||
|
save_path = save_output(report, args.emit, args.save_dir, suffix=args.save_suffix or "")
|
||||||
|
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
print(rendered)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -7,11 +7,13 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import signal
|
||||||
import shutil
|
import shutil
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from . import http, log, subproc
|
from . import http, log
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
@@ -166,51 +168,62 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
|||||||
"--json",
|
"--json",
|
||||||
]
|
]
|
||||||
|
|
||||||
pid_holder: list[int] = []
|
# Use process groups for clean cleanup on timeout/kill
|
||||||
|
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||||
|
|
||||||
def _register(pid: int) -> None:
|
try:
|
||||||
pid_holder.append(pid)
|
proc = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
preexec_fn=preexec,
|
||||||
|
env=_subprocess_env(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Register for cleanup tracking (if available)
|
||||||
try:
|
try:
|
||||||
from last30days import register_child_pid
|
from last30days import register_child_pid, unregister_child_pid
|
||||||
register_child_pid(pid)
|
register_child_pid(proc.pid)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = subproc.run_with_timeout(
|
stdout, stderr = proc.communicate(timeout=timeout)
|
||||||
cmd,
|
except subprocess.TimeoutExpired:
|
||||||
timeout=timeout,
|
# Kill the entire process group
|
||||||
env=_subprocess_env(),
|
try:
|
||||||
on_pid=_register,
|
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||||
)
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
except subproc.SubprocTimeout:
|
proc.kill()
|
||||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
proc.wait(timeout=5)
|
||||||
except Exception as e:
|
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||||
return {"error": str(e), "items": []}
|
finally:
|
||||||
finally:
|
|
||||||
if pid_holder:
|
|
||||||
try:
|
try:
|
||||||
from last30days import unregister_child_pid
|
from last30days import unregister_child_pid
|
||||||
unregister_child_pid(pid_holder[0])
|
unregister_child_pid(proc.pid)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if result.returncode != 0:
|
if proc.returncode != 0:
|
||||||
error = result.stderr.strip() or "Bird search failed"
|
error = stderr.strip() if stderr else "Bird search failed"
|
||||||
return {"error": error, "items": []}
|
return {"error": error, "items": []}
|
||||||
|
|
||||||
output = result.stdout.strip()
|
output = stdout.strip() if stdout else ""
|
||||||
if not output:
|
if not output:
|
||||||
return {"items": []}
|
return {"items": []}
|
||||||
|
|
||||||
try:
|
|
||||||
parsed = json.loads(output)
|
parsed = json.loads(output)
|
||||||
|
if isinstance(parsed, list):
|
||||||
|
return {"items": parsed}
|
||||||
|
return parsed
|
||||||
|
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError as e:
|
||||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||||
|
except Exception as e:
|
||||||
if isinstance(parsed, list):
|
return {"error": str(e), "items": []}
|
||||||
return {"items": parsed}
|
|
||||||
return parsed
|
|
||||||
|
|
||||||
|
|
||||||
def search_x(
|
def search_x(
|
||||||
@@ -317,29 +330,47 @@ def search_handles(
|
|||||||
"--json",
|
"--json",
|
||||||
]
|
]
|
||||||
|
|
||||||
try:
|
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
|
||||||
except subproc.SubprocTimeout:
|
|
||||||
_log(f"Handle search timed out for @{handle}")
|
|
||||||
return []
|
|
||||||
except OSError as e:
|
|
||||||
_log(f"Handle search error for @{handle}: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
output = result.stdout.strip()
|
|
||||||
if not output:
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
preexec_fn=preexec,
|
||||||
|
env=_subprocess_env(),
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
stdout, stderr = proc.communicate(timeout=15)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||||
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
|
proc.kill()
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
_log(f"Handle search timed out for @{handle}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
if proc.returncode != 0:
|
||||||
|
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
output = (stdout or "").strip()
|
||||||
|
if not output:
|
||||||
|
return []
|
||||||
|
|
||||||
response = json.loads(output)
|
response = json.loads(output)
|
||||||
|
return parse_bird_response(response, query=core_topic)
|
||||||
|
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
_log(f"Invalid JSON from handle search for @{handle}")
|
_log(f"Invalid JSON from handle search for @{handle}")
|
||||||
return []
|
except (OSError, subprocess.SubprocessError) as e:
|
||||||
return parse_bird_response(response, query=core_topic)
|
_log(f"Handle search error for @{handle}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
|
||||||
@@ -39,14 +39,11 @@ def normalize_text(text: str) -> str:
|
|||||||
return re.sub(r"\s+", " ", text).strip()
|
return re.sub(r"\s+", " ", text).strip()
|
||||||
|
|
||||||
|
|
||||||
def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
|
|
||||||
if len(norm) < n:
|
|
||||||
return {norm} if norm else set()
|
|
||||||
return {norm[index:index + n] for index in range(len(norm) - n + 1)}
|
|
||||||
|
|
||||||
|
|
||||||
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
||||||
return _ngrams_of_normalized(normalize_text(text), n)
|
text = normalize_text(text)
|
||||||
|
if len(text) < n:
|
||||||
|
return {text} if text else set()
|
||||||
|
return {text[index:index + n] for index in range(len(text) - n + 1)}
|
||||||
|
|
||||||
|
|
||||||
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
||||||
@@ -93,7 +90,7 @@ class _PreparedText:
|
|||||||
|
|
||||||
def __init__(self, raw: str) -> None:
|
def __init__(self, raw: str) -> None:
|
||||||
norm = normalize_text(raw)
|
norm = normalize_text(raw)
|
||||||
self.ngrams = _ngrams_of_normalized(norm)
|
self.ngrams = get_ngrams(norm) if norm else set()
|
||||||
self.tokens = _tokenize(norm)
|
self.tokens = _tokenize(norm)
|
||||||
|
|
||||||
|
|
||||||
@@ -356,10 +356,6 @@ def get_x_source_with_method(config: dict[str, Any]) -> tuple[str | None, str]:
|
|||||||
if config.get("AUTH_TOKEN") and config.get("CT0"):
|
if config.get("AUTH_TOKEN") and config.get("CT0"):
|
||||||
method = config.get("_AUTH_TOKEN_SOURCE", "env")
|
method = config.get("_AUTH_TOKEN_SOURCE", "env")
|
||||||
return "bird", method
|
return "bird", method
|
||||||
# Fall back to xurl CLI (official X API v2, OAuth2, free developer app)
|
|
||||||
from . import xurl_x
|
|
||||||
if xurl_x.is_available():
|
|
||||||
return "xurl", "oauth2"
|
|
||||||
return None, "none"
|
return None, "none"
|
||||||
|
|
||||||
|
|
||||||
@@ -372,6 +368,14 @@ def config_exists() -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_reddit_available(config: dict[str, Any]) -> bool:
|
||||||
|
"""Check if Reddit search is available.
|
||||||
|
|
||||||
|
v3 uses ScrapeCreators only.
|
||||||
|
"""
|
||||||
|
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||||
|
|
||||||
|
|
||||||
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
||||||
"""Determine which Reddit backend to use.
|
"""Determine which Reddit backend to use.
|
||||||
|
|
||||||
@@ -397,7 +401,6 @@ def get_x_source(config: dict[str, Any]) -> str | None:
|
|||||||
Returns:
|
Returns:
|
||||||
'bird' if Bird is installed and explicit cookies are configured,
|
'bird' if Bird is installed and explicit cookies are configured,
|
||||||
'xai' if XAI_API_KEY is configured,
|
'xai' if XAI_API_KEY is configured,
|
||||||
'xurl' if xurl CLI is installed and authenticated,
|
|
||||||
None if no X source available.
|
None if no X source available.
|
||||||
"""
|
"""
|
||||||
# Import here to avoid circular dependency
|
# Import here to avoid circular dependency
|
||||||
@@ -418,11 +421,6 @@ def get_x_source(config: dict[str, Any]) -> str | None:
|
|||||||
if has_bird_creds and bird_x.is_bird_installed():
|
if has_bird_creds and bird_x.is_bird_installed():
|
||||||
return 'bird'
|
return 'bird'
|
||||||
|
|
||||||
# Fall back to xurl CLI (official X API v2, OAuth2, free developer app)
|
|
||||||
from . import xurl_x
|
|
||||||
if xurl_x.is_available():
|
|
||||||
return 'xurl'
|
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -604,18 +602,14 @@ def get_x_source_status(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
elif xai_available:
|
elif xai_available:
|
||||||
source = 'xai'
|
source = 'xai'
|
||||||
else:
|
else:
|
||||||
# Fall back to xurl CLI
|
source = None
|
||||||
from . import xurl_x as _xurl_check
|
|
||||||
source = 'xurl' if _xurl_check.is_available() else None
|
|
||||||
|
|
||||||
from . import xurl_x as _xurl_x
|
|
||||||
return {
|
return {
|
||||||
"source": source,
|
"source": source,
|
||||||
"bird_installed": bird_status["installed"],
|
"bird_installed": bird_status["installed"],
|
||||||
"bird_authenticated": bird_status["authenticated"],
|
"bird_authenticated": bird_status["authenticated"],
|
||||||
"bird_username": bird_status["username"],
|
"bird_username": bird_status["username"],
|
||||||
"xai_available": xai_available,
|
"xai_available": xai_available,
|
||||||
"xurl_available": _xurl_x.is_available(),
|
|
||||||
"can_install_bird": bird_status["can_install"],
|
"can_install_bird": bird_status["can_install"],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,8 +116,6 @@ def weighted_rrf(
|
|||||||
"""Fuse ranked lists into a single candidate pool."""
|
"""Fuse ranked lists into a single candidate pool."""
|
||||||
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
||||||
candidates: dict[str, schema.Candidate] = {}
|
candidates: dict[str, schema.Candidate] = {}
|
||||||
# Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
|
|
||||||
seen_source_items: dict[str, set[tuple[str, str]]] = {}
|
|
||||||
|
|
||||||
for (label, source), items in streams.items():
|
for (label, source), items in streams.items():
|
||||||
subquery = subqueries[label]
|
subquery = subqueries[label]
|
||||||
@@ -156,7 +154,6 @@ def weighted_rrf(
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
seen_source_items[key] = {(item.source, item.item_id)}
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
candidate = candidates[key]
|
candidate = candidates[key]
|
||||||
@@ -182,9 +179,7 @@ def weighted_rrf(
|
|||||||
candidate.subquery_labels.append(label)
|
candidate.subquery_labels.append(label)
|
||||||
if item.source not in candidate.sources:
|
if item.source not in candidate.sources:
|
||||||
candidate.sources.append(item.source)
|
candidate.sources.append(item.source)
|
||||||
source_item_key = (item.source, item.item_id)
|
if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
|
||||||
if source_item_key not in seen_source_items[key]:
|
|
||||||
seen_source_items[key].add(source_item_key)
|
|
||||||
candidate.source_items.append(item)
|
candidate.source_items.append(item)
|
||||||
candidate.metadata.setdefault("provenance", []).append(
|
candidate.metadata.setdefault("provenance", []).append(
|
||||||
{
|
{
|
||||||
@@ -12,6 +12,11 @@ import sys
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests as _requests
|
||||||
|
except ImportError:
|
||||||
|
_requests = None
|
||||||
|
|
||||||
from . import dates, http, log
|
from . import dates, http, log
|
||||||
|
|
||||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||||
@@ -231,17 +236,30 @@ def _user_reels(
|
|||||||
"""
|
"""
|
||||||
_log(f"User reels: @{handle}")
|
_log(f"User reels: @{handle}")
|
||||||
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
|
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
try:
|
||||||
reels_url,
|
from urllib.parse import urlencode
|
||||||
params={"handle": handle},
|
params = urlencode({"handle": handle})
|
||||||
headers=http.scrapecreators_headers(token),
|
url = f"{reels_url}?{params}"
|
||||||
timeout=30,
|
headers = http.scrapecreators_headers(token)
|
||||||
retries=2,
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
)
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"User reels error for @{handle}: {e}")
|
_log(f"User reels error (urllib) for @{handle}: {e}")
|
||||||
return []
|
return []
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
reels_url,
|
||||||
|
params={"handle": handle},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"User reels error for @{handle}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
|
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
|
||||||
_log(f" -> {len(raw_items)} reels from @{handle}")
|
_log(f" -> {len(raw_items)} reels from @{handle}")
|
||||||
@@ -275,17 +293,31 @@ def search_instagram(
|
|||||||
|
|
||||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||||
|
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
_log("requests library not installed, falling back to urllib")
|
||||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
try:
|
||||||
params={"query": core_topic},
|
from urllib.parse import urlencode
|
||||||
headers=http.scrapecreators_headers(token),
|
params = urlencode({"query": core_topic})
|
||||||
timeout=30,
|
url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
|
||||||
retries=2,
|
headers = http.scrapecreators_headers(token)
|
||||||
)
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
except Exception as e:
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
_log(f"ScrapeCreators error: {e}")
|
except Exception as e:
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
_log(f"ScrapeCreators error (urllib): {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||||
|
params={"query": core_topic},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"ScrapeCreators error: {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
|
||||||
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
||||||
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
||||||
@@ -335,7 +367,7 @@ def fetch_captions(
|
|||||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||||
max_captions = config["max_captions"]
|
max_captions = config["max_captions"]
|
||||||
|
|
||||||
if not video_items or not token:
|
if not video_items or not token or not _requests:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
top_items = video_items[:max_captions]
|
top_items = video_items[:max_captions]
|
||||||
@@ -360,24 +392,26 @@ def fetch_captions(
|
|||||||
if not url:
|
if not url:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
data = http.get(
|
resp = _requests.get(
|
||||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||||
params={"url": url},
|
params={"url": url},
|
||||||
headers=http.scrapecreators_headers(token),
|
headers=http.scrapecreators_headers(token),
|
||||||
timeout=15,
|
timeout=15,
|
||||||
retries=1,
|
|
||||||
)
|
)
|
||||||
transcripts = data.get("transcripts") or []
|
if resp.status_code == 200:
|
||||||
if transcripts and isinstance(transcripts, list):
|
data = resp.json()
|
||||||
transcript_text = " ".join(
|
transcripts = data.get("transcripts") or []
|
||||||
t.get("text", "") for t in transcripts
|
if transcripts and isinstance(transcripts, list):
|
||||||
if isinstance(t, dict) and t.get("text")
|
# Combine all transcript segments
|
||||||
)
|
transcript_text = " ".join(
|
||||||
if transcript_text:
|
t.get("text", "") for t in transcripts
|
||||||
words = transcript_text.split()
|
if isinstance(t, dict) and t.get("text")
|
||||||
if len(words) > CAPTION_MAX_WORDS:
|
)
|
||||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
if transcript_text:
|
||||||
captions[vid] = transcript_text
|
words = transcript_text.split()
|
||||||
|
if len(words) > CAPTION_MAX_WORDS:
|
||||||
|
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||||
|
captions[vid] = transcript_text
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Transcript fetch failed for {vid}: {e}")
|
_log(f"Transcript fetch failed for {vid}: {e}")
|
||||||
|
|
||||||
@@ -49,7 +49,6 @@ def normalize_source_items(
|
|||||||
"xquik": _normalize_x,
|
"xquik": _normalize_x,
|
||||||
"pinterest": _normalize_pinterest,
|
"pinterest": _normalize_pinterest,
|
||||||
"polymarket": _normalize_polymarket,
|
"polymarket": _normalize_polymarket,
|
||||||
"digg": _normalize_digg,
|
|
||||||
"grounding": _normalize_grounding,
|
"grounding": _normalize_grounding,
|
||||||
"xiaohongshu": _normalize_grounding,
|
"xiaohongshu": _normalize_grounding,
|
||||||
"github": _normalize_github,
|
"github": _normalize_github,
|
||||||
@@ -111,19 +110,6 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
|
|||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
def _join_comment_excerpts(
|
|
||||||
top_comments: list[Any],
|
|
||||||
key: str,
|
|
||||||
limit: int = 3,
|
|
||||||
) -> str:
|
|
||||||
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
|
|
||||||
return " ".join(
|
|
||||||
str(comment.get(key) or "").strip()
|
|
||||||
for comment in top_comments[:limit]
|
|
||||||
if isinstance(comment, dict)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _domain_from_url(url: str) -> str | None:
|
def _domain_from_url(url: str) -> str | None:
|
||||||
if not url:
|
if not url:
|
||||||
return None
|
return None
|
||||||
@@ -183,7 +169,11 @@ def _normalize_reddit(
|
|||||||
to_date: str,
|
to_date: str,
|
||||||
) -> schema.SourceItem:
|
) -> schema.SourceItem:
|
||||||
top_comments = item.get("top_comments") or []
|
top_comments = item.get("top_comments") or []
|
||||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
comment_text = " ".join(
|
||||||
|
str(comment.get("excerpt") or "").strip()
|
||||||
|
for comment in top_comments[:3]
|
||||||
|
if isinstance(comment, dict)
|
||||||
|
)
|
||||||
body = "\n".join(
|
body = "\n".join(
|
||||||
part
|
part
|
||||||
for part in [
|
for part in [
|
||||||
@@ -348,7 +338,11 @@ def _normalize_hackernews(
|
|||||||
to_date: str,
|
to_date: str,
|
||||||
) -> schema.SourceItem:
|
) -> schema.SourceItem:
|
||||||
top_comments = item.get("top_comments") or []
|
top_comments = item.get("top_comments") or []
|
||||||
comment_text = _join_comment_excerpts(top_comments, "text")
|
comment_text = " ".join(
|
||||||
|
str(comment.get("text") or "").strip()
|
||||||
|
for comment in top_comments[:3]
|
||||||
|
if isinstance(comment, dict)
|
||||||
|
)
|
||||||
title = str(item.get("title") or "").strip()
|
title = str(item.get("title") or "").strip()
|
||||||
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
||||||
return _source_item(
|
return _source_item(
|
||||||
@@ -400,53 +394,6 @@ def _normalize_microblog(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _normalize_digg(
|
|
||||||
source: str,
|
|
||||||
item: dict[str, Any],
|
|
||||||
index: int,
|
|
||||||
from_date: str,
|
|
||||||
to_date: str,
|
|
||||||
) -> schema.SourceItem:
|
|
||||||
"""Normalizer for Digg AI 1000 clusters.
|
|
||||||
|
|
||||||
Each cluster is one item. The TLDR carries the most useful body for
|
|
||||||
rerank and synthesis. Top-ranked X posts attached at search time are
|
|
||||||
passed through under metadata['posts'] so render can emit them as
|
|
||||||
inline 'via Digg' quotes.
|
|
||||||
"""
|
|
||||||
title = str(item.get("title") or "").strip()
|
|
||||||
tldr = str(item.get("tldr") or "").strip()
|
|
||||||
body = "\n\n".join(part for part in [title, tldr] if part)
|
|
||||||
posts = item.get("posts") or []
|
|
||||||
if not isinstance(posts, list):
|
|
||||||
posts = []
|
|
||||||
cluster_url_id = str(item.get("id") or f"DG{index + 1}")
|
|
||||||
return _source_item(
|
|
||||||
item_id=cluster_url_id,
|
|
||||||
source=source,
|
|
||||||
title=title or f"Digg cluster {index + 1}",
|
|
||||||
body=body,
|
|
||||||
url=str(item.get("url") or f"https://di.gg/ai/{cluster_url_id}"),
|
|
||||||
author="",
|
|
||||||
container="Digg",
|
|
||||||
published_at=item.get("date"),
|
|
||||||
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
|
|
||||||
engagement=item.get("engagement") or {},
|
|
||||||
relevance_hint=item.get("relevance", 0.5),
|
|
||||||
why_relevant=str(item.get("why_relevant") or ""),
|
|
||||||
snippet=tldr[:400],
|
|
||||||
metadata={
|
|
||||||
"clusterUrlId": cluster_url_id,
|
|
||||||
"tldr": tldr,
|
|
||||||
"rank": (item.get("engagement") or {}).get("rank"),
|
|
||||||
"uniqueAuthors": (item.get("engagement") or {}).get("uniqueAuthors"),
|
|
||||||
"postCount": (item.get("engagement") or {}).get("postCount"),
|
|
||||||
"firstPostAge": item.get("first_post_age"),
|
|
||||||
"posts": posts,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_polymarket(
|
def _normalize_polymarket(
|
||||||
source: str,
|
source: str,
|
||||||
item: dict[str, Any],
|
item: dict[str, Any],
|
||||||
@@ -494,7 +441,11 @@ def _normalize_github(
|
|||||||
title = str(item.get("title") or "").strip()
|
title = str(item.get("title") or "").strip()
|
||||||
snippet_text = str(item.get("snippet") or "").strip()
|
snippet_text = str(item.get("snippet") or "").strip()
|
||||||
top_comments = item.get("metadata", {}).get("top_comments") or []
|
top_comments = item.get("metadata", {}).get("top_comments") or []
|
||||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
comment_text = " ".join(
|
||||||
|
str(comment.get("excerpt") or "").strip()
|
||||||
|
for comment in top_comments[:3]
|
||||||
|
if isinstance(comment, dict)
|
||||||
|
)
|
||||||
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
||||||
metadata = item.get("metadata") or {}
|
metadata = item.get("metadata") or {}
|
||||||
return _source_item(
|
return _source_item(
|
||||||
@@ -11,6 +11,11 @@ import re
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests as _requests
|
||||||
|
except ImportError:
|
||||||
|
_requests = None
|
||||||
|
|
||||||
from . import dates, http, log
|
from . import dates, http, log
|
||||||
|
|
||||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
|
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
|
||||||
@@ -135,17 +140,31 @@ def search_pinterest(
|
|||||||
|
|
||||||
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||||
|
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
_log("requests library not installed, falling back to urllib")
|
||||||
f"{SCRAPECREATORS_BASE}/search",
|
try:
|
||||||
params={"keyword": core_topic},
|
from urllib.parse import urlencode
|
||||||
headers=http.scrapecreators_headers(token),
|
params = urlencode({"keyword": core_topic})
|
||||||
timeout=30,
|
url = f"{SCRAPECREATORS_BASE}/search?{params}"
|
||||||
retries=2,
|
headers = http.scrapecreators_headers(token)
|
||||||
)
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
except Exception as e:
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
_log(f"ScrapeCreators error: {e}")
|
except Exception as e:
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
_log(f"ScrapeCreators error (urllib): {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/search",
|
||||||
|
params={"keyword": core_topic},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"ScrapeCreators error: {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
|
||||||
# Extract items from response - try common SC response shapes
|
# Extract items from response - try common SC response shapes
|
||||||
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
|
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
|
||||||
@@ -15,7 +15,6 @@ from . import (
|
|||||||
bluesky,
|
bluesky,
|
||||||
dates,
|
dates,
|
||||||
dedupe,
|
dedupe,
|
||||||
digg,
|
|
||||||
entity_extract,
|
entity_extract,
|
||||||
env,
|
env,
|
||||||
github,
|
github,
|
||||||
@@ -31,7 +30,6 @@ from . import (
|
|||||||
query,
|
query,
|
||||||
reddit,
|
reddit,
|
||||||
reddit_public,
|
reddit_public,
|
||||||
relevance,
|
|
||||||
rerank,
|
rerank,
|
||||||
schema,
|
schema,
|
||||||
signals,
|
signals,
|
||||||
@@ -42,7 +40,6 @@ from . import (
|
|||||||
xai_x,
|
xai_x,
|
||||||
xiaohongshu_api,
|
xiaohongshu_api,
|
||||||
xquik,
|
xquik,
|
||||||
xurl_x,
|
|
||||||
youtube_yt,
|
youtube_yt,
|
||||||
)
|
)
|
||||||
from .cluster import cluster_candidates
|
from .cluster import cluster_candidates
|
||||||
@@ -80,7 +77,6 @@ MOCK_AVAILABLE_SOURCES = [
|
|||||||
"github",
|
"github",
|
||||||
"perplexity",
|
"perplexity",
|
||||||
"xquik",
|
"xquik",
|
||||||
"digg",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -108,8 +104,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
|||||||
available.extend(["hackernews", "polymarket"])
|
available.extend(["hackernews", "polymarket"])
|
||||||
if config.get("GITHUB_TOKEN") or which("gh"):
|
if config.get("GITHUB_TOKEN") or which("gh"):
|
||||||
available.append("github")
|
available.append("github")
|
||||||
if which("digg-pp-cli"):
|
|
||||||
available.append("digg")
|
|
||||||
if env.is_bluesky_available(config):
|
if env.is_bluesky_available(config):
|
||||||
available.append("bluesky")
|
available.append("bluesky")
|
||||||
if env.is_truthsocial_available(config):
|
if env.is_truthsocial_available(config):
|
||||||
@@ -183,7 +177,6 @@ def run(
|
|||||||
lookback_days: int = 30,
|
lookback_days: int = 30,
|
||||||
github_user: str | None = None,
|
github_user: str | None = None,
|
||||||
github_repos: list[str] | None = None,
|
github_repos: list[str] | None = None,
|
||||||
internal_subrun: bool = False,
|
|
||||||
) -> schema.Report:
|
) -> schema.Report:
|
||||||
settings = DEPTH_SETTINGS[depth]
|
settings = DEPTH_SETTINGS[depth]
|
||||||
requested_sources = normalize_requested_sources(requested_sources)
|
requested_sources = normalize_requested_sources(requested_sources)
|
||||||
@@ -221,7 +214,6 @@ def run(
|
|||||||
provider=None if mock else reasoning_provider,
|
provider=None if mock else reasoning_provider,
|
||||||
model=None if mock else runtime.planner_model,
|
model=None if mock else runtime.planner_model,
|
||||||
context=config.get("_auto_resolve_context", ""),
|
context=config.get("_auto_resolve_context", ""),
|
||||||
internal_subrun=internal_subrun,
|
|
||||||
)
|
)
|
||||||
# Source labelling: the fallback path annotates notes with "fallback-plan"
|
# Source labelling: the fallback path annotates notes with "fallback-plan"
|
||||||
# or "deterministic-comparison-plan"; anything else came from the LLM.
|
# or "deterministic-comparison-plan"; anything else came from the LLM.
|
||||||
@@ -448,7 +440,7 @@ def run(
|
|||||||
if bundle.items_by_source.get(source):
|
if bundle.items_by_source.get(source):
|
||||||
del bundle.errors_by_source[source]
|
del bundle.errors_by_source[source]
|
||||||
|
|
||||||
items_by_source = _finalize_items_by_source(bundle.items_by_source, topic=topic, config=config)
|
items_by_source = _finalize_items_by_source(bundle.items_by_source, topic=topic)
|
||||||
candidates = weighted_rrf(bundle.items_by_source_and_query, plan, pool_limit=settings["pool_limit"])
|
candidates = weighted_rrf(bundle.items_by_source_and_query, plan, pool_limit=settings["pool_limit"])
|
||||||
ranked_candidates = rerank.rerank_candidates(
|
ranked_candidates = rerank.rerank_candidates(
|
||||||
topic=topic,
|
topic=topic,
|
||||||
@@ -505,19 +497,17 @@ def _normalize_score_dedupe(
|
|||||||
source, raw_items, from_date, to_date,
|
source, raw_items, from_date, to_date,
|
||||||
freshness_mode=freshness_mode,
|
freshness_mode=freshness_mode,
|
||||||
)
|
)
|
||||||
prepared_query = relevance.PreparedQuery(ranking_query)
|
normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
|
||||||
normalized = signals.annotate_stream(normalized, prepared_query, freshness_mode)
|
|
||||||
normalized = signals.prune_low_relevance(normalized)
|
normalized = signals.prune_low_relevance(normalized)
|
||||||
normalized = dedupe.dedupe_items(normalized)
|
normalized = dedupe.dedupe_items(normalized)
|
||||||
for item in normalized:
|
for item in normalized:
|
||||||
item.snippet = snippet.extract_best_snippet(item, prepared_query)
|
item.snippet = snippet.extract_best_snippet(item, ranking_query)
|
||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
def _finalize_items_by_source(
|
def _finalize_items_by_source(
|
||||||
items_by_source_raw: dict[str, list[schema.SourceItem]],
|
items_by_source_raw: dict[str, list[schema.SourceItem]],
|
||||||
topic: str = "",
|
topic: str = "",
|
||||||
config: dict | None = None,
|
|
||||||
) -> dict[str, list[schema.SourceItem]]:
|
) -> dict[str, list[schema.SourceItem]]:
|
||||||
finalized = {}
|
finalized = {}
|
||||||
for source, items in items_by_source_raw.items():
|
for source, items in items_by_source_raw.items():
|
||||||
@@ -530,17 +520,6 @@ def _finalize_items_by_source(
|
|||||||
# (e.g., WTI crude oil, Elon tweet counts) before footer emission.
|
# (e.g., WTI crude oil, Elon tweet counts) before footer emission.
|
||||||
if source == "polymarket" and topic:
|
if source == "polymarket" and topic:
|
||||||
items = polymarket.filter_items_against_topic(topic, items)
|
items = polymarket.filter_items_against_topic(topic, items)
|
||||||
# --polymarket-keywords (via config): additional keyword filter
|
|
||||||
# for ambiguous single-token topics (e.g., "Warriors" → nba,gsw).
|
|
||||||
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
|
|
||||||
if keywords:
|
|
||||||
items = polymarket.filter_items_against_keywords(items, keywords)
|
|
||||||
if source == "digg" and items:
|
|
||||||
# Pull top-ranked X posts only for the survivors that will appear
|
|
||||||
# in the brief. Spending the enrichment budget here (rather than
|
|
||||||
# at retrieval time) keeps the inline 'via Digg' quotes
|
|
||||||
# paired with the clusters dedupe actually kept.
|
|
||||||
digg.enrich_source_items(items, top_k=3)
|
|
||||||
finalized[source] = items
|
finalized[source] = items
|
||||||
return finalized
|
return finalized
|
||||||
|
|
||||||
@@ -916,9 +895,6 @@ def _retrieve_stream(
|
|||||||
depth=depth,
|
depth=depth,
|
||||||
)
|
)
|
||||||
return xai_x.parse_x_response(result), {}
|
return xai_x.parse_x_response(result), {}
|
||||||
if backend == "xurl":
|
|
||||||
result = xurl_x.search_x(subquery.search_query, depth=depth)
|
|
||||||
return xurl_x.parse_x_response(result, topic=subquery.search_query), {}
|
|
||||||
raise RuntimeError("No X backend is available.")
|
raise RuntimeError("No X backend is available.")
|
||||||
if source == "youtube":
|
if source == "youtube":
|
||||||
# Use raw_topic so expand_youtube_queries() generates diverse variants
|
# Use raw_topic so expand_youtube_queries() generates diverse variants
|
||||||
@@ -976,13 +952,6 @@ def _retrieve_stream(
|
|||||||
if source == "hackernews":
|
if source == "hackernews":
|
||||||
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
||||||
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
||||||
if source == "digg":
|
|
||||||
result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
|
|
||||||
items = digg.parse_digg_response(result, query=subquery.search_query)
|
|
||||||
# Enrichment with attached X posts is deferred to
|
|
||||||
# _finalize_items_by_source so it runs on the items that actually
|
|
||||||
# survive dedupe rather than on top-K of the raw fanout.
|
|
||||||
return items, {}
|
|
||||||
if source == "bluesky":
|
if source == "bluesky":
|
||||||
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
||||||
return bluesky.parse_bluesky_response(result), {}
|
return bluesky.parse_bluesky_response(result), {}
|
||||||
@@ -1075,45 +1044,6 @@ def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[d
|
|||||||
"why_relevant": "Brave web search",
|
"why_relevant": "Brave web search",
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"digg": [
|
|
||||||
{
|
|
||||||
"id": "mock1abc",
|
|
||||||
"title": f"Digg cluster about {subquery.search_query}",
|
|
||||||
"url": "https://di.gg/ai/mock1abc",
|
|
||||||
"tldr": f"Curated cluster summarizing recent {subquery.search_query} discussion across the AI 1000.",
|
|
||||||
"author": "",
|
|
||||||
"date": dates.get_date_range(3)[0],
|
|
||||||
"engagement": {"postCount": 8, "uniqueAuthors": 5, "rank": 2, "rank_score": 49.0},
|
|
||||||
"first_post_age": "3d",
|
|
||||||
"posts": [
|
|
||||||
{
|
|
||||||
"username": "exampledev",
|
|
||||||
"display_name": "Example Dev",
|
|
||||||
"category": "Engineer",
|
|
||||||
"rank": 142,
|
|
||||||
"body": f"Quote from the AI 1000 about {subquery.search_query}.",
|
|
||||||
"post_type": "tweet",
|
|
||||||
"x_url": "https://x.com/exampledev/status/1",
|
|
||||||
"posted_at": dates.get_date_range(3)[0],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
"relevance": 0.84,
|
|
||||||
"why_relevant": "Mock Digg cluster",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "mock2def",
|
|
||||||
"title": f"Second Digg cluster on {subquery.search_query}",
|
|
||||||
"url": "https://di.gg/ai/mock2def",
|
|
||||||
"tldr": f"Another angle on {subquery.search_query}.",
|
|
||||||
"author": "",
|
|
||||||
"date": dates.get_date_range(8)[0],
|
|
||||||
"engagement": {"postCount": 3, "uniqueAuthors": 2, "rank": 18, "rank_score": 33.0},
|
|
||||||
"first_post_age": "8d",
|
|
||||||
"posts": [],
|
|
||||||
"relevance": 0.71,
|
|
||||||
"why_relevant": "Mock Digg cluster",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
}
|
||||||
if source == "grounding":
|
if source == "grounding":
|
||||||
return payloads.get(source, []), {
|
return payloads.get(source, []), {
|
||||||
@@ -67,7 +67,6 @@ SOURCE_CAPABILITIES = {
|
|||||||
"bluesky": {"discussion", "social"},
|
"bluesky": {"discussion", "social"},
|
||||||
"truthsocial": {"discussion", "social"},
|
"truthsocial": {"discussion", "social"},
|
||||||
"polymarket": {"market"},
|
"polymarket": {"market"},
|
||||||
"digg": {"discussion", "social", "link"},
|
|
||||||
"xiaohongshu": {"video", "video_shortform", "social"},
|
"xiaohongshu": {"video", "video_shortform", "social"},
|
||||||
"github": {"discussion", "link"},
|
"github": {"discussion", "link"},
|
||||||
"grounding": {"web", "reference", "link"},
|
"grounding": {"web", "reference", "link"},
|
||||||
@@ -87,16 +86,9 @@ def plan_query(
|
|||||||
provider: providers.ReasoningClient | None,
|
provider: providers.ReasoningClient | None,
|
||||||
model: str | None,
|
model: str | None,
|
||||||
context: str = "",
|
context: str = "",
|
||||||
internal_subrun: bool = False,
|
|
||||||
) -> schema.QueryPlan:
|
) -> schema.QueryPlan:
|
||||||
"""Create a query plan. Comparison queries with extractable entities use a
|
"""Create a query plan. Comparison queries with extractable entities use a
|
||||||
deterministic plan; other intents prefer the configured reasoning provider.
|
deterministic plan; other intents prefer the configured reasoning provider."""
|
||||||
|
|
||||||
internal_subrun: when True, suppress the LAW 7 "No --plan passed" stderr
|
|
||||||
warning. LAW 7 targets the hosting-reasoning-model path; competitor
|
|
||||||
fan-out sub-runs are engine-internal and the warning is a false positive
|
|
||||||
there. Default False preserves the warning on every user-facing invocation.
|
|
||||||
"""
|
|
||||||
if _should_force_deterministic_plan(topic):
|
if _should_force_deterministic_plan(topic):
|
||||||
return _fallback_plan(
|
return _fallback_plan(
|
||||||
topic,
|
topic,
|
||||||
@@ -130,17 +122,16 @@ def plan_query(
|
|||||||
# planner credentials - NOT a prerequisite the caller needs. If you are
|
# planner credentials - NOT a prerequisite the caller needs. If you are
|
||||||
# the hosting reasoning model, YOU are the provider. LAW 7 / 2026-04-19
|
# the hosting reasoning model, YOU are the provider. LAW 7 / 2026-04-19
|
||||||
# Hermes Agent Use Cases failure mode.
|
# Hermes Agent Use Cases failure mode.
|
||||||
if not internal_subrun:
|
import sys
|
||||||
import sys
|
print(
|
||||||
print(
|
"[Planner] No --plan passed. If you are the reasoning model hosting "
|
||||||
"[Planner] No --plan passed. If you are the reasoning model hosting "
|
"this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime), "
|
||||||
"this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime), "
|
"YOU ARE the planner: generate a JSON query plan yourself and pass it "
|
||||||
"YOU ARE the planner: generate a JSON query plan yourself and pass it "
|
"via --plan. You do not need an API key or credentials; you ARE the "
|
||||||
"via --plan. You do not need an API key or credentials; you ARE the "
|
"LLM. The deterministic fallback below is the headless/cron path only. "
|
||||||
"LLM. The deterministic fallback below is the headless/cron path only. "
|
"See LAW 7 in SKILL.md and Step 0.75 for the plan schema.",
|
||||||
"See LAW 7 in SKILL.md and Step 0.75 for the plan schema.",
|
file=sys.stderr,
|
||||||
file=sys.stderr,
|
)
|
||||||
)
|
|
||||||
return _fallback_plan(topic, available_sources, requested_sources, depth)
|
return _fallback_plan(topic, available_sources, requested_sources, depth)
|
||||||
|
|
||||||
|
|
||||||
@@ -232,39 +232,6 @@ def filter_items_against_topic(topic: str, items: List[Any]) -> List[Any]:
|
|||||||
return filtered
|
return filtered
|
||||||
|
|
||||||
|
|
||||||
def filter_items_against_keywords(items: List[Any], keywords: List[str]) -> List[Any]:
|
|
||||||
"""Keep only items whose title contains at least one keyword (case-insensitive).
|
|
||||||
|
|
||||||
Intended for disambiguating ambiguous single-token topics like 'Warriors'
|
|
||||||
via --polymarket-keywords (e.g., 'nba,gsw,golden-state') to filter out
|
|
||||||
Glasgow Warriors rugby, Honor of Kings Rogue Warriors markets that share
|
|
||||||
the 'Warriors' token but are not the target entity.
|
|
||||||
"""
|
|
||||||
if not keywords:
|
|
||||||
return items
|
|
||||||
normalized_keywords = [kw.strip().lower() for kw in keywords if kw and kw.strip()]
|
|
||||||
if not normalized_keywords:
|
|
||||||
return items
|
|
||||||
|
|
||||||
filtered = []
|
|
||||||
for item in items:
|
|
||||||
title = getattr(item, "title", None)
|
|
||||||
if title is None and isinstance(item, dict):
|
|
||||||
title = item.get("title", "")
|
|
||||||
title = (title or "").lower()
|
|
||||||
if any(kw in title for kw in normalized_keywords):
|
|
||||||
filtered.append(item)
|
|
||||||
|
|
||||||
dropped = len(items) - len(filtered)
|
|
||||||
if dropped:
|
|
||||||
_log(
|
|
||||||
f"Keyword filter dropped {dropped} Polymarket items; "
|
|
||||||
f"kept {len(filtered)} matching {normalized_keywords}"
|
|
||||||
)
|
|
||||||
|
|
||||||
return filtered
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_domain_queries(topic: str, events: List[Dict]) -> List[str]:
|
def _extract_domain_queries(topic: str, events: List[Dict]) -> List[str]:
|
||||||
"""Extract domain-indicator search terms from first-pass event tags.
|
"""Extract domain-indicator search terms from first-pass event tags.
|
||||||
|
|
||||||
@@ -93,6 +93,13 @@ class GeminiClient(ReasoningClient):
|
|||||||
)
|
)
|
||||||
return extract_gemini_text(payload)
|
return extract_gemini_text(payload)
|
||||||
|
|
||||||
|
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
|
||||||
|
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
|
||||||
|
|
||||||
|
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
|
||||||
|
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
|
||||||
|
|
||||||
|
|
||||||
class OpenAIClient(ReasoningClient):
|
class OpenAIClient(ReasoningClient):
|
||||||
name = "openai"
|
name = "openai"
|
||||||
|
|
||||||
@@ -71,29 +71,8 @@ def _normalize_phrase(text: str) -> str:
|
|||||||
return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split())
|
return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split())
|
||||||
|
|
||||||
|
|
||||||
class PreparedQuery:
|
|
||||||
"""Precomputed query shape reused across items in a stream.
|
|
||||||
|
|
||||||
Built once per ranking_query; reused by token_overlap_relevance so the
|
|
||||||
per-item normalize/score loops don't re-tokenize the same query N times.
|
|
||||||
"""
|
|
||||||
|
|
||||||
__slots__ = ("raw", "q_tokens", "informative_q_tokens", "normalized_phrase")
|
|
||||||
|
|
||||||
def __init__(self, query: str) -> None:
|
|
||||||
self.raw = query
|
|
||||||
self.q_tokens = tokenize(query)
|
|
||||||
informative = {t for t in self.q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
|
||||||
self.informative_q_tokens = informative or self.q_tokens
|
|
||||||
self.normalized_phrase = _normalize_phrase(query)
|
|
||||||
|
|
||||||
|
|
||||||
def _as_prepared(query: "str | PreparedQuery") -> PreparedQuery:
|
|
||||||
return query if isinstance(query, PreparedQuery) else PreparedQuery(query)
|
|
||||||
|
|
||||||
|
|
||||||
def token_overlap_relevance(
|
def token_overlap_relevance(
|
||||||
query: "str | PreparedQuery",
|
query: str,
|
||||||
text: str,
|
text: str,
|
||||||
hashtags: Optional[List[str]] = None,
|
hashtags: Optional[List[str]] = None,
|
||||||
) -> float:
|
) -> float:
|
||||||
@@ -116,8 +95,7 @@ def token_overlap_relevance(
|
|||||||
Returns:
|
Returns:
|
||||||
Float between 0.0 and 1.0 (0.5 for empty queries)
|
Float between 0.0 and 1.0 (0.5 for empty queries)
|
||||||
"""
|
"""
|
||||||
prepared = _as_prepared(query)
|
q_tokens = tokenize(query)
|
||||||
q_tokens = prepared.q_tokens
|
|
||||||
|
|
||||||
# Combine text and hashtags for matching
|
# Combine text and hashtags for matching
|
||||||
combined = text
|
combined = text
|
||||||
@@ -141,7 +119,9 @@ def token_overlap_relevance(
|
|||||||
if overlap == 0:
|
if overlap == 0:
|
||||||
return 0.0
|
return 0.0
|
||||||
|
|
||||||
informative_q_tokens = prepared.informative_q_tokens
|
informative_q_tokens = {t for t in q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
||||||
|
if not informative_q_tokens:
|
||||||
|
informative_q_tokens = q_tokens
|
||||||
|
|
||||||
coverage = overlap / len(q_tokens)
|
coverage = overlap / len(q_tokens)
|
||||||
informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens)
|
informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens)
|
||||||
@@ -149,7 +129,7 @@ def token_overlap_relevance(
|
|||||||
precision = overlap / precision_denominator
|
precision = overlap / precision_denominator
|
||||||
|
|
||||||
phrase_bonus = 0.0
|
phrase_bonus = 0.0
|
||||||
normalized_query = prepared.normalized_phrase
|
normalized_query = _normalize_phrase(query)
|
||||||
normalized_text = _normalize_phrase(combined)
|
normalized_text = _normalize_phrase(combined)
|
||||||
if normalized_query and normalized_query in normalized_text:
|
if normalized_query and normalized_query in normalized_text:
|
||||||
phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
|
phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
|
||||||
@@ -4,7 +4,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
@@ -12,47 +11,21 @@ from urllib.parse import urlparse
|
|||||||
from . import dates, schema
|
from . import dates, schema
|
||||||
|
|
||||||
|
|
||||||
_VERSION_RE = re.compile(
|
|
||||||
r'''^version:\s*(?:"([^"]+)"|'([^']+)'|(\S+))\s*$''',
|
|
||||||
re.MULTILINE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _skill_version() -> str:
|
def _skill_version() -> str:
|
||||||
"""Read plugin version from .claude-plugin/plugin.json, falling back to SKILL.md frontmatter.
|
"""Read plugin version from .claude-plugin/plugin.json if available.
|
||||||
|
|
||||||
Per-harness skill install dirs (`~/.claude/skills`, `~/.codex/skills`, `~/.agents/skills`,
|
Tries nearest plugin.json by walking up from render.py's own location.
|
||||||
Hermes, etc.) do not always carry `.claude-plugin/plugin.json` — that file ships with
|
Falls back to "?" if not found. This keeps the badge emission from
|
||||||
plugin-cache installs but not with per-harness skill installs. SKILL.md frontmatter is
|
crashing on non-plugin-cache installs (repo checkout, Gemini, Codex).
|
||||||
the fallback that keeps the badge from emitting v? on those installs. Returns "?" only
|
|
||||||
if both sources are missing.
|
|
||||||
|
|
||||||
A corrupt manifest at one ancestor does not shadow a valid manifest at a deeper one
|
|
||||||
(continue, not break). YAML frontmatter accepts double-quoted, single-quoted, or
|
|
||||||
unquoted version scalars.
|
|
||||||
"""
|
"""
|
||||||
here = pathlib.Path(__file__).resolve()
|
here = pathlib.Path(__file__).resolve()
|
||||||
for parent in here.parents:
|
for parent in [here.parent, *here.parents]:
|
||||||
manifest = parent / ".claude-plugin" / "plugin.json"
|
candidate = parent / ".claude-plugin" / "plugin.json"
|
||||||
if manifest.is_file():
|
if candidate.is_file():
|
||||||
try:
|
try:
|
||||||
version = json.loads(manifest.read_text()).get("version")
|
return json.loads(candidate.read_text()).get("version", "?")
|
||||||
except (json.JSONDecodeError, OSError):
|
except (json.JSONDecodeError, OSError):
|
||||||
continue
|
return "?"
|
||||||
if version:
|
|
||||||
return version
|
|
||||||
|
|
||||||
# No usable manifest found at any ancestor — fall back to SKILL.md frontmatter.
|
|
||||||
for parent in here.parents:
|
|
||||||
skill_md = parent / "SKILL.md"
|
|
||||||
if skill_md.is_file():
|
|
||||||
try:
|
|
||||||
match = _VERSION_RE.search(skill_md.read_text())
|
|
||||||
except (OSError, UnicodeDecodeError):
|
|
||||||
break
|
|
||||||
if match:
|
|
||||||
return next(g for g in match.groups() if g is not None)
|
|
||||||
break
|
|
||||||
return "?"
|
return "?"
|
||||||
|
|
||||||
|
|
||||||
@@ -79,7 +52,6 @@ SOURCE_LABELS = {
|
|||||||
"xiaohongshu": "Xiaohongshu",
|
"xiaohongshu": "Xiaohongshu",
|
||||||
"x": "X",
|
"x": "X",
|
||||||
"github": "GitHub",
|
"github": "GitHub",
|
||||||
"digg": "Digg",
|
|
||||||
"perplexity": "Perplexity",
|
"perplexity": "Perplexity",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,168 +170,6 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
|||||||
return "\n".join(lines).strip() + "\n"
|
return "\n".join(lines).strip() + "\n"
|
||||||
|
|
||||||
|
|
||||||
def render_for_html(
|
|
||||||
report: schema.Report,
|
|
||||||
synthesis_md: str | None = None,
|
|
||||||
*,
|
|
||||||
save_path: str | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Render markdown intended for shareable HTML conversion.
|
|
||||||
|
|
||||||
This output keeps the public badge, compact source/date metadata, an
|
|
||||||
optional one-line data quality note, optional synthesized brief markdown,
|
|
||||||
and the engine footer. It deliberately omits the debug file header,
|
|
||||||
model-facing safety note, and evidence scratchpad emitted by
|
|
||||||
render_compact().
|
|
||||||
|
|
||||||
When synthesis_md is None, the body is intentionally sparse: badge,
|
|
||||||
metadata, optional data quality note, and engine footer only.
|
|
||||||
"""
|
|
||||||
lines = [
|
|
||||||
*_render_badge(),
|
|
||||||
*_render_html_metadata(report),
|
|
||||||
]
|
|
||||||
if synthesis_md:
|
|
||||||
lines.extend(["", synthesis_md.strip()])
|
|
||||||
# Data quality warnings are NOT rendered into the HTML artifact. The HTML
|
|
||||||
# is meant to be shared (Slack, email, Notion); recipients haven't asked
|
|
||||||
# for technical commentary about how the run was produced. Generators see
|
|
||||||
# the same warnings via collect_html_warnings() routed to stderr by the
|
|
||||||
# CLI, so they can fix quality issues before sharing.
|
|
||||||
_append_html_footer(lines, report, save_path)
|
|
||||||
return "\n".join(lines).strip() + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def render_for_html_comparison(
|
|
||||||
entity_reports: list[tuple[str, schema.Report]],
|
|
||||||
synthesis_md: str | None = None,
|
|
||||||
*,
|
|
||||||
save_path: str | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Render comparison markdown intended for shareable HTML conversion.
|
|
||||||
|
|
||||||
Same semantics as render_for_html(), but metadata and data quality notes
|
|
||||||
are aggregated across the compared entities.
|
|
||||||
"""
|
|
||||||
if not entity_reports:
|
|
||||||
raise ValueError("render_for_html_comparison requires at least one report")
|
|
||||||
|
|
||||||
entities = [label for label, _ in entity_reports]
|
|
||||||
main_report = entity_reports[0][1]
|
|
||||||
meta = (
|
|
||||||
f"<!-- META: {main_report.range_from} to {main_report.range_to} "
|
|
||||||
f"· comparing {len(entities)}: {', '.join(entities)} -->"
|
|
||||||
)
|
|
||||||
lines = [
|
|
||||||
*_render_badge(),
|
|
||||||
meta,
|
|
||||||
]
|
|
||||||
if synthesis_md:
|
|
||||||
lines.extend(["", synthesis_md.strip()])
|
|
||||||
# Comparison data quality notes also go to stderr, not into the artifact.
|
|
||||||
_append_html_footer(lines, main_report, save_path)
|
|
||||||
return "\n".join(lines).strip() + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def collect_html_warnings(report: schema.Report) -> list[str]:
|
|
||||||
"""Collect data quality warnings for stderr output (NOT for the HTML artifact).
|
|
||||||
|
|
||||||
Returns a list of human-readable warning strings. Empty list if the run
|
|
||||||
was clean. Used by the CLI to emit diagnostics to stderr after writing
|
|
||||||
the HTML to stdout/file.
|
|
||||||
"""
|
|
||||||
notes: list[str] = []
|
|
||||||
if _render_degraded_run_warning(report):
|
|
||||||
notes.append("Run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
|
||||||
elif _render_pre_research_warning(report):
|
|
||||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
|
||||||
freshness_warning = _assess_data_freshness(report)
|
|
||||||
if freshness_warning:
|
|
||||||
notes.append(freshness_warning)
|
|
||||||
notes.extend(report.warnings)
|
|
||||||
return _dedupe_notes(notes)
|
|
||||||
|
|
||||||
|
|
||||||
def collect_html_warnings_comparison(
|
|
||||||
entity_reports: list[tuple[str, schema.Report]],
|
|
||||||
) -> list[str]:
|
|
||||||
"""Collect comparison-mode warnings, prefixed by entity label."""
|
|
||||||
notes: list[str] = []
|
|
||||||
for label, report in entity_reports:
|
|
||||||
for w in collect_html_warnings(report):
|
|
||||||
notes.append(f"{label}: {w}")
|
|
||||||
return notes
|
|
||||||
|
|
||||||
|
|
||||||
def _render_html_metadata(report: schema.Report) -> list[str]:
|
|
||||||
"""Inline metadata as an HTML comment marker.
|
|
||||||
|
|
||||||
html_render.py post-processes ``<!-- META: ... -->`` markers into a
|
|
||||||
``<div class="meta">`` after markdown conversion, so the metadata escapes
|
|
||||||
the markdown converter's HTML-escaping pass cleanly. Same pattern as the
|
|
||||||
PASS_THROUGH_FOOTER marker used for the engine tree.
|
|
||||||
"""
|
|
||||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
|
||||||
if non_empty:
|
|
||||||
sources = ", ".join(_source_label(s) for s in non_empty)
|
|
||||||
else:
|
|
||||||
sources = "no active sources"
|
|
||||||
return [
|
|
||||||
f"<!-- META: {report.range_from} to {report.range_to} · {sources} -->",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _render_html_data_quality_note(report: schema.Report) -> str | None:
|
|
||||||
notes: list[str] = []
|
|
||||||
degraded_warning = _render_degraded_run_warning(report)
|
|
||||||
if degraded_warning:
|
|
||||||
notes.append("This run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
|
||||||
pre_research_warning = _render_pre_research_warning(report)
|
|
||||||
if pre_research_warning and not degraded_warning:
|
|
||||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
|
||||||
freshness_warning = _assess_data_freshness(report)
|
|
||||||
if freshness_warning:
|
|
||||||
notes.append(freshness_warning)
|
|
||||||
notes.extend(report.warnings)
|
|
||||||
if not notes:
|
|
||||||
return None
|
|
||||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
|
||||||
|
|
||||||
|
|
||||||
def _render_html_comparison_data_quality_note(
|
|
||||||
entity_reports: list[tuple[str, schema.Report]],
|
|
||||||
) -> str | None:
|
|
||||||
notes: list[str] = []
|
|
||||||
for label, report in entity_reports:
|
|
||||||
note = _render_html_data_quality_note(report)
|
|
||||||
if note:
|
|
||||||
clean = note.removeprefix("> **Data quality note:** ").strip()
|
|
||||||
notes.append(f"{label}: {clean}")
|
|
||||||
if not notes:
|
|
||||||
return None
|
|
||||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
|
||||||
|
|
||||||
|
|
||||||
def _dedupe_notes(notes: list[str]) -> list[str]:
|
|
||||||
out: list[str] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
for note in notes:
|
|
||||||
normalized = " ".join(str(note).split())
|
|
||||||
if not normalized or normalized in seen:
|
|
||||||
continue
|
|
||||||
seen.add(normalized)
|
|
||||||
out.append(normalized)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _append_html_footer(lines: list[str], report: schema.Report, save_path: str | None) -> None:
|
|
||||||
footer = _render_emoji_footer(report, save_path)
|
|
||||||
lines.append("")
|
|
||||||
lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")
|
|
||||||
lines.extend(footer)
|
|
||||||
lines.append("<!-- END PASS-THROUGH FOOTER -->")
|
|
||||||
|
|
||||||
|
|
||||||
def _render_canonical_boundary() -> list[str]:
|
def _render_canonical_boundary() -> list[str]:
|
||||||
"""Emit the explicit END-OF-CANONICAL-OUTPUT boundary.
|
"""Emit the explicit END-OF-CANONICAL-OUTPUT boundary.
|
||||||
|
|
||||||
@@ -582,220 +392,6 @@ def _render_comparison_scaffold(topic: str) -> list[str]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def render_comparison_multi(
|
|
||||||
entity_reports: list[tuple[str, schema.Report]],
|
|
||||||
*,
|
|
||||||
cluster_limit: int = 4,
|
|
||||||
fun_level: str = "medium",
|
|
||||||
save_path: str | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Render N (entity, Report) pairs as a single comparison output.
|
|
||||||
|
|
||||||
Reuses _render_comparison_scaffold for the synthesis table and emits
|
|
||||||
per-entity evidence sections inside one EVIDENCE FOR SYNTHESIS envelope.
|
|
||||||
The single-Report render_compact path is unchanged.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
entity_reports: Ordered (label, Report) pairs. The first pair is the
|
|
||||||
user's main topic; the remainder are discovered/explicit competitors.
|
|
||||||
cluster_limit: Max clusters to surface per entity (kept lower than the
|
|
||||||
single-entity default to keep N-way comparisons readable).
|
|
||||||
fun_level: Same fun-level knob as render_compact, applied to each
|
|
||||||
entity's best-takes block.
|
|
||||||
save_path: Optional save-path display string for the footer.
|
|
||||||
"""
|
|
||||||
if not entity_reports:
|
|
||||||
raise ValueError("render_comparison_multi requires at least one report")
|
|
||||||
|
|
||||||
entities = [label for label, _ in entity_reports]
|
|
||||||
main_label, main_report = entity_reports[0]
|
|
||||||
synthesized_topic = " vs ".join(entities)
|
|
||||||
|
|
||||||
lines: list[str] = [
|
|
||||||
*_render_badge(),
|
|
||||||
f"# last30days v3.0.0: {synthesized_topic}",
|
|
||||||
"",
|
|
||||||
*_assistant_safety_lines(),
|
|
||||||
f"- Comparison mode: {len(entities)} entities ({', '.join(entities)})",
|
|
||||||
f"- Date range: {main_report.range_from} to {main_report.range_to}",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
|
|
||||||
aggregated_warnings: list[str] = []
|
|
||||||
for label, report in entity_reports:
|
|
||||||
aggregated_warnings.extend(f"[{label}] {w}" for w in report.warnings)
|
|
||||||
if aggregated_warnings:
|
|
||||||
lines.append("## Warnings")
|
|
||||||
lines.extend(f"- {w}" for w in aggregated_warnings)
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
lines.append(
|
|
||||||
"<!-- EVIDENCE FOR SYNTHESIS: read this, do not emit verbatim. Transform into "
|
|
||||||
"`What I learned:` prose per LAW 2. Each entity has its own evidence subsection. -->"
|
|
||||||
)
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
resolved_block = _render_resolved_entities_block(entity_reports)
|
|
||||||
if resolved_block:
|
|
||||||
lines.extend(resolved_block)
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
fun_params = _FUN_LEVELS.get(fun_level, _FUN_LEVELS["medium"])
|
|
||||||
for label, report in entity_reports:
|
|
||||||
lines.extend(_render_entity_evidence_block(
|
|
||||||
label=label,
|
|
||||||
report=report,
|
|
||||||
cluster_limit=cluster_limit,
|
|
||||||
fun_params=fun_params,
|
|
||||||
))
|
|
||||||
|
|
||||||
lines.append("<!-- END EVIDENCE FOR SYNTHESIS -->")
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
# Reuse the existing comparison scaffold by feeding it the synthesized
|
|
||||||
# topic. _parse_comparison_entities splits on " vs " so the scaffold
|
|
||||||
# picks up all N entities automatically.
|
|
||||||
scaffold = _render_comparison_scaffold(synthesized_topic)
|
|
||||||
lines.extend(scaffold)
|
|
||||||
|
|
||||||
footer = _render_emoji_footer(main_report, save_path)
|
|
||||||
if footer:
|
|
||||||
lines.append("")
|
|
||||||
lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")
|
|
||||||
lines.extend(footer)
|
|
||||||
lines.append("<!-- END PASS-THROUGH FOOTER -->")
|
|
||||||
|
|
||||||
lines.extend(_render_canonical_boundary())
|
|
||||||
|
|
||||||
return "\n".join(lines).strip() + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def _render_resolved_entities_block(
|
|
||||||
entity_reports: list[tuple[str, schema.Report]],
|
|
||||||
) -> list[str]:
|
|
||||||
"""Emit a visible per-entity Step 0.55 resolution summary.
|
|
||||||
|
|
||||||
Reads `resolved` dicts from each Report's artifacts. Returns an empty
|
|
||||||
list when no entity has a resolved payload (mock mode, no web backend,
|
|
||||||
or artifacts not populated). Missing per-entity fields render as `-`.
|
|
||||||
Context strings truncate at 120 chars.
|
|
||||||
"""
|
|
||||||
any_resolved = any(
|
|
||||||
isinstance(report.artifacts.get("resolved"), dict)
|
|
||||||
for _label, report in entity_reports
|
|
||||||
)
|
|
||||||
if not any_resolved:
|
|
||||||
return []
|
|
||||||
|
|
||||||
out: list[str] = ["## Resolved Entities", ""]
|
|
||||||
for label, report in entity_reports:
|
|
||||||
resolved = report.artifacts.get("resolved") or {}
|
|
||||||
x_handle = resolved.get("x_handle") or ""
|
|
||||||
subs = resolved.get("subreddits") or []
|
|
||||||
gh_user = resolved.get("github_user") or ""
|
|
||||||
gh_repos = resolved.get("github_repos") or []
|
|
||||||
context = resolved.get("context") or ""
|
|
||||||
|
|
||||||
x_display = f"@{x_handle}" if x_handle else "-"
|
|
||||||
subs_display = (
|
|
||||||
", ".join(f"r/{s}" for s in subs[:5]) + (
|
|
||||||
f" (+{len(subs) - 5})" if len(subs) > 5 else ""
|
|
||||||
)
|
|
||||||
) if subs else "-"
|
|
||||||
gh_display = f"@{gh_user}" if gh_user else "-"
|
|
||||||
if gh_repos:
|
|
||||||
gh_display += f" ({', '.join(gh_repos[:3])}" + (
|
|
||||||
f" +{len(gh_repos) - 3}" if len(gh_repos) > 3 else ""
|
|
||||||
) + ")"
|
|
||||||
context_display = _truncate(context, 120) if context else "-"
|
|
||||||
|
|
||||||
out.append(
|
|
||||||
f"- **{label}**: X {x_display} | Subs {subs_display} | "
|
|
||||||
f"GitHub {gh_display} | Context: {context_display}"
|
|
||||||
)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _render_entity_evidence_block(
|
|
||||||
*,
|
|
||||||
label: str,
|
|
||||||
report: schema.Report,
|
|
||||||
cluster_limit: int,
|
|
||||||
fun_params: dict,
|
|
||||||
) -> list[str]:
|
|
||||||
"""Render one entity's clusters and best-takes inside the evidence envelope."""
|
|
||||||
candidate_by_id = {c.candidate_id: c for c in report.ranked_candidates}
|
|
||||||
out: list[str] = [f"## {label}", ""]
|
|
||||||
|
|
||||||
if not report.clusters:
|
|
||||||
out.append("(no significant discussion this month)")
|
|
||||||
out.append("")
|
|
||||||
return out
|
|
||||||
|
|
||||||
out.append("### Ranked Evidence Clusters")
|
|
||||||
out.append("")
|
|
||||||
for index, cluster in enumerate(report.clusters[:cluster_limit], start=1):
|
|
||||||
out.append(
|
|
||||||
f"#### {index}. {cluster.title} "
|
|
||||||
f"(score {cluster.score:.0f}, {len(cluster.candidate_ids)} item"
|
|
||||||
f"{'s' if len(cluster.candidate_ids) != 1 else ''}, "
|
|
||||||
f"sources: {', '.join(_source_label(s) for s in cluster.sources)})"
|
|
||||||
)
|
|
||||||
if cluster.uncertainty:
|
|
||||||
out.append(f"- Uncertainty: {cluster.uncertainty}")
|
|
||||||
for rep_index, candidate_id in enumerate(cluster.representative_ids, start=1):
|
|
||||||
candidate = candidate_by_id.get(candidate_id)
|
|
||||||
if not candidate:
|
|
||||||
continue
|
|
||||||
out.extend(_render_candidate(candidate, prefix=f"{rep_index}."))
|
|
||||||
out.append("")
|
|
||||||
|
|
||||||
best_takes = _render_best_takes(
|
|
||||||
report.ranked_candidates,
|
|
||||||
limit=fun_params["limit"],
|
|
||||||
threshold=fun_params["threshold"],
|
|
||||||
)
|
|
||||||
if best_takes:
|
|
||||||
out.extend(best_takes)
|
|
||||||
out.append("")
|
|
||||||
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def render_comparison_multi_context(
|
|
||||||
entity_reports: list[tuple[str, schema.Report]],
|
|
||||||
cluster_limit: int = 4,
|
|
||||||
) -> str:
|
|
||||||
"""Context-mode rendering for the multi-entity comparison."""
|
|
||||||
if not entity_reports:
|
|
||||||
raise ValueError("render_comparison_multi_context requires at least one report")
|
|
||||||
|
|
||||||
entities = [label for label, _ in entity_reports]
|
|
||||||
lines = [
|
|
||||||
f"Comparison: {' vs '.join(entities)}",
|
|
||||||
f"Entities: {len(entities)}",
|
|
||||||
_AI_SAFETY_NOTE,
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
resolved_block = _render_resolved_entities_block(entity_reports)
|
|
||||||
if resolved_block:
|
|
||||||
lines.extend(resolved_block)
|
|
||||||
lines.append("")
|
|
||||||
for label, report in entity_reports:
|
|
||||||
lines.append(f"## {label}")
|
|
||||||
lines.append(f"Intent: {report.query_plan.intent}")
|
|
||||||
if not report.clusters:
|
|
||||||
lines.append("- (no significant discussion this month)")
|
|
||||||
else:
|
|
||||||
for cluster in report.clusters[:cluster_limit]:
|
|
||||||
lines.append(
|
|
||||||
f"- {cluster.title} "
|
|
||||||
f"[{', '.join(_source_label(s) for s in cluster.sources)}]"
|
|
||||||
)
|
|
||||||
lines.append("")
|
|
||||||
return "\n".join(lines).strip() + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def render_full(report: schema.Report) -> str:
|
def render_full(report: schema.Report) -> str:
|
||||||
"""Full data dump: ALL clusters + ALL items by source. For saved files and debugging."""
|
"""Full data dump: ALL clusters + ALL items by source. For saved files and debugging."""
|
||||||
# Start with the same header as compact
|
# Start with the same header as compact
|
||||||
@@ -814,17 +410,6 @@ def render_full(report: schema.Report) -> str:
|
|||||||
lines.extend(f"- {warning}" for warning in report.warnings)
|
lines.extend(f"- {warning}" for warning in report.warnings)
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# When this Report is a per-entity sub-run from vs-mode / --competitors,
|
|
||||||
# include the single-row Resolved Entities block so the saved file is
|
|
||||||
# self-describing. The artifact is populated by last30days.py's
|
|
||||||
# _competitor_runner and _main_runner closures.
|
|
||||||
resolved = report.artifacts.get("resolved")
|
|
||||||
if isinstance(resolved, dict) and resolved.get("entity"):
|
|
||||||
single_row = _render_resolved_entities_block([(resolved["entity"], report)])
|
|
||||||
if single_row:
|
|
||||||
lines.extend(single_row)
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
# ALL clusters (no limit)
|
# ALL clusters (no limit)
|
||||||
lines.append("## Ranked Evidence Clusters")
|
lines.append("## Ranked Evidence Clusters")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
@@ -853,7 +438,7 @@ def render_full(report: schema.Report) -> str:
|
|||||||
lines.append("## All Items by Source")
|
lines.append("## All Items by Source")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest",
|
source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest",
|
||||||
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "digg", "perplexity"]
|
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "perplexity"]
|
||||||
for source in source_order:
|
for source in source_order:
|
||||||
items = report.items_by_source.get(source, [])
|
items = report.items_by_source.get(source, [])
|
||||||
if not items:
|
if not items:
|
||||||
@@ -877,11 +462,7 @@ def render_full(report: schema.Report) -> str:
|
|||||||
for tc in top_comments[:3]:
|
for tc in top_comments[:3]:
|
||||||
excerpt = tc.get("excerpt", tc.get("text", ""))[:200]
|
excerpt = tc.get("excerpt", tc.get("text", ""))[:200]
|
||||||
tc_score = tc.get("score", "")
|
tc_score = tc.get("score", "")
|
||||||
attribution = _comment_attribution(item.source, tc.get("author"))
|
lines.append(f" Top comment ({tc_score} {vote_label}): {excerpt}")
|
||||||
lines.append(f" Top comment {attribution} ({tc_score} {vote_label}): {excerpt}")
|
|
||||||
# Digg: inline X-post quotes attached to the cluster.
|
|
||||||
for post in _digg_posts_for(item, limit=3):
|
|
||||||
lines.append(f" > {_format_digg_quote(post)}")
|
|
||||||
# Comment insights for Reddit
|
# Comment insights for Reddit
|
||||||
insights = item.metadata.get("comment_insights", [])
|
insights = item.metadata.get("comment_insights", [])
|
||||||
if insights:
|
if insights:
|
||||||
@@ -1000,11 +581,7 @@ def _render_candidate(candidate: schema.Candidate, prefix: str) -> list[str]:
|
|||||||
excerpt = tc.get("excerpt") or tc.get("text") or ""
|
excerpt = tc.get("excerpt") or tc.get("text") or ""
|
||||||
score = tc.get("score", "")
|
score = tc.get("score", "")
|
||||||
vote_label = _vote_label_for(primary.source) if primary else "upvotes"
|
vote_label = _vote_label_for(primary.source) if primary else "upvotes"
|
||||||
source = primary.source if primary else None
|
lines.append(f" - Comment ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
|
||||||
attribution = _comment_attribution(source, tc.get("author"))
|
|
||||||
lines.append(f" - {attribution} ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
|
|
||||||
for post in _digg_posts_for(primary):
|
|
||||||
lines.append(f" - {_format_digg_quote(post)}")
|
|
||||||
insight = _comment_insight(primary)
|
insight = _comment_insight(primary)
|
||||||
if insight:
|
if insight:
|
||||||
lines.append(f" - Insight: {_truncate(insight, 220)}")
|
lines.append(f" - Insight: {_truncate(insight, 220)}")
|
||||||
@@ -1255,7 +832,6 @@ _FOOTER_SOURCES: list[tuple[str, str, str, str, list[tuple[str, str]]]] = [
|
|||||||
("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||||
("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||||
("github", "🐙", "GitHub", "item", [("reactions", "reactions"), ("comments", "comments")]),
|
("github", "🐙", "GitHub", "item", [("reactions", "reactions"), ("comments", "comments")]),
|
||||||
("digg", "⛏️", "Digg", "cluster", [("postCount", "posts"), ("uniqueAuthors", "authors")]),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -1513,7 +1089,6 @@ ENGAGEMENT_DISPLAY: dict[str, list[tuple[str, str]]] = {
|
|||||||
"polymarket": [],
|
"polymarket": [],
|
||||||
"github": [("reactions", "react"), ("comments", "cmt")],
|
"github": [("reactions", "react"), ("comments", "cmt")],
|
||||||
"perplexity": [("citations", "cite")],
|
"perplexity": [("citations", "cite")],
|
||||||
"digg": [("postCount", "posts"), ("uniqueAuthors", "auth")],
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1657,33 +1232,6 @@ def _vote_label_for(source: str) -> str:
|
|||||||
return _TOP_COMMENT_VOTE_LABEL.get(source, "votes")
|
return _TOP_COMMENT_VOTE_LABEL.get(source, "votes")
|
||||||
|
|
||||||
|
|
||||||
# Handle prefixes for commenter attribution. Reddit uses `u/`; everyone else
|
|
||||||
# uses `@`. Missing source or unknown platform falls back to plain-text so
|
|
||||||
# we never emit `u/` or `@` with no handle attached.
|
|
||||||
_HANDLE_PREFIX: dict[str, str] = {
|
|
||||||
"reddit": "u/",
|
|
||||||
"tiktok": "@",
|
|
||||||
"youtube": "@",
|
|
||||||
"instagram": "@",
|
|
||||||
"bluesky": "@",
|
|
||||||
"x": "@",
|
|
||||||
"threads": "@",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _comment_attribution(source: str | None, author: str | None) -> str:
|
|
||||||
"""Build the attribution prefix for a top comment line.
|
|
||||||
|
|
||||||
Returns a string like ``u/Cyrisaurus`` or ``@moosanoormahomed`` when an
|
|
||||||
author is captured, or the legacy ``Comment`` marker when the author is
|
|
||||||
missing, empty, deleted, or removed.
|
|
||||||
"""
|
|
||||||
if not author or author in ("[deleted]", "[removed]"):
|
|
||||||
return "Comment"
|
|
||||||
prefix = _HANDLE_PREFIX.get(source or "", "")
|
|
||||||
return f"{prefix}{author}" if prefix else author
|
|
||||||
|
|
||||||
|
|
||||||
def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score: int | None = None) -> list[dict]:
|
def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score: int | None = None) -> list[dict]:
|
||||||
"""Return up to `limit` top comments with score at or above the source's minimum.
|
"""Return up to `limit` top comments with score at or above the source's minimum.
|
||||||
|
|
||||||
@@ -1701,6 +1249,16 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score
|
|||||||
return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]
|
return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None:
|
||||||
|
if not item:
|
||||||
|
return None
|
||||||
|
comments = item.metadata.get("top_comments") or []
|
||||||
|
if not comments or not isinstance(comments[0], dict):
|
||||||
|
return None
|
||||||
|
top = comments[0]
|
||||||
|
return str(top.get("excerpt") or top.get("text") or "").strip() or None
|
||||||
|
|
||||||
|
|
||||||
def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
||||||
if not item:
|
if not item:
|
||||||
return None
|
return None
|
||||||
@@ -1710,39 +1268,6 @@ def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
|||||||
return str(insights[0]).strip() or None
|
return str(insights[0]).strip() or None
|
||||||
|
|
||||||
|
|
||||||
def _digg_posts_for(item: schema.SourceItem | None, limit: int = 3) -> list[dict]:
|
|
||||||
"""Return up to `limit` parsed Digg posts attached as enrichment to a cluster.
|
|
||||||
|
|
||||||
Returns an empty list for non-digg sources or clusters without enrichment.
|
|
||||||
"""
|
|
||||||
if not item or item.source != "digg":
|
|
||||||
return []
|
|
||||||
posts = item.metadata.get("posts") or []
|
|
||||||
if not isinstance(posts, list):
|
|
||||||
return []
|
|
||||||
out: list[dict] = []
|
|
||||||
for entry in posts:
|
|
||||||
if isinstance(entry, dict) and entry.get("body") and entry.get("username"):
|
|
||||||
out.append(entry)
|
|
||||||
if len(out) >= limit:
|
|
||||||
break
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _format_digg_quote(post: dict, body_limit: int = 200) -> str:
|
|
||||||
"""Format a Digg-attached X post as an inline 'via Digg' quote line."""
|
|
||||||
handle = post.get("username") or ""
|
|
||||||
x_url = post.get("x_url") or ""
|
|
||||||
body = (post.get("body") or "").replace("\n", " ").strip()
|
|
||||||
if len(body) > body_limit:
|
|
||||||
body = body[: body_limit - 1].rstrip() + "…"
|
|
||||||
if x_url and handle:
|
|
||||||
return f"[@{handle}]({x_url}) via Digg: {body}"
|
|
||||||
if handle:
|
|
||||||
return f"@{handle} via Digg: {body}"
|
|
||||||
return f"via Digg: {body}"
|
|
||||||
|
|
||||||
|
|
||||||
def _transcript_highlights(item: schema.SourceItem | None) -> list[str]:
|
def _transcript_highlights(item: schema.SourceItem | None) -> list[str]:
|
||||||
if not item or item.source != "youtube":
|
if not item or item.source != "youtube":
|
||||||
return []
|
return []
|
||||||
@@ -11,64 +11,14 @@ import re
|
|||||||
import sys
|
import sys
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from . import categories, dates, grounding
|
from . import dates, grounding
|
||||||
|
|
||||||
MAX_SUBS = 10
|
|
||||||
|
|
||||||
|
|
||||||
def _log(msg: str) -> None:
|
def _log(msg: str) -> None:
|
||||||
print(f"[Resolve] {msg}", file=sys.stderr)
|
print(f"[Resolve] {msg}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
def _merge_category_peers(topic: str, subreddits: list[str]) -> tuple[list[str], Optional[str]]:
|
|
||||||
"""Extend the WebSearch-extracted subreddit list with category peers.
|
|
||||||
|
|
||||||
Classifies the topic, fetches the category's peer subs, dedupes
|
|
||||||
case-insensitively against the existing list, and appends missing
|
|
||||||
peers in priority order. Caps the final list at MAX_SUBS, preserving
|
|
||||||
every WebSearch-returned sub (they are the freshest signal) and
|
|
||||||
trimming from the peer-additions end.
|
|
||||||
|
|
||||||
Returns a tuple of (merged_subs, matched_category_id_or_None).
|
|
||||||
Emits a [Resolve] Matched category log line only when peers were
|
|
||||||
actually added (not when every peer was already in the WebSearch set).
|
|
||||||
|
|
||||||
Classification failures degrade to "no match" — the unwidened list
|
|
||||||
is returned and a warning is logged.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
category = categories.detect_category(topic)
|
|
||||||
except Exception as exc:
|
|
||||||
_log(f"Category classification failed: {exc}")
|
|
||||||
return list(subreddits)[:MAX_SUBS], None
|
|
||||||
|
|
||||||
if category is None:
|
|
||||||
return list(subreddits)[:MAX_SUBS], None
|
|
||||||
|
|
||||||
peers = categories.peer_subs_for(category)
|
|
||||||
if not peers:
|
|
||||||
return list(subreddits)[:MAX_SUBS], category
|
|
||||||
|
|
||||||
existing_lower = {s.lower() for s in subreddits}
|
|
||||||
merged = list(subreddits)
|
|
||||||
added: list[str] = []
|
|
||||||
for peer in peers:
|
|
||||||
if len(merged) >= MAX_SUBS:
|
|
||||||
break
|
|
||||||
if peer.lower() in existing_lower:
|
|
||||||
continue
|
|
||||||
merged.append(peer)
|
|
||||||
existing_lower.add(peer.lower())
|
|
||||||
added.append(peer)
|
|
||||||
|
|
||||||
if added:
|
|
||||||
_log(f"Matched category={category}, adding peers: {', '.join(added)}")
|
|
||||||
|
|
||||||
return merged, category
|
|
||||||
|
|
||||||
|
|
||||||
def _has_backend(config: dict) -> bool:
|
def _has_backend(config: dict) -> bool:
|
||||||
"""Check if any web search backend is available."""
|
"""Check if any web search backend is available."""
|
||||||
return bool(
|
return bool(
|
||||||
@@ -184,19 +134,10 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
|||||||
config: Dict with API keys (BRAVE_API_KEY, EXA_API_KEY, SERPER_API_KEY).
|
config: Dict with API keys (BRAVE_API_KEY, EXA_API_KEY, SERPER_API_KEY).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict with keys: subreddits, x_handle, github_user, github_repos,
|
Dict with keys: subreddits, x_handle, context, searches_run.
|
||||||
context, category, searches_run. Returns empty result if no web
|
Returns empty result if no web search backend is available.
|
||||||
search backend is available.
|
|
||||||
"""
|
"""
|
||||||
empty = {
|
empty = {"subreddits": [], "x_handle": "", "context": "", "searches_run": 0}
|
||||||
"subreddits": [],
|
|
||||||
"x_handle": "",
|
|
||||||
"github_user": "",
|
|
||||||
"github_repos": [],
|
|
||||||
"context": "",
|
|
||||||
"category": None,
|
|
||||||
"searches_run": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
if not _has_backend(config):
|
if not _has_backend(config):
|
||||||
_log("No web search backend available, skipping resolve")
|
_log("No web search backend available, skipping resolve")
|
||||||
@@ -243,9 +184,7 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
|||||||
github_repos = _extract_github_repos(results.get("github", []))
|
github_repos = _extract_github_repos(results.get("github", []))
|
||||||
context = _build_context_summary(results.get("news", []))
|
context = _build_context_summary(results.get("news", []))
|
||||||
|
|
||||||
subreddits, category = _merge_category_peers(topic, subreddits)
|
_log(f"Resolved {len(subreddits)} subreddits, x_handle={x_handle!r}, github_user={github_user!r}, github_repos={github_repos!r}, context_len={len(context)}")
|
||||||
|
|
||||||
_log(f"Resolved {len(subreddits)} subreddits, x_handle={x_handle!r}, github_user={github_user!r}, github_repos={github_repos!r}, context_len={len(context)}, category={category!r}")
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"subreddits": subreddits,
|
"subreddits": subreddits,
|
||||||
@@ -253,6 +192,5 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
|||||||
"github_user": github_user,
|
"github_user": github_user,
|
||||||
"github_repos": github_repos,
|
"github_repos": github_repos,
|
||||||
"context": context,
|
"context": context,
|
||||||
"category": category,
|
|
||||||
"searches_run": searches_run,
|
"searches_run": searches_run,
|
||||||
}
|
}
|
||||||
@@ -12,7 +12,6 @@ SOURCE_QUALITY = {
|
|||||||
"xiaohongshu": 0.7,
|
"xiaohongshu": 0.7,
|
||||||
"hackernews": 0.8,
|
"hackernews": 0.8,
|
||||||
"youtube": 0.85,
|
"youtube": 0.85,
|
||||||
"digg": 0.85,
|
|
||||||
"reddit": 0.6,
|
"reddit": 0.6,
|
||||||
"x": 0.68,
|
"x": 0.68,
|
||||||
"bluesky": 0.66,
|
"bluesky": 0.66,
|
||||||
@@ -27,10 +26,7 @@ def source_quality(source: str) -> float:
|
|||||||
return SOURCE_QUALITY.get(source, 0.6)
|
return SOURCE_QUALITY.get(source, 0.6)
|
||||||
|
|
||||||
|
|
||||||
def local_relevance(
|
def local_relevance(item: schema.SourceItem, ranking_query: str) -> float:
|
||||||
item: schema.SourceItem,
|
|
||||||
ranking_query: "str | relevance.PreparedQuery",
|
|
||||||
) -> float:
|
|
||||||
text = "\n".join(
|
text = "\n".join(
|
||||||
part
|
part
|
||||||
for part in [item.title, item.body, item.snippet]
|
for part in [item.title, item.body, item.snippet]
|
||||||
@@ -96,7 +92,6 @@ ENGAGEMENT_WEIGHTS: dict[str, list[tuple[str, float]]] = {
|
|||||||
"bluesky": [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)],
|
"bluesky": [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)],
|
||||||
"truthsocial": [("likes", 0.45), ("reposts", 0.30), ("replies", 0.25)],
|
"truthsocial": [("likes", 0.45), ("reposts", 0.30), ("replies", 0.25)],
|
||||||
"polymarket": [("volume", 0.60), ("liquidity", 0.40)],
|
"polymarket": [("volume", 0.60), ("liquidity", 0.40)],
|
||||||
"digg": [("postCount", 0.40), ("uniqueAuthors", 0.30), ("rank_score", 0.30)],
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -180,14 +175,13 @@ def normalize(values: list[float | None]) -> list[int | None]:
|
|||||||
|
|
||||||
def annotate_stream(
|
def annotate_stream(
|
||||||
items: list[schema.SourceItem],
|
items: list[schema.SourceItem],
|
||||||
ranking_query: "str | relevance.PreparedQuery",
|
ranking_query: str,
|
||||||
freshness_mode: str,
|
freshness_mode: str,
|
||||||
) -> list[schema.SourceItem]:
|
) -> list[schema.SourceItem]:
|
||||||
"""Attach local scoring metadata and return items sorted by local_rank_score."""
|
"""Attach local scoring metadata and return items sorted by local_rank_score."""
|
||||||
prepared_query = ranking_query if isinstance(ranking_query, relevance.PreparedQuery) else relevance.PreparedQuery(ranking_query)
|
|
||||||
engagement_scores = normalize([engagement_raw(item) for item in items])
|
engagement_scores = normalize([engagement_raw(item) for item in items])
|
||||||
for item, eng_score in zip(items, engagement_scores, strict=True):
|
for item, eng_score in zip(items, engagement_scores, strict=True):
|
||||||
item.local_relevance = local_relevance(item, prepared_query)
|
item.local_relevance = local_relevance(item, ranking_query)
|
||||||
item.freshness = freshness(item, freshness_mode)
|
item.freshness = freshness(item, freshness_mode)
|
||||||
item.engagement_score = eng_score
|
item.engagement_score = eng_score
|
||||||
item.source_quality = source_quality(item.source)
|
item.source_quality = source_quality(item.source)
|
||||||
@@ -26,7 +26,7 @@ def _windows(words: list[str], size: int, overlap: int) -> list[str]:
|
|||||||
|
|
||||||
def extract_best_snippet(
|
def extract_best_snippet(
|
||||||
item: schema.SourceItem,
|
item: schema.SourceItem,
|
||||||
ranking_query: "str | relevance.PreparedQuery",
|
ranking_query: str,
|
||||||
max_words: int = 120,
|
max_words: int = 120,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Prefer existing snippets, else extract the best matching evidence window."""
|
"""Prefer existing snippets, else extract the best matching evidence window."""
|
||||||
@@ -43,9 +43,8 @@ def extract_best_snippet(
|
|||||||
if not candidates:
|
if not candidates:
|
||||||
return _truncate_words(body, max_words)
|
return _truncate_words(body, max_words)
|
||||||
|
|
||||||
prepared_query = ranking_query if isinstance(ranking_query, relevance.PreparedQuery) else relevance.PreparedQuery(ranking_query)
|
|
||||||
best = max(
|
best = max(
|
||||||
candidates,
|
candidates,
|
||||||
key=lambda candidate: relevance.token_overlap_relevance(prepared_query, candidate),
|
key=lambda candidate: relevance.token_overlap_relevance(ranking_query, candidate),
|
||||||
)
|
)
|
||||||
return _truncate_words(best, max_words)
|
return _truncate_words(best, max_words)
|
||||||
@@ -152,16 +152,35 @@ def search_threads(
|
|||||||
_log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
|
_log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = http.get(
|
import requests as _requests
|
||||||
f"{SCRAPECREATORS_BASE}/search",
|
except ImportError:
|
||||||
params={"keyword": core_topic},
|
_requests = None
|
||||||
headers=http.scrapecreators_headers(token),
|
|
||||||
timeout=30,
|
if not _requests:
|
||||||
retries=2,
|
_log("requests library not installed, falling back to urllib")
|
||||||
)
|
try:
|
||||||
except Exception as e:
|
from urllib.parse import urlencode
|
||||||
_log(f"ScrapeCreators error: {e}")
|
params = urlencode({"keyword": core_topic})
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
url = f"{SCRAPECREATORS_BASE}/search?{params}"
|
||||||
|
headers = http.scrapecreators_headers(token)
|
||||||
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"ScrapeCreators error (urllib): {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/search",
|
||||||
|
params={"keyword": core_topic},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"ScrapeCreators error: {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
|
||||||
# Extract items from response (try common SC response shapes)
|
# Extract items from response (try common SC response shapes)
|
||||||
raw_items = (
|
raw_items = (
|
||||||
@@ -11,6 +11,11 @@ import re
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests as _requests
|
||||||
|
except ImportError:
|
||||||
|
_requests = None
|
||||||
|
|
||||||
from . import dates, http, log
|
from . import dates, http, log
|
||||||
|
|
||||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
|
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
|
||||||
@@ -209,17 +214,30 @@ def _hashtag_search(
|
|||||||
List of raw TikTok item dicts (aweme_info format).
|
List of raw TikTok item dicts (aweme_info format).
|
||||||
"""
|
"""
|
||||||
_log(f"Hashtag search: #{hashtag}")
|
_log(f"Hashtag search: #{hashtag}")
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
try:
|
||||||
f"{SCRAPECREATORS_BASE}/search/hashtag",
|
from urllib.parse import urlencode
|
||||||
params={"hashtag": hashtag},
|
params = urlencode({"hashtag": hashtag})
|
||||||
headers=http.scrapecreators_headers(token),
|
url = f"{SCRAPECREATORS_BASE}/search/hashtag?{params}"
|
||||||
timeout=30,
|
headers = http.scrapecreators_headers(token)
|
||||||
retries=2,
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
)
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Hashtag search error for #{hashtag}: {e}")
|
_log(f"Hashtag search error (urllib) for #{hashtag}: {e}")
|
||||||
return []
|
return []
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/search/hashtag",
|
||||||
|
params={"hashtag": hashtag},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"Hashtag search error for #{hashtag}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
raw_items = data.get("aweme_list") or data.get("data") or []
|
raw_items = data.get("aweme_list") or data.get("data") or []
|
||||||
_log(f" -> {len(raw_items)} results for #{hashtag}")
|
_log(f" -> {len(raw_items)} results for #{hashtag}")
|
||||||
@@ -243,17 +261,30 @@ def _profile_videos(
|
|||||||
"""
|
"""
|
||||||
_log(f"Profile videos: @{handle}")
|
_log(f"Profile videos: @{handle}")
|
||||||
profile_url = "https://api.scrapecreators.com/v3/tiktok/profile/videos"
|
profile_url = "https://api.scrapecreators.com/v3/tiktok/profile/videos"
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
try:
|
||||||
profile_url,
|
from urllib.parse import urlencode
|
||||||
params={"handle": handle, "sort_by": "latest"},
|
params = urlencode({"handle": handle, "sort_by": "latest"})
|
||||||
headers=http.scrapecreators_headers(token),
|
url = f"{profile_url}?{params}"
|
||||||
timeout=30,
|
headers = http.scrapecreators_headers(token)
|
||||||
retries=2,
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
)
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Profile videos error for @{handle}: {e}")
|
_log(f"Profile videos error (urllib) for @{handle}: {e}")
|
||||||
return []
|
return []
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
profile_url,
|
||||||
|
params={"handle": handle, "sort_by": "latest"},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"Profile videos error for @{handle}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
raw_items = data.get("aweme_list") or data.get("data") or []
|
raw_items = data.get("aweme_list") or data.get("data") or []
|
||||||
_log(f" -> {len(raw_items)} videos from @{handle}")
|
_log(f" -> {len(raw_items)} videos from @{handle}")
|
||||||
@@ -287,17 +318,31 @@ def search_tiktok(
|
|||||||
|
|
||||||
_log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
_log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||||
|
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
_log("requests library not installed, falling back to urllib")
|
||||||
f"{SCRAPECREATORS_BASE}/search/keyword",
|
try:
|
||||||
params={"query": core_topic, "sort_by": "relevance"},
|
from urllib.parse import urlencode
|
||||||
headers=http.scrapecreators_headers(token),
|
params = urlencode({"query": core_topic, "sort_by": "relevance"})
|
||||||
timeout=30,
|
url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
|
||||||
retries=2,
|
headers = http.scrapecreators_headers(token)
|
||||||
)
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
except Exception as e:
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
_log(f"ScrapeCreators error: {e}")
|
except Exception as e:
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
_log(f"ScrapeCreators error (urllib): {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/search/keyword",
|
||||||
|
params={"query": core_topic, "sort_by": "relevance"},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"ScrapeCreators error: {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
|
||||||
# Items are nested under aweme_info
|
# Items are nested under aweme_info
|
||||||
raw_entries = data.get("search_item_list") or data.get("data") or []
|
raw_entries = data.get("search_item_list") or data.get("data") or []
|
||||||
@@ -352,7 +397,7 @@ def fetch_captions(
|
|||||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||||
max_captions = config["max_captions"]
|
max_captions = config["max_captions"]
|
||||||
|
|
||||||
if not video_items or not token:
|
if not video_items or not token or not _requests:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
top_items = video_items[:max_captions]
|
top_items = video_items[:max_captions]
|
||||||
@@ -377,23 +422,24 @@ def fetch_captions(
|
|||||||
if not url:
|
if not url:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
data = http.get(
|
resp = _requests.get(
|
||||||
f"{SCRAPECREATORS_BASE}/video/transcript",
|
f"{SCRAPECREATORS_BASE}/video/transcript",
|
||||||
params={"url": url},
|
params={"url": url},
|
||||||
headers=http.scrapecreators_headers(token),
|
headers=http.scrapecreators_headers(token),
|
||||||
timeout=15,
|
timeout=15,
|
||||||
retries=1,
|
|
||||||
)
|
)
|
||||||
transcript = data.get("transcript")
|
if resp.status_code == 200:
|
||||||
if transcript:
|
data = resp.json()
|
||||||
if isinstance(transcript, list):
|
transcript = data.get("transcript")
|
||||||
transcript = " ".join(str(s) for s in transcript)
|
|
||||||
transcript = _clean_webvtt(transcript)
|
|
||||||
if transcript:
|
if transcript:
|
||||||
words = transcript.split()
|
if isinstance(transcript, list):
|
||||||
if len(words) > CAPTION_MAX_WORDS:
|
transcript = " ".join(str(s) for s in transcript)
|
||||||
transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
transcript = _clean_webvtt(transcript)
|
||||||
captions[vid] = transcript
|
if transcript:
|
||||||
|
words = transcript.split()
|
||||||
|
if len(words) > CAPTION_MAX_WORDS:
|
||||||
|
transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||||
|
captions[vid] = transcript
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Transcript fetch failed for {vid}: {e}")
|
_log(f"Transcript fetch failed for {vid}: {e}")
|
||||||
|
|
||||||
@@ -574,17 +620,30 @@ def _fetch_post_comments(
|
|||||||
List of comment dicts with author, text, digg_count (likes), date.
|
List of comment dicts with author, text, digg_count (likes), date.
|
||||||
Empty list on any error — comment failures never crash the pipeline.
|
Empty list on any error — comment failures never crash the pipeline.
|
||||||
"""
|
"""
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
try:
|
||||||
f"{SCRAPECREATORS_BASE}/video/comments",
|
from urllib.parse import urlencode
|
||||||
params={"url": post_url, "trim": "true"},
|
params = urlencode({"url": post_url, "trim": "true"})
|
||||||
headers=http.scrapecreators_headers(token),
|
url = f"{SCRAPECREATORS_BASE}/video/comments?{params}"
|
||||||
timeout=30,
|
headers = http.scrapecreators_headers(token)
|
||||||
retries=2,
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
)
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_log(f"Comment fetch error for {post_url}: {exc}")
|
_log(f"Comment fetch error (urllib) for {post_url}: {exc}")
|
||||||
return []
|
return []
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/video/comments",
|
||||||
|
params={"url": post_url, "trim": "true"},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as exc:
|
||||||
|
_log(f"Comment fetch error for {post_url}: {exc}")
|
||||||
|
return []
|
||||||
|
|
||||||
raw_comments = data.get("comments") or data.get("data") or []
|
raw_comments = data.get("comments") or data.get("data") or []
|
||||||
# Sort by digg_count desc so normalize sees the highest-signal first.
|
# Sort by digg_count desc so normalize sees the highest-signal first.
|
||||||
@@ -599,9 +658,7 @@ def _fetch_post_comments(
|
|||||||
if not text:
|
if not text:
|
||||||
continue
|
continue
|
||||||
user = c.get("user") if isinstance(c.get("user"), dict) else {}
|
user = c.get("user") if isinstance(c.get("user"), dict) else {}
|
||||||
# Prefer unique_id (the @handle) over nickname (display name) so
|
author = user.get("nickname") or user.get("unique_id") or ""
|
||||||
# downstream render can cite @handle consistently across platforms.
|
|
||||||
author = user.get("unique_id") or user.get("nickname") or ""
|
|
||||||
create_time = c.get("create_time")
|
create_time = c.get("create_time")
|
||||||
date_str = ""
|
date_str = ""
|
||||||
if create_time:
|
if create_time:
|
||||||
@@ -124,7 +124,6 @@ SOURCE_COMPLETION_ORDER = [
|
|||||||
"polymarket",
|
"polymarket",
|
||||||
"grounding",
|
"grounding",
|
||||||
"xiaohongshu",
|
"xiaohongshu",
|
||||||
"digg",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
SOURCE_COMPLETION_META = {
|
SOURCE_COMPLETION_META = {
|
||||||
@@ -139,7 +138,6 @@ SOURCE_COMPLETION_META = {
|
|||||||
"polymarket": ("Polymarket", "market", "markets", Colors.GREEN),
|
"polymarket": ("Polymarket", "market", "markets", Colors.GREEN),
|
||||||
"grounding": ("Web", "result", "results", Colors.GREEN),
|
"grounding": ("Web", "result", "results", Colors.GREEN),
|
||||||
"xiaohongshu": ("Xiaohongshu", "post", "posts", Colors.RED),
|
"xiaohongshu": ("Xiaohongshu", "post", "posts", Colors.RED),
|
||||||
"digg": ("Digg", "cluster", "clusters", Colors.YELLOW),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||