Compare commits

..

1 Commits

Author SHA1 Message Date
Matt Van Horn b31ec05c74 marketing: v3.1 launch video — 30s Remotion render
Six-scene 30-second MP4 announcing v3.1 (competitors mode) for X.

Lives at marketing/v3.1-launch/. Renders to out/last30days-v3.1-launch.mp4
(gitignored). 1920×1080 @ 30fps, H.264, ~3MB final.

Six scenes:
1. Hook (0-3s): badge + "What if one search ran 3 at once?"
2. Old way (3-8s): single terminal /last30days OpenAI
3. Fan-out (8-14s): --competitors splits into 3 parallel panes
4. Comparison (14-21s): 3 panes collapse into Head-to-Head table
5. How (21-26s): "You pick the topic. Agent picks peers. Engine fans out."
6. CTA (26-30s): install command + repo URL

Reusable components: TerminalWindow, TypedLine, BadgeBar,
ComparisonTable. Scene timing in src/lib/timing.ts as single source of
truth — one edit to retime everything.

Marketing version 3.1 ≠ engine code version (still 3.0.14). 3.1 is the
launch label that bundles 3.0.11-3.0.14 into one shippable narrative.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 21:48:02 -07:00
211 changed files with 2900 additions and 3761 deletions
-20
View File
@@ -1,20 +0,0 @@
{
"name": "last30days-skill",
"interface": {
"displayName": "Last 30 Days"
},
"plugins": [
{
"name": "last30days",
"source": {
"source": "local",
"path": "./"
},
"policy": {
"installation": "AVAILABLE",
"authentication": "ON_INSTALL"
},
"category": "Research"
}
]
}
+4 -5
View File
@@ -1,17 +1,16 @@
{ {
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
"name": "last30days-skill", "name": "last30days-skill",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
"owner": { "owner": {
"name": "Matt Van Horn", "name": "Matt Van Horn",
"url": "https://github.com/mvanhorn" "url": "https://github.com/mvanhorn"
}, },
"metadata": {
"description": "Marketplace hosting the Last 30 Days research plugin."
},
"plugins": [ "plugins": [
{ {
"name": "last30days", "name": "last30days",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.", "description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
"version": "3.2.3", "version": "3.0.9",
"author": { "author": {
"name": "Matt Van Horn", "name": "Matt Van Horn",
"url": "https://github.com/mvanhorn" "url": "https://github.com/mvanhorn"
+3 -2
View File
@@ -1,6 +1,6 @@
{ {
"name": "last30days", "name": "last30days",
"version": "3.2.3", "version": "3.0.14",
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.", "description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
"author": { "author": {
"name": "Matt Van Horn", "name": "Matt Van Horn",
@@ -10,5 +10,6 @@
"homepage": "https://github.com/mvanhorn/last30days-skill", "homepage": "https://github.com/mvanhorn/last30days-skill",
"repository": "https://github.com/mvanhorn/last30days-skill", "repository": "https://github.com/mvanhorn/last30days-skill",
"license": "MIT", "license": "MIT",
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"] "keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"],
"hooks": {}
} }
+3
View File
@@ -0,0 +1,3 @@
{
"name": "last30days"
}
+4 -4
View File
@@ -1,6 +1,5 @@
# Exclude non-runtime files from `git archive` output. # Exclude non-runtime files from `git archive` output.
# Used by skills/last30days/scripts/build-skill.sh to produce a # Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md. # See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
# Anthropic canonical skill-packaging excludes # Anthropic canonical skill-packaging excludes
@@ -33,8 +32,9 @@ release-notes.md export-ignore
CHANGELOG.md export-ignore CHANGELOG.md export-ignore
uv.lock export-ignore uv.lock export-ignore
# Platform adapters are kept in git archives because Claude Code and Codex # Platform adapters - skill-upload path is platform-agnostic
# plugin installs use the same repository archive as their source payload. .agents/ export-ignore
.codex-plugin/ export-ignore
.hermes-plugin/ export-ignore .hermes-plugin/ export-ignore
# CI workflows - repo-only, not needed at skill runtime # CI workflows - repo-only, not needed at skill runtime
-53
View File
@@ -1,53 +0,0 @@
name: Bug Report
description: Report a bug or unexpected behavior
labels: [bug]
body:
- type: textarea
id: summary
attributes:
label: Summary
description: What happened?
placeholder: Describe the bug in 1-2 sentences.
validations:
required: true
- type: textarea
id: repro
attributes:
label: Steps to Reproduce
description: How can we reproduce this?
placeholder: |
1. Run `python3 scripts/last30days.py "topic" --emit compact`
2. ...
validations:
required: true
- type: textarea
id: expected
attributes:
label: Expected Behavior
description: What should have happened?
validations:
required: true
- type: textarea
id: traceback
attributes:
label: Error / Traceback
description: Paste the full traceback or error output.
render: text
- type: dropdown
id: install
attributes:
label: Install Method
options:
- Claude Code plugin
- Gemini CLI extension
- Codex plugin
- Hermes skill
- Manual (git clone)
- Other
validations:
required: true
- type: input
id: os
attributes:
label: OS
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
@@ -1,24 +0,0 @@
name: Feature Request
description: Suggest a new feature or improvement
labels: [enhancement]
body:
- type: textarea
id: problem
attributes:
label: Problem
description: What problem does this solve?
placeholder: When I try to ..., I can't ...
validations:
required: true
- type: textarea
id: solution
attributes:
label: Proposed Solution
description: How should this work?
validations:
required: true
- type: textarea
id: alternatives
attributes:
label: Alternatives Considered
description: Other approaches you thought of (optional).
-19
View File
@@ -1,19 +0,0 @@
## Summary
<!-- What does this PR do? 1-3 sentences. -->
## Changes
<!-- Bullet list of what changed. Reference files if helpful. -->
-
## Testing
<!-- How did you verify this works? -->
- [ ] Ran `uv run python -m pytest -q --tb=short`
## Related Issues
<!-- Link issues: Fixes #123 or Relates to #456 -->
+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
- name: Build .skill artifact - name: Build .skill artifact
run: | run: |
bash skills/last30days/scripts/build-skill.sh bash scripts/build-skill.sh
test -f dist/last30days.skill test -f dist/last30days.skill
- name: Create GitHub release - name: Create GitHub release
-26
View File
@@ -1,26 +0,0 @@
name: Validate
on:
pull_request:
push:
branches:
- main
permissions:
contents: read
jobs:
plugin-contract:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v5
- name: Set up Python
run: uv python install 3.12
- name: Run plugin contract tests
run: uv run pytest tests/test_plugin_contract.py tests/test_version_consistency.py
+6
View File
@@ -28,3 +28,9 @@ htmlcov/
# Internal planning docs (ce:plan output) — keep local, don't publish # Internal planning docs (ce:plan output) — keep local, don't publish
docs/plans/ docs/plans/
# Marketing video build artifacts
marketing/v3.1-launch/node_modules/
marketing/v3.1-launch/out/
marketing/*/node_modules/
marketing/*/out/
-1
View File
@@ -1 +0,0 @@
@CLAUDE.md
-47
View File
@@ -5,52 +5,6 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Changed
- Rename "Digg AI 1000" to just "Digg" in user-facing output (footer line, source label, inline-quote suffix, why_relevant, container attribution). Internal references to the upstream Digg AI 1000 product remain in code comments and docstrings.
- Bump `POSTS_PER_CLUSTER` from 3 to 5 and the render-side display limit from 2 to 3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous 3/2 caps routinely truncated cluster context (e.g. dropped a Jason Calacanis quote tweet on a `cli-printing-press` run).
- Rewrite SKILL.md path resolution. STEP 0 narrows from a global canonical-path enforcement to a Claude-Code-marketplaces-only stale-clone guard. Step 1 SKILL_ROOT resolver walks a single precedence list (Claude plugin cache, then `~/.codex/skills/`, `~/.agents/skills/`, repo checkout, `./.skills/last30days` for `npx skills add`, CWD, Gemini). Adds SKILL.md frontmatter fallback to `render.py::_skill_version` so the badge no longer prints `v?` on installs that don't include `.claude-plugin/plugin.json`.
- Switch SKILL.md's `--plan` and `--competitors-plan` invocation templates from inline single-quoted JSON to heredoc-written tmpfiles. Apostrophes in resolved context strings ("McDonald's", "people's choice", "developer's") previously closed the outer single-quote and broke shell parsing before the engine started — observed in a Codex run during PR #400 testing. The engine's `parse_plan()` / `parse_competitors_plan()` already supported file paths (via `os.path.isfile()` probe); only the template prose changed. Fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403).
### Removed
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1. Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
- **`skills/last30days/scripts/sync.sh`.** The maintainer dev-deploy script is gone. Every job it did has a better replacement: `npx skills add . -g -y` symlinks the working tree into every detected harness's skill dir (better than sync.sh's copy model — edits propagate live), `hermes skills install mvanhorn/last30days-skill --force` handles Hermes, `clawhub install last30days-official` handles OpenClaw, and the Claude marketplace cache target was a "test against the official install path" hack we shouldn't have been recommending in the first place. The `test_sync_cache_path_uses_skill_version` test was dropped along with it. CLAUDE.md, HERMES_SETUP.md, the PR template, and a render.py docstring were updated to drop references; CHANGELOG and historical docs (release notes, plan files) keep their existing mentions as accurate history.
## [3.2.0] - 2026-05-09
### Added
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
## [3.1.1] - 2026-04-24
### Fixed
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
## [3.1.0] - 2026-04-22
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
### Added
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
### Fixed
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
## [3.0.14] - 2026-04-22 ## [3.0.14] - 2026-04-22
### Changed ### Changed
@@ -107,7 +61,6 @@ Individual changelog entries for 3.0.10 through 3.0.14 below document the increm
### Fixed ### Fixed
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier. - **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
### Behavior fallback ### Behavior fallback
+7 -7
View File
@@ -4,20 +4,20 @@ Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
Python scripts with multi-source search aggregation. Python scripts with multi-source search aggregation.
## Structure ## Structure
- `skills/last30days/SKILL.md` — canonical skill definition - `scripts/last30days.py` — main research engine
- `skills/last30days/scripts/last30days.py` — main research engine - `scripts/lib/`search, enrichment, rendering modules
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules - `scripts/lib/vendor/bird-search/` — vendored X search client
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client - `SKILL.md` — skill definition (deployed to ~/.claude/skills/last30days/)
## Commands ## Commands
```bash ```bash
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact python3 scripts/last30days.py "test query" --emit=compact # Run research
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir bash scripts/sync.sh # Deploy to ~/.claude, ~/.agents, ~/.codex
``` ```
## Rules ## Rules
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports) - `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed. - After edits: run `bash scripts/sync.sh` to deploy
- Git remote: origin = public (`mvanhorn/last30days-skill`) - Git remote: origin = public (`mvanhorn/last30days-skill`)
## Beta channel ## Beta channel
+22 -12
View File
@@ -10,20 +10,28 @@ This guide covers installing last30days on Hermes AI Agent.
## Installation ## Installation
### Option 1: Via sync.sh (Recommended)
```bash ```bash
hermes skills install mvanhorn/last30days-skill --force # Clone the repo
git clone https://github.com/mvanhorn/last30days-skill.git
cd last30days-skill
# Run the sync script
bash scripts/sync.sh
``` ```
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy. This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days/`
### Developer / live-edit alternative ### Option 2: Manual Copy
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
```bash ```bash
git clone https://github.com/mvanhorn/last30days-skill.git # Create directory
mkdir -p ~/.hermes/skills/research mkdir -p ~/.hermes/skills/research/last30days
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
# Copy files
cp -r scripts ~/.hermes/skills/research/last30days/
cp .hermes-plugin/SKILL.md ~/.hermes/skills/research/last30days/
``` ```
## Usage ## Usage
@@ -98,11 +106,13 @@ python3.12 scripts/last30days.py --diagnose
## Updating ## Updating
```bash To update to the latest version:
hermes skills install mvanhorn/last30days-skill --force
```
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step. ```bash
cd last30days-skill
git pull
bash scripts/sync.sh
```
## Support ## Support
+35 -88
View File
@@ -12,20 +12,23 @@
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.** **An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
This README tracks the current v3 pipeline. The runtime skill spec lives in [SKILL.md](SKILL.md), which is the source of truth for the latest command and setup behavior. This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
**Claude Code (recommended — auto-updates via marketplace):** Claude Code:
``` ```
/plugin marketplace add mvanhorn/last30days-skill /plugin marketplace add mvanhorn/last30days-skill
``` ```
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:** OpenClaw:
``` ```
npx skills add mvanhorn/last30days-skill -g clawhub install last30days-official
``` ```
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below. Hermes:
```
# The skill auto-deploys when you run sync.sh
# Or manually copy to ~/.hermes/skills/research/last30days/
```
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds. Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
@@ -65,7 +68,6 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. | | **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. | | **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. | | **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. | | **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. | | **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. | | **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
@@ -94,28 +96,6 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
## What v3 Changed ## What v3 Changed
### Shareable HTML briefs
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
```
/last30days OpenClaw --emit=html
```
or just ask in plain language:
```
/last30days OpenClaw, give me a shareable HTML brief
/last30days Cursor IDE for slack
/last30days Anthropic earnings export as html
```
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
### Intelligent search: the killer feature ### Intelligent search: the killer feature
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities. The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
@@ -165,61 +145,12 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
## Install ## Install
| Surface | Install | Updates | | Surface | Install |
|---------|---------|---------| |---------|---------|
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` | | **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + |
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` | | **Claude Code** | `/plugin marketplace add mvanhorn/last30days-skill` |
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload | | **OpenClaw** | `clawhub install last30days-official` |
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` | | **Gemini CLI** | Clone then `gemini extensions install ./last30days-skill` (see below) |
### Claude Code (recommended)
```
/plugin marketplace add mvanhorn/last30days-skill
```
Recommended because the Claude Code marketplace handles updates for you — the plugin cache is versioned and auto-refreshes when a new release publishes. Run `claude plugin update last30days@last30days-skill` to force a check.
If you'd rather use the agent-skills install path on Claude Code, that's also supported:
```
npx skills add mvanhorn/last30days-skill -g -a claude-code
```
The native plugin and the `npx skills` install can coexist; Claude Code dedupes the slash command.
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
```bash
npx skills add mvanhorn/last30days-skill -g
```
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
```bash
npx skills add mvanhorn/last30days-skill -g -a codex
npx skills add mvanhorn/last30days-skill -g -a cursor
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
```
Update later with:
```bash
npx skills update last30days -g
```
Or update everything you've installed globally via `npx skills`:
```bash
npx skills update -g
```
List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
### claude.ai (web) ### claude.ai (web)
@@ -227,7 +158,15 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities) 2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
3. Click the `+` button in the Skills panel and drop the file in 3. Click the `+` button in the Skills panel and drop the file in
Enable "Code execution and file creation" under Capabilities first skills won't run without it. Enable "Code execution and file creation" under Capabilities first - skills won't run without it.
### Claude Code
```
/plugin marketplace add mvanhorn/last30days-skill
```
Update later with `claude plugin update last30days@last30days-skill`.
### OpenClaw ### OpenClaw
@@ -235,14 +174,22 @@ Enable "Code execution and file creation" under Capabilities first — skills wo
clawhub install last30days-official clawhub install last30days-official
``` ```
### Gemini CLI
Gemini CLI v0.9.0 has an upstream installer bug that can fail with `Configuration file not found at /tmp/gemini-extensionXXXXXX/gemini-extension.json` ([upstream issue](https://github.com/google-gemini/gemini-cli/issues/11452)). Workaround:
```bash
git clone https://github.com/mvanhorn/last30days-skill
gemini extensions install ./last30days-skill
```
### Manual (developer) ### Manual (developer)
```bash ```bash
git clone https://github.com/mvanhorn/last30days-skill.git git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
``` ```
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`. Or build the claude.ai `.skill` file from source: `bash scripts/build-skill.sh` produces `dist/last30days.skill`.
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds. Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
+37 -144
View File
@@ -1,6 +1,6 @@
--- ---
name: last30days name: last30days
version: "3.2.3" version: "3.0.1"
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web." description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react' argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
@@ -45,7 +45,6 @@ metadata:
- instagram - instagram
- hackernews - hackernews
- polymarket - polymarket
- digg
- bluesky - bluesky
- truthsocial - truthsocial
- trends - trends
@@ -60,32 +59,23 @@ metadata:
- clawhub - clawhub
--- ---
# STEP 0: STALE-CLONE SELF-CHECK — RUN BEFORE READING BELOW # STEP 0: CANONICAL PATH SELF-CHECK — RUN BEFORE READING BELOW
Before reading anything else in this file, check whether you loaded SKILL.md from the one known stale-clone location: Claude Code's marketplaces directory. Before reading anything else in this file, verify that the SKILL.md you just loaded is the versioned cache copy, not a stale marketplace clone:
**Run this check:**
```bash ```bash
CLAUDE_CACHE_LATEST=$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1) CANONICAL_ROOT="$HOME/.claude/plugins/cache/last30days-skill/last30days"
# Two cache layouts ship in the wild — nested ({version}/skills/last30days/SKILL.md) CANONICAL_LATEST=$(ls -d "$CANONICAL_ROOT"/*/ 2>/dev/null | sort -V | tail -1)
# and flat ({version}/SKILL.md). Resolve to whichever shape actually exists. CANONICAL_LATEST="${CANONICAL_LATEST%/}"
CLAUDE_CACHE_SKILL_MD=""
if [ -n "$CLAUDE_CACHE_LATEST" ]; then
if [ -f "$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md" ]; then
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md"
elif [ -f "$CLAUDE_CACHE_LATEST/SKILL.md" ]; then
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/SKILL.md"
fi
fi
echo "CLAUDE_CACHE_SKILL_MD=$CLAUDE_CACHE_SKILL_MD"
``` ```
If the SKILL.md path you just Read contains `/.claude/plugins/marketplaces/` AND `$CLAUDE_CACHE_SKILL_MD` is non-empty, STOP and re-read `$CLAUDE_CACHE_SKILL_MD` before proceeding. Otherwise the SKILL.md you have is fine — continue. If the SKILL.md file you just read is NOT under `$CANONICAL_LATEST/`, STOP and re-read `$CANONICAL_LATEST/SKILL.md` before proceeding. Do not run `--help`, do not plan, do not invoke the engine against any other path.
**Why this specific check:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test. STEP 0 defends against that one Claude Code-specific bug. **Why:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test.
**Other install paths are fine:** `~/.codex/skills/`, `~/.agents/skills/`, an `npx skills add` install dir, or a repo checkout are all valid load points - the resolver in Step 1 picks them up. Do NOT abort or hop on those paths. **How to self-check:** the file path you used in your last Read tool call should match `$CANONICAL_LATEST/SKILL.md`. If it contains `marketplaces/` or any other prefix, that is the stale-path failure mode. Re-read from `$CANONICAL_LATEST/SKILL.md` and restart this contract from the top.
The same pinned resolver appears later in Step 1 for the engine Bash invocation. That guard is necessary but insufficient — by the time you reach Step 1, you may have already internalized an out-of-date flag list from the stale SKILL.md above it. This STEP 0 runs first so the CONTRACT itself is read from the right file.
--- ---
@@ -97,7 +87,7 @@ You are inside the `/last30days` SKILL. This is a specific research tool with a
**How v3.0.7 fixes it:** three structural anchors. **How v3.0.7 fixes it:** three structural anchors.
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section. 1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
2. **The SKILL_ROOT resolver** in the engine Bash calls walks a precedence list of known install locations and picks the highest-versioned freshest copy, never `~/.openclaw/` or other stale copies. 2. **The pinned SKILL_ROOT resolution** in the engine Bash calls always points to the public plugin cache, never `~/.openclaw/` or other stale copies.
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom. 3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response. If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
@@ -114,7 +104,7 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
🌐 last30days v{VERSION} · synced {YYYY-MM-DD} 🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
``` ```
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_ROOT/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins. Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1. **Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
@@ -186,13 +176,13 @@ The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do." Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
``` ```
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan "$QUERY_PLAN_FILE"` (the path to a tmpfile you wrote via heredoc — see Step 1 for the pattern; never inline `--plan '$JSON'`, apostrophes in search/ranking strings break shell parsing). **LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan '$JSON'`.
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan "$QUERY_PLAN_FILE"` (or any path the engine can read). A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema). Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan '$JSON'`. A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step. **Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan "$QUERY_PLAN_FILE"` (or another path the engine can read)? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan, then write it to a tmpfile per the Step 1 pattern. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider. **Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan '$JSON'`? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself. **LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
@@ -243,7 +233,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
--- ---
# last30days v3.2.3: Research Any Topic from the Last 30 Days # last30days v3.0.1: Research Any Topic from the Last 30 Days
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section. > **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
@@ -327,7 +317,6 @@ Common patterns:
- Always active: Reddit, Hacker News, Polymarket - Always active: Reddit, Hacker News, Polymarket
- If gh CLI is installed (check `which gh`): add GitHub - If gh CLI is installed (check `which gh`): add GitHub
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X - If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
- If yt-dlp is installed (check `which yt-dlp`): add YouTube - If yt-dlp is installed (check `which yt-dlp`): add YouTube
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok - If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
@@ -592,56 +581,18 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe
**Invocation:** **Invocation:**
```bash ```bash
# Comparison mode skips Step 1, so resolve SKILL_ROOT inline here (same precedence
# walk as Step 1 — keep the two in sync if you edit either).
SKILL_ROOT=""
CLAUDE_PLUGIN_ROOT="$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)"
if [ -n "$CLAUDE_PLUGIN_ROOT" ]; then
if [ -f "$CLAUDE_PLUGIN_ROOT/skills/last30days/scripts/last30days.py" ]; then
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT/skills/last30days"
elif [ -f "$CLAUDE_PLUGIN_ROOT/scripts/last30days.py" ]; then
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT"
fi
fi
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
for dir in \
"$HOME/.codex/skills/last30days" \
"$HOME/.agents/skills/last30days" \
"./skills/last30days" \
"./.skills/last30days" \
"." \
"${GEMINI_EXTENSION_DIR:-}"; do
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
done
fi
# Write the per-entity plan to a tmpfile and pass the path to the engine.
# The engine's parse_competitors_plan() reads file paths transparently. This
# avoids the inline-single-quoted-JSON apostrophe trap (resolved context
# strings like "people's choice" or "McDonald's" otherwise close the outer
# single-quote and break shell parsing before the engine is even invoked).
# Trailing XXXXXX (no .json suffix) so BSD/macOS mktemp works the same as
# GNU; BSD only substitutes X's at the end of the template.
COMPETITORS_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-competitors.XXXXXX")
trap 'rm -f "$COMPETITORS_PLAN_FILE"' EXIT
cat > "$COMPETITORS_PLAN_FILE" <<'PLAN_EOF'
{
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
}
PLAN_EOF
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \ "${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
--emit=compact \ --emit=compact \
--save-dir="${LAST30DAYS_MEMORY_DIR}" \ --save-dir="${LAST30DAYS_MEMORY_DIR}" \
--save-suffix=v3 \ --save-suffix=v3 \
--x-handle={TOPIC_A_HANDLE} \ --x-handle={TOPIC_A_HANDLE} \
--subreddits={TOPIC_A_SUBS} \ --subreddits={TOPIC_A_SUBS} \
--competitors-plan "$COMPETITORS_PLAN_FILE" --competitors-plan '{
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
}'
``` ```
**The quoted heredoc marker `'PLAN_EOF'` is load-bearing** — quoting suppresses shell interpolation so apostrophes, `$`, backticks, etc. pass through verbatim. If you ever switch to an unquoted `<<PLAN_EOF`, every variable reference and apostrophe inside the JSON becomes a parse hazard.
Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive). Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive).
**Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan. **Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan.
@@ -873,7 +824,7 @@ Only show lines for platforms where something was resolved. Skip empty lines. On
- For how_to: prioritize YouTube (tutorials) and Reddit (guides) - For how_to: prioritize YouTube (tutorials) and Reddit (guides)
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5 - Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key), digg (Digg clusters - only if `digg-pp-cli` is on PATH) **Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key)
**Intent → freshness_mode mapping:** **Intent → freshness_mode mapping:**
- breaking_news, prediction → `strict_recent` - breaking_news, prediction → `strict_recent`
@@ -916,65 +867,34 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).** **IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
```bash ```bash
# Resolve SKILL_ROOT by walking a precedence list of known install locations. # PIN SKILL_ROOT to the public plugin cache (highest-version dir wins on upgrade).
# Claude Code plugin cache wins when present (highest version dir picked on upgrade), # DO NOT write your own path-discovery loop. The 2026-04-18 Peter Steinberger run 1
# then common per-harness skill dirs, then a repo checkout. # regression was caused by a custom discovery loop landing on ~/.openclaw/skills/last30days/
SKILL_ROOT="" # (a stale copy from a private-repo sync pattern). That path contains a pre-plan-007
# engine and produces non-canonical output. This pinned resolution ignores every stale
# copy (~/.openclaw/, ~/.agents/, ~/.codex/) and picks the plugin cache exclusively.
SKILL_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
SKILL_ROOT="${SKILL_ROOT%/}"
# 1. Claude Code plugin cache (versioned, sort -V picks freshest). Two cache layouts ship in the wild: # Fallback for repo checkout / Gemini / Codex hosts where the plugin cache does not exist.
# nested ({cache}/{version}/skills/last30days/scripts/...) and flat ({cache}/{version}/scripts/...). # Only runs if the public plugin cache is missing entirely.
# `find` (not `ls + glob`) because zsh errors on globs that match nothing, leaking
# noisy "no matches found" stderr in Codex/zsh sessions even with 2>/dev/null.
CLAUDE_PLUGIN_ROOT="$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)"
if [ -n "$CLAUDE_PLUGIN_ROOT" ]; then
if [ -f "$CLAUDE_PLUGIN_ROOT/skills/last30days/scripts/last30days.py" ]; then
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT/skills/last30days"
elif [ -f "$CLAUDE_PLUGIN_ROOT/scripts/last30days.py" ]; then
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT"
fi
fi
# 2. Common per-harness skill dirs and repo checkout (npx skills, Codex, Agents, Gemini, etc).
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
for dir in \ for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
"$HOME/.codex/skills/last30days" \
"$HOME/.agents/skills/last30days" \
"./skills/last30days" \
"./.skills/last30days" \
"." \
"${GEMINI_EXTENSION_DIR:-}"; do
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break [ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
done done
fi fi
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
echo "ERROR: Could not find scripts/last30days.py in any known install location" >&2 echo "ERROR: Could not find scripts/last30days.py in public plugin cache or repo checkout" >&2
echo "Searched: ~/.claude/plugins/cache/, ~/.codex/skills/, ~/.agents/skills/, ./skills/last30days, ./.skills/last30days, ." >&2 echo "Expected: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/scripts/last30days.py" >&2
exit 1 exit 1
fi fi
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3 "${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
``` ```
**If you ran Steps 0.55 and 0.75 (agent planning), pass the plan via a tmpfile and add the targeting flags:** **If you ran Steps 0.55 and 0.75 (agent planning), add these flags:**
- `--plan 'QUERY_PLAN_JSON'` (replace with actual JSON from Step 0.75)
```bash
# Write QUERY_PLAN_JSON to a tmpfile before the engine invocation above.
# parse_plan() reads file paths transparently; this avoids inline-JSON
# shell-quoting hazards (apostrophes in search_query / ranking_query
# strings break single-quoted command-line JSON). Trailing XXXXXX (no
# .json suffix) for BSD/macOS portability — BSD mktemp only substitutes
# X's at the end of the template.
QUERY_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-plan.XXXXXX")
trap 'rm -f "$QUERY_PLAN_FILE"' EXIT
cat > "$QUERY_PLAN_FILE" <<'PLAN_EOF'
{QUERY_PLAN_JSON_FROM_STEP_0.75}
PLAN_EOF
```
Then add to the engine command:
- `--plan "$QUERY_PLAN_FILE"` (path to the file you just wrote)
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5) - `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55) - `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55) - `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
@@ -1577,33 +1497,6 @@ Close with `I have all the links to the {N} {source list} I pulled from. Just as
--- ---
## SHAREABLE HTML BRIEF (when the user asked for one)
**This section fires if EITHER trigger is true:**
- `$ARGUMENTS` contains `--emit=html`, `--emit:html`, or `--html` as a flag
- The user's natural-language request asks for an HTML brief, shareable doc, or file for sharing (Slack, email, Notion, "export as HTML", etc). Use your judgment for phrasing variants.
**If neither trigger fires, skip this entire section and proceed to WAIT FOR USER'S RESPONSE.** No HTML save flow, no reference read needed.
**When triggered, you MUST:**
- Read `references/save-html-brief.md` BEFORE proceeding to WAIT FOR USER'S RESPONSE
- Follow that file's instructions exactly - it is the canonical source for the save flow
- Append the confirmation line (`📎 Shareable brief saved to <path>`) to your already-emitted chat response
**You MUST NOT:**
- Improvise the HTML save flow from memory or from instructions you've seen before
- Skip the reference read because the steps "look familiar"
- Save to a different path than the reference specifies
- Add data quality warnings, debug headers, or safety notes to the saved HTML
- Re-research the topic for the HTML render - the engine cache covers the second invocation
**Why the directive is forceful:** the reference file is the only source of truth for the save flow. Skipping it produces broken artifacts - wrong path conventions, missing synthesis content, leaked engine debug output, or warnings that don't belong in shareable docs.
---
## WAIT FOR USER'S RESPONSE ## WAIT FOR USER'S RESPONSE
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`. **STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.

Before

Width:  |  Height:  |  Size: 2.7 MiB

After

Width:  |  Height:  |  Size: 2.7 MiB

Before

Width:  |  Height:  |  Size: 2.3 MiB

After

Width:  |  Height:  |  Size: 2.3 MiB

Before

Width:  |  Height:  |  Size: 3.8 MiB

After

Width:  |  Height:  |  Size: 3.8 MiB

Before

Width:  |  Height:  |  Size: 2.6 MiB

After

Width:  |  Height:  |  Size: 2.6 MiB

@@ -0,0 +1,396 @@
---
title: "marketing: v3.1 launch video — 30s Remotion piece for X"
type: feat
status: active
date: 2026-04-22
---
# marketing: v3.1 launch video — 30s Remotion piece for X
## Overview
Build a 30-second Remotion-rendered MP4 marketing video announcing **v3.1: Competitors mode** for `/last30days`. Posts to X. Frames the new vs-mode-with-auto-discovery as the headline — three full passes, three save files, one comparison — without burying it in CLI minutiae.
Marketing release tag is **v3.1** (rebrands the 3.0.11-3.0.14 bundle into one shippable narrative for the launch tweet). Engine version stays 3.0.14 — `3.1` is the marketing version, not a code version bump.
## Script (60 frames per second × 30 seconds = 900 frames; this version assumes 30fps × 30s = 900 frames at 30fps)
Total runtime: 30.0 seconds @ 30fps = 900 frames. Six scenes, on-screen text only (silent autoplay-friendly).
### Scene 1 — Hook (0:00 0:03 | frames 0-90)
**Visual:** Black background. The `/last30days` badge animates in (the literal `🌐 last30days v3.1 · synced 2026-04-22` line) with the spring scale-in used in slick devtool intros.
**Caption (overlay, large):**
```
What if one search
ran 3 at once?
```
### Scene 2 — Set the world (0:03 0:08 | frames 90-240)
**Visual:** Single mac terminal window center-stage. Type-on animation:
```
$ /last30days OpenAI
```
Below it, a simple result card stub appears (Reddit upvote count + X likes), then static.
**Caption (small bottom-left):**
```
The old way: one topic.
```
### Scene 3 — The reveal (0:08 0:14 | frames 240-420)
**Visual:** The terminal types one more flag:
```
$ /last30days OpenAI --competitors
```
Hard cut → the single terminal **splits into 3 panes** side by side. Each pane shows a different topic header animating in, in this order:
- Left: `OpenAI`
- Middle: `vs Anthropic`
- Right: `vs xAI`
Pane content scrolls fake "search progress" lines (Reddit, X, YouTube indicators) in parallel, like a live fan-out.
**Caption (top center):**
```
Now it discovers competitors
and runs all 3.
```
### Scene 4 — Result reveal (0:14 0:21 | frames 420-630)
**Visual:** The 3 panes collapse into a single comparison surface — the `## Head-to-Head` table from the actual engine output, with rows fading in one by one (What it is, Streams, Best for, Trajectory). Each entity column lights up as its row populates.
**Caption (bottom):**
```
3 full passes. 3 save files.
1 comparison.
```
### Scene 5 — How it's special (0:21 0:26 | frames 630-780)
**Visual:** Cut to a clean text card, large mono font:
```
You pick the topic.
The agent picks the peers.
The engine fans out.
```
Each line fades in 1.5s apart.
### Scene 6 — CTA (0:26 0:30 | frames 780-900)
**Visual:** Black background. Centered:
- Top: `🌐 last30days v3.1`
- Middle: `/last30days {topic} --competitors`
- Bottom: `github.com/mvanhorn/last30days-skill`
Subtle pulse on the install line.
**End frame holds for ~0.5s.**
## Problem Frame
The 3.0.11-3.0.14 release bundle ships a transformative feature (per-entity vs-mode fanout + `--competitors` shortcut + per-entity save files), but the value lands flat in a tweet thread or screenshot. A 30s video does what static text cannot: shows the fan-out happening in real time and the 3 → 1 collapse into a comparison. Higher tweet engagement, easier to RT/QT.
## Requirements Trace
- R1. Final artifact: a single MP4 file, 1920×1080, 30fps, ~30 seconds (±0.5s), under 30MB, suitable for direct X upload.
- R2. Six-scene script as defined above, with text/visuals/timing matching to within 5 frames.
- R3. Branded look: matches the `🌐 last30days v3.1` badge style (terminal aesthetic, mono font, dark background).
- R4. Silent — no voiceover, no music in v1. Captions baked in. Designed for autoplay-muted feeds.
- R5. Reproducible: another contributor (or a future-me) can re-render with one command. Project lives in-repo so the source is versioned.
## Scope Boundaries
- No voiceover. Text-on-screen only. (Voice can be a v1.1 if engagement is high.)
- No background music in the rendered MP4. (Music can be added in post via QuickTime/iMovie if desired before posting.)
- No localization. English captions only.
- No A/B test variants. One video.
- No 9:16 vertical version. 16:9 only. (Vertical can be a separate render after launch validates the format.)
### Deferred to Separate Tasks
- Voiceover variant: defer to follow-up if v1 lands well.
- 9:16 mobile cut: defer; same source compositions can re-render at 1080×1920 in a follow-up.
- Animated GIF for embedding in README.md: defer; can be ffmpeg-extracted from the MP4.
## Context & Research
### Relevant Code and Patterns
- `SKILL.md` — the comparison render scaffold (`## Head-to-Head` table) is the visual reference for Scene 4's table look.
- `scripts/lib/render.py` `_render_comparison_scaffold` — emits the 9-axis table whose visual style we're recreating in a more polished form.
- `CHANGELOG.md` 3.0.11-3.0.14 entries — the prose source for the script's beats.
- `.claude-plugin/plugin.json` version 3.0.14 — current code version (marketing version is 3.1 for the launch).
### External References
- Remotion 4.x docs (https://www.remotion.dev/docs/) — current API for compositions, sequences, springs, and render CLI.
- X video specs 2026: max 2 min 20 s, ≤512MB, MP4 with H.264 + AAC, recommended 1920×1080 for landscape autoplay.
### Institutional Learnings
- No prior `marketing/` dir or video plans in `docs/plans/`. This is a greenfield asset directory.
## Key Technical Decisions
- **Remotion, not ffmpeg-only.** Remotion's React-based compositions handle the typed-on terminal effect, spring-animated badges, and scene transitions far more cleanly than raw ffmpeg filtergraphs. Render output is still MP4 via Remotion's bundled ffmpeg.
- **In-repo asset directory at `marketing/v3.1-launch/`.** Lives with the product so future versions can fork the project. Adds `marketing/` to `.gitignore` exceptions only for source files; rendered MP4 stays out of git (uploaded separately).
- **16:9 1920×1080 @ 30fps.** Best fit for X landscape autoplay on desktop and mobile feed. 30fps is plenty for typed-text + UI animation; 60fps doubles render time without obvious quality gain.
- **Silent + captions.** X autoplay defaults to muted. Sound-off is the realistic viewing condition. Captions baked into the visual.
- **Six scenes, one composition.** Single Remotion composition with sequenced child compositions per scene. Easier to re-time than scene-files. Frame-numbered timing in the script enables precise edits.
- **Mono font (JetBrains Mono or Geist Mono).** Matches the terminal aesthetic of the actual `/last30days` output. Available via Google Fonts or @remotion/google-fonts.
- **Marketing version 3.1 ≠ engine version 3.0.14.** `3.1` is the launch label. Engine stays 3.0.14. Avoids confusion in CHANGELOG.
## Open Questions
### Resolved During Planning
- **Aspect ratio?** 16:9 1080p. Best X autoplay format; vertical can be a follow-up.
- **Voiceover or silent?** Silent + on-screen captions. Autoplay-muted is the realistic condition.
- **In-repo or separate repo?** In-repo at `marketing/v3.1-launch/`. Rendered MP4 not committed; source compositions are.
- **Length?** Exactly 30s (900 frames @ 30fps). No flex.
- **Marketing version label?** `v3.1`. Engine code version stays 3.0.14.
### Deferred to Implementation
- Exact animation easing curves per scene — pick during build via Remotion preview iteration.
- Whether the comparison-table mock in Scene 4 uses canned text or pulls from the actual saved `*-raw.md` files. Probably canned for visual control.
- Whether Scene 3's "search progress" lines are typed individually or use a marquee scroll. Pick during preview.
- Exact accent color palette beyond "terminal dark." Iterate against preview.
## Output Structure
marketing/
v3.1-launch/
package.json # Remotion dependency manifest
tsconfig.json # TypeScript config
remotion.config.ts # Remotion render config (codec, fps, resolution)
src/
index.ts # Remotion entry — registers compositions
Root.tsx # Root composition definition
LaunchVideo.tsx # Main 30s composition that sequences scenes
scenes/
Scene1Hook.tsx
Scene2OldWay.tsx
Scene3FanOut.tsx
Scene4Comparison.tsx
Scene5HowItWorks.tsx
Scene6CTA.tsx
components/
TerminalWindow.tsx # Reusable mac-style terminal frame
TypedLine.tsx # Type-on animation primitive
ComparisonTable.tsx # The Head-to-Head table mock
BadgeBar.tsx # The 🌐 last30days v3.1 badge
lib/
timing.ts # Frame ranges per scene (single source of truth)
colors.ts # Brand palette
public/ # Static assets (logo, fonts if local)
out/ # Rendered MP4 lives here (gitignored)
README.md # How to preview/render
## High-Level Technical Design
> *Directional guidance for review — not implementation specification.*
```
LaunchVideo (durationInFrames = 900)
├── <Sequence from={0} durationInFrames={90}> <Scene1Hook />
├── <Sequence from={90} durationInFrames={150}> <Scene2OldWay />
├── <Sequence from={240} durationInFrames={180}> <Scene3FanOut />
├── <Sequence from={420} durationInFrames={210}> <Scene4Comparison />
├── <Sequence from={630} durationInFrames={150}> <Scene5HowItWorks />
└── <Sequence from={780} durationInFrames={120}> <Scene6CTA />
```
Per-scene components use `useCurrentFrame()` + `interpolate()` + `spring()` for timing. `TerminalWindow` is the dominant motif across scenes 2-4.
## Implementation Units
- [ ] **Unit 1: Remotion project scaffold**
**Goal:** Spin up a working Remotion project at `marketing/v3.1-launch/` that previews a blank composition and renders to MP4.
**Requirements:** R1, R5
**Files:**
- Create: `marketing/v3.1-launch/package.json`
- Create: `marketing/v3.1-launch/tsconfig.json`
- Create: `marketing/v3.1-launch/remotion.config.ts`
- Create: `marketing/v3.1-launch/src/index.ts`
- Create: `marketing/v3.1-launch/src/Root.tsx`
- Create: `marketing/v3.1-launch/README.md`
- Modify: `.gitignore` (add `marketing/v3.1-launch/out/`, `marketing/v3.1-launch/node_modules/`)
**Approach:**
- Use `npx create-video@latest --blank` (Remotion 4.x scaffolding) targeting `marketing/v3.1-launch/`. Strip the demo composition.
- Configure: 1920×1080, 30fps, H.264, AAC (audio codec needed even for silent — empty track).
- README documents `npm install`, `npm run preview` (Remotion Studio), `npm run render` (one-command MP4).
**Test scenarios:**
- Test expectation: none for scaffold, but verification = `npm run preview` opens Remotion Studio with a blank 30s composition; `npm run render` produces a black MP4 at the right resolution.
**Verification:**
- Studio loads at localhost:3000 with the empty `LaunchVideo` composition listed.
- A render produces `out/launch-video.mp4` at 1920×1080, 30s, valid MP4.
- [ ] **Unit 2: Reusable components (Terminal, TypedLine, BadgeBar)**
**Goal:** Build the three primitive components scenes 2-6 will compose. Each is independently previewable.
**Requirements:** R3, R5
**Dependencies:** Unit 1
**Files:**
- Create: `marketing/v3.1-launch/src/components/TerminalWindow.tsx`
- Create: `marketing/v3.1-launch/src/components/TypedLine.tsx`
- Create: `marketing/v3.1-launch/src/components/BadgeBar.tsx`
- Create: `marketing/v3.1-launch/src/lib/colors.ts`
- Create: `marketing/v3.1-launch/src/lib/timing.ts`
**Approach:**
- `TerminalWindow`: mac-style traffic-light header, dark gradient background, mono content area. Accepts children.
- `TypedLine`: takes a string and a `startFrame`, renders character-by-character at ~30 chars/sec. Reuses Remotion's `interpolate(useCurrentFrame() - startFrame, [0, lengthFrames], [0, text.length])` clamped.
- `BadgeBar`: renders the literal `🌐 last30days v3.1 · synced 2026-04-22` line in mono with the same gradient color treatment as the engine's compact emit.
- `colors.ts`: 5-7 brand colors (terminal-bg, terminal-fg, accent-cyan, accent-magenta, muted, success-green, warning-amber).
- `timing.ts`: exports the scene frame ranges as named constants. Single source of truth for any retiming.
**Test scenarios:**
- Test expectation: none — visual components verified in Remotion Studio.
**Verification:**
- Each component renders standalone in Studio when wrapped in a temporary preview composition.
- TypedLine animates character-by-character without flicker.
- [ ] **Unit 3: Scenes 1-3 (Hook, Old way, Fan-out reveal)**
**Goal:** Build the first half of the video (frames 0-420). The narrative arc up through the visual fan-out.
**Requirements:** R2
**Dependencies:** Unit 2
**Files:**
- Create: `marketing/v3.1-launch/src/scenes/Scene1Hook.tsx`
- Create: `marketing/v3.1-launch/src/scenes/Scene2OldWay.tsx`
- Create: `marketing/v3.1-launch/src/scenes/Scene3FanOut.tsx`
- Modify: `marketing/v3.1-launch/src/Root.tsx` (register sequences)
**Approach:**
- Scene 1: spring-in BadgeBar, large overlay caption, 3-second hold.
- Scene 2: TerminalWindow with TypedLine (`$ /last30days OpenAI`), then a single result-card mock fading in.
- Scene 3: typing animation appends `--competitors`, hard cut, three TerminalWindow components arranged in a row with staggered fade-in. Each pane shows a different entity header + scrolling progress lines.
**Test scenarios:**
- Test expectation: none — verified visually in Studio.
**Verification:**
- Scrub the 0-14s range in Studio; visuals match the script timing within 5 frames.
- The fan-out moment (frame 240) lands cleanly; no jank in the transition from 1 → 3 panes.
- [ ] **Unit 4: Scenes 4-6 (Comparison reveal, How it works, CTA)**
**Goal:** Build the back half of the video (frames 420-900). Resolution + payoff + call to action.
**Requirements:** R2
**Dependencies:** Unit 2
**Files:**
- Create: `marketing/v3.1-launch/src/scenes/Scene4Comparison.tsx`
- Create: `marketing/v3.1-launch/src/scenes/Scene5HowItWorks.tsx`
- Create: `marketing/v3.1-launch/src/scenes/Scene6CTA.tsx`
- Create: `marketing/v3.1-launch/src/components/ComparisonTable.tsx`
**Approach:**
- Scene 4: ComparisonTable component renders 3-column markdown-style table; rows fade in one by one (stagger 15-20 frames). Uses canned data — OpenAI / Anthropic / xAI with believable cell content drawn from real 3.0.13 outputs.
- Scene 5: three-line text card; lines fade in 45 frames apart.
- Scene 6: three centered text blocks; install line gets a 1Hz subtle opacity pulse for emphasis.
**Test scenarios:**
- Test expectation: none — verified visually.
**Verification:**
- Scrub 14-30s; table reveal feels paced (not too slow, not strobed); CTA holds long enough to read (~3-4s).
- ComparisonTable cells are legible at 1920×1080 (mono font ≥ 28px).
- [ ] **Unit 5: Final composition wiring + render**
**Goal:** Wire the six scenes into the master `LaunchVideo` composition, render to MP4, verify against X upload constraints.
**Requirements:** R1, R2, R5
**Dependencies:** Units 3, 4
**Files:**
- Modify: `marketing/v3.1-launch/src/LaunchVideo.tsx` (sequence all 6 scenes)
- Modify: `marketing/v3.1-launch/README.md` (add render command + verification checklist)
**Approach:**
- `LaunchVideo` is a single Composition that imports `Scene1Hook``Scene6CTA` and wraps each in `<Sequence from=… durationInFrames=…>` matching `lib/timing.ts`.
- Run `npx remotion render LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18`.
- Verify output: 30.0s ±0.1s, 1920×1080, file size <30MB, opens in QuickTime, plays without dropped frames.
**Test scenarios:**
- Test expectation: none — verification is the rendered MP4 itself.
**Verification:**
- `ffprobe out/last30days-v3.1-launch.mp4` reports 1920×1080, 30fps, ~30.0s, h264, faststart-friendly.
- Manual play-through end-to-end in QuickTime feels coherent and on-pace.
- File <30MB so X upload is instant.
- [ ] **Unit 6: Polish pass + ship**
**Goal:** Watch the full render, fix obvious jank, do a second render, and stage for X posting.
**Requirements:** R1, R2, R3
**Dependencies:** Unit 5
**Files:**
- Possibly modify: any scene file based on watch-through findings.
**Approach:**
- Watch the rendered MP4 at full size. Note: timing felt off, transitions too fast, captions overflow, color clash, anything visibly broken.
- Iterate: edit scene component → re-preview in Studio → re-render full MP4.
- Cap at 2 polish passes; ship the better of the two renders.
- Final MP4 sits at `marketing/v3.1-launch/out/last30days-v3.1-launch.mp4` ready for X upload.
**Test scenarios:**
- Test expectation: none — pure subjective polish.
**Verification:**
- User watches the final render and approves.
- No glaring visual bugs (overflowing text, frozen frames, color clashes).
## System-Wide Impact
- **Interaction graph:** None — this is a standalone marketing artifact. Doesn't touch the Python engine, doesn't change any user-facing behavior.
- **State lifecycle risks:** None.
- **API surface parity:** N/A.
- **Unchanged invariants:** The shipped 3.0.14 engine is untouched.
## Risks & Dependencies
| Risk | Mitigation |
|------|------------|
| Remotion install pulls 200MB+ of node_modules. | `marketing/v3.1-launch/node_modules/` in `.gitignore`; checked-in source stays small. |
| Render time blows past patience (>5 min for 30s @ 1080p30). | Bun-based render or `--concurrency` flag. Default Remotion is fast enough on M-series Macs. If slow, lower preview to 720p, render final at 1080p. |
| Captions overflow the 1920px width on certain fonts. | Use a known mono font with a measured per-character width; cap caption lines at 36 chars. |
| The "fan-out" visual in Scene 3 looks confusing instead of magical. | Polish pass (Unit 6) is the safety net; if still bad, fall back to a simpler "1 → 3 panes wipe" instead of typed split. |
| File size >30MB hits X upload friction. | Use `--crf=18` (high quality, reasonable size); fall back to `--crf=23` if over. 30s @ 1080p30 H.264 is normally 5-15MB. |
## Documentation / Operational Notes
- README at `marketing/v3.1-launch/README.md` documents preview / render commands.
- After render, the MP4 is uploaded directly to X. Tweet copy is the user's call (this plan stops at the rendered file).
## Sources & References
- Related code: `scripts/lib/render.py` (`_render_comparison_scaffold` is the visual model for Scene 4); `SKILL.md` Competitor mode section (the narrative source).
- Related PRs: #308, #311, #312 (the 3.0.11 → 3.0.14 release bundle this video markets as "v3.1").
- External docs: https://www.remotion.dev/docs/ (Remotion 4.x API).
- X video specs: https://help.x.com/en/using-x/twitter-videos.
+37
View File
@@ -0,0 +1,37 @@
# /last30days v3.1 Launch Video
30-second Remotion-rendered MP4 announcing **v3.1: Competitors mode** for posting on X.
## Quick start
```bash
cd marketing/v3.1-launch
npm install # one-time, ~200MB of node_modules
npm run preview # opens Remotion Studio at localhost:3000 to scrub frames
npm run render # writes out/last30days-v3.1-launch.mp4 (high quality, CRF 18)
npm run render:fast # writes a CRF 23 preview for fast iteration
```
## Specs
- 1920×1080, 30fps, 30 seconds (900 frames)
- H.264 / MP4
- Silent (autoplay-muted-friendly; captions baked in)
- Marketing label `v3.1` (engine code version stays 3.0.14)
## Scene timing (single source of truth: `src/lib/timing.ts`)
| Scene | Frames | Time | What |
|-------|--------|------|------|
| 1. Hook | 0-89 | 0.0-3.0s | Badge animates in + "What if one search ran 3 at once?" |
| 2. Old way | 90-239 | 3.0-8.0s | Single terminal: `/last30days OpenAI` |
| 3. Fan-out | 240-419 | 8.0-14.0s | `--competitors` types in → splits into 3 panes |
| 4. Comparison | 420-629 | 14.0-21.0s | 3 panes collapse into Head-to-Head table |
| 5. How | 630-779 | 21.0-26.0s | 3-line text card |
| 6. CTA | 780-899 | 26.0-30.0s | Install command + repo URL |
Edit `src/lib/timing.ts` to retime scenes; the `LaunchVideo` composition reads from there.
## Output
Rendered MP4 lives at `out/last30days-v3.1-launch.mp4` (gitignored). Upload directly to X.
+23
View File
@@ -0,0 +1,23 @@
{
"name": "last30days-v3-1-launch-video",
"version": "0.1.0",
"private": true,
"description": "30s Remotion launch video for /last30days v3.1 (competitors mode).",
"scripts": {
"preview": "remotion studio src/index.ts",
"render": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18",
"render:fast": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch-preview.mp4 --codec=h264 --crf=23"
},
"dependencies": {
"react": "19.0.0",
"react-dom": "19.0.0",
"remotion": "4.0.250",
"@remotion/cli": "4.0.250",
"@remotion/google-fonts": "4.0.250"
},
"devDependencies": {
"@types/react": "19.0.0",
"@types/node": "22.10.0",
"typescript": "5.6.3"
}
}
+6
View File
@@ -0,0 +1,6 @@
import { Config } from "@remotion/cli/config";
Config.setVideoImageFormat("jpeg");
Config.setOverwriteOutput(true);
Config.setConcurrency(null);
Config.setCodec("h264");
+35
View File
@@ -0,0 +1,35 @@
import React from "react";
import { AbsoluteFill, Sequence } from "remotion";
import { SCENES } from "./lib/timing";
import { Scene1Hook } from "./scenes/Scene1Hook";
import { Scene2OldWay } from "./scenes/Scene2OldWay";
import { Scene3FanOut } from "./scenes/Scene3FanOut";
import { Scene4Comparison } from "./scenes/Scene4Comparison";
import { Scene5HowItWorks } from "./scenes/Scene5HowItWorks";
import { Scene6CTA } from "./scenes/Scene6CTA";
import { COLORS } from "./lib/colors";
export const LaunchVideo: React.FC = () => {
return (
<AbsoluteFill style={{ background: COLORS.bgDeep }}>
<Sequence from={SCENES.hook.from} durationInFrames={SCENES.hook.durationInFrames}>
<Scene1Hook />
</Sequence>
<Sequence from={SCENES.oldWay.from} durationInFrames={SCENES.oldWay.durationInFrames}>
<Scene2OldWay />
</Sequence>
<Sequence from={SCENES.fanOut.from} durationInFrames={SCENES.fanOut.durationInFrames}>
<Scene3FanOut />
</Sequence>
<Sequence from={SCENES.comparison.from} durationInFrames={SCENES.comparison.durationInFrames}>
<Scene4Comparison />
</Sequence>
<Sequence from={SCENES.howItWorks.from} durationInFrames={SCENES.howItWorks.durationInFrames}>
<Scene5HowItWorks />
</Sequence>
<Sequence from={SCENES.cta.from} durationInFrames={SCENES.cta.durationInFrames}>
<Scene6CTA />
</Sequence>
</AbsoluteFill>
);
};
+19
View File
@@ -0,0 +1,19 @@
import React from "react";
import { Composition } from "remotion";
import { LaunchVideo } from "./LaunchVideo";
import { FPS, TOTAL_FRAMES } from "./lib/timing";
export const RemotionRoot: React.FC = () => {
return (
<>
<Composition
id="LaunchVideo"
component={LaunchVideo}
durationInFrames={TOTAL_FRAMES}
fps={FPS}
width={1920}
height={1080}
/>
</>
);
};
@@ -0,0 +1,68 @@
import React from "react";
import { spring, useCurrentFrame, useVideoConfig } from "remotion";
import { COLORS, FONT_MONO } from "../lib/colors";
type Props = {
startFrame?: number;
size?: "small" | "large";
};
export const BadgeBar: React.FC<Props> = ({ startFrame = 0, size = "large" }) => {
const frame = useCurrentFrame();
const { fps } = useVideoConfig();
const elapsed = Math.max(0, frame - startFrame);
const scale = spring({
frame: elapsed,
fps,
config: { damping: 12, stiffness: 90 },
from: 0.85,
to: 1,
});
const opacity = spring({
frame: elapsed,
fps,
config: { damping: 20 },
from: 0,
to: 1,
});
const fontSize = size === "large" ? 56 : 28;
return (
<div
style={{
display: "flex",
alignItems: "center",
justifyContent: "center",
transform: `scale(${scale})`,
opacity,
fontFamily: FONT_MONO,
fontSize,
color: COLORS.fgPrimary,
letterSpacing: 0.5,
}}
>
<span style={{ fontSize: fontSize * 1.1, marginRight: 16 }}>🌐</span>
<span>last30days</span>
<span
style={{
marginLeft: 14,
color: COLORS.accentCyan,
fontWeight: 600,
}}
>
v3.1
</span>
<span
style={{
marginLeft: 16,
color: COLORS.fgDim,
fontSize: fontSize * 0.55,
}}
>
· synced 2026-04-22
</span>
</div>
);
};
@@ -0,0 +1,135 @@
import React from "react";
import { interpolate, useCurrentFrame } from "remotion";
import { COLORS, FONT_MONO } from "../lib/colors";
type Row = {
dimension: string;
cells: [string, string, string];
};
type Props = {
startFrame: number;
entities: [string, string, string];
rows: Row[];
rowStaggerFrames?: number;
};
export const ComparisonTable: React.FC<Props> = ({
startFrame,
entities,
rows,
rowStaggerFrames = 18,
}) => {
const frame = useCurrentFrame();
const headerOpacity = interpolate(
frame - startFrame,
[0, 12],
[0, 1],
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
);
const colTemplate = "1.4fr 1fr 1fr 1fr";
const cellPad = "16px 22px";
return (
<div
style={{
width: "100%",
background: COLORS.bgPanel,
borderRadius: 16,
border: `1px solid ${COLORS.border}`,
overflow: "hidden",
fontFamily: FONT_MONO,
boxShadow: "0 24px 60px rgba(0,0,0,0.6)",
}}
>
<div
style={{
display: "grid",
gridTemplateColumns: colTemplate,
background: COLORS.bgPanelSoft,
borderBottom: `1px solid ${COLORS.border}`,
opacity: headerOpacity,
}}
>
<div
style={{
padding: cellPad,
color: COLORS.fgMuted,
fontSize: 22,
fontWeight: 500,
}}
>
Dimension
</div>
{entities.map((entity, idx) => (
<div
key={entity}
style={{
padding: cellPad,
color: idx === 0 ? COLORS.accentCyan : COLORS.fgPrimary,
fontSize: 26,
fontWeight: 600,
borderLeft: `1px solid ${COLORS.border}`,
}}
>
{entity}
</div>
))}
</div>
{rows.map((row, idx) => {
const rowStart = startFrame + 12 + idx * rowStaggerFrames;
const rowOpacity = interpolate(
frame - rowStart,
[0, 14],
[0, 1],
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
);
const rowSlide = interpolate(
frame - rowStart,
[0, 14],
[12, 0],
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
);
return (
<div
key={row.dimension}
style={{
display: "grid",
gridTemplateColumns: colTemplate,
borderBottom:
idx === rows.length - 1 ? "none" : `1px solid ${COLORS.border}`,
opacity: rowOpacity,
transform: `translateY(${rowSlide}px)`,
}}
>
<div
style={{
padding: cellPad,
color: COLORS.fgMuted,
fontSize: 22,
}}
>
{row.dimension}
</div>
{row.cells.map((cell, cellIdx) => (
<div
key={cellIdx}
style={{
padding: cellPad,
color: COLORS.fgPrimary,
fontSize: 22,
borderLeft: `1px solid ${COLORS.border}`,
lineHeight: 1.35,
}}
>
{cell}
</div>
))}
</div>
);
})}
</div>
);
};
@@ -0,0 +1,95 @@
import React from "react";
import { COLORS, FONT_MONO } from "../lib/colors";
type Props = {
title?: string;
width?: number | string;
height?: number | string;
children?: React.ReactNode;
glow?: boolean;
};
export const TerminalWindow: React.FC<Props> = ({
title = "/last30days",
width = "100%",
height = "100%",
children,
glow = false,
}) => {
return (
<div
style={{
width,
height,
background: COLORS.bgPanel,
borderRadius: 16,
border: `1px solid ${COLORS.border}`,
boxShadow: glow
? `0 0 60px ${COLORS.accentCyan}33, 0 24px 60px rgba(0,0,0,0.6)`
: "0 24px 60px rgba(0,0,0,0.6)",
overflow: "hidden",
display: "flex",
flexDirection: "column",
fontFamily: FONT_MONO,
}}
>
<div
style={{
height: 36,
background: COLORS.bgPanelSoft,
borderBottom: `1px solid ${COLORS.border}`,
display: "flex",
alignItems: "center",
padding: "0 16px",
gap: 8,
}}
>
<span
style={{
width: 12,
height: 12,
borderRadius: 12,
background: COLORS.trafficRed,
}}
/>
<span
style={{
width: 12,
height: 12,
borderRadius: 12,
background: COLORS.trafficYellow,
}}
/>
<span
style={{
width: 12,
height: 12,
borderRadius: 12,
background: COLORS.trafficGreen,
}}
/>
<span
style={{
marginLeft: 16,
color: COLORS.fgMuted,
fontSize: 14,
fontFamily: FONT_MONO,
letterSpacing: 0.5,
}}
>
{title}
</span>
</div>
<div
style={{
flex: 1,
padding: "20px 28px",
color: COLORS.fgPrimary,
overflow: "hidden",
}}
>
{children}
</div>
</div>
);
};
@@ -0,0 +1,71 @@
import React from "react";
import { interpolate, useCurrentFrame } from "remotion";
import { COLORS, FONT_MONO } from "../lib/colors";
type Props = {
text: string;
startFrame: number;
charsPerSec?: number;
fontSize?: number;
color?: string;
prefix?: string;
prefixColor?: string;
showCursor?: boolean;
fps?: number;
};
export const TypedLine: React.FC<Props> = ({
text,
startFrame,
charsPerSec = 28,
fontSize = 32,
color = COLORS.fgPrimary,
prefix,
prefixColor = COLORS.accentGreen,
showCursor = true,
fps = 30,
}) => {
const frame = useCurrentFrame();
const elapsed = Math.max(0, frame - startFrame);
const totalChars = text.length;
const lengthFrames = Math.ceil((totalChars / charsPerSec) * fps);
const visibleChars = Math.round(
interpolate(elapsed, [0, lengthFrames], [0, totalChars], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
}),
);
const visible = text.slice(0, visibleChars);
const done = visibleChars >= totalChars;
const cursorOn = showCursor && Math.floor(frame / 15) % 2 === 0;
return (
<div
style={{
fontFamily: FONT_MONO,
fontSize,
color,
whiteSpace: "pre",
lineHeight: 1.4,
}}
>
{prefix ? (
<span style={{ color: prefixColor, marginRight: 12 }}>{prefix}</span>
) : null}
<span>{visible}</span>
{(!done || cursorOn) && (
<span
style={{
display: "inline-block",
width: fontSize * 0.55,
height: fontSize * 0.95,
background: color,
verticalAlign: "text-bottom",
marginLeft: 2,
opacity: cursorOn ? 0.85 : 0,
}}
/>
)}
</div>
);
};
+4
View File
@@ -0,0 +1,4 @@
import { registerRoot } from "remotion";
import { RemotionRoot } from "./Root";
registerRoot(RemotionRoot);
+20
View File
@@ -0,0 +1,20 @@
export const COLORS = {
bgDeep: "#0a0e14",
bgPanel: "#11161d",
bgPanelSoft: "#161c25",
fgPrimary: "#e6e6e6",
fgMuted: "#8a93a3",
fgDim: "#5b6573",
border: "#2a3340",
accentCyan: "#36d6f7",
accentMagenta: "#ff55a3",
accentGreen: "#5fff9f",
accentAmber: "#ffc857",
trafficRed: "#ff5f57",
trafficYellow: "#febc2e",
trafficGreen: "#28c840",
} as const;
export const FONT_MONO = '"JetBrains Mono", "SF Mono", "Menlo", monospace';
export const FONT_SANS =
'"Inter", "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif';
+13
View File
@@ -0,0 +1,13 @@
// Single source of truth for scene frame ranges.
// 30fps × 30s = 900 frames total.
export const FPS = 30;
export const TOTAL_FRAMES = 900;
export const SCENES = {
hook: { from: 0, durationInFrames: 90 },
oldWay: { from: 90, durationInFrames: 150 },
fanOut: { from: 240, durationInFrames: 180 },
comparison: { from: 420, durationInFrames: 210 },
howItWorks: { from: 630, durationInFrames: 150 },
cta: { from: 780, durationInFrames: 120 },
} as const;
@@ -0,0 +1,61 @@
import React from "react";
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
import { BadgeBar } from "../components/BadgeBar";
import { COLORS, FONT_SANS } from "../lib/colors";
export const Scene1Hook: React.FC = () => {
const frame = useCurrentFrame();
const { fps } = useVideoConfig();
const captionOpacity = interpolate(frame, [20, 35, 75, 90], [0, 1, 1, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const captionLift = spring({
frame: frame - 20,
fps,
config: { damping: 15, stiffness: 70 },
from: 16,
to: 0,
});
const badgeFadeOut = interpolate(frame, [70, 90], [1, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<AbsoluteFill
style={{
background: `radial-gradient(circle at 50% 40%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 60%)`,
display: "flex",
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
gap: 64,
}}
>
<div style={{ opacity: badgeFadeOut }}>
<BadgeBar />
</div>
<div
style={{
opacity: captionOpacity,
transform: `translateY(${captionLift}px)`,
fontFamily: FONT_SANS,
fontSize: 96,
fontWeight: 600,
color: COLORS.fgPrimary,
textAlign: "center",
letterSpacing: -1.5,
lineHeight: 1.1,
maxWidth: 1400,
}}
>
What if one search
<br />
ran <span style={{ color: COLORS.accentCyan }}>3 at once?</span>
</div>
</AbsoluteFill>
);
};
@@ -0,0 +1,100 @@
import React from "react";
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
import { TerminalWindow } from "../components/TerminalWindow";
import { TypedLine } from "../components/TypedLine";
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
export const Scene2OldWay: React.FC = () => {
const frame = useCurrentFrame();
const enter = interpolate(frame, [0, 20], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const slide = interpolate(frame, [0, 20], [40, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
// Result card fades in after type completes (~70 frames)
const resultFade = interpolate(frame, [70, 95], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
// Caption appears late
const captionFade = interpolate(frame, [110, 130, 150], [0, 1, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<AbsoluteFill
style={{
background: COLORS.bgDeep,
padding: 80,
display: "flex",
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
}}
>
<div
style={{
width: 1400,
height: 520,
opacity: enter,
transform: `translateY(${slide}px)`,
}}
>
<TerminalWindow title="bash">
<TypedLine
text="/last30days OpenAI"
startFrame={20}
prefix="$"
fontSize={42}
/>
<div
style={{
opacity: resultFade,
marginTop: 36,
padding: "20px 24px",
background: COLORS.bgPanelSoft,
borderRadius: 12,
border: `1px solid ${COLORS.border}`,
fontFamily: FONT_MONO,
fontSize: 26,
color: COLORS.fgPrimary,
lineHeight: 1.6,
}}
>
<div style={{ color: COLORS.accentGreen }}>
All agents reported back!
</div>
<div style={{ color: COLORS.fgMuted, marginTop: 6 }}>
🟠 Reddit: 14 threads
</div>
<div style={{ color: COLORS.fgMuted }}>
🔵 X: 22 posts
</div>
<div style={{ color: COLORS.fgMuted }}>
🟡 HN: 1 story
</div>
</div>
</TerminalWindow>
</div>
<div
style={{
opacity: captionFade,
marginTop: 60,
fontFamily: FONT_SANS,
fontSize: 38,
color: COLORS.fgMuted,
}}
>
The old way: <span style={{ color: COLORS.fgPrimary }}>one topic.</span>
</div>
</AbsoluteFill>
);
};
@@ -0,0 +1,221 @@
import React from "react";
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
import { TerminalWindow } from "../components/TerminalWindow";
import { TypedLine } from "../components/TypedLine";
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
const PROGRESS_LINES = [
{ source: "Reddit", color: "#ff6a3d" },
{ source: "X", color: "#36d6f7" },
{ source: "YouTube", color: "#ff5757" },
{ source: "TikTok", color: "#5fff9f" },
{ source: "Instagram", color: "#ff55a3" },
];
const ENTITIES: { label: string; tag: string; accent: string }[] = [
{ label: "OpenAI", tag: "$ /last30days OpenAI", accent: COLORS.accentCyan },
{ label: "Anthropic", tag: "$ /last30days Anthropic", accent: COLORS.accentMagenta },
{ label: "xAI", tag: "$ /last30days xAI", accent: COLORS.accentAmber },
];
const FanPane: React.FC<{
label: string;
tag: string;
accent: string;
panelStart: number;
}> = ({ label, tag, accent, panelStart }) => {
const frame = useCurrentFrame();
const { fps } = useVideoConfig();
const localFrame = Math.max(0, frame - panelStart);
const enter = spring({
frame: localFrame,
fps,
config: { damping: 18, stiffness: 80 },
from: 0,
to: 1,
});
const slide = interpolate(localFrame, [0, 20], [40, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<div
style={{
flex: 1,
opacity: enter,
transform: `translateY(${slide}px)`,
height: 480,
}}
>
<TerminalWindow title={label} glow>
<div
style={{
color: accent,
fontSize: 18,
fontFamily: FONT_MONO,
marginBottom: 14,
}}
>
{tag}
</div>
<div
style={{
fontSize: 22,
color: COLORS.accentGreen,
fontFamily: FONT_MONO,
marginBottom: 12,
}}
>
[Competitors] running...
</div>
{PROGRESS_LINES.map((line, idx) => {
const lineStart = panelStart + 16 + idx * 6;
const lineFade = interpolate(
frame - lineStart,
[0, 8],
[0, 1],
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
);
// pulse the in-progress dot
const dotOn = Math.floor((frame - lineStart) / 6) % 2 === 0;
return (
<div
key={line.source}
style={{
opacity: lineFade,
fontFamily: FONT_MONO,
fontSize: 20,
color: COLORS.fgMuted,
display: "flex",
alignItems: "center",
marginBottom: 6,
}}
>
<span
style={{
display: "inline-block",
width: 10,
height: 10,
borderRadius: 10,
background: dotOn ? line.color : COLORS.bgPanelSoft,
marginRight: 12,
boxShadow: dotOn ? `0 0 10px ${line.color}` : "none",
}}
/>
<span style={{ color: line.color, marginRight: 8 }}>
</span>
<span>{line.source}</span>
<span style={{ marginLeft: "auto", color: COLORS.fgDim }}>
{dotOn ? "..." : "·"}
</span>
</div>
);
})}
</TerminalWindow>
</div>
);
};
export const Scene3FanOut: React.FC = () => {
const frame = useCurrentFrame();
// Phase 1 (0-30 frames): single terminal types --competitors flag
// Phase 2 (30+): split into 3 panes
const splitProgress = interpolate(frame, [30, 50], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const singleOpacity = interpolate(frame, [0, 8, 30, 45], [0, 1, 1, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
// Caption fades in shortly after panes settle so it has time to read.
const captionFade = interpolate(frame, [60, 80], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<AbsoluteFill
style={{
background: COLORS.bgDeep,
padding: 60,
display: "flex",
flexDirection: "column",
}}
>
<div
style={{
opacity: captionFade,
textAlign: "center",
fontFamily: FONT_SANS,
fontSize: 42,
color: COLORS.fgPrimary,
marginBottom: 32,
}}
>
Now it discovers competitors
<br />
<span style={{ color: COLORS.accentCyan }}>and runs all 3.</span>
</div>
<div
style={{
flex: 1,
position: "relative",
}}
>
{/* Single terminal during phase 1, fades out as panes appear */}
<div
style={{
position: "absolute",
inset: 0,
display: "flex",
alignItems: "center",
justifyContent: "center",
opacity: singleOpacity,
}}
>
<div style={{ width: 1200, height: 380 }}>
<TerminalWindow title="bash">
<TypedLine
text="/last30days OpenAI --competitors"
startFrame={0}
prefix="$"
fontSize={42}
/>
</TerminalWindow>
</div>
</div>
{/* Three panes fade in starting frame ~30 */}
<div
style={{
position: "absolute",
inset: 0,
display: "flex",
alignItems: "center",
justifyContent: "center",
gap: 24,
opacity: splitProgress,
}}
>
{ENTITIES.map((entity, idx) => (
<FanPane
key={entity.label}
label={entity.label}
tag={entity.tag}
accent={entity.accent}
panelStart={45 + idx * 8}
/>
))}
</div>
</div>
</AbsoluteFill>
);
};
@@ -0,0 +1,105 @@
import React from "react";
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
import { ComparisonTable } from "../components/ComparisonTable";
import { COLORS, FONT_SANS } from "../lib/colors";
const ROWS = [
{
dimension: "What it is",
cells: [
"GPT-5 leader, Plus + API",
"Claude 4, safety-first",
"Grok, X-native, fast",
] as [string, string, string],
},
{
dimension: "30-day momentum",
cells: [
"GPT-5 launch wave",
"Claude 4.7 1M context",
"Grok 5 reveal",
] as [string, string, string],
},
{
dimension: "Community vibe",
cells: [
"Defensive but deep",
"Quiet, devs-only",
"Loud, meme-rich",
] as [string, string, string],
},
{
dimension: "Best for",
cells: [
"Mainstream + tools",
"Long-context coding",
"Live X intel",
] as [string, string, string],
},
];
export const Scene4Comparison: React.FC = () => {
const frame = useCurrentFrame();
const captionFade = interpolate(frame, [0, 12, 180, 210], [0, 1, 1, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const captionLift = interpolate(frame, [0, 14], [16, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const tableFade = interpolate(frame, [16, 32], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<AbsoluteFill
style={{
background: COLORS.bgDeep,
padding: "48px 80px",
display: "flex",
flexDirection: "column",
alignItems: "center",
}}
>
<div
style={{
opacity: captionFade,
transform: `translateY(${captionLift}px)`,
fontFamily: FONT_SANS,
fontSize: 38,
color: COLORS.fgMuted,
textAlign: "center",
marginBottom: 36,
}}
>
<span style={{ color: COLORS.accentCyan, fontWeight: 600 }}>
3 full passes.
</span>
<span style={{ marginLeft: 18, color: COLORS.accentMagenta, fontWeight: 600 }}>
3 save files.
</span>
<span style={{ marginLeft: 18, color: COLORS.fgPrimary, fontWeight: 600 }}>
1 comparison.
</span>
</div>
<div
style={{
width: "100%",
maxWidth: 1640,
opacity: tableFade,
}}
>
<ComparisonTable
startFrame={20}
entities={["OpenAI", "Anthropic", "xAI"]}
rows={ROWS}
rowStaggerFrames={22}
/>
</div>
</AbsoluteFill>
);
};
@@ -0,0 +1,54 @@
import React from "react";
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
import { COLORS, FONT_SANS } from "../lib/colors";
const LINES = [
{ text: "You pick the topic.", color: COLORS.fgPrimary },
{ text: "The agent picks the peers.", color: COLORS.accentCyan },
{ text: "The engine fans out.", color: COLORS.accentMagenta },
];
export const Scene5HowItWorks: React.FC = () => {
const frame = useCurrentFrame();
return (
<AbsoluteFill
style={{
background: `radial-gradient(circle at 50% 60%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
display: "flex",
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
gap: 48,
}}
>
{LINES.map((line, idx) => {
const start = 10 + idx * 28;
const fade = interpolate(frame, [start, start + 14], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const slide = interpolate(frame, [start, start + 18], [24, 0], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
return (
<div
key={line.text}
style={{
opacity: fade,
transform: `translateY(${slide}px)`,
fontFamily: FONT_SANS,
fontSize: 78,
fontWeight: 600,
color: line.color,
letterSpacing: -1,
}}
>
{line.text}
</div>
);
})}
</AbsoluteFill>
);
};
@@ -0,0 +1,73 @@
import React from "react";
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
import { BadgeBar } from "../components/BadgeBar";
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
export const Scene6CTA: React.FC = () => {
const frame = useCurrentFrame();
const { fps } = useVideoConfig();
const installFade = interpolate(frame, [20, 40], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
// Stronger 1Hz pulse on the install line for end-of-video emphasis
const pulse = 0.8 + 0.2 * Math.sin((frame / fps) * 2 * Math.PI);
const glowPulse = 0.4 + 0.4 * Math.sin((frame / fps) * 2 * Math.PI);
const repoFade = interpolate(frame, [50, 70], [0, 1], {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
});
const enterScale = spring({
frame,
fps,
config: { damping: 18, stiffness: 90 },
from: 0.95,
to: 1,
});
return (
<AbsoluteFill
style={{
background: `radial-gradient(circle at 50% 50%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
display: "flex",
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
gap: 56,
transform: `scale(${enterScale})`,
}}
>
<BadgeBar />
<div
style={{
opacity: installFade * pulse,
padding: "18px 36px",
background: COLORS.bgPanel,
border: `1px solid ${COLORS.accentCyan}`,
borderRadius: 14,
boxShadow: `0 0 ${40 + glowPulse * 60}px ${COLORS.accentCyan}${Math.round(40 + glowPulse * 80).toString(16)}`,
fontFamily: FONT_MONO,
fontSize: 44,
color: COLORS.fgPrimary,
}}
>
<span style={{ color: COLORS.accentGreen, marginRight: 18 }}>$</span>
/last30days <span style={{ color: COLORS.fgMuted }}>{"{topic}"}</span>{" "}
<span style={{ color: COLORS.accentCyan }}>--competitors</span>
</div>
<div
style={{
opacity: repoFade,
fontFamily: FONT_SANS,
fontSize: 28,
color: COLORS.fgMuted,
}}
>
github.com/mvanhorn/last30days-skill
</div>
</AbsoluteFill>
);
};
+19
View File
@@ -0,0 +1,19 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"jsx": "react-jsx",
"strict": true,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"types": ["node"]
},
"include": ["src/**/*"]
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 MiB

+8 -5
View File
@@ -1,10 +1,12 @@
[project] [project]
name = "last30days-skill" name = "last30days-skill"
version = "3.2.3" version = "3.0.0"
description = "Multi-source last-30-days research skill" description = "Multi-source last-30-days research skill"
readme = "README.md" readme = "README.md"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [] dependencies = [
"requests>=2.32,<3",
]
[dependency-groups] [dependency-groups]
dev = [ dev = [
@@ -22,9 +24,9 @@ addopts = [
[tool.coverage.run] [tool.coverage.run]
branch = true branch = true
source = ["skills/last30days/scripts", "tests"] source = ["scripts", "tests"]
omit = [ omit = [
"skills/last30days/scripts/lib/vendor/*", "scripts/lib/vendor/*",
"dist/*", "dist/*",
] ]
@@ -32,6 +34,7 @@ omit = [
skip_empty = true skip_empty = true
show_missing = true show_missing = true
omit = [ omit = [
"skills/last30days/scripts/lib/vendor/*", "scripts/lib/vendor/*",
"dist/*", "dist/*",
] ]
+1 -1
View File
@@ -58,7 +58,7 @@ OpenClaw:
clawhub install last30days-official clawhub install last30days-official
``` ```
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`. OpenAI Codex CLI: run `codex` from a checkout of this repo and v3's skill at `.agents/skills/last30days/SKILL.md` will be discovered automatically. Or copy `SKILL.md` to `~/.agents/skills/last30days/SKILL.md` for a global install.
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds. Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
@@ -1,14 +1,13 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file # build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root) # Usage: bash scripts/build-skill.sh (run from repo root)
# #
# Produces dist/last30days.skill, a zip with a single top-level `last30days/` # Produces dist/last30days.skill, a zip with a single top-level `last30days/`
# directory containing SKILL.md and the scripts/ runtime from skills/last30days. # directory containing SKILL.md and the scripts/ runtime. See
# See
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md. # docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
set -euo pipefail set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$REPO_ROOT" cd "$REPO_ROOT"
if ! git diff --quiet || ! git diff --cached --quiet; then if ! git diff --quiet || ! git diff --cached --quiet; then
@@ -18,7 +17,14 @@ fi
mkdir -p dist mkdir -p dist
OUT="dist/last30days.skill" OUT="dist/last30days.skill"
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD
# claude.ai's .skill bundle only needs the root SKILL.md + scripts/ runtime.
# Claude Code needs skills/ and .claude-plugin/ in the git archive
# (that's why they're NOT in .gitattributes export-ignore), but the .skill
# bundle must strip them to keep a single canonical SKILL.md and stay under
# the 200-file cap.
zip -d "$OUT" "last30days/skills/*" "last30days/.claude-plugin/*" > /dev/null 2>&1 || true
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}') COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
SIZE=$(du -h "$OUT" | cut -f1) SIZE=$(du -h "$OUT" | cut -f1)
@@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
# A/B test runner: public release vs private beta # A/B test runner: public release vs private beta
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West" # Usage: bash scripts/compare.sh "Kanye West"
# #
# Runs /last30days (public release) and /last30days-beta (private beta) # Runs /last30days (public release) and /last30days-beta (private beta)
# sequentially with a 30s gap, saves raw results with distinct suffixes, # sequentially with a 30s gap, saves raw results with distinct suffixes,
@@ -9,8 +9,8 @@
set -e set -e
if [ $# -eq 0 ]; then if [ $# -eq 0 ]; then
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>" echo "Usage: bash scripts/compare.sh <topic>"
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose" echo " Example: bash scripts/compare.sh Kevin Rose"
exit 1 exit 1
fi fi
TOPIC="$*" TOPIC="$*"
@@ -22,8 +22,7 @@ from lib import env as envlib
from lib import schema from lib import schema
SKILL_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parent.parent
REPO_ROOT = Path(__file__).resolve().parents[3]
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json" EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
@@ -308,10 +307,7 @@ def create_eval_env() -> dict[str, str]:
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]: def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py" cmd = [sys.executable, "scripts/last30days.py", topic, "--emit=json"]
if not engine.exists():
engine = repo_dir / "scripts" / "last30days.py"
cmd = [sys.executable, str(engine), topic, "--emit=json"]
if search: if search:
cmd.extend(["--search", search]) cmd.extend(["--search", search])
if quick: if quick:
@@ -41,7 +41,7 @@ if os.name == "nt":
SCRIPT_DIR = Path(__file__).parent.resolve() SCRIPT_DIR = Path(__file__).parent.resolve()
sys.path.insert(0, str(SCRIPT_DIR)) sys.path.insert(0, str(SCRIPT_DIR))
from lib import env, html_render, pipeline, render, schema, ui from lib import env, pipeline, render, schema, ui
_child_pids: set[int] = set() _child_pids: set[int] = set()
_child_pids_lock = threading.Lock() _child_pids_lock = threading.Lock()
@@ -91,46 +91,30 @@ def slugify(value: str) -> str:
return slug or "last30days" return slug or "last30days"
def save_output( def save_output(report: schema.Report, emit: str, save_dir: str, suffix: str = "") -> Path:
report: schema.Report,
emit: str,
save_dir: str,
suffix: str = "",
synthesis_md: str | None = None,
) -> Path:
from datetime import datetime from datetime import datetime
path = Path(save_dir).expanduser().resolve() path = Path(save_dir).expanduser().resolve()
path.mkdir(parents=True, exist_ok=True) path.mkdir(parents=True, exist_ok=True)
slug = slugify(report.topic) slug = slugify(report.topic)
extension = "json" if emit == "json" else "html" if emit == "html" else "md" extension = "json" if emit == "json" else "md"
raw_label = "raw-html" if emit == "html" else "raw"
suffix_part = f"-{suffix}" if suffix else "" suffix_part = f"-{suffix}" if suffix else ""
out_path = path / f"{slug}-{raw_label}{suffix_part}.{extension}" out_path = path / f"{slug}-raw{suffix_part}.{extension}"
if out_path.exists(): if out_path.exists():
out_path = path / f"{slug}-{raw_label}{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}" out_path = path / f"{slug}-raw{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
# Markdown saves keep the complete debug artifact. JSON and HTML preserve # Always save the FULL dump to disk (all items, all sources, transcripts).
# their requested wire format so file extensions match their content. # Claude sees compact clusters via --emit=compact on stdout.
if emit in {"json", "html"}: # The saved file is the complete debug artifact.
content = emit_output(report, emit, synthesis_md=synthesis_md) if emit == "json":
content = emit_output(report, emit)
else: else:
content = render.render_full(report) content = render.render_full(report)
out_path.write_text(content, encoding="utf-8") out_path.write_text(content, encoding="utf-8")
return out_path return out_path
def emit_output( def emit_output(report: schema.Report, emit: str, fun_level: str = "medium", save_path: str | None = None) -> str:
report: schema.Report,
emit: str,
fun_level: str = "medium",
save_path: str | None = None,
synthesis_md: str | None = None,
) -> str:
if emit == "json": if emit == "json":
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True) return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
if emit == "html":
return html_render.render_html(
report, fun_level=fun_level, save_path=save_path, synthesis_md=synthesis_md,
)
if emit in {"compact", "md"}: if emit in {"compact", "md"}:
return render.render_compact(report, fun_level=fun_level, save_path=save_path) return render.render_compact(report, fun_level=fun_level, save_path=save_path)
if emit == "context": if emit == "context":
@@ -143,7 +127,6 @@ def emit_comparison_output(
emit: str, emit: str,
fun_level: str = "medium", fun_level: str = "medium",
save_path: str | None = None, save_path: str | None = None,
synthesis_md: str | None = None,
) -> str: ) -> str:
if emit == "json": if emit == "json":
payload = { payload = {
@@ -155,13 +138,6 @@ def emit_comparison_output(
], ],
} }
return json.dumps(payload, indent=2, sort_keys=True) return json.dumps(payload, indent=2, sort_keys=True)
if emit == "html":
return html_render.render_html_comparison(
entity_reports,
fun_level=fun_level,
save_path=save_path,
synthesis_md=synthesis_md,
)
if emit in {"compact", "md"}: if emit in {"compact", "md"}:
return render.render_comparison_multi( return render.render_comparison_multi(
entity_reports, fun_level=fun_level, save_path=save_path, entity_reports, fun_level=fun_level, save_path=save_path,
@@ -180,10 +156,9 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
from pathlib import Path as _Path from pathlib import Path as _Path
path = _Path(save_dir).expanduser().resolve() path = _Path(save_dir).expanduser().resolve()
slug = slugify(topic) slug = slugify(topic)
extension = "json" if emit == "json" else "html" if emit == "html" else "md" extension = "json" if emit == "json" else "md"
raw_label = "raw-html" if emit == "html" else "raw"
suffix_part = f"-{suffix}" if suffix else "" suffix_part = f"-{suffix}" if suffix else ""
raw = path / f"{slug}-{raw_label}{suffix_part}.{extension}" raw = path / f"{slug}-raw{suffix_part}.{extension}"
try: try:
home = _Path.home().resolve() home = _Path.home().resolve()
relative = raw.relative_to(home) relative = raw.relative_to(home)
@@ -192,14 +167,6 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
return str(raw) return str(raw)
def read_synthesis_file(path: str) -> str:
try:
return Path(path).expanduser().read_text(encoding="utf-8")
except OSError as exc:
sys.stderr.write(f"[last30days] Cannot read --synthesis-file: {exc}\n")
raise SystemExit(2)
def persist_report(report: schema.Report) -> dict[str, int]: def persist_report(report: schema.Report) -> dict[str, int]:
import store import store
@@ -226,7 +193,7 @@ def persist_report(report: schema.Report) -> dict[str, int]:
def build_parser() -> argparse.ArgumentParser: def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.") parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
parser.add_argument("topic", nargs="*", help="Research topic") parser.add_argument("topic", nargs="*", help="Research topic")
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md", "html"]) parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md"])
parser.add_argument("--search", help="Comma-separated source list") parser.add_argument("--search", help="Comma-separated source list")
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile") parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile") parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
@@ -234,7 +201,6 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures") parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability") parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output") parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
parser.add_argument("--synthesis-file", help="Markdown synthesis to embed in --emit=html output")
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store") parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
parser.add_argument("--x-handle", help="X handle for targeted supplemental search") parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)") parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
@@ -571,13 +537,6 @@ def main() -> int:
parser.print_usage(sys.stderr) parser.print_usage(sys.stderr)
return 2 return 2
synthesis_md = None
if args.synthesis_file:
if args.emit == "html":
synthesis_md = read_synthesis_file(args.synthesis_file)
else:
sys.stderr.write("[last30days] Warning: --synthesis-file is only used with --emit=html; ignoring.\n")
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"): if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
from lib import preflight from lib import preflight
refuse_msg = preflight.check_class_1_trap(topic) refuse_msg = preflight.check_class_1_trap(topic)
@@ -895,29 +854,15 @@ def main() -> int:
if entity_reports: if entity_reports:
rendered = emit_comparison_output( rendered = emit_comparison_output(
entity_reports, entity_reports, args.emit, fun_level=fun_level, save_path=footer_save_path,
args.emit,
fun_level=fun_level,
save_path=footer_save_path,
synthesis_md=synthesis_md,
) )
else: else:
rendered = emit_output( rendered = emit_output(
report, report, args.emit, fun_level=fun_level, save_path=footer_save_path,
args.emit,
fun_level=fun_level,
save_path=footer_save_path,
synthesis_md=synthesis_md,
) )
if args.save_dir: if args.save_dir:
# Save the main topic's raw file (single-entity or comparison main). # Save the main topic's raw file (single-entity or comparison main).
save_path = save_output( save_path = save_output(report, args.emit, args.save_dir, suffix=args.save_suffix or "")
report,
args.emit,
args.save_dir,
suffix=args.save_suffix or "",
synthesis_md=synthesis_md,
)
sys.stderr.write(f"[last30days] Saved output to {save_path}\n") sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
# Competitor / vs-mode: also save a per-entity raw file for each peer. # Competitor / vs-mode: also save a per-entity raw file for each peer.
# Matches historical vs-mode behavior (N passes → N save files). # Matches historical vs-mode behavior (N passes → N save files).
@@ -926,7 +871,6 @@ def main() -> int:
peer_path = save_output( peer_path = save_output(
entity_report, args.emit, args.save_dir, entity_report, args.emit, args.save_dir,
suffix=args.save_suffix or "", suffix=args.save_suffix or "",
synthesis_md=synthesis_md,
) )
sys.stderr.write(f"[last30days] Saved output to {peer_path}\n") sys.stderr.write(f"[last30days] Saved output to {peer_path}\n")
sys.stderr.flush() sys.stderr.flush()
@@ -7,11 +7,13 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
import json import json
import os import os
import signal
import shutil import shutil
import subprocess
import sys import sys
from pathlib import Path from pathlib import Path
from . import http, log, subproc from . import http, log
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
@@ -166,51 +168,62 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
"--json", "--json",
] ]
pid_holder: list[int] = [] # Use process groups for clean cleanup on timeout/kill
preexec = os.setsid if hasattr(os, 'setsid') else None
def _register(pid: int) -> None: try:
pid_holder.append(pid) proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
preexec_fn=preexec,
env=_subprocess_env(),
)
# Register for cleanup tracking (if available)
try: try:
from last30days import register_child_pid from last30days import register_child_pid, unregister_child_pid
register_child_pid(pid) register_child_pid(proc.pid)
except ImportError: except ImportError:
pass pass
try: try:
result = subproc.run_with_timeout( stdout, stderr = proc.communicate(timeout=timeout)
cmd, except subprocess.TimeoutExpired:
timeout=timeout, # Kill the entire process group
env=_subprocess_env(), try:
on_pid=_register, os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
) except (ProcessLookupError, PermissionError, OSError):
except subproc.SubprocTimeout: proc.kill()
return {"error": f"Search timed out after {timeout}s", "items": []} proc.wait(timeout=5)
except Exception as e: return {"error": f"Search timed out after {timeout}s", "items": []}
return {"error": str(e), "items": []} finally:
finally:
if pid_holder:
try: try:
from last30days import unregister_child_pid from last30days import unregister_child_pid
unregister_child_pid(pid_holder[0]) unregister_child_pid(proc.pid)
except Exception: except Exception:
pass pass
if result.returncode != 0: if proc.returncode != 0:
error = result.stderr.strip() or "Bird search failed" error = stderr.strip() if stderr else "Bird search failed"
return {"error": error, "items": []} return {"error": error, "items": []}
output = result.stdout.strip() output = stdout.strip() if stdout else ""
if not output: if not output:
return {"items": []} return {"items": []}
try:
parsed = json.loads(output) parsed = json.loads(output)
if isinstance(parsed, list):
return {"items": parsed}
return parsed
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
return {"error": f"Invalid JSON response: {e}", "items": []} return {"error": f"Invalid JSON response: {e}", "items": []}
except Exception as e:
if isinstance(parsed, list): return {"error": str(e), "items": []}
return {"items": parsed}
return parsed
def search_x( def search_x(
@@ -317,29 +330,47 @@ def search_handles(
"--json", "--json",
] ]
try: preexec = os.setsid if hasattr(os, 'setsid') else None
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
except subproc.SubprocTimeout:
_log(f"Handle search timed out for @{handle}")
return []
except OSError as e:
_log(f"Handle search error for @{handle}: {e}")
return []
if result.returncode != 0:
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
return []
output = result.stdout.strip()
if not output:
return []
try: try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
preexec_fn=preexec,
env=_subprocess_env(),
)
try:
stdout, stderr = proc.communicate(timeout=15)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except (ProcessLookupError, PermissionError, OSError):
proc.kill()
proc.wait(timeout=5)
_log(f"Handle search timed out for @{handle}")
return []
if proc.returncode != 0:
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
return []
output = (stdout or "").strip()
if not output:
return []
response = json.loads(output) response = json.loads(output)
return parse_bird_response(response, query=core_topic)
except json.JSONDecodeError: except json.JSONDecodeError:
_log(f"Invalid JSON from handle search for @{handle}") _log(f"Invalid JSON from handle search for @{handle}")
return [] except (OSError, subprocess.SubprocessError) as e:
return parse_bird_response(response, query=core_topic) _log(f"Handle search error for @{handle}: {e}")
return []
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -39,14 +39,11 @@ def normalize_text(text: str) -> str:
return re.sub(r"\s+", " ", text).strip() return re.sub(r"\s+", " ", text).strip()
def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
if len(norm) < n:
return {norm} if norm else set()
return {norm[index:index + n] for index in range(len(norm) - n + 1)}
def get_ngrams(text: str, n: int = 3) -> set[str]: def get_ngrams(text: str, n: int = 3) -> set[str]:
return _ngrams_of_normalized(normalize_text(text), n) text = normalize_text(text)
if len(text) < n:
return {text} if text else set()
return {text[index:index + n] for index in range(len(text) - n + 1)}
def jaccard_similarity(left: set[str], right: set[str]) -> float: def jaccard_similarity(left: set[str], right: set[str]) -> float:
@@ -93,7 +90,7 @@ class _PreparedText:
def __init__(self, raw: str) -> None: def __init__(self, raw: str) -> None:
norm = normalize_text(raw) norm = normalize_text(raw)
self.ngrams = _ngrams_of_normalized(norm) self.ngrams = get_ngrams(norm) if norm else set()
self.tokens = _tokenize(norm) self.tokens = _tokenize(norm)
@@ -372,6 +372,14 @@ def config_exists() -> bool:
return False return False
def is_reddit_available(config: dict[str, Any]) -> bool:
"""Check if Reddit search is available.
v3 uses ScrapeCreators only.
"""
return bool(config.get('SCRAPECREATORS_API_KEY'))
def get_reddit_source(config: dict[str, Any]) -> str | None: def get_reddit_source(config: dict[str, Any]) -> str | None:
"""Determine which Reddit backend to use. """Determine which Reddit backend to use.
@@ -116,8 +116,6 @@ def weighted_rrf(
"""Fuse ranked lists into a single candidate pool.""" """Fuse ranked lists into a single candidate pool."""
subqueries = {subquery.label: subquery for subquery in plan.subqueries} subqueries = {subquery.label: subquery for subquery in plan.subqueries}
candidates: dict[str, schema.Candidate] = {} candidates: dict[str, schema.Candidate] = {}
# Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
seen_source_items: dict[str, set[tuple[str, str]]] = {}
for (label, source), items in streams.items(): for (label, source), items in streams.items():
subquery = subqueries[label] subquery = subqueries[label]
@@ -156,7 +154,6 @@ def weighted_rrf(
] ]
}, },
) )
seen_source_items[key] = {(item.source, item.item_id)}
continue continue
candidate = candidates[key] candidate = candidates[key]
@@ -182,9 +179,7 @@ def weighted_rrf(
candidate.subquery_labels.append(label) candidate.subquery_labels.append(label)
if item.source not in candidate.sources: if item.source not in candidate.sources:
candidate.sources.append(item.source) candidate.sources.append(item.source)
source_item_key = (item.source, item.item_id) if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
if source_item_key not in seen_source_items[key]:
seen_source_items[key].add(source_item_key)
candidate.source_items.append(item) candidate.source_items.append(item)
candidate.metadata.setdefault("provenance", []).append( candidate.metadata.setdefault("provenance", []).append(
{ {
@@ -12,6 +12,11 @@ import sys
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List, Optional, Set from typing import Any, Dict, List, Optional, Set
try:
import requests as _requests
except ImportError:
_requests = None
from . import dates, http, log from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com" SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
@@ -231,17 +236,30 @@ def _user_reels(
""" """
_log(f"User reels: @{handle}") _log(f"User reels: @{handle}")
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels" reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
try: if not _requests:
data = http.get( try:
reels_url, from urllib.parse import urlencode
params={"handle": handle}, params = urlencode({"handle": handle})
headers=http.scrapecreators_headers(token), url = f"{reels_url}?{params}"
timeout=30, headers = http.scrapecreators_headers(token)
retries=2, headers["User-Agent"] = http.USER_AGENT
) data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e: except Exception as e:
_log(f"User reels error for @{handle}: {e}") _log(f"User reels error (urllib) for @{handle}: {e}")
return [] return []
else:
try:
resp = _requests.get(
reels_url,
params={"handle": handle},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"User reels error for @{handle}: {e}")
return []
raw_items = data.get("items") or data.get("reels") or data.get("data") or [] raw_items = data.get("items") or data.get("reels") or data.get("data") or []
_log(f" -> {len(raw_items)} reels from @{handle}") _log(f" -> {len(raw_items)} reels from @{handle}")
@@ -275,17 +293,31 @@ def search_instagram(
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})") _log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
try: if not _requests:
data = http.get( _log("requests library not installed, falling back to urllib")
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search", try:
params={"query": core_topic}, from urllib.parse import urlencode
headers=http.scrapecreators_headers(token), params = urlencode({"query": core_topic})
timeout=30, url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
retries=2, headers = http.scrapecreators_headers(token)
) headers["User-Agent"] = http.USER_AGENT
except Exception as e: data = http.get(url, headers=headers, timeout=30, retries=2)
_log(f"ScrapeCreators error: {e}") except Exception as e:
return {"items": [], "error": f"{type(e).__name__}: {e}"} _log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
params={"query": core_topic},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Items are in the 'reels' array (ScrapeCreators v2 response) # Items are in the 'reels' array (ScrapeCreators v2 response)
raw_items = data.get("reels") or data.get("items") or data.get("data") or [] raw_items = data.get("reels") or data.get("items") or data.get("data") or []
@@ -335,7 +367,7 @@ def fetch_captions(
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
max_captions = config["max_captions"] max_captions = config["max_captions"]
if not video_items or not token: if not video_items or not token or not _requests:
return {} return {}
top_items = video_items[:max_captions] top_items = video_items[:max_captions]
@@ -360,24 +392,26 @@ def fetch_captions(
if not url: if not url:
continue continue
try: try:
data = http.get( resp = _requests.get(
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript", f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
params={"url": url}, params={"url": url},
headers=http.scrapecreators_headers(token), headers=http.scrapecreators_headers(token),
timeout=15, timeout=15,
retries=1,
) )
transcripts = data.get("transcripts") or [] if resp.status_code == 200:
if transcripts and isinstance(transcripts, list): data = resp.json()
transcript_text = " ".join( transcripts = data.get("transcripts") or []
t.get("text", "") for t in transcripts if transcripts and isinstance(transcripts, list):
if isinstance(t, dict) and t.get("text") # Combine all transcript segments
) transcript_text = " ".join(
if transcript_text: t.get("text", "") for t in transcripts
words = transcript_text.split() if isinstance(t, dict) and t.get("text")
if len(words) > CAPTION_MAX_WORDS: )
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...' if transcript_text:
captions[vid] = transcript_text words = transcript_text.split()
if len(words) > CAPTION_MAX_WORDS:
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
captions[vid] = transcript_text
except Exception as e: except Exception as e:
_log(f"Transcript fetch failed for {vid}: {e}") _log(f"Transcript fetch failed for {vid}: {e}")
@@ -49,7 +49,6 @@ def normalize_source_items(
"xquik": _normalize_x, "xquik": _normalize_x,
"pinterest": _normalize_pinterest, "pinterest": _normalize_pinterest,
"polymarket": _normalize_polymarket, "polymarket": _normalize_polymarket,
"digg": _normalize_digg,
"grounding": _normalize_grounding, "grounding": _normalize_grounding,
"xiaohongshu": _normalize_grounding, "xiaohongshu": _normalize_grounding,
"github": _normalize_github, "github": _normalize_github,
@@ -111,19 +110,6 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
return default return default
def _join_comment_excerpts(
top_comments: list[Any],
key: str,
limit: int = 3,
) -> str:
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
return " ".join(
str(comment.get(key) or "").strip()
for comment in top_comments[:limit]
if isinstance(comment, dict)
)
def _domain_from_url(url: str) -> str | None: def _domain_from_url(url: str) -> str | None:
if not url: if not url:
return None return None
@@ -183,7 +169,11 @@ def _normalize_reddit(
to_date: str, to_date: str,
) -> schema.SourceItem: ) -> schema.SourceItem:
top_comments = item.get("top_comments") or [] top_comments = item.get("top_comments") or []
comment_text = _join_comment_excerpts(top_comments, "excerpt") comment_text = " ".join(
str(comment.get("excerpt") or "").strip()
for comment in top_comments[:3]
if isinstance(comment, dict)
)
body = "\n".join( body = "\n".join(
part part
for part in [ for part in [
@@ -348,7 +338,11 @@ def _normalize_hackernews(
to_date: str, to_date: str,
) -> schema.SourceItem: ) -> schema.SourceItem:
top_comments = item.get("top_comments") or [] top_comments = item.get("top_comments") or []
comment_text = _join_comment_excerpts(top_comments, "text") comment_text = " ".join(
str(comment.get("text") or "").strip()
for comment in top_comments[:3]
if isinstance(comment, dict)
)
title = str(item.get("title") or "").strip() title = str(item.get("title") or "").strip()
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part) body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
return _source_item( return _source_item(
@@ -400,53 +394,6 @@ def _normalize_microblog(
) )
def _normalize_digg(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> schema.SourceItem:
"""Normalizer for Digg AI 1000 clusters.
Each cluster is one item. The TLDR carries the most useful body for
rerank and synthesis. Top-ranked X posts attached at search time are
passed through under metadata['posts'] so render can emit them as
inline 'via Digg' quotes.
"""
title = str(item.get("title") or "").strip()
tldr = str(item.get("tldr") or "").strip()
body = "\n\n".join(part for part in [title, tldr] if part)
posts = item.get("posts") or []
if not isinstance(posts, list):
posts = []
cluster_url_id = str(item.get("id") or f"DG{index + 1}")
return _source_item(
item_id=cluster_url_id,
source=source,
title=title or f"Digg cluster {index + 1}",
body=body,
url=str(item.get("url") or f"https://di.gg/ai/{cluster_url_id}"),
author="",
container="Digg",
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=tldr[:400],
metadata={
"clusterUrlId": cluster_url_id,
"tldr": tldr,
"rank": (item.get("engagement") or {}).get("rank"),
"uniqueAuthors": (item.get("engagement") or {}).get("uniqueAuthors"),
"postCount": (item.get("engagement") or {}).get("postCount"),
"firstPostAge": item.get("first_post_age"),
"posts": posts,
},
)
def _normalize_polymarket( def _normalize_polymarket(
source: str, source: str,
item: dict[str, Any], item: dict[str, Any],
@@ -494,7 +441,11 @@ def _normalize_github(
title = str(item.get("title") or "").strip() title = str(item.get("title") or "").strip()
snippet_text = str(item.get("snippet") or "").strip() snippet_text = str(item.get("snippet") or "").strip()
top_comments = item.get("metadata", {}).get("top_comments") or [] top_comments = item.get("metadata", {}).get("top_comments") or []
comment_text = _join_comment_excerpts(top_comments, "excerpt") comment_text = " ".join(
str(comment.get("excerpt") or "").strip()
for comment in top_comments[:3]
if isinstance(comment, dict)
)
body = "\n".join(part for part in [title, snippet_text, comment_text] if part) body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
metadata = item.get("metadata") or {} metadata = item.get("metadata") or {}
return _source_item( return _source_item(
@@ -11,6 +11,11 @@ import re
import sys import sys
from typing import Any, Dict, List, Optional, Set from typing import Any, Dict, List, Optional, Set
try:
import requests as _requests
except ImportError:
_requests = None
from . import dates, http, log from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest" SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
@@ -135,17 +140,31 @@ def search_pinterest(
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})") _log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
try: if not _requests:
data = http.get( _log("requests library not installed, falling back to urllib")
f"{SCRAPECREATORS_BASE}/search", try:
params={"keyword": core_topic}, from urllib.parse import urlencode
headers=http.scrapecreators_headers(token), params = urlencode({"keyword": core_topic})
timeout=30, url = f"{SCRAPECREATORS_BASE}/search?{params}"
retries=2, headers = http.scrapecreators_headers(token)
) headers["User-Agent"] = http.USER_AGENT
except Exception as e: data = http.get(url, headers=headers, timeout=30, retries=2)
_log(f"ScrapeCreators error: {e}") except Exception as e:
return {"items": [], "error": f"{type(e).__name__}: {e}"} _log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search",
params={"keyword": core_topic},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Extract items from response - try common SC response shapes # Extract items from response - try common SC response shapes
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or [] raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
@@ -15,7 +15,6 @@ from . import (
bluesky, bluesky,
dates, dates,
dedupe, dedupe,
digg,
entity_extract, entity_extract,
env, env,
github, github,
@@ -31,7 +30,6 @@ from . import (
query, query,
reddit, reddit,
reddit_public, reddit_public,
relevance,
rerank, rerank,
schema, schema,
signals, signals,
@@ -80,7 +78,6 @@ MOCK_AVAILABLE_SOURCES = [
"github", "github",
"perplexity", "perplexity",
"xquik", "xquik",
"digg",
] ]
@@ -108,8 +105,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
available.extend(["hackernews", "polymarket"]) available.extend(["hackernews", "polymarket"])
if config.get("GITHUB_TOKEN") or which("gh"): if config.get("GITHUB_TOKEN") or which("gh"):
available.append("github") available.append("github")
if which("digg-pp-cli"):
available.append("digg")
if env.is_bluesky_available(config): if env.is_bluesky_available(config):
available.append("bluesky") available.append("bluesky")
if env.is_truthsocial_available(config): if env.is_truthsocial_available(config):
@@ -505,12 +500,11 @@ def _normalize_score_dedupe(
source, raw_items, from_date, to_date, source, raw_items, from_date, to_date,
freshness_mode=freshness_mode, freshness_mode=freshness_mode,
) )
prepared_query = relevance.PreparedQuery(ranking_query) normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
normalized = signals.annotate_stream(normalized, prepared_query, freshness_mode)
normalized = signals.prune_low_relevance(normalized) normalized = signals.prune_low_relevance(normalized)
normalized = dedupe.dedupe_items(normalized) normalized = dedupe.dedupe_items(normalized)
for item in normalized: for item in normalized:
item.snippet = snippet.extract_best_snippet(item, prepared_query) item.snippet = snippet.extract_best_snippet(item, ranking_query)
return normalized return normalized
@@ -535,12 +529,6 @@ def _finalize_items_by_source(
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
if keywords: if keywords:
items = polymarket.filter_items_against_keywords(items, keywords) items = polymarket.filter_items_against_keywords(items, keywords)
if source == "digg" and items:
# Pull top-ranked X posts only for the survivors that will appear
# in the brief. Spending the enrichment budget here (rather than
# at retrieval time) keeps the inline 'via Digg' quotes
# paired with the clusters dedupe actually kept.
digg.enrich_source_items(items, top_k=3)
finalized[source] = items finalized[source] = items
return finalized return finalized
@@ -976,13 +964,6 @@ def _retrieve_stream(
if source == "hackernews": if source == "hackernews":
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth) result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {} return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
if source == "digg":
result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
items = digg.parse_digg_response(result, query=subquery.search_query)
# Enrichment with attached X posts is deferred to
# _finalize_items_by_source so it runs on the items that actually
# survive dedupe rather than on top-K of the raw fanout.
return items, {}
if source == "bluesky": if source == "bluesky":
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config) result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
return bluesky.parse_bluesky_response(result), {} return bluesky.parse_bluesky_response(result), {}
@@ -1075,45 +1056,6 @@ def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[d
"why_relevant": "Brave web search", "why_relevant": "Brave web search",
} }
], ],
"digg": [
{
"id": "mock1abc",
"title": f"Digg cluster about {subquery.search_query}",
"url": "https://di.gg/ai/mock1abc",
"tldr": f"Curated cluster summarizing recent {subquery.search_query} discussion across the AI 1000.",
"author": "",
"date": dates.get_date_range(3)[0],
"engagement": {"postCount": 8, "uniqueAuthors": 5, "rank": 2, "rank_score": 49.0},
"first_post_age": "3d",
"posts": [
{
"username": "exampledev",
"display_name": "Example Dev",
"category": "Engineer",
"rank": 142,
"body": f"Quote from the AI 1000 about {subquery.search_query}.",
"post_type": "tweet",
"x_url": "https://x.com/exampledev/status/1",
"posted_at": dates.get_date_range(3)[0],
},
],
"relevance": 0.84,
"why_relevant": "Mock Digg cluster",
},
{
"id": "mock2def",
"title": f"Second Digg cluster on {subquery.search_query}",
"url": "https://di.gg/ai/mock2def",
"tldr": f"Another angle on {subquery.search_query}.",
"author": "",
"date": dates.get_date_range(8)[0],
"engagement": {"postCount": 3, "uniqueAuthors": 2, "rank": 18, "rank_score": 33.0},
"first_post_age": "8d",
"posts": [],
"relevance": 0.71,
"why_relevant": "Mock Digg cluster",
},
],
} }
if source == "grounding": if source == "grounding":
return payloads.get(source, []), { return payloads.get(source, []), {
@@ -67,7 +67,6 @@ SOURCE_CAPABILITIES = {
"bluesky": {"discussion", "social"}, "bluesky": {"discussion", "social"},
"truthsocial": {"discussion", "social"}, "truthsocial": {"discussion", "social"},
"polymarket": {"market"}, "polymarket": {"market"},
"digg": {"discussion", "social", "link"},
"xiaohongshu": {"video", "video_shortform", "social"}, "xiaohongshu": {"video", "video_shortform", "social"},
"github": {"discussion", "link"}, "github": {"discussion", "link"},
"grounding": {"web", "reference", "link"}, "grounding": {"web", "reference", "link"},
@@ -93,6 +93,13 @@ class GeminiClient(ReasoningClient):
) )
return extract_gemini_text(payload) return extract_gemini_text(payload)
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
class OpenAIClient(ReasoningClient): class OpenAIClient(ReasoningClient):
name = "openai" name = "openai"
@@ -71,29 +71,8 @@ def _normalize_phrase(text: str) -> str:
return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split()) return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split())
class PreparedQuery:
"""Precomputed query shape reused across items in a stream.
Built once per ranking_query; reused by token_overlap_relevance so the
per-item normalize/score loops don't re-tokenize the same query N times.
"""
__slots__ = ("raw", "q_tokens", "informative_q_tokens", "normalized_phrase")
def __init__(self, query: str) -> None:
self.raw = query
self.q_tokens = tokenize(query)
informative = {t for t in self.q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
self.informative_q_tokens = informative or self.q_tokens
self.normalized_phrase = _normalize_phrase(query)
def _as_prepared(query: "str | PreparedQuery") -> PreparedQuery:
return query if isinstance(query, PreparedQuery) else PreparedQuery(query)
def token_overlap_relevance( def token_overlap_relevance(
query: "str | PreparedQuery", query: str,
text: str, text: str,
hashtags: Optional[List[str]] = None, hashtags: Optional[List[str]] = None,
) -> float: ) -> float:
@@ -116,8 +95,7 @@ def token_overlap_relevance(
Returns: Returns:
Float between 0.0 and 1.0 (0.5 for empty queries) Float between 0.0 and 1.0 (0.5 for empty queries)
""" """
prepared = _as_prepared(query) q_tokens = tokenize(query)
q_tokens = prepared.q_tokens
# Combine text and hashtags for matching # Combine text and hashtags for matching
combined = text combined = text
@@ -141,7 +119,9 @@ def token_overlap_relevance(
if overlap == 0: if overlap == 0:
return 0.0 return 0.0
informative_q_tokens = prepared.informative_q_tokens informative_q_tokens = {t for t in q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
if not informative_q_tokens:
informative_q_tokens = q_tokens
coverage = overlap / len(q_tokens) coverage = overlap / len(q_tokens)
informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens) informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens)
@@ -149,7 +129,7 @@ def token_overlap_relevance(
precision = overlap / precision_denominator precision = overlap / precision_denominator
phrase_bonus = 0.0 phrase_bonus = 0.0
normalized_query = prepared.normalized_phrase normalized_query = _normalize_phrase(query)
normalized_text = _normalize_phrase(combined) normalized_text = _normalize_phrase(combined)
if normalized_query and normalized_query in normalized_text: if normalized_query and normalized_query in normalized_text:
phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16 phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
@@ -4,7 +4,6 @@ from __future__ import annotations
import json import json
import pathlib import pathlib
import re
from collections import Counter from collections import Counter
from datetime import date from datetime import date
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -12,47 +11,21 @@ from urllib.parse import urlparse
from . import dates, schema from . import dates, schema
_VERSION_RE = re.compile(
r'''^version:\s*(?:"([^"]+)"|'([^']+)'|(\S+))\s*$''',
re.MULTILINE,
)
def _skill_version() -> str: def _skill_version() -> str:
"""Read plugin version from .claude-plugin/plugin.json, falling back to SKILL.md frontmatter. """Read plugin version from .claude-plugin/plugin.json if available.
Per-harness skill install dirs (`~/.claude/skills`, `~/.codex/skills`, `~/.agents/skills`, Tries nearest plugin.json by walking up from render.py's own location.
Hermes, etc.) do not always carry `.claude-plugin/plugin.json` that file ships with Falls back to "?" if not found. This keeps the badge emission from
plugin-cache installs but not with per-harness skill installs. SKILL.md frontmatter is crashing on non-plugin-cache installs (repo checkout, Gemini, Codex).
the fallback that keeps the badge from emitting v? on those installs. Returns "?" only
if both sources are missing.
A corrupt manifest at one ancestor does not shadow a valid manifest at a deeper one
(continue, not break). YAML frontmatter accepts double-quoted, single-quoted, or
unquoted version scalars.
""" """
here = pathlib.Path(__file__).resolve() here = pathlib.Path(__file__).resolve()
for parent in here.parents: for parent in [here.parent, *here.parents]:
manifest = parent / ".claude-plugin" / "plugin.json" candidate = parent / ".claude-plugin" / "plugin.json"
if manifest.is_file(): if candidate.is_file():
try: try:
version = json.loads(manifest.read_text()).get("version") return json.loads(candidate.read_text()).get("version", "?")
except (json.JSONDecodeError, OSError): except (json.JSONDecodeError, OSError):
continue return "?"
if version:
return version
# No usable manifest found at any ancestor — fall back to SKILL.md frontmatter.
for parent in here.parents:
skill_md = parent / "SKILL.md"
if skill_md.is_file():
try:
match = _VERSION_RE.search(skill_md.read_text())
except (OSError, UnicodeDecodeError):
break
if match:
return next(g for g in match.groups() if g is not None)
break
return "?" return "?"
@@ -79,7 +52,6 @@ SOURCE_LABELS = {
"xiaohongshu": "Xiaohongshu", "xiaohongshu": "Xiaohongshu",
"x": "X", "x": "X",
"github": "GitHub", "github": "GitHub",
"digg": "Digg",
"perplexity": "Perplexity", "perplexity": "Perplexity",
} }
@@ -198,168 +170,6 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
return "\n".join(lines).strip() + "\n" return "\n".join(lines).strip() + "\n"
def render_for_html(
report: schema.Report,
synthesis_md: str | None = None,
*,
save_path: str | None = None,
) -> str:
"""Render markdown intended for shareable HTML conversion.
This output keeps the public badge, compact source/date metadata, an
optional one-line data quality note, optional synthesized brief markdown,
and the engine footer. It deliberately omits the debug file header,
model-facing safety note, and evidence scratchpad emitted by
render_compact().
When synthesis_md is None, the body is intentionally sparse: badge,
metadata, optional data quality note, and engine footer only.
"""
lines = [
*_render_badge(),
*_render_html_metadata(report),
]
if synthesis_md:
lines.extend(["", synthesis_md.strip()])
# Data quality warnings are NOT rendered into the HTML artifact. The HTML
# is meant to be shared (Slack, email, Notion); recipients haven't asked
# for technical commentary about how the run was produced. Generators see
# the same warnings via collect_html_warnings() routed to stderr by the
# CLI, so they can fix quality issues before sharing.
_append_html_footer(lines, report, save_path)
return "\n".join(lines).strip() + "\n"
def render_for_html_comparison(
entity_reports: list[tuple[str, schema.Report]],
synthesis_md: str | None = None,
*,
save_path: str | None = None,
) -> str:
"""Render comparison markdown intended for shareable HTML conversion.
Same semantics as render_for_html(), but metadata and data quality notes
are aggregated across the compared entities.
"""
if not entity_reports:
raise ValueError("render_for_html_comparison requires at least one report")
entities = [label for label, _ in entity_reports]
main_report = entity_reports[0][1]
meta = (
f"<!-- META: {main_report.range_from} to {main_report.range_to} "
f"· comparing {len(entities)}: {', '.join(entities)} -->"
)
lines = [
*_render_badge(),
meta,
]
if synthesis_md:
lines.extend(["", synthesis_md.strip()])
# Comparison data quality notes also go to stderr, not into the artifact.
_append_html_footer(lines, main_report, save_path)
return "\n".join(lines).strip() + "\n"
def collect_html_warnings(report: schema.Report) -> list[str]:
"""Collect data quality warnings for stderr output (NOT for the HTML artifact).
Returns a list of human-readable warning strings. Empty list if the run
was clean. Used by the CLI to emit diagnostics to stderr after writing
the HTML to stdout/file.
"""
notes: list[str] = []
if _render_degraded_run_warning(report):
notes.append("Run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
elif _render_pre_research_warning(report):
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
freshness_warning = _assess_data_freshness(report)
if freshness_warning:
notes.append(freshness_warning)
notes.extend(report.warnings)
return _dedupe_notes(notes)
def collect_html_warnings_comparison(
entity_reports: list[tuple[str, schema.Report]],
) -> list[str]:
"""Collect comparison-mode warnings, prefixed by entity label."""
notes: list[str] = []
for label, report in entity_reports:
for w in collect_html_warnings(report):
notes.append(f"{label}: {w}")
return notes
def _render_html_metadata(report: schema.Report) -> list[str]:
"""Inline metadata as an HTML comment marker.
html_render.py post-processes ``<!-- META: ... -->`` markers into a
``<div class="meta">`` after markdown conversion, so the metadata escapes
the markdown converter's HTML-escaping pass cleanly. Same pattern as the
PASS_THROUGH_FOOTER marker used for the engine tree.
"""
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
if non_empty:
sources = ", ".join(_source_label(s) for s in non_empty)
else:
sources = "no active sources"
return [
f"<!-- META: {report.range_from} to {report.range_to} · {sources} -->",
]
def _render_html_data_quality_note(report: schema.Report) -> str | None:
notes: list[str] = []
degraded_warning = _render_degraded_run_warning(report)
if degraded_warning:
notes.append("This run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
pre_research_warning = _render_pre_research_warning(report)
if pre_research_warning and not degraded_warning:
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
freshness_warning = _assess_data_freshness(report)
if freshness_warning:
notes.append(freshness_warning)
notes.extend(report.warnings)
if not notes:
return None
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
def _render_html_comparison_data_quality_note(
entity_reports: list[tuple[str, schema.Report]],
) -> str | None:
notes: list[str] = []
for label, report in entity_reports:
note = _render_html_data_quality_note(report)
if note:
clean = note.removeprefix("> **Data quality note:** ").strip()
notes.append(f"{label}: {clean}")
if not notes:
return None
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
def _dedupe_notes(notes: list[str]) -> list[str]:
out: list[str] = []
seen: set[str] = set()
for note in notes:
normalized = " ".join(str(note).split())
if not normalized or normalized in seen:
continue
seen.add(normalized)
out.append(normalized)
return out
def _append_html_footer(lines: list[str], report: schema.Report, save_path: str | None) -> None:
footer = _render_emoji_footer(report, save_path)
lines.append("")
lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")
lines.extend(footer)
lines.append("<!-- END PASS-THROUGH FOOTER -->")
def _render_canonical_boundary() -> list[str]: def _render_canonical_boundary() -> list[str]:
"""Emit the explicit END-OF-CANONICAL-OUTPUT boundary. """Emit the explicit END-OF-CANONICAL-OUTPUT boundary.
@@ -853,7 +663,7 @@ def render_full(report: schema.Report) -> str:
lines.append("## All Items by Source") lines.append("## All Items by Source")
lines.append("") lines.append("")
source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest", source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest",
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "digg", "perplexity"] "hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "perplexity"]
for source in source_order: for source in source_order:
items = report.items_by_source.get(source, []) items = report.items_by_source.get(source, [])
if not items: if not items:
@@ -879,9 +689,6 @@ def render_full(report: schema.Report) -> str:
tc_score = tc.get("score", "") tc_score = tc.get("score", "")
attribution = _comment_attribution(item.source, tc.get("author")) attribution = _comment_attribution(item.source, tc.get("author"))
lines.append(f" Top comment {attribution} ({tc_score} {vote_label}): {excerpt}") lines.append(f" Top comment {attribution} ({tc_score} {vote_label}): {excerpt}")
# Digg: inline X-post quotes attached to the cluster.
for post in _digg_posts_for(item, limit=3):
lines.append(f" > {_format_digg_quote(post)}")
# Comment insights for Reddit # Comment insights for Reddit
insights = item.metadata.get("comment_insights", []) insights = item.metadata.get("comment_insights", [])
if insights: if insights:
@@ -1003,8 +810,6 @@ def _render_candidate(candidate: schema.Candidate, prefix: str) -> list[str]:
source = primary.source if primary else None source = primary.source if primary else None
attribution = _comment_attribution(source, tc.get("author")) attribution = _comment_attribution(source, tc.get("author"))
lines.append(f" - {attribution} ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}") lines.append(f" - {attribution} ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
for post in _digg_posts_for(primary):
lines.append(f" - {_format_digg_quote(post)}")
insight = _comment_insight(primary) insight = _comment_insight(primary)
if insight: if insight:
lines.append(f" - Insight: {_truncate(insight, 220)}") lines.append(f" - Insight: {_truncate(insight, 220)}")
@@ -1255,7 +1060,6 @@ _FOOTER_SOURCES: list[tuple[str, str, str, str, list[tuple[str, str]]]] = [
("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]), ("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),
("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]), ("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]),
("github", "🐙", "GitHub", "item", [("reactions", "reactions"), ("comments", "comments")]), ("github", "🐙", "GitHub", "item", [("reactions", "reactions"), ("comments", "comments")]),
("digg", "⛏️", "Digg", "cluster", [("postCount", "posts"), ("uniqueAuthors", "authors")]),
] ]
@@ -1513,7 +1317,6 @@ ENGAGEMENT_DISPLAY: dict[str, list[tuple[str, str]]] = {
"polymarket": [], "polymarket": [],
"github": [("reactions", "react"), ("comments", "cmt")], "github": [("reactions", "react"), ("comments", "cmt")],
"perplexity": [("citations", "cite")], "perplexity": [("citations", "cite")],
"digg": [("postCount", "posts"), ("uniqueAuthors", "auth")],
} }
@@ -1701,6 +1504,16 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score
return [c for c in comments if (c.get("score") or 0) >= min_score][:limit] return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]
def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None:
if not item:
return None
comments = item.metadata.get("top_comments") or []
if not comments or not isinstance(comments[0], dict):
return None
top = comments[0]
return str(top.get("excerpt") or top.get("text") or "").strip() or None
def _comment_insight(item: schema.SourceItem | None) -> str | None: def _comment_insight(item: schema.SourceItem | None) -> str | None:
if not item: if not item:
return None return None
@@ -1710,39 +1523,6 @@ def _comment_insight(item: schema.SourceItem | None) -> str | None:
return str(insights[0]).strip() or None return str(insights[0]).strip() or None
def _digg_posts_for(item: schema.SourceItem | None, limit: int = 3) -> list[dict]:
"""Return up to `limit` parsed Digg posts attached as enrichment to a cluster.
Returns an empty list for non-digg sources or clusters without enrichment.
"""
if not item or item.source != "digg":
return []
posts = item.metadata.get("posts") or []
if not isinstance(posts, list):
return []
out: list[dict] = []
for entry in posts:
if isinstance(entry, dict) and entry.get("body") and entry.get("username"):
out.append(entry)
if len(out) >= limit:
break
return out
def _format_digg_quote(post: dict, body_limit: int = 200) -> str:
"""Format a Digg-attached X post as an inline 'via Digg' quote line."""
handle = post.get("username") or ""
x_url = post.get("x_url") or ""
body = (post.get("body") or "").replace("\n", " ").strip()
if len(body) > body_limit:
body = body[: body_limit - 1].rstrip() + ""
if x_url and handle:
return f"[@{handle}]({x_url}) via Digg: {body}"
if handle:
return f"@{handle} via Digg: {body}"
return f"via Digg: {body}"
def _transcript_highlights(item: schema.SourceItem | None) -> list[str]: def _transcript_highlights(item: schema.SourceItem | None) -> list[str]:
if not item or item.source != "youtube": if not item or item.source != "youtube":
return [] return []
@@ -12,7 +12,6 @@ SOURCE_QUALITY = {
"xiaohongshu": 0.7, "xiaohongshu": 0.7,
"hackernews": 0.8, "hackernews": 0.8,
"youtube": 0.85, "youtube": 0.85,
"digg": 0.85,
"reddit": 0.6, "reddit": 0.6,
"x": 0.68, "x": 0.68,
"bluesky": 0.66, "bluesky": 0.66,
@@ -27,10 +26,7 @@ def source_quality(source: str) -> float:
return SOURCE_QUALITY.get(source, 0.6) return SOURCE_QUALITY.get(source, 0.6)
def local_relevance( def local_relevance(item: schema.SourceItem, ranking_query: str) -> float:
item: schema.SourceItem,
ranking_query: "str | relevance.PreparedQuery",
) -> float:
text = "\n".join( text = "\n".join(
part part
for part in [item.title, item.body, item.snippet] for part in [item.title, item.body, item.snippet]
@@ -96,7 +92,6 @@ ENGAGEMENT_WEIGHTS: dict[str, list[tuple[str, float]]] = {
"bluesky": [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)], "bluesky": [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)],
"truthsocial": [("likes", 0.45), ("reposts", 0.30), ("replies", 0.25)], "truthsocial": [("likes", 0.45), ("reposts", 0.30), ("replies", 0.25)],
"polymarket": [("volume", 0.60), ("liquidity", 0.40)], "polymarket": [("volume", 0.60), ("liquidity", 0.40)],
"digg": [("postCount", 0.40), ("uniqueAuthors", 0.30), ("rank_score", 0.30)],
} }
@@ -180,14 +175,13 @@ def normalize(values: list[float | None]) -> list[int | None]:
def annotate_stream( def annotate_stream(
items: list[schema.SourceItem], items: list[schema.SourceItem],
ranking_query: "str | relevance.PreparedQuery", ranking_query: str,
freshness_mode: str, freshness_mode: str,
) -> list[schema.SourceItem]: ) -> list[schema.SourceItem]:
"""Attach local scoring metadata and return items sorted by local_rank_score.""" """Attach local scoring metadata and return items sorted by local_rank_score."""
prepared_query = ranking_query if isinstance(ranking_query, relevance.PreparedQuery) else relevance.PreparedQuery(ranking_query)
engagement_scores = normalize([engagement_raw(item) for item in items]) engagement_scores = normalize([engagement_raw(item) for item in items])
for item, eng_score in zip(items, engagement_scores, strict=True): for item, eng_score in zip(items, engagement_scores, strict=True):
item.local_relevance = local_relevance(item, prepared_query) item.local_relevance = local_relevance(item, ranking_query)
item.freshness = freshness(item, freshness_mode) item.freshness = freshness(item, freshness_mode)
item.engagement_score = eng_score item.engagement_score = eng_score
item.source_quality = source_quality(item.source) item.source_quality = source_quality(item.source)
@@ -26,7 +26,7 @@ def _windows(words: list[str], size: int, overlap: int) -> list[str]:
def extract_best_snippet( def extract_best_snippet(
item: schema.SourceItem, item: schema.SourceItem,
ranking_query: "str | relevance.PreparedQuery", ranking_query: str,
max_words: int = 120, max_words: int = 120,
) -> str: ) -> str:
"""Prefer existing snippets, else extract the best matching evidence window.""" """Prefer existing snippets, else extract the best matching evidence window."""
@@ -43,9 +43,8 @@ def extract_best_snippet(
if not candidates: if not candidates:
return _truncate_words(body, max_words) return _truncate_words(body, max_words)
prepared_query = ranking_query if isinstance(ranking_query, relevance.PreparedQuery) else relevance.PreparedQuery(ranking_query)
best = max( best = max(
candidates, candidates,
key=lambda candidate: relevance.token_overlap_relevance(prepared_query, candidate), key=lambda candidate: relevance.token_overlap_relevance(ranking_query, candidate),
) )
return _truncate_words(best, max_words) return _truncate_words(best, max_words)
@@ -152,16 +152,35 @@ def search_threads(
_log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})") _log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
try: try:
data = http.get( import requests as _requests
f"{SCRAPECREATORS_BASE}/search", except ImportError:
params={"keyword": core_topic}, _requests = None
headers=http.scrapecreators_headers(token),
timeout=30, if not _requests:
retries=2, _log("requests library not installed, falling back to urllib")
) try:
except Exception as e: from urllib.parse import urlencode
_log(f"ScrapeCreators error: {e}") params = urlencode({"keyword": core_topic})
return {"items": [], "error": f"{type(e).__name__}: {e}"} url = f"{SCRAPECREATORS_BASE}/search?{params}"
headers = http.scrapecreators_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search",
params={"keyword": core_topic},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Extract items from response (try common SC response shapes) # Extract items from response (try common SC response shapes)
raw_items = ( raw_items = (
@@ -11,6 +11,11 @@ import re
import sys import sys
from typing import Any, Dict, List, Optional, Set from typing import Any, Dict, List, Optional, Set
try:
import requests as _requests
except ImportError:
_requests = None
from . import dates, http, log from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok" SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
@@ -209,17 +214,30 @@ def _hashtag_search(
List of raw TikTok item dicts (aweme_info format). List of raw TikTok item dicts (aweme_info format).
""" """
_log(f"Hashtag search: #{hashtag}") _log(f"Hashtag search: #{hashtag}")
try: if not _requests:
data = http.get( try:
f"{SCRAPECREATORS_BASE}/search/hashtag", from urllib.parse import urlencode
params={"hashtag": hashtag}, params = urlencode({"hashtag": hashtag})
headers=http.scrapecreators_headers(token), url = f"{SCRAPECREATORS_BASE}/search/hashtag?{params}"
timeout=30, headers = http.scrapecreators_headers(token)
retries=2, headers["User-Agent"] = http.USER_AGENT
) data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e: except Exception as e:
_log(f"Hashtag search error for #{hashtag}: {e}") _log(f"Hashtag search error (urllib) for #{hashtag}: {e}")
return [] return []
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search/hashtag",
params={"hashtag": hashtag},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"Hashtag search error for #{hashtag}: {e}")
return []
raw_items = data.get("aweme_list") or data.get("data") or [] raw_items = data.get("aweme_list") or data.get("data") or []
_log(f" -> {len(raw_items)} results for #{hashtag}") _log(f" -> {len(raw_items)} results for #{hashtag}")
@@ -243,17 +261,30 @@ def _profile_videos(
""" """
_log(f"Profile videos: @{handle}") _log(f"Profile videos: @{handle}")
profile_url = "https://api.scrapecreators.com/v3/tiktok/profile/videos" profile_url = "https://api.scrapecreators.com/v3/tiktok/profile/videos"
try: if not _requests:
data = http.get( try:
profile_url, from urllib.parse import urlencode
params={"handle": handle, "sort_by": "latest"}, params = urlencode({"handle": handle, "sort_by": "latest"})
headers=http.scrapecreators_headers(token), url = f"{profile_url}?{params}"
timeout=30, headers = http.scrapecreators_headers(token)
retries=2, headers["User-Agent"] = http.USER_AGENT
) data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e: except Exception as e:
_log(f"Profile videos error for @{handle}: {e}") _log(f"Profile videos error (urllib) for @{handle}: {e}")
return [] return []
else:
try:
resp = _requests.get(
profile_url,
params={"handle": handle, "sort_by": "latest"},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"Profile videos error for @{handle}: {e}")
return []
raw_items = data.get("aweme_list") or data.get("data") or [] raw_items = data.get("aweme_list") or data.get("data") or []
_log(f" -> {len(raw_items)} videos from @{handle}") _log(f" -> {len(raw_items)} videos from @{handle}")
@@ -287,17 +318,31 @@ def search_tiktok(
_log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})") _log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
try: if not _requests:
data = http.get( _log("requests library not installed, falling back to urllib")
f"{SCRAPECREATORS_BASE}/search/keyword", try:
params={"query": core_topic, "sort_by": "relevance"}, from urllib.parse import urlencode
headers=http.scrapecreators_headers(token), params = urlencode({"query": core_topic, "sort_by": "relevance"})
timeout=30, url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
retries=2, headers = http.scrapecreators_headers(token)
) headers["User-Agent"] = http.USER_AGENT
except Exception as e: data = http.get(url, headers=headers, timeout=30, retries=2)
_log(f"ScrapeCreators error: {e}") except Exception as e:
return {"items": [], "error": f"{type(e).__name__}: {e}"} _log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search/keyword",
params={"query": core_topic, "sort_by": "relevance"},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Items are nested under aweme_info # Items are nested under aweme_info
raw_entries = data.get("search_item_list") or data.get("data") or [] raw_entries = data.get("search_item_list") or data.get("data") or []
@@ -352,7 +397,7 @@ def fetch_captions(
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
max_captions = config["max_captions"] max_captions = config["max_captions"]
if not video_items or not token: if not video_items or not token or not _requests:
return {} return {}
top_items = video_items[:max_captions] top_items = video_items[:max_captions]
@@ -377,23 +422,24 @@ def fetch_captions(
if not url: if not url:
continue continue
try: try:
data = http.get( resp = _requests.get(
f"{SCRAPECREATORS_BASE}/video/transcript", f"{SCRAPECREATORS_BASE}/video/transcript",
params={"url": url}, params={"url": url},
headers=http.scrapecreators_headers(token), headers=http.scrapecreators_headers(token),
timeout=15, timeout=15,
retries=1,
) )
transcript = data.get("transcript") if resp.status_code == 200:
if transcript: data = resp.json()
if isinstance(transcript, list): transcript = data.get("transcript")
transcript = " ".join(str(s) for s in transcript)
transcript = _clean_webvtt(transcript)
if transcript: if transcript:
words = transcript.split() if isinstance(transcript, list):
if len(words) > CAPTION_MAX_WORDS: transcript = " ".join(str(s) for s in transcript)
transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...' transcript = _clean_webvtt(transcript)
captions[vid] = transcript if transcript:
words = transcript.split()
if len(words) > CAPTION_MAX_WORDS:
transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
captions[vid] = transcript
except Exception as e: except Exception as e:
_log(f"Transcript fetch failed for {vid}: {e}") _log(f"Transcript fetch failed for {vid}: {e}")
@@ -574,17 +620,30 @@ def _fetch_post_comments(
List of comment dicts with author, text, digg_count (likes), date. List of comment dicts with author, text, digg_count (likes), date.
Empty list on any error comment failures never crash the pipeline. Empty list on any error comment failures never crash the pipeline.
""" """
try: if not _requests:
data = http.get( try:
f"{SCRAPECREATORS_BASE}/video/comments", from urllib.parse import urlencode
params={"url": post_url, "trim": "true"}, params = urlencode({"url": post_url, "trim": "true"})
headers=http.scrapecreators_headers(token), url = f"{SCRAPECREATORS_BASE}/video/comments?{params}"
timeout=30, headers = http.scrapecreators_headers(token)
retries=2, headers["User-Agent"] = http.USER_AGENT
) data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as exc: except Exception as exc:
_log(f"Comment fetch error for {post_url}: {exc}") _log(f"Comment fetch error (urllib) for {post_url}: {exc}")
return [] return []
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/video/comments",
params={"url": post_url, "trim": "true"},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as exc:
_log(f"Comment fetch error for {post_url}: {exc}")
return []
raw_comments = data.get("comments") or data.get("data") or [] raw_comments = data.get("comments") or data.get("data") or []
# Sort by digg_count desc so normalize sees the highest-signal first. # Sort by digg_count desc so normalize sees the highest-signal first.
@@ -124,7 +124,6 @@ SOURCE_COMPLETION_ORDER = [
"polymarket", "polymarket",
"grounding", "grounding",
"xiaohongshu", "xiaohongshu",
"digg",
] ]
SOURCE_COMPLETION_META = { SOURCE_COMPLETION_META = {
@@ -139,7 +138,6 @@ SOURCE_COMPLETION_META = {
"polymarket": ("Polymarket", "market", "markets", Colors.GREEN), "polymarket": ("Polymarket", "market", "markets", Colors.GREEN),
"grounding": ("Web", "result", "results", Colors.GREEN), "grounding": ("Web", "result", "results", Colors.GREEN),
"xiaohongshu": ("Xiaohongshu", "post", "posts", Colors.RED), "xiaohongshu": ("Xiaohongshu", "post", "posts", Colors.RED),
"digg": ("Digg", "cluster", "clusters", Colors.YELLOW),
} }

Some files were not shown because too many files have changed in this diff Show More