Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Matt Van Horn
|
80392061d4 | ||
|
Matt Van Horn
|
c04bd67922 | ||
|
Trevin Chow
|
b1773be8f3 | ||
|
Ilia Alshanetsky
|
5b87cca886 | ||
|
Ilia Alshanetsky
|
bbf892aecc | ||
|
Ilia Alshanetsky
|
2acbf8a869 | ||
|
Ilia Alshanetsky
|
e6b89f2644 | ||
|
Ilia Alshanetsky
|
2c2755b49c | ||
|
Ilia Alshanetsky
|
18b5658674 | ||
|
Matt Van Horn
|
145adc9f56 | ||
|
Trevin Chow
|
b100caf2df | ||
|
Trevin Chow
|
dc0cb9850b | ||
|
Trevin Chow
|
ceec99b24c | ||
|
Dave Morin
|
d1823a2d05 | ||
|
Claire Novotny
|
17caa0526d | ||
|
Claire Novotny
|
f03cb866aa | ||
|
Claire Novotny
|
72495c1c14 | ||
|
Matt Van Horn
|
1f7e85a03f |
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Research"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,16 +1,17 @@
|
||||
{
|
||||
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "last30days-skill",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"owner": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"version": "3.0.9",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.2.0",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.0.14",
|
||||
"version": "3.2.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
@@ -10,6 +10,5 @@
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"],
|
||||
"hooks": {}
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
||||
}
|
||||
|
||||
@@ -1,3 +1,43 @@
|
||||
{
|
||||
"name": "last30days"
|
||||
"name": "last30days",
|
||||
"version": "3.2.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"email": "mvanhorn@gmail.com",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"youtube",
|
||||
"tiktok",
|
||||
"instagram",
|
||||
"trends",
|
||||
"polymarket",
|
||||
"github",
|
||||
"hacker-news"
|
||||
],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days",
|
||||
"shortDescription": "Research recent discussion across social and web sources",
|
||||
"longDescription": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"developerName": "Matt Van Horn",
|
||||
"category": "Research",
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Read",
|
||||
"Write"
|
||||
],
|
||||
"websiteURL": "https://github.com/mvanhorn/last30days-skill",
|
||||
"privacyPolicyURL": "https://docs.github.com/en/site-policy/privacy-policies/github-general-privacy-statement",
|
||||
"termsOfServiceURL": "https://docs.github.com/en/site-policy/github-terms/github-terms-of-service",
|
||||
"defaultPrompt": "Use Last 30 Days to research this topic from the last 30 days across Reddit, X, YouTube, and web.",
|
||||
"brandColor": "#FF6B35"
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -1,5 +1,6 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||
# Used by skills/last30days/scripts/build-skill.sh to produce a
|
||||
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
|
||||
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
@@ -32,9 +33,8 @@ release-notes.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
uv.lock export-ignore
|
||||
|
||||
# Platform adapters - skill-upload path is platform-agnostic
|
||||
.agents/ export-ignore
|
||||
.codex-plugin/ export-ignore
|
||||
# Platform adapters are kept in git archives because Claude Code and Codex
|
||||
# plugin installs use the same repository archive as their source payload.
|
||||
.hermes-plugin/ export-ignore
|
||||
|
||||
# CI workflows - repo-only, not needed at skill runtime
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
name: Bug Report
|
||||
description: Report a bug or unexpected behavior
|
||||
labels: [bug]
|
||||
body:
|
||||
- type: textarea
|
||||
id: summary
|
||||
attributes:
|
||||
label: Summary
|
||||
description: What happened?
|
||||
placeholder: Describe the bug in 1-2 sentences.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: How can we reproduce this?
|
||||
placeholder: |
|
||||
1. Run `python3 scripts/last30days.py "topic" --emit compact`
|
||||
2. ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected Behavior
|
||||
description: What should have happened?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: traceback
|
||||
attributes:
|
||||
label: Error / Traceback
|
||||
description: Paste the full traceback or error output.
|
||||
render: text
|
||||
- type: dropdown
|
||||
id: install
|
||||
attributes:
|
||||
label: Install Method
|
||||
options:
|
||||
- Claude Code plugin
|
||||
- Gemini CLI extension
|
||||
- Codex plugin
|
||||
- Hermes skill
|
||||
- Manual (git clone)
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: OS
|
||||
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Feature Request
|
||||
description: Suggest a new feature or improvement
|
||||
labels: [enhancement]
|
||||
body:
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem
|
||||
description: What problem does this solve?
|
||||
placeholder: When I try to ..., I can't ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: How should this work?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
description: Other approaches you thought of (optional).
|
||||
@@ -0,0 +1,20 @@
|
||||
## Summary
|
||||
|
||||
<!-- What does this PR do? 1-3 sentences. -->
|
||||
|
||||
## Changes
|
||||
|
||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
||||
|
||||
-
|
||||
|
||||
## Testing
|
||||
|
||||
<!-- How did you verify this works? -->
|
||||
|
||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
||||
- [ ] Ran `bash scripts/sync.sh` (if scripts/ changed)
|
||||
|
||||
## Related Issues
|
||||
|
||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash scripts/build-skill.sh
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
name: Validate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
plugin-contract:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run plugin contract tests
|
||||
run: uv run pytest tests/test_plugin_contract.py tests/test_version_consistency.py
|
||||
@@ -28,9 +28,3 @@ htmlcov/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
|
||||
# Marketing video build artifacts
|
||||
marketing/v3.1-launch/node_modules/
|
||||
marketing/v3.1-launch/out/
|
||||
marketing/*/node_modules/
|
||||
marketing/*/out/
|
||||
|
||||
@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [3.2.0] - 2026-05-09
|
||||
|
||||
### Added
|
||||
|
||||
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
|
||||
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
|
||||
|
||||
## [3.1.1] - 2026-04-24
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
|
||||
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
|
||||
|
||||
## [3.1.0] - 2026-04-22
|
||||
|
||||
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
|
||||
|
||||
### Added
|
||||
|
||||
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
|
||||
|
||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||
|
||||
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
|
||||
|
||||
## [3.0.14] - 2026-04-22
|
||||
|
||||
### Changed
|
||||
@@ -61,6 +94,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
### Fixed
|
||||
|
||||
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
||||
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
|
||||
|
||||
### Behavior fallback
|
||||
|
||||
|
||||
@@ -4,20 +4,20 @@ Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||
Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `scripts/last30days.py` — main research engine
|
||||
- `scripts/lib/` — search, enrichment, rendering modules
|
||||
- `scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `SKILL.md` — skill definition (deployed to ~/.claude/skills/last30days/)
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
python3 scripts/last30days.py "test query" --emit=compact # Run research
|
||||
bash scripts/sync.sh # Deploy to ~/.claude, ~/.agents, ~/.codex
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
bash skills/last30days/scripts/sync.sh
|
||||
```
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- After edits: run `bash scripts/sync.sh` to deploy
|
||||
- After edits: run `bash skills/last30days/scripts/sync.sh` to deploy
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Beta channel
|
||||
|
||||
+4
-4
@@ -18,7 +18,7 @@ git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
cd last30days-skill
|
||||
|
||||
# Run the sync script
|
||||
bash scripts/sync.sh
|
||||
bash skills/last30days/scripts/sync.sh
|
||||
```
|
||||
|
||||
This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days/`
|
||||
@@ -30,8 +30,8 @@ This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days
|
||||
mkdir -p ~/.hermes/skills/research/last30days
|
||||
|
||||
# Copy files
|
||||
cp -r scripts ~/.hermes/skills/research/last30days/
|
||||
cp .hermes-plugin/SKILL.md ~/.hermes/skills/research/last30days/
|
||||
cp skills/last30days/SKILL.md ~/.hermes/skills/research/last30days/
|
||||
cp -r skills/last30days/scripts ~/.hermes/skills/research/last30days/
|
||||
```
|
||||
|
||||
## Usage
|
||||
@@ -111,7 +111,7 @@ To update to the latest version:
|
||||
```bash
|
||||
cd last30days-skill
|
||||
git pull
|
||||
bash scripts/sync.sh
|
||||
bash skills/last30days/scripts/sync.sh
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
||||
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [SKILL.md](SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
|
||||
Claude Code:
|
||||
```
|
||||
@@ -68,6 +68,7 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||
| **Digg AI 1000** | Curated story clusters from ~1000 high-signal AI accounts on X, with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||
@@ -96,6 +97,28 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
## What v3 Changed
|
||||
|
||||
### Shareable HTML briefs
|
||||
|
||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
||||
|
||||
```
|
||||
/last30days OpenClaw --emit=html
|
||||
```
|
||||
|
||||
or just ask in plain language:
|
||||
|
||||
```
|
||||
/last30days OpenClaw, give me a shareable HTML brief
|
||||
/last30days Cursor IDE for slack
|
||||
/last30days Anthropic earnings export as html
|
||||
```
|
||||
|
||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
||||
|
||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
||||
|
||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
||||
|
||||
### Intelligent search: the killer feature
|
||||
|
||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
@@ -189,7 +212,7 @@ gemini extensions install ./last30days-skill
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
Or build the claude.ai `.skill` file from source: `bash scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
Or build the claude.ai `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||
|
||||
|
||||
@@ -1,396 +0,0 @@
|
||||
---
|
||||
title: "marketing: v3.1 launch video — 30s Remotion piece for X"
|
||||
type: feat
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
---
|
||||
|
||||
# marketing: v3.1 launch video — 30s Remotion piece for X
|
||||
|
||||
## Overview
|
||||
|
||||
Build a 30-second Remotion-rendered MP4 marketing video announcing **v3.1: Competitors mode** for `/last30days`. Posts to X. Frames the new vs-mode-with-auto-discovery as the headline — three full passes, three save files, one comparison — without burying it in CLI minutiae.
|
||||
|
||||
Marketing release tag is **v3.1** (rebrands the 3.0.11-3.0.14 bundle into one shippable narrative for the launch tweet). Engine version stays 3.0.14 — `3.1` is the marketing version, not a code version bump.
|
||||
|
||||
## Script (60 frames per second × 30 seconds = 900 frames; this version assumes 30fps × 30s = 900 frames at 30fps)
|
||||
|
||||
Total runtime: 30.0 seconds @ 30fps = 900 frames. Six scenes, on-screen text only (silent autoplay-friendly).
|
||||
|
||||
### Scene 1 — Hook (0:00 – 0:03 | frames 0-90)
|
||||
|
||||
**Visual:** Black background. The `/last30days` badge animates in (the literal `🌐 last30days v3.1 · synced 2026-04-22` line) with the spring scale-in used in slick devtool intros.
|
||||
|
||||
**Caption (overlay, large):**
|
||||
```
|
||||
What if one search
|
||||
ran 3 at once?
|
||||
```
|
||||
|
||||
### Scene 2 — Set the world (0:03 – 0:08 | frames 90-240)
|
||||
|
||||
**Visual:** Single mac terminal window center-stage. Type-on animation:
|
||||
```
|
||||
$ /last30days OpenAI
|
||||
```
|
||||
Below it, a simple result card stub appears (Reddit upvote count + X likes), then static.
|
||||
|
||||
**Caption (small bottom-left):**
|
||||
```
|
||||
The old way: one topic.
|
||||
```
|
||||
|
||||
### Scene 3 — The reveal (0:08 – 0:14 | frames 240-420)
|
||||
|
||||
**Visual:** The terminal types one more flag:
|
||||
```
|
||||
$ /last30days OpenAI --competitors
|
||||
```
|
||||
Hard cut → the single terminal **splits into 3 panes** side by side. Each pane shows a different topic header animating in, in this order:
|
||||
- Left: `OpenAI`
|
||||
- Middle: `vs Anthropic`
|
||||
- Right: `vs xAI`
|
||||
|
||||
Pane content scrolls fake "search progress" lines (Reddit, X, YouTube indicators) in parallel, like a live fan-out.
|
||||
|
||||
**Caption (top center):**
|
||||
```
|
||||
Now it discovers competitors
|
||||
and runs all 3.
|
||||
```
|
||||
|
||||
### Scene 4 — Result reveal (0:14 – 0:21 | frames 420-630)
|
||||
|
||||
**Visual:** The 3 panes collapse into a single comparison surface — the `## Head-to-Head` table from the actual engine output, with rows fading in one by one (What it is, Streams, Best for, Trajectory). Each entity column lights up as its row populates.
|
||||
|
||||
**Caption (bottom):**
|
||||
```
|
||||
3 full passes. 3 save files.
|
||||
1 comparison.
|
||||
```
|
||||
|
||||
### Scene 5 — How it's special (0:21 – 0:26 | frames 630-780)
|
||||
|
||||
**Visual:** Cut to a clean text card, large mono font:
|
||||
```
|
||||
You pick the topic.
|
||||
The agent picks the peers.
|
||||
The engine fans out.
|
||||
```
|
||||
Each line fades in 1.5s apart.
|
||||
|
||||
### Scene 6 — CTA (0:26 – 0:30 | frames 780-900)
|
||||
|
||||
**Visual:** Black background. Centered:
|
||||
- Top: `🌐 last30days v3.1`
|
||||
- Middle: `/last30days {topic} --competitors`
|
||||
- Bottom: `github.com/mvanhorn/last30days-skill`
|
||||
|
||||
Subtle pulse on the install line.
|
||||
|
||||
**End frame holds for ~0.5s.**
|
||||
|
||||
## Problem Frame
|
||||
|
||||
The 3.0.11-3.0.14 release bundle ships a transformative feature (per-entity vs-mode fanout + `--competitors` shortcut + per-entity save files), but the value lands flat in a tweet thread or screenshot. A 30s video does what static text cannot: shows the fan-out happening in real time and the 3 → 1 collapse into a comparison. Higher tweet engagement, easier to RT/QT.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. Final artifact: a single MP4 file, 1920×1080, 30fps, ~30 seconds (±0.5s), under 30MB, suitable for direct X upload.
|
||||
- R2. Six-scene script as defined above, with text/visuals/timing matching to within 5 frames.
|
||||
- R3. Branded look: matches the `🌐 last30days v3.1` badge style (terminal aesthetic, mono font, dark background).
|
||||
- R4. Silent — no voiceover, no music in v1. Captions baked in. Designed for autoplay-muted feeds.
|
||||
- R5. Reproducible: another contributor (or a future-me) can re-render with one command. Project lives in-repo so the source is versioned.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No voiceover. Text-on-screen only. (Voice can be a v1.1 if engagement is high.)
|
||||
- No background music in the rendered MP4. (Music can be added in post via QuickTime/iMovie if desired before posting.)
|
||||
- No localization. English captions only.
|
||||
- No A/B test variants. One video.
|
||||
- No 9:16 vertical version. 16:9 only. (Vertical can be a separate render after launch validates the format.)
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Voiceover variant: defer to follow-up if v1 lands well.
|
||||
- 9:16 mobile cut: defer; same source compositions can re-render at 1080×1920 in a follow-up.
|
||||
- Animated GIF for embedding in README.md: defer; can be ffmpeg-extracted from the MP4.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `SKILL.md` — the comparison render scaffold (`## Head-to-Head` table) is the visual reference for Scene 4's table look.
|
||||
- `scripts/lib/render.py` `_render_comparison_scaffold` — emits the 9-axis table whose visual style we're recreating in a more polished form.
|
||||
- `CHANGELOG.md` 3.0.11-3.0.14 entries — the prose source for the script's beats.
|
||||
- `.claude-plugin/plugin.json` version 3.0.14 — current code version (marketing version is 3.1 for the launch).
|
||||
|
||||
### External References
|
||||
|
||||
- Remotion 4.x docs (https://www.remotion.dev/docs/) — current API for compositions, sequences, springs, and render CLI.
|
||||
- X video specs 2026: max 2 min 20 s, ≤512MB, MP4 with H.264 + AAC, recommended 1920×1080 for landscape autoplay.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- No prior `marketing/` dir or video plans in `docs/plans/`. This is a greenfield asset directory.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Remotion, not ffmpeg-only.** Remotion's React-based compositions handle the typed-on terminal effect, spring-animated badges, and scene transitions far more cleanly than raw ffmpeg filtergraphs. Render output is still MP4 via Remotion's bundled ffmpeg.
|
||||
- **In-repo asset directory at `marketing/v3.1-launch/`.** Lives with the product so future versions can fork the project. Adds `marketing/` to `.gitignore` exceptions only for source files; rendered MP4 stays out of git (uploaded separately).
|
||||
- **16:9 1920×1080 @ 30fps.** Best fit for X landscape autoplay on desktop and mobile feed. 30fps is plenty for typed-text + UI animation; 60fps doubles render time without obvious quality gain.
|
||||
- **Silent + captions.** X autoplay defaults to muted. Sound-off is the realistic viewing condition. Captions baked into the visual.
|
||||
- **Six scenes, one composition.** Single Remotion composition with sequenced child compositions per scene. Easier to re-time than scene-files. Frame-numbered timing in the script enables precise edits.
|
||||
- **Mono font (JetBrains Mono or Geist Mono).** Matches the terminal aesthetic of the actual `/last30days` output. Available via Google Fonts or @remotion/google-fonts.
|
||||
- **Marketing version 3.1 ≠ engine version 3.0.14.** `3.1` is the launch label. Engine stays 3.0.14. Avoids confusion in CHANGELOG.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **Aspect ratio?** 16:9 1080p. Best X autoplay format; vertical can be a follow-up.
|
||||
- **Voiceover or silent?** Silent + on-screen captions. Autoplay-muted is the realistic condition.
|
||||
- **In-repo or separate repo?** In-repo at `marketing/v3.1-launch/`. Rendered MP4 not committed; source compositions are.
|
||||
- **Length?** Exactly 30s (900 frames @ 30fps). No flex.
|
||||
- **Marketing version label?** `v3.1`. Engine code version stays 3.0.14.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact animation easing curves per scene — pick during build via Remotion preview iteration.
|
||||
- Whether the comparison-table mock in Scene 4 uses canned text or pulls from the actual saved `*-raw.md` files. Probably canned for visual control.
|
||||
- Whether Scene 3's "search progress" lines are typed individually or use a marquee scroll. Pick during preview.
|
||||
- Exact accent color palette beyond "terminal dark." Iterate against preview.
|
||||
|
||||
## Output Structure
|
||||
|
||||
marketing/
|
||||
v3.1-launch/
|
||||
package.json # Remotion dependency manifest
|
||||
tsconfig.json # TypeScript config
|
||||
remotion.config.ts # Remotion render config (codec, fps, resolution)
|
||||
src/
|
||||
index.ts # Remotion entry — registers compositions
|
||||
Root.tsx # Root composition definition
|
||||
LaunchVideo.tsx # Main 30s composition that sequences scenes
|
||||
scenes/
|
||||
Scene1Hook.tsx
|
||||
Scene2OldWay.tsx
|
||||
Scene3FanOut.tsx
|
||||
Scene4Comparison.tsx
|
||||
Scene5HowItWorks.tsx
|
||||
Scene6CTA.tsx
|
||||
components/
|
||||
TerminalWindow.tsx # Reusable mac-style terminal frame
|
||||
TypedLine.tsx # Type-on animation primitive
|
||||
ComparisonTable.tsx # The Head-to-Head table mock
|
||||
BadgeBar.tsx # The 🌐 last30days v3.1 badge
|
||||
lib/
|
||||
timing.ts # Frame ranges per scene (single source of truth)
|
||||
colors.ts # Brand palette
|
||||
public/ # Static assets (logo, fonts if local)
|
||||
out/ # Rendered MP4 lives here (gitignored)
|
||||
README.md # How to preview/render
|
||||
|
||||
## High-Level Technical Design
|
||||
|
||||
> *Directional guidance for review — not implementation specification.*
|
||||
|
||||
```
|
||||
LaunchVideo (durationInFrames = 900)
|
||||
├── <Sequence from={0} durationInFrames={90}> <Scene1Hook />
|
||||
├── <Sequence from={90} durationInFrames={150}> <Scene2OldWay />
|
||||
├── <Sequence from={240} durationInFrames={180}> <Scene3FanOut />
|
||||
├── <Sequence from={420} durationInFrames={210}> <Scene4Comparison />
|
||||
├── <Sequence from={630} durationInFrames={150}> <Scene5HowItWorks />
|
||||
└── <Sequence from={780} durationInFrames={120}> <Scene6CTA />
|
||||
```
|
||||
|
||||
Per-scene components use `useCurrentFrame()` + `interpolate()` + `spring()` for timing. `TerminalWindow` is the dominant motif across scenes 2-4.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: Remotion project scaffold**
|
||||
|
||||
**Goal:** Spin up a working Remotion project at `marketing/v3.1-launch/` that previews a blank composition and renders to MP4.
|
||||
|
||||
**Requirements:** R1, R5
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/package.json`
|
||||
- Create: `marketing/v3.1-launch/tsconfig.json`
|
||||
- Create: `marketing/v3.1-launch/remotion.config.ts`
|
||||
- Create: `marketing/v3.1-launch/src/index.ts`
|
||||
- Create: `marketing/v3.1-launch/src/Root.tsx`
|
||||
- Create: `marketing/v3.1-launch/README.md`
|
||||
- Modify: `.gitignore` (add `marketing/v3.1-launch/out/`, `marketing/v3.1-launch/node_modules/`)
|
||||
|
||||
**Approach:**
|
||||
- Use `npx create-video@latest --blank` (Remotion 4.x scaffolding) targeting `marketing/v3.1-launch/`. Strip the demo composition.
|
||||
- Configure: 1920×1080, 30fps, H.264, AAC (audio codec needed even for silent — empty track).
|
||||
- README documents `npm install`, `npm run preview` (Remotion Studio), `npm run render` (one-command MP4).
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none for scaffold, but verification = `npm run preview` opens Remotion Studio with a blank 30s composition; `npm run render` produces a black MP4 at the right resolution.
|
||||
|
||||
**Verification:**
|
||||
- Studio loads at localhost:3000 with the empty `LaunchVideo` composition listed.
|
||||
- A render produces `out/launch-video.mp4` at 1920×1080, 30s, valid MP4.
|
||||
|
||||
- [ ] **Unit 2: Reusable components (Terminal, TypedLine, BadgeBar)**
|
||||
|
||||
**Goal:** Build the three primitive components scenes 2-6 will compose. Each is independently previewable.
|
||||
|
||||
**Requirements:** R3, R5
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/components/TerminalWindow.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/TypedLine.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/BadgeBar.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/lib/colors.ts`
|
||||
- Create: `marketing/v3.1-launch/src/lib/timing.ts`
|
||||
|
||||
**Approach:**
|
||||
- `TerminalWindow`: mac-style traffic-light header, dark gradient background, mono content area. Accepts children.
|
||||
- `TypedLine`: takes a string and a `startFrame`, renders character-by-character at ~30 chars/sec. Reuses Remotion's `interpolate(useCurrentFrame() - startFrame, [0, lengthFrames], [0, text.length])` clamped.
|
||||
- `BadgeBar`: renders the literal `🌐 last30days v3.1 · synced 2026-04-22` line in mono with the same gradient color treatment as the engine's compact emit.
|
||||
- `colors.ts`: 5-7 brand colors (terminal-bg, terminal-fg, accent-cyan, accent-magenta, muted, success-green, warning-amber).
|
||||
- `timing.ts`: exports the scene frame ranges as named constants. Single source of truth for any retiming.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — visual components verified in Remotion Studio.
|
||||
|
||||
**Verification:**
|
||||
- Each component renders standalone in Studio when wrapped in a temporary preview composition.
|
||||
- TypedLine animates character-by-character without flicker.
|
||||
|
||||
- [ ] **Unit 3: Scenes 1-3 (Hook, Old way, Fan-out reveal)**
|
||||
|
||||
**Goal:** Build the first half of the video (frames 0-420). The narrative arc up through the visual fan-out.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 2
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene1Hook.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene2OldWay.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene3FanOut.tsx`
|
||||
- Modify: `marketing/v3.1-launch/src/Root.tsx` (register sequences)
|
||||
|
||||
**Approach:**
|
||||
- Scene 1: spring-in BadgeBar, large overlay caption, 3-second hold.
|
||||
- Scene 2: TerminalWindow with TypedLine (`$ /last30days OpenAI`), then a single result-card mock fading in.
|
||||
- Scene 3: typing animation appends `--competitors`, hard cut, three TerminalWindow components arranged in a row with staggered fade-in. Each pane shows a different entity header + scrolling progress lines.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verified visually in Studio.
|
||||
|
||||
**Verification:**
|
||||
- Scrub the 0-14s range in Studio; visuals match the script timing within 5 frames.
|
||||
- The fan-out moment (frame 240) lands cleanly; no jank in the transition from 1 → 3 panes.
|
||||
|
||||
- [ ] **Unit 4: Scenes 4-6 (Comparison reveal, How it works, CTA)**
|
||||
|
||||
**Goal:** Build the back half of the video (frames 420-900). Resolution + payoff + call to action.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 2
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene4Comparison.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene5HowItWorks.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene6CTA.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/ComparisonTable.tsx`
|
||||
|
||||
**Approach:**
|
||||
- Scene 4: ComparisonTable component renders 3-column markdown-style table; rows fade in one by one (stagger 15-20 frames). Uses canned data — OpenAI / Anthropic / xAI with believable cell content drawn from real 3.0.13 outputs.
|
||||
- Scene 5: three-line text card; lines fade in 45 frames apart.
|
||||
- Scene 6: three centered text blocks; install line gets a 1Hz subtle opacity pulse for emphasis.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verified visually.
|
||||
|
||||
**Verification:**
|
||||
- Scrub 14-30s; table reveal feels paced (not too slow, not strobed); CTA holds long enough to read (~3-4s).
|
||||
- ComparisonTable cells are legible at 1920×1080 (mono font ≥ 28px).
|
||||
|
||||
- [ ] **Unit 5: Final composition wiring + render**
|
||||
|
||||
**Goal:** Wire the six scenes into the master `LaunchVideo` composition, render to MP4, verify against X upload constraints.
|
||||
|
||||
**Requirements:** R1, R2, R5
|
||||
|
||||
**Dependencies:** Units 3, 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `marketing/v3.1-launch/src/LaunchVideo.tsx` (sequence all 6 scenes)
|
||||
- Modify: `marketing/v3.1-launch/README.md` (add render command + verification checklist)
|
||||
|
||||
**Approach:**
|
||||
- `LaunchVideo` is a single Composition that imports `Scene1Hook` … `Scene6CTA` and wraps each in `<Sequence from=… durationInFrames=…>` matching `lib/timing.ts`.
|
||||
- Run `npx remotion render LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18`.
|
||||
- Verify output: 30.0s ±0.1s, 1920×1080, file size <30MB, opens in QuickTime, plays without dropped frames.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verification is the rendered MP4 itself.
|
||||
|
||||
**Verification:**
|
||||
- `ffprobe out/last30days-v3.1-launch.mp4` reports 1920×1080, 30fps, ~30.0s, h264, faststart-friendly.
|
||||
- Manual play-through end-to-end in QuickTime feels coherent and on-pace.
|
||||
- File <30MB so X upload is instant.
|
||||
|
||||
- [ ] **Unit 6: Polish pass + ship**
|
||||
|
||||
**Goal:** Watch the full render, fix obvious jank, do a second render, and stage for X posting.
|
||||
|
||||
**Requirements:** R1, R2, R3
|
||||
|
||||
**Dependencies:** Unit 5
|
||||
|
||||
**Files:**
|
||||
- Possibly modify: any scene file based on watch-through findings.
|
||||
|
||||
**Approach:**
|
||||
- Watch the rendered MP4 at full size. Note: timing felt off, transitions too fast, captions overflow, color clash, anything visibly broken.
|
||||
- Iterate: edit scene component → re-preview in Studio → re-render full MP4.
|
||||
- Cap at 2 polish passes; ship the better of the two renders.
|
||||
- Final MP4 sits at `marketing/v3.1-launch/out/last30days-v3.1-launch.mp4` ready for X upload.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — pure subjective polish.
|
||||
|
||||
**Verification:**
|
||||
- User watches the final render and approves.
|
||||
- No glaring visual bugs (overflowing text, frozen frames, color clashes).
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** None — this is a standalone marketing artifact. Doesn't touch the Python engine, doesn't change any user-facing behavior.
|
||||
- **State lifecycle risks:** None.
|
||||
- **API surface parity:** N/A.
|
||||
- **Unchanged invariants:** The shipped 3.0.14 engine is untouched.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Remotion install pulls 200MB+ of node_modules. | `marketing/v3.1-launch/node_modules/` in `.gitignore`; checked-in source stays small. |
|
||||
| Render time blows past patience (>5 min for 30s @ 1080p30). | Bun-based render or `--concurrency` flag. Default Remotion is fast enough on M-series Macs. If slow, lower preview to 720p, render final at 1080p. |
|
||||
| Captions overflow the 1920px width on certain fonts. | Use a known mono font with a measured per-character width; cap caption lines at 36 chars. |
|
||||
| The "fan-out" visual in Scene 3 looks confusing instead of magical. | Polish pass (Unit 6) is the safety net; if still bad, fall back to a simpler "1 → 3 panes wipe" instead of typed split. |
|
||||
| File size >30MB hits X upload friction. | Use `--crf=18` (high quality, reasonable size); fall back to `--crf=23` if over. 30s @ 1080p30 H.264 is normally 5-15MB. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- README at `marketing/v3.1-launch/README.md` documents preview / render commands.
|
||||
- After render, the MP4 is uploaded directly to X. Tweet copy is the user's call (this plan stops at the rendered file).
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Related code: `scripts/lib/render.py` (`_render_comparison_scaffold` is the visual model for Scene 4); `SKILL.md` Competitor mode section (the narrative source).
|
||||
- Related PRs: #308, #311, #312 (the 3.0.11 → 3.0.14 release bundle this video markets as "v3.1").
|
||||
- External docs: https://www.remotion.dev/docs/ (Remotion 4.x API).
|
||||
- X video specs: https://help.x.com/en/using-x/twitter-videos.
|
||||
@@ -1,37 +0,0 @@
|
||||
# /last30days v3.1 Launch Video
|
||||
|
||||
30-second Remotion-rendered MP4 announcing **v3.1: Competitors mode** for posting on X.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
cd marketing/v3.1-launch
|
||||
npm install # one-time, ~200MB of node_modules
|
||||
npm run preview # opens Remotion Studio at localhost:3000 to scrub frames
|
||||
npm run render # writes out/last30days-v3.1-launch.mp4 (high quality, CRF 18)
|
||||
npm run render:fast # writes a CRF 23 preview for fast iteration
|
||||
```
|
||||
|
||||
## Specs
|
||||
|
||||
- 1920×1080, 30fps, 30 seconds (900 frames)
|
||||
- H.264 / MP4
|
||||
- Silent (autoplay-muted-friendly; captions baked in)
|
||||
- Marketing label `v3.1` (engine code version stays 3.0.14)
|
||||
|
||||
## Scene timing (single source of truth: `src/lib/timing.ts`)
|
||||
|
||||
| Scene | Frames | Time | What |
|
||||
|-------|--------|------|------|
|
||||
| 1. Hook | 0-89 | 0.0-3.0s | Badge animates in + "What if one search ran 3 at once?" |
|
||||
| 2. Old way | 90-239 | 3.0-8.0s | Single terminal: `/last30days OpenAI` |
|
||||
| 3. Fan-out | 240-419 | 8.0-14.0s | `--competitors` types in → splits into 3 panes |
|
||||
| 4. Comparison | 420-629 | 14.0-21.0s | 3 panes collapse into Head-to-Head table |
|
||||
| 5. How | 630-779 | 21.0-26.0s | 3-line text card |
|
||||
| 6. CTA | 780-899 | 26.0-30.0s | Install command + repo URL |
|
||||
|
||||
Edit `src/lib/timing.ts` to retime scenes; the `LaunchVideo` composition reads from there.
|
||||
|
||||
## Output
|
||||
|
||||
Rendered MP4 lives at `out/last30days-v3.1-launch.mp4` (gitignored). Upload directly to X.
|
||||
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"name": "last30days-v3-1-launch-video",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "30s Remotion launch video for /last30days v3.1 (competitors mode).",
|
||||
"scripts": {
|
||||
"preview": "remotion studio src/index.ts",
|
||||
"render": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18",
|
||||
"render:fast": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch-preview.mp4 --codec=h264 --crf=23"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "19.0.0",
|
||||
"react-dom": "19.0.0",
|
||||
"remotion": "4.0.250",
|
||||
"@remotion/cli": "4.0.250",
|
||||
"@remotion/google-fonts": "4.0.250"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "19.0.0",
|
||||
"@types/node": "22.10.0",
|
||||
"typescript": "5.6.3"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
import { Config } from "@remotion/cli/config";
|
||||
|
||||
Config.setVideoImageFormat("jpeg");
|
||||
Config.setOverwriteOutput(true);
|
||||
Config.setConcurrency(null);
|
||||
Config.setCodec("h264");
|
||||
@@ -1,35 +0,0 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, Sequence } from "remotion";
|
||||
import { SCENES } from "./lib/timing";
|
||||
import { Scene1Hook } from "./scenes/Scene1Hook";
|
||||
import { Scene2OldWay } from "./scenes/Scene2OldWay";
|
||||
import { Scene3FanOut } from "./scenes/Scene3FanOut";
|
||||
import { Scene4Comparison } from "./scenes/Scene4Comparison";
|
||||
import { Scene5HowItWorks } from "./scenes/Scene5HowItWorks";
|
||||
import { Scene6CTA } from "./scenes/Scene6CTA";
|
||||
import { COLORS } from "./lib/colors";
|
||||
|
||||
export const LaunchVideo: React.FC = () => {
|
||||
return (
|
||||
<AbsoluteFill style={{ background: COLORS.bgDeep }}>
|
||||
<Sequence from={SCENES.hook.from} durationInFrames={SCENES.hook.durationInFrames}>
|
||||
<Scene1Hook />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.oldWay.from} durationInFrames={SCENES.oldWay.durationInFrames}>
|
||||
<Scene2OldWay />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.fanOut.from} durationInFrames={SCENES.fanOut.durationInFrames}>
|
||||
<Scene3FanOut />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.comparison.from} durationInFrames={SCENES.comparison.durationInFrames}>
|
||||
<Scene4Comparison />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.howItWorks.from} durationInFrames={SCENES.howItWorks.durationInFrames}>
|
||||
<Scene5HowItWorks />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.cta.from} durationInFrames={SCENES.cta.durationInFrames}>
|
||||
<Scene6CTA />
|
||||
</Sequence>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -1,19 +0,0 @@
|
||||
import React from "react";
|
||||
import { Composition } from "remotion";
|
||||
import { LaunchVideo } from "./LaunchVideo";
|
||||
import { FPS, TOTAL_FRAMES } from "./lib/timing";
|
||||
|
||||
export const RemotionRoot: React.FC = () => {
|
||||
return (
|
||||
<>
|
||||
<Composition
|
||||
id="LaunchVideo"
|
||||
component={LaunchVideo}
|
||||
durationInFrames={TOTAL_FRAMES}
|
||||
fps={FPS}
|
||||
width={1920}
|
||||
height={1080}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -1,68 +0,0 @@
|
||||
import React from "react";
|
||||
import { spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
startFrame?: number;
|
||||
size?: "small" | "large";
|
||||
};
|
||||
|
||||
export const BadgeBar: React.FC<Props> = ({ startFrame = 0, size = "large" }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
const elapsed = Math.max(0, frame - startFrame);
|
||||
|
||||
const scale = spring({
|
||||
frame: elapsed,
|
||||
fps,
|
||||
config: { damping: 12, stiffness: 90 },
|
||||
from: 0.85,
|
||||
to: 1,
|
||||
});
|
||||
const opacity = spring({
|
||||
frame: elapsed,
|
||||
fps,
|
||||
config: { damping: 20 },
|
||||
from: 0,
|
||||
to: 1,
|
||||
});
|
||||
|
||||
const fontSize = size === "large" ? 56 : 28;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
transform: `scale(${scale})`,
|
||||
opacity,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize,
|
||||
color: COLORS.fgPrimary,
|
||||
letterSpacing: 0.5,
|
||||
}}
|
||||
>
|
||||
<span style={{ fontSize: fontSize * 1.1, marginRight: 16 }}>🌐</span>
|
||||
<span>last30days</span>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 14,
|
||||
color: COLORS.accentCyan,
|
||||
fontWeight: 600,
|
||||
}}
|
||||
>
|
||||
v3.1
|
||||
</span>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 16,
|
||||
color: COLORS.fgDim,
|
||||
fontSize: fontSize * 0.55,
|
||||
}}
|
||||
>
|
||||
· synced 2026-04-22
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -1,135 +0,0 @@
|
||||
import React from "react";
|
||||
import { interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Row = {
|
||||
dimension: string;
|
||||
cells: [string, string, string];
|
||||
};
|
||||
|
||||
type Props = {
|
||||
startFrame: number;
|
||||
entities: [string, string, string];
|
||||
rows: Row[];
|
||||
rowStaggerFrames?: number;
|
||||
};
|
||||
|
||||
export const ComparisonTable: React.FC<Props> = ({
|
||||
startFrame,
|
||||
entities,
|
||||
rows,
|
||||
rowStaggerFrames = 18,
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const headerOpacity = interpolate(
|
||||
frame - startFrame,
|
||||
[0, 12],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
|
||||
const colTemplate = "1.4fr 1fr 1fr 1fr";
|
||||
const cellPad = "16px 22px";
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
background: COLORS.bgPanel,
|
||||
borderRadius: 16,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
overflow: "hidden",
|
||||
fontFamily: FONT_MONO,
|
||||
boxShadow: "0 24px 60px rgba(0,0,0,0.6)",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: colTemplate,
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderBottom: `1px solid ${COLORS.border}`,
|
||||
opacity: headerOpacity,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 22,
|
||||
fontWeight: 500,
|
||||
}}
|
||||
>
|
||||
Dimension
|
||||
</div>
|
||||
{entities.map((entity, idx) => (
|
||||
<div
|
||||
key={entity}
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: idx === 0 ? COLORS.accentCyan : COLORS.fgPrimary,
|
||||
fontSize: 26,
|
||||
fontWeight: 600,
|
||||
borderLeft: `1px solid ${COLORS.border}`,
|
||||
}}
|
||||
>
|
||||
{entity}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{rows.map((row, idx) => {
|
||||
const rowStart = startFrame + 12 + idx * rowStaggerFrames;
|
||||
const rowOpacity = interpolate(
|
||||
frame - rowStart,
|
||||
[0, 14],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
const rowSlide = interpolate(
|
||||
frame - rowStart,
|
||||
[0, 14],
|
||||
[12, 0],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
return (
|
||||
<div
|
||||
key={row.dimension}
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: colTemplate,
|
||||
borderBottom:
|
||||
idx === rows.length - 1 ? "none" : `1px solid ${COLORS.border}`,
|
||||
opacity: rowOpacity,
|
||||
transform: `translateY(${rowSlide}px)`,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 22,
|
||||
}}
|
||||
>
|
||||
{row.dimension}
|
||||
</div>
|
||||
{row.cells.map((cell, cellIdx) => (
|
||||
<div
|
||||
key={cellIdx}
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgPrimary,
|
||||
fontSize: 22,
|
||||
borderLeft: `1px solid ${COLORS.border}`,
|
||||
lineHeight: 1.35,
|
||||
}}
|
||||
>
|
||||
{cell}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -1,95 +0,0 @@
|
||||
import React from "react";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
title?: string;
|
||||
width?: number | string;
|
||||
height?: number | string;
|
||||
children?: React.ReactNode;
|
||||
glow?: boolean;
|
||||
};
|
||||
|
||||
export const TerminalWindow: React.FC<Props> = ({
|
||||
title = "/last30days",
|
||||
width = "100%",
|
||||
height = "100%",
|
||||
children,
|
||||
glow = false,
|
||||
}) => {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width,
|
||||
height,
|
||||
background: COLORS.bgPanel,
|
||||
borderRadius: 16,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
boxShadow: glow
|
||||
? `0 0 60px ${COLORS.accentCyan}33, 0 24px 60px rgba(0,0,0,0.6)`
|
||||
: "0 24px 60px rgba(0,0,0,0.6)",
|
||||
overflow: "hidden",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
fontFamily: FONT_MONO,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
height: 36,
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderBottom: `1px solid ${COLORS.border}`,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
padding: "0 16px",
|
||||
gap: 8,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficRed,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficYellow,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficGreen,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 16,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 14,
|
||||
fontFamily: FONT_MONO,
|
||||
letterSpacing: 0.5,
|
||||
}}
|
||||
>
|
||||
{title}
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
padding: "20px 28px",
|
||||
color: COLORS.fgPrimary,
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -1,71 +0,0 @@
|
||||
import React from "react";
|
||||
import { interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
text: string;
|
||||
startFrame: number;
|
||||
charsPerSec?: number;
|
||||
fontSize?: number;
|
||||
color?: string;
|
||||
prefix?: string;
|
||||
prefixColor?: string;
|
||||
showCursor?: boolean;
|
||||
fps?: number;
|
||||
};
|
||||
|
||||
export const TypedLine: React.FC<Props> = ({
|
||||
text,
|
||||
startFrame,
|
||||
charsPerSec = 28,
|
||||
fontSize = 32,
|
||||
color = COLORS.fgPrimary,
|
||||
prefix,
|
||||
prefixColor = COLORS.accentGreen,
|
||||
showCursor = true,
|
||||
fps = 30,
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
const elapsed = Math.max(0, frame - startFrame);
|
||||
const totalChars = text.length;
|
||||
const lengthFrames = Math.ceil((totalChars / charsPerSec) * fps);
|
||||
const visibleChars = Math.round(
|
||||
interpolate(elapsed, [0, lengthFrames], [0, totalChars], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
}),
|
||||
);
|
||||
const visible = text.slice(0, visibleChars);
|
||||
const done = visibleChars >= totalChars;
|
||||
const cursorOn = showCursor && Math.floor(frame / 15) % 2 === 0;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize,
|
||||
color,
|
||||
whiteSpace: "pre",
|
||||
lineHeight: 1.4,
|
||||
}}
|
||||
>
|
||||
{prefix ? (
|
||||
<span style={{ color: prefixColor, marginRight: 12 }}>{prefix}</span>
|
||||
) : null}
|
||||
<span>{visible}</span>
|
||||
{(!done || cursorOn) && (
|
||||
<span
|
||||
style={{
|
||||
display: "inline-block",
|
||||
width: fontSize * 0.55,
|
||||
height: fontSize * 0.95,
|
||||
background: color,
|
||||
verticalAlign: "text-bottom",
|
||||
marginLeft: 2,
|
||||
opacity: cursorOn ? 0.85 : 0,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -1,4 +0,0 @@
|
||||
import { registerRoot } from "remotion";
|
||||
import { RemotionRoot } from "./Root";
|
||||
|
||||
registerRoot(RemotionRoot);
|
||||
@@ -1,20 +0,0 @@
|
||||
export const COLORS = {
|
||||
bgDeep: "#0a0e14",
|
||||
bgPanel: "#11161d",
|
||||
bgPanelSoft: "#161c25",
|
||||
fgPrimary: "#e6e6e6",
|
||||
fgMuted: "#8a93a3",
|
||||
fgDim: "#5b6573",
|
||||
border: "#2a3340",
|
||||
accentCyan: "#36d6f7",
|
||||
accentMagenta: "#ff55a3",
|
||||
accentGreen: "#5fff9f",
|
||||
accentAmber: "#ffc857",
|
||||
trafficRed: "#ff5f57",
|
||||
trafficYellow: "#febc2e",
|
||||
trafficGreen: "#28c840",
|
||||
} as const;
|
||||
|
||||
export const FONT_MONO = '"JetBrains Mono", "SF Mono", "Menlo", monospace';
|
||||
export const FONT_SANS =
|
||||
'"Inter", "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif';
|
||||
@@ -1,13 +0,0 @@
|
||||
// Single source of truth for scene frame ranges.
|
||||
// 30fps × 30s = 900 frames total.
|
||||
export const FPS = 30;
|
||||
export const TOTAL_FRAMES = 900;
|
||||
|
||||
export const SCENES = {
|
||||
hook: { from: 0, durationInFrames: 90 },
|
||||
oldWay: { from: 90, durationInFrames: 150 },
|
||||
fanOut: { from: 240, durationInFrames: 180 },
|
||||
comparison: { from: 420, durationInFrames: 210 },
|
||||
howItWorks: { from: 630, durationInFrames: 150 },
|
||||
cta: { from: 780, durationInFrames: 120 },
|
||||
} as const;
|
||||
@@ -1,61 +0,0 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { BadgeBar } from "../components/BadgeBar";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene1Hook: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const captionOpacity = interpolate(frame, [20, 35, 75, 90], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const captionLift = spring({
|
||||
frame: frame - 20,
|
||||
fps,
|
||||
config: { damping: 15, stiffness: 70 },
|
||||
from: 16,
|
||||
to: 0,
|
||||
});
|
||||
|
||||
const badgeFadeOut = interpolate(frame, [70, 90], [1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 40%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 60%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 64,
|
||||
}}
|
||||
>
|
||||
<div style={{ opacity: badgeFadeOut }}>
|
||||
<BadgeBar />
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionOpacity,
|
||||
transform: `translateY(${captionLift}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 96,
|
||||
fontWeight: 600,
|
||||
color: COLORS.fgPrimary,
|
||||
textAlign: "center",
|
||||
letterSpacing: -1.5,
|
||||
lineHeight: 1.1,
|
||||
maxWidth: 1400,
|
||||
}}
|
||||
>
|
||||
What if one search
|
||||
<br />
|
||||
ran <span style={{ color: COLORS.accentCyan }}>3 at once?</span>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -1,100 +0,0 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { TerminalWindow } from "../components/TerminalWindow";
|
||||
import { TypedLine } from "../components/TypedLine";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene2OldWay: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const enter = interpolate(frame, [0, 20], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const slide = interpolate(frame, [0, 20], [40, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Result card fades in after type completes (~70 frames)
|
||||
const resultFade = interpolate(frame, [70, 95], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Caption appears late
|
||||
const captionFade = interpolate(frame, [110, 130, 150], [0, 1, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: 80,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 1400,
|
||||
height: 520,
|
||||
opacity: enter,
|
||||
transform: `translateY(${slide}px)`,
|
||||
}}
|
||||
>
|
||||
<TerminalWindow title="bash">
|
||||
<TypedLine
|
||||
text="/last30days OpenAI"
|
||||
startFrame={20}
|
||||
prefix="$"
|
||||
fontSize={42}
|
||||
/>
|
||||
<div
|
||||
style={{
|
||||
opacity: resultFade,
|
||||
marginTop: 36,
|
||||
padding: "20px 24px",
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderRadius: 12,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 26,
|
||||
color: COLORS.fgPrimary,
|
||||
lineHeight: 1.6,
|
||||
}}
|
||||
>
|
||||
<div style={{ color: COLORS.accentGreen }}>
|
||||
✅ All agents reported back!
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted, marginTop: 6 }}>
|
||||
├─ 🟠 Reddit: 14 threads
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted }}>
|
||||
├─ 🔵 X: 22 posts
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted }}>
|
||||
└─ 🟡 HN: 1 story
|
||||
</div>
|
||||
</div>
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
marginTop: 60,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 38,
|
||||
color: COLORS.fgMuted,
|
||||
}}
|
||||
>
|
||||
The old way: <span style={{ color: COLORS.fgPrimary }}>one topic.</span>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -1,221 +0,0 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { TerminalWindow } from "../components/TerminalWindow";
|
||||
import { TypedLine } from "../components/TypedLine";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const PROGRESS_LINES = [
|
||||
{ source: "Reddit", color: "#ff6a3d" },
|
||||
{ source: "X", color: "#36d6f7" },
|
||||
{ source: "YouTube", color: "#ff5757" },
|
||||
{ source: "TikTok", color: "#5fff9f" },
|
||||
{ source: "Instagram", color: "#ff55a3" },
|
||||
];
|
||||
|
||||
const ENTITIES: { label: string; tag: string; accent: string }[] = [
|
||||
{ label: "OpenAI", tag: "$ /last30days OpenAI", accent: COLORS.accentCyan },
|
||||
{ label: "Anthropic", tag: "$ /last30days Anthropic", accent: COLORS.accentMagenta },
|
||||
{ label: "xAI", tag: "$ /last30days xAI", accent: COLORS.accentAmber },
|
||||
];
|
||||
|
||||
const FanPane: React.FC<{
|
||||
label: string;
|
||||
tag: string;
|
||||
accent: string;
|
||||
panelStart: number;
|
||||
}> = ({ label, tag, accent, panelStart }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
const localFrame = Math.max(0, frame - panelStart);
|
||||
|
||||
const enter = spring({
|
||||
frame: localFrame,
|
||||
fps,
|
||||
config: { damping: 18, stiffness: 80 },
|
||||
from: 0,
|
||||
to: 1,
|
||||
});
|
||||
const slide = interpolate(localFrame, [0, 20], [40, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
opacity: enter,
|
||||
transform: `translateY(${slide}px)`,
|
||||
height: 480,
|
||||
}}
|
||||
>
|
||||
<TerminalWindow title={label} glow>
|
||||
<div
|
||||
style={{
|
||||
color: accent,
|
||||
fontSize: 18,
|
||||
fontFamily: FONT_MONO,
|
||||
marginBottom: 14,
|
||||
}}
|
||||
>
|
||||
{tag}
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
fontSize: 22,
|
||||
color: COLORS.accentGreen,
|
||||
fontFamily: FONT_MONO,
|
||||
marginBottom: 12,
|
||||
}}
|
||||
>
|
||||
[Competitors] running...
|
||||
</div>
|
||||
{PROGRESS_LINES.map((line, idx) => {
|
||||
const lineStart = panelStart + 16 + idx * 6;
|
||||
const lineFade = interpolate(
|
||||
frame - lineStart,
|
||||
[0, 8],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
// pulse the in-progress dot
|
||||
const dotOn = Math.floor((frame - lineStart) / 6) % 2 === 0;
|
||||
return (
|
||||
<div
|
||||
key={line.source}
|
||||
style={{
|
||||
opacity: lineFade,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 20,
|
||||
color: COLORS.fgMuted,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
marginBottom: 6,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
display: "inline-block",
|
||||
width: 10,
|
||||
height: 10,
|
||||
borderRadius: 10,
|
||||
background: dotOn ? line.color : COLORS.bgPanelSoft,
|
||||
marginRight: 12,
|
||||
boxShadow: dotOn ? `0 0 10px ${line.color}` : "none",
|
||||
}}
|
||||
/>
|
||||
<span style={{ color: line.color, marginRight: 8 }}>
|
||||
►
|
||||
</span>
|
||||
<span>{line.source}</span>
|
||||
<span style={{ marginLeft: "auto", color: COLORS.fgDim }}>
|
||||
{dotOn ? "..." : "·"}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const Scene3FanOut: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
// Phase 1 (0-30 frames): single terminal types --competitors flag
|
||||
// Phase 2 (30+): split into 3 panes
|
||||
const splitProgress = interpolate(frame, [30, 50], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const singleOpacity = interpolate(frame, [0, 8, 30, 45], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Caption fades in shortly after panes settle so it has time to read.
|
||||
const captionFade = interpolate(frame, [60, 80], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: 60,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
textAlign: "center",
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 42,
|
||||
color: COLORS.fgPrimary,
|
||||
marginBottom: 32,
|
||||
}}
|
||||
>
|
||||
Now it discovers competitors
|
||||
<br />
|
||||
<span style={{ color: COLORS.accentCyan }}>and runs all 3.</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
position: "relative",
|
||||
}}
|
||||
>
|
||||
{/* Single terminal during phase 1, fades out as panes appear */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: 0,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
opacity: singleOpacity,
|
||||
}}
|
||||
>
|
||||
<div style={{ width: 1200, height: 380 }}>
|
||||
<TerminalWindow title="bash">
|
||||
<TypedLine
|
||||
text="/last30days OpenAI --competitors"
|
||||
startFrame={0}
|
||||
prefix="$"
|
||||
fontSize={42}
|
||||
/>
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Three panes fade in starting frame ~30 */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: 0,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 24,
|
||||
opacity: splitProgress,
|
||||
}}
|
||||
>
|
||||
{ENTITIES.map((entity, idx) => (
|
||||
<FanPane
|
||||
key={entity.label}
|
||||
label={entity.label}
|
||||
tag={entity.tag}
|
||||
accent={entity.accent}
|
||||
panelStart={45 + idx * 8}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -1,105 +0,0 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { ComparisonTable } from "../components/ComparisonTable";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const ROWS = [
|
||||
{
|
||||
dimension: "What it is",
|
||||
cells: [
|
||||
"GPT-5 leader, Plus + API",
|
||||
"Claude 4, safety-first",
|
||||
"Grok, X-native, fast",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "30-day momentum",
|
||||
cells: [
|
||||
"GPT-5 launch wave",
|
||||
"Claude 4.7 1M context",
|
||||
"Grok 5 reveal",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "Community vibe",
|
||||
cells: [
|
||||
"Defensive but deep",
|
||||
"Quiet, devs-only",
|
||||
"Loud, meme-rich",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "Best for",
|
||||
cells: [
|
||||
"Mainstream + tools",
|
||||
"Long-context coding",
|
||||
"Live X intel",
|
||||
] as [string, string, string],
|
||||
},
|
||||
];
|
||||
|
||||
export const Scene4Comparison: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const captionFade = interpolate(frame, [0, 12, 180, 210], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const captionLift = interpolate(frame, [0, 14], [16, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const tableFade = interpolate(frame, [16, 32], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: "48px 80px",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
transform: `translateY(${captionLift}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 38,
|
||||
color: COLORS.fgMuted,
|
||||
textAlign: "center",
|
||||
marginBottom: 36,
|
||||
}}
|
||||
>
|
||||
<span style={{ color: COLORS.accentCyan, fontWeight: 600 }}>
|
||||
3 full passes.
|
||||
</span>
|
||||
<span style={{ marginLeft: 18, color: COLORS.accentMagenta, fontWeight: 600 }}>
|
||||
3 save files.
|
||||
</span>
|
||||
<span style={{ marginLeft: 18, color: COLORS.fgPrimary, fontWeight: 600 }}>
|
||||
1 comparison.
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
maxWidth: 1640,
|
||||
opacity: tableFade,
|
||||
}}
|
||||
>
|
||||
<ComparisonTable
|
||||
startFrame={20}
|
||||
entities={["OpenAI", "Anthropic", "xAI"]}
|
||||
rows={ROWS}
|
||||
rowStaggerFrames={22}
|
||||
/>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -1,54 +0,0 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const LINES = [
|
||||
{ text: "You pick the topic.", color: COLORS.fgPrimary },
|
||||
{ text: "The agent picks the peers.", color: COLORS.accentCyan },
|
||||
{ text: "The engine fans out.", color: COLORS.accentMagenta },
|
||||
];
|
||||
|
||||
export const Scene5HowItWorks: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 60%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 48,
|
||||
}}
|
||||
>
|
||||
{LINES.map((line, idx) => {
|
||||
const start = 10 + idx * 28;
|
||||
const fade = interpolate(frame, [start, start + 14], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const slide = interpolate(frame, [start, start + 18], [24, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
return (
|
||||
<div
|
||||
key={line.text}
|
||||
style={{
|
||||
opacity: fade,
|
||||
transform: `translateY(${slide}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 78,
|
||||
fontWeight: 600,
|
||||
color: line.color,
|
||||
letterSpacing: -1,
|
||||
}}
|
||||
>
|
||||
{line.text}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -1,73 +0,0 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { BadgeBar } from "../components/BadgeBar";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene6CTA: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const installFade = interpolate(frame, [20, 40], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
// Stronger 1Hz pulse on the install line for end-of-video emphasis
|
||||
const pulse = 0.8 + 0.2 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||
const glowPulse = 0.4 + 0.4 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||
|
||||
const repoFade = interpolate(frame, [50, 70], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const enterScale = spring({
|
||||
frame,
|
||||
fps,
|
||||
config: { damping: 18, stiffness: 90 },
|
||||
from: 0.95,
|
||||
to: 1,
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 50%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 56,
|
||||
transform: `scale(${enterScale})`,
|
||||
}}
|
||||
>
|
||||
<BadgeBar />
|
||||
<div
|
||||
style={{
|
||||
opacity: installFade * pulse,
|
||||
padding: "18px 36px",
|
||||
background: COLORS.bgPanel,
|
||||
border: `1px solid ${COLORS.accentCyan}`,
|
||||
borderRadius: 14,
|
||||
boxShadow: `0 0 ${40 + glowPulse * 60}px ${COLORS.accentCyan}${Math.round(40 + glowPulse * 80).toString(16)}`,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 44,
|
||||
color: COLORS.fgPrimary,
|
||||
}}
|
||||
>
|
||||
<span style={{ color: COLORS.accentGreen, marginRight: 18 }}>$</span>
|
||||
/last30days <span style={{ color: COLORS.fgMuted }}>{"{topic}"}</span>{" "}
|
||||
<span style={{ color: COLORS.accentCyan }}>--competitors</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
opacity: repoFade,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 28,
|
||||
color: COLORS.fgMuted,
|
||||
}}
|
||||
>
|
||||
github.com/mvanhorn/last30days-skill
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
+4
-5
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.0.0"
|
||||
version = "3.2.0"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -24,9 +24,9 @@ addopts = [
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["scripts", "tests"]
|
||||
source = ["skills/last30days/scripts", "tests"]
|
||||
omit = [
|
||||
"scripts/lib/vendor/*",
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
@@ -34,7 +34,6 @@ omit = [
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
omit = [
|
||||
"scripts/lib/vendor/*",
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ OpenClaw:
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
OpenAI Codex CLI: run `codex` from a checkout of this repo and v3's skill at `.agents/skills/last30days/SKILL.md` will be discovered automatically. Or copy `SKILL.md` to `~/.agents/skills/last30days/SKILL.md` for a global install.
|
||||
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`.
|
||||
|
||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: last30days
|
||||
version: "3.0.1"
|
||||
version: "3.2.0"
|
||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
@@ -45,6 +45,7 @@ metadata:
|
||||
- instagram
|
||||
- hackernews
|
||||
- polymarket
|
||||
- digg
|
||||
- bluesky
|
||||
- truthsocial
|
||||
- trends
|
||||
@@ -104,7 +105,7 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
```
|
||||
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/../../.codex-plugin/plugin.json" 2>/dev/null || jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
|
||||
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
||||
|
||||
@@ -233,7 +234,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
||||
|
||||
---
|
||||
|
||||
# last30days v3.0.1: Research Any Topic from the Last 30 Days
|
||||
# last30days v3.2.0: Research Any Topic from the Last 30 Days
|
||||
|
||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||
|
||||
@@ -317,6 +318,7 @@ Common patterns:
|
||||
|
||||
- Always active: Reddit, Hacker News, Polymarket
|
||||
- If gh CLI is installed (check `which gh`): add GitHub
|
||||
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg AI 1000
|
||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
||||
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
||||
@@ -824,7 +826,7 @@ Only show lines for platforms where something was resolved. Skip empty lines. On
|
||||
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
||||
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
||||
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key)
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key), digg (Digg AI 1000 clusters - only if `digg-pp-cli` is on PATH)
|
||||
|
||||
**Intent → freshness_mode mapping:**
|
||||
- breaking_news, prediction → `strict_recent`
|
||||
@@ -867,26 +869,36 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
||||
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||
|
||||
```bash
|
||||
# PIN SKILL_ROOT to the public plugin cache (highest-version dir wins on upgrade).
|
||||
# DO NOT write your own path-discovery loop. The 2026-04-18 Peter Steinberger run 1
|
||||
# regression was caused by a custom discovery loop landing on ~/.openclaw/skills/last30days/
|
||||
# (a stale copy from a private-repo sync pattern). That path contains a pre-plan-007
|
||||
# engine and produces non-canonical output. This pinned resolution ignores every stale
|
||||
# copy (~/.openclaw/, ~/.agents/, ~/.codex/) and picks the plugin cache exclusively.
|
||||
SKILL_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
|
||||
# PIN SKILL_ROOT to an installed plugin cache first (highest-version dir wins on upgrade).
|
||||
# Prefer Codex's skill package path when installed as a Codex plugin. Keep the Claude
|
||||
# plugin-root fallback for other hosts, then fall back to a repo checkout.
|
||||
SKILL_ROOT="$(ls -d "$HOME/.codex/plugins/cache/"*/last30days/*/skills/last30days/ 2>/dev/null | sort -V | tail -1)"
|
||||
SKILL_ROOT="${SKILL_ROOT%/}"
|
||||
|
||||
# Fallback for repo checkout / Gemini / Codex hosts where the plugin cache does not exist.
|
||||
# Only runs if the public plugin cache is missing entirely.
|
||||
# Fallback for Claude plugin cache.
|
||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
|
||||
CLAUDE_PLUGIN_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
|
||||
CLAUDE_PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT%/}"
|
||||
if [ -n "$CLAUDE_PLUGIN_ROOT" ]; then
|
||||
if [ -f "$CLAUDE_PLUGIN_ROOT/skills/last30days/scripts/last30days.py" ]; then
|
||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT/skills/last30days"
|
||||
elif [ -f "$CLAUDE_PLUGIN_ROOT/scripts/last30days.py" ]; then
|
||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Fallback for repo checkout / Gemini / local development hosts where the plugin cache does not exist.
|
||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
for dir in "." "./skills/last30days" "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
echo "ERROR: Could not find scripts/last30days.py in public plugin cache or repo checkout" >&2
|
||||
echo "Expected: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/scripts/last30days.py" >&2
|
||||
echo "ERROR: Could not find scripts/last30days.py in Codex/Claude plugin cache or repo checkout" >&2
|
||||
echo "Expected Codex: $HOME/.codex/plugins/cache/{MARKETPLACE}/last30days/{VERSION}/skills/last30days/scripts/last30days.py" >&2
|
||||
echo "Expected Claude: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/skills/last30days/scripts/last30days.py" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -1497,6 +1509,33 @@ Close with `I have all the links to the {N} {source list} I pulled from. Just as
|
||||
|
||||
---
|
||||
|
||||
## SHAREABLE HTML BRIEF (when the user asked for one)
|
||||
|
||||
**This section fires if EITHER trigger is true:**
|
||||
|
||||
- `$ARGUMENTS` contains `--emit=html`, `--emit:html`, or `--html` as a flag
|
||||
- The user's natural-language request asks for an HTML brief, shareable doc, or file for sharing (Slack, email, Notion, "export as HTML", etc). Use your judgment for phrasing variants.
|
||||
|
||||
**If neither trigger fires, skip this entire section and proceed to WAIT FOR USER'S RESPONSE.** No HTML save flow, no reference read needed.
|
||||
|
||||
**When triggered, you MUST:**
|
||||
|
||||
- Read `references/save-html-brief.md` BEFORE proceeding to WAIT FOR USER'S RESPONSE
|
||||
- Follow that file's instructions exactly - it is the canonical source for the save flow
|
||||
- Append the confirmation line (`📎 Shareable brief saved to <path>`) to your already-emitted chat response
|
||||
|
||||
**You MUST NOT:**
|
||||
|
||||
- Improvise the HTML save flow from memory or from instructions you've seen before
|
||||
- Skip the reference read because the steps "look familiar"
|
||||
- Save to a different path than the reference specifies
|
||||
- Add data quality warnings, debug headers, or safety notes to the saved HTML
|
||||
- Re-research the topic for the HTML render - the engine cache covers the second invocation
|
||||
|
||||
**Why the directive is forceful:** the reference file is the only source of truth for the save flow. Skipping it produces broken artifacts - wrong path conventions, missing synthesis content, leaked engine debug output, or warnings that don't belong in shareable docs.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S RESPONSE
|
||||
|
||||
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.
|
||||
|
Before Width: | Height: | Size: 2.7 MiB After Width: | Height: | Size: 2.7 MiB |
|
Before Width: | Height: | Size: 2.3 MiB After Width: | Height: | Size: 2.3 MiB |
|
Before Width: | Height: | Size: 3.8 MiB After Width: | Height: | Size: 3.8 MiB |
|
Before Width: | Height: | Size: 2.6 MiB After Width: | Height: | Size: 2.6 MiB |
@@ -0,0 +1,90 @@
|
||||
# Save shareable HTML brief
|
||||
|
||||
This reference file is loaded by the main `SKILL.md` when the user asked for an HTML brief (either explicitly via `--emit=html` / `--emit:html` / `--html`, or in natural language - "give me a shareable HTML brief", "for Slack", "for Notion", "export as HTML", etc.). The detection happens in `SKILL.md` so that the common no-HTML path stays short; the implementation lives here.
|
||||
|
||||
The contract: the synthesis still appears in chat as the primary output. The HTML is an additional artifact saved to disk for sharing. Both happen in the same turn.
|
||||
|
||||
## When to fire this flow
|
||||
|
||||
- After you have already emitted the full chat response: badge, "What I learned:" (or comparison title), bold-lead-in paragraphs with citations, KEY PATTERNS list, engine footer pass-through, invitation block.
|
||||
- BEFORE the WAIT FOR USER'S RESPONSE pause.
|
||||
- ONLY if the user asked. Do NOT save HTML when the user didn't ask for it.
|
||||
|
||||
## How to fire it
|
||||
|
||||
```bash
|
||||
# 1. Write your synthesis prose VERBATIM to a temp file. The synthesis is the
|
||||
# "What I learned:" prose label, the bold-lead-in paragraphs with their
|
||||
# inline citations as you wrote them in chat, and the "KEY PATTERNS from
|
||||
# the research:" numbered list. Do NOT include the badge or the engine
|
||||
# footer in the temp file - the engine adds those when it renders the HTML.
|
||||
# Use the EXACT text you just wrote in chat. Do not paraphrase, do not
|
||||
# summarize, do not reorder. The HTML must read identically to the chat
|
||||
# response in voice and citations.
|
||||
SYNTHESIS_FILE="/tmp/last30days-synthesis-${CLAUDE_SESSION_ID}.md"
|
||||
cat > "$SYNTHESIS_FILE" <<'SYNTHESIS_EOF'
|
||||
What I learned:
|
||||
|
||||
**{First headline}** - {body with [name](url) inline citations}
|
||||
|
||||
**{Second headline}** - {body}
|
||||
|
||||
**{Third headline}** - {body}
|
||||
|
||||
KEY PATTERNS from the research:
|
||||
1. {pattern} - per [@handle](url)
|
||||
2. {pattern} - per [r/sub](url)
|
||||
3. {pattern} - per [@handle](url)
|
||||
SYNTHESIS_EOF
|
||||
|
||||
# 2. Convert the synthesis to a self-contained HTML file via the engine.
|
||||
# The engine reuses the cache from your earlier engine run (same topic
|
||||
# + plan), so this second invocation is typically <1s on cache hit.
|
||||
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/^-//;s/-$//')
|
||||
HTML_PATH="${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html"
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "${TOPIC}" \
|
||||
--emit=html \
|
||||
--synthesis-file "$SYNTHESIS_FILE" \
|
||||
> "$HTML_PATH"
|
||||
|
||||
# 3. Append ONE line to your already-emitted chat response, after the
|
||||
# invitation block. Use a paperclip emoji as a visible signal that an
|
||||
# artifact was produced:
|
||||
echo "📎 Shareable brief saved to $HTML_PATH"
|
||||
```
|
||||
|
||||
## What ends up in the HTML file
|
||||
|
||||
The engine's `--emit=html` renderer combines:
|
||||
|
||||
- The badge (`🌐 last30days vX.Y.Z · synced YYYY-MM-DD`) at the top
|
||||
- A single inline metadata line (`{date range} · {active sources}`) below the badge
|
||||
- Your synthesis verbatim, with prose labels promoted to `<h2>` and bold lead-ins preserved
|
||||
- All `[name](url)` citations rendered as `<a>` tags
|
||||
- The engine footer (`✅ All agents reported back!` tree) preserved verbatim in monospace
|
||||
- A colophon with the topic and a re-run hint
|
||||
|
||||
The renderer strips engine-internal noise that doesn't belong in a shareable artifact: the `# last30days vX.Y.Z: TOPIC` debug file header, the model-facing `> Safety note:` blockquote, and the `I'm now an expert on X` invitation block. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs - they never leak into the share-ready file.
|
||||
|
||||
## Comparison mode
|
||||
|
||||
Same flow when the topic is `X vs Y` (or `X vs Y vs Z`). The engine routes through `render_for_html_comparison` internally; you don't need to do anything special. The synthesis temp file should still contain the comparison-shaped synthesis you wrote in chat (`## Quick Verdict`, `## {Entity}` per entity, `## Head-to-Head` table, `## The Bottom Line`, `## The emerging stack` per LAW 4 comparison exception).
|
||||
|
||||
## Follow-up turn
|
||||
|
||||
If the user runs `/last30days OpenClaw` normally, sees the synthesis in chat, and THEN says "save that as HTML" or "give me a shareable version" in a follow-up turn, do the same save flow on the synthesis you wrote in the previous turn. Do not re-research; the synthesis is already in the conversation history. Just write it to the temp file and call the engine with `--emit=html --synthesis-file`.
|
||||
|
||||
## What NOT to do
|
||||
|
||||
- Do NOT save HTML if the user didn't ask. The sparse mode (no synthesis) produces a thin file; not useful as a shareable.
|
||||
- Do NOT add content to the temp file beyond your synthesis prose. The badge / footer / colophon come from the engine.
|
||||
- Do NOT change the file path convention. `${LAST30DAYS_MEMORY_DIR}/${SLUG}-brief.html` is the canonical location.
|
||||
- Do NOT silently overwrite an existing file without telling the user. If `$HTML_PATH` already exists from a prior run, the engine will pick a date-suffixed name (`{slug}-brief-YYYY-MM-DD.html`) automatically; just print whichever path the redirect produced.
|
||||
- Do NOT include the data quality warning text in the temp file or in your final chat line. Warnings are an engine-stderr concern, not an artifact concern.
|
||||
|
||||
## Edge cases
|
||||
|
||||
- **Topic with shell-special characters** (quotes, ampersands): the temp filename uses a slugified version, but the engine receives the raw topic. The `cat <<'SYNTHESIS_EOF'` quoted heredoc form handles arbitrary content without expansion. Your synthesis text can include any character.
|
||||
- **Very long synthesis**: no upper bound. The engine handles long markdown bodies. Just paste verbatim.
|
||||
- **Synthesis with images or non-ASCII**: emoji and Unicode pass through. Image tags pass through as raw HTML; the renderer doesn't transform them. If you didn't include images in chat, don't add them here.
|
||||
- **No `${LAST30DAYS_MEMORY_DIR}` set**: defaults to `~/Documents/Last30Days/` per the SKILL.md `Configuration` section.
|
||||
@@ -1,13 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
||||
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root)
|
||||
#
|
||||
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
||||
# directory containing SKILL.md and the scripts/ runtime. See
|
||||
# directory containing SKILL.md and the scripts/ runtime from skills/last30days.
|
||||
# See
|
||||
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
@@ -17,14 +18,7 @@ fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/last30days.skill"
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD
|
||||
|
||||
# claude.ai's .skill bundle only needs the root SKILL.md + scripts/ runtime.
|
||||
# Claude Code needs skills/ and .claude-plugin/ in the git archive
|
||||
# (that's why they're NOT in .gitattributes export-ignore), but the .skill
|
||||
# bundle must strip them to keep a single canonical SKILL.md and stay under
|
||||
# the 200-file cap.
|
||||
zip -d "$OUT" "last30days/skills/*" "last30days/.claude-plugin/*" > /dev/null 2>&1 || true
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# A/B test runner: public release vs private beta
|
||||
# Usage: bash scripts/compare.sh "Kanye West"
|
||||
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West"
|
||||
#
|
||||
# Runs /last30days (public release) and /last30days-beta (private beta)
|
||||
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
||||
@@ -9,8 +9,8 @@
|
||||
set -e
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: bash scripts/compare.sh <topic>"
|
||||
echo " Example: bash scripts/compare.sh Kevin Rose"
|
||||
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>"
|
||||
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose"
|
||||
exit 1
|
||||
fi
|
||||
TOPIC="$*"
|
||||
+6
-2
@@ -22,7 +22,8 @@ from lib import env as envlib
|
||||
from lib import schema
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
||||
|
||||
|
||||
@@ -307,7 +308,10 @@ def create_eval_env() -> dict[str, str]:
|
||||
|
||||
|
||||
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
||||
cmd = [sys.executable, "scripts/last30days.py", topic, "--emit=json"]
|
||||
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py"
|
||||
if not engine.exists():
|
||||
engine = repo_dir / "scripts" / "last30days.py"
|
||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
||||
if search:
|
||||
cmd.extend(["--search", search])
|
||||
if quick:
|
||||
@@ -41,7 +41,7 @@ if os.name == "nt":
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from lib import env, pipeline, render, schema, ui
|
||||
from lib import env, html_render, pipeline, render, schema, ui
|
||||
|
||||
_child_pids: set[int] = set()
|
||||
_child_pids_lock = threading.Lock()
|
||||
@@ -91,30 +91,46 @@ def slugify(value: str) -> str:
|
||||
return slug or "last30days"
|
||||
|
||||
|
||||
def save_output(report: schema.Report, emit: str, save_dir: str, suffix: str = "") -> Path:
|
||||
def save_output(
|
||||
report: schema.Report,
|
||||
emit: str,
|
||||
save_dir: str,
|
||||
suffix: str = "",
|
||||
synthesis_md: str | None = None,
|
||||
) -> Path:
|
||||
from datetime import datetime
|
||||
path = Path(save_dir).expanduser().resolve()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
slug = slugify(report.topic)
|
||||
extension = "json" if emit == "json" else "md"
|
||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
||||
raw_label = "raw-html" if emit == "html" else "raw"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
out_path = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
out_path = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
||||
if out_path.exists():
|
||||
out_path = path / f"{slug}-raw{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Always save the FULL dump to disk (all items, all sources, transcripts).
|
||||
# Claude sees compact clusters via --emit=compact on stdout.
|
||||
# The saved file is the complete debug artifact.
|
||||
if emit == "json":
|
||||
content = emit_output(report, emit)
|
||||
out_path = path / f"{slug}-{raw_label}{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Markdown saves keep the complete debug artifact. JSON and HTML preserve
|
||||
# their requested wire format so file extensions match their content.
|
||||
if emit in {"json", "html"}:
|
||||
content = emit_output(report, emit, synthesis_md=synthesis_md)
|
||||
else:
|
||||
content = render.render_full(report)
|
||||
out_path.write_text(content, encoding="utf-8")
|
||||
return out_path
|
||||
|
||||
|
||||
def emit_output(report: schema.Report, emit: str, fun_level: str = "medium", save_path: str | None = None) -> str:
|
||||
def emit_output(
|
||||
report: schema.Report,
|
||||
emit: str,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
if emit == "json":
|
||||
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
|
||||
if emit == "html":
|
||||
return html_render.render_html(
|
||||
report, fun_level=fun_level, save_path=save_path, synthesis_md=synthesis_md,
|
||||
)
|
||||
if emit in {"compact", "md"}:
|
||||
return render.render_compact(report, fun_level=fun_level, save_path=save_path)
|
||||
if emit == "context":
|
||||
@@ -127,6 +143,7 @@ def emit_comparison_output(
|
||||
emit: str,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
if emit == "json":
|
||||
payload = {
|
||||
@@ -138,6 +155,13 @@ def emit_comparison_output(
|
||||
],
|
||||
}
|
||||
return json.dumps(payload, indent=2, sort_keys=True)
|
||||
if emit == "html":
|
||||
return html_render.render_html_comparison(
|
||||
entity_reports,
|
||||
fun_level=fun_level,
|
||||
save_path=save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
if emit in {"compact", "md"}:
|
||||
return render.render_comparison_multi(
|
||||
entity_reports, fun_level=fun_level, save_path=save_path,
|
||||
@@ -156,9 +180,10 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
||||
from pathlib import Path as _Path
|
||||
path = _Path(save_dir).expanduser().resolve()
|
||||
slug = slugify(topic)
|
||||
extension = "json" if emit == "json" else "md"
|
||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
||||
raw_label = "raw-html" if emit == "html" else "raw"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
raw = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
raw = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
||||
try:
|
||||
home = _Path.home().resolve()
|
||||
relative = raw.relative_to(home)
|
||||
@@ -167,6 +192,14 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
||||
return str(raw)
|
||||
|
||||
|
||||
def read_synthesis_file(path: str) -> str:
|
||||
try:
|
||||
return Path(path).expanduser().read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
sys.stderr.write(f"[last30days] Cannot read --synthesis-file: {exc}\n")
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
def persist_report(report: schema.Report) -> dict[str, int]:
|
||||
import store
|
||||
|
||||
@@ -193,7 +226,7 @@ def persist_report(report: schema.Report) -> dict[str, int]:
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
|
||||
parser.add_argument("topic", nargs="*", help="Research topic")
|
||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md"])
|
||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md", "html"])
|
||||
parser.add_argument("--search", help="Comma-separated source list")
|
||||
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
|
||||
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
|
||||
@@ -201,6 +234,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
||||
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
||||
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
||||
parser.add_argument("--synthesis-file", help="Markdown synthesis to embed in --emit=html output")
|
||||
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
|
||||
parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
|
||||
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
|
||||
@@ -537,6 +571,13 @@ def main() -> int:
|
||||
parser.print_usage(sys.stderr)
|
||||
return 2
|
||||
|
||||
synthesis_md = None
|
||||
if args.synthesis_file:
|
||||
if args.emit == "html":
|
||||
synthesis_md = read_synthesis_file(args.synthesis_file)
|
||||
else:
|
||||
sys.stderr.write("[last30days] Warning: --synthesis-file is only used with --emit=html; ignoring.\n")
|
||||
|
||||
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
|
||||
from lib import preflight
|
||||
refuse_msg = preflight.check_class_1_trap(topic)
|
||||
@@ -854,15 +895,29 @@ def main() -> int:
|
||||
|
||||
if entity_reports:
|
||||
rendered = emit_comparison_output(
|
||||
entity_reports, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||
entity_reports,
|
||||
args.emit,
|
||||
fun_level=fun_level,
|
||||
save_path=footer_save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
else:
|
||||
rendered = emit_output(
|
||||
report, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||
report,
|
||||
args.emit,
|
||||
fun_level=fun_level,
|
||||
save_path=footer_save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
if args.save_dir:
|
||||
# Save the main topic's raw file (single-entity or comparison main).
|
||||
save_path = save_output(report, args.emit, args.save_dir, suffix=args.save_suffix or "")
|
||||
save_path = save_output(
|
||||
report,
|
||||
args.emit,
|
||||
args.save_dir,
|
||||
suffix=args.save_suffix or "",
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
||||
# Competitor / vs-mode: also save a per-entity raw file for each peer.
|
||||
# Matches historical vs-mode behavior (N passes → N save files).
|
||||
@@ -871,6 +926,7 @@ def main() -> int:
|
||||
peer_path = save_output(
|
||||
entity_report, args.emit, args.save_dir,
|
||||
suffix=args.save_suffix or "",
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
sys.stderr.write(f"[last30days] Saved output to {peer_path}\n")
|
||||
sys.stderr.flush()
|
||||
@@ -7,13 +7,11 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log
|
||||
from . import http, log, subproc
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -168,62 +166,51 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"--json",
|
||||
]
|
||||
|
||||
# Use process groups for clean cleanup on timeout/kill
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
pid_holder: list[int] = []
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
# Register for cleanup tracking (if available)
|
||||
def _register(pid: int) -> None:
|
||||
pid_holder.append(pid)
|
||||
try:
|
||||
from last30days import register_child_pid, unregister_child_pid
|
||||
register_child_pid(proc.pid)
|
||||
from last30days import register_child_pid
|
||||
register_child_pid(pid)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
# Kill the entire process group
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
finally:
|
||||
try:
|
||||
result = subproc.run_with_timeout(
|
||||
cmd,
|
||||
timeout=timeout,
|
||||
env=_subprocess_env(),
|
||||
on_pid=_register,
|
||||
)
|
||||
except subproc.SubprocTimeout:
|
||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "items": []}
|
||||
finally:
|
||||
if pid_holder:
|
||||
try:
|
||||
from last30days import unregister_child_pid
|
||||
unregister_child_pid(proc.pid)
|
||||
unregister_child_pid(pid_holder[0])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if proc.returncode != 0:
|
||||
error = stderr.strip() if stderr else "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
output = stdout.strip() if stdout else ""
|
||||
if not output:
|
||||
return {"items": []}
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "items": []}
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
|
||||
def search_x(
|
||||
@@ -330,47 +317,29 @@ def search_handles(
|
||||
"--json",
|
||||
]
|
||||
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
||||
except subproc.SubprocTimeout:
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
except OSError as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
if result.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return []
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
|
||||
if proc.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||
return []
|
||||
|
||||
output = (stdout or "").strip()
|
||||
if not output:
|
||||
return []
|
||||
|
||||
response = json.loads(output)
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
return []
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
@@ -39,11 +39,14 @@ def normalize_text(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
|
||||
if len(norm) < n:
|
||||
return {norm} if norm else set()
|
||||
return {norm[index:index + n] for index in range(len(norm) - n + 1)}
|
||||
|
||||
|
||||
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
||||
text = normalize_text(text)
|
||||
if len(text) < n:
|
||||
return {text} if text else set()
|
||||
return {text[index:index + n] for index in range(len(text) - n + 1)}
|
||||
return _ngrams_of_normalized(normalize_text(text), n)
|
||||
|
||||
|
||||
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
||||
@@ -90,7 +93,7 @@ class _PreparedText:
|
||||
|
||||
def __init__(self, raw: str) -> None:
|
||||
norm = normalize_text(raw)
|
||||
self.ngrams = get_ngrams(norm) if norm else set()
|
||||
self.ngrams = _ngrams_of_normalized(norm)
|
||||
self.tokens = _tokenize(norm)
|
||||
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
"""Digg AI 1000 source for last30days.
|
||||
|
||||
Shells out to ``digg-pp-cli`` (read-only, no auth required) to surface
|
||||
clustered stories curated from ~1000 high-signal AI accounts on X. Each
|
||||
cluster carries a published TLDR, a curatorial rank, and a list of X
|
||||
posts that can be fetched as inline quotes.
|
||||
|
||||
Activation gate: this source is only available when ``digg-pp-cli`` is
|
||||
on PATH. ``pipeline.available_sources`` checks ``shutil.which`` before
|
||||
including ``digg`` in the source list. The functions below also detect
|
||||
the missing-binary case as a defensive fallback.
|
||||
|
||||
Primary path: ``digg-pp-cli search <topic> --since 30d --agent --limit N``.
|
||||
Optional enrichment: ``digg-pp-cli posts <clusterUrlId> --agent --by rank
|
||||
--limit M`` for the top K clusters in default/deep depth, attaching the
|
||||
top-ranked X posts to each cluster's ``posts`` field.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import log, subproc
|
||||
from .relevance import token_overlap_relevance
|
||||
|
||||
|
||||
CLI_BIN = "digg-pp-cli"
|
||||
|
||||
# Per-depth knobs.
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 8,
|
||||
"default": 20,
|
||||
"deep": 40,
|
||||
}
|
||||
|
||||
# How many top-ranked clusters get post enrichment, per depth. Quick mode
|
||||
# skips enrichment to keep latency low (clusters already carry a TLDR).
|
||||
ENRICH_CONFIG = {
|
||||
"quick": 0,
|
||||
"default": 3,
|
||||
"deep": 5,
|
||||
}
|
||||
|
||||
# X posts pulled per enriched cluster.
|
||||
POSTS_PER_CLUSTER = 3
|
||||
|
||||
SEARCH_TIMEOUT = 30
|
||||
POSTS_TIMEOUT = 15
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
log.source_log("Digg", msg)
|
||||
|
||||
|
||||
def _is_available() -> bool:
|
||||
"""True when the digg-pp-cli binary is on PATH."""
|
||||
return shutil.which(CLI_BIN) is not None
|
||||
|
||||
|
||||
def _today() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _parse_first_post_age(age: Optional[str], today: Optional[datetime] = None) -> Optional[str]:
|
||||
"""Convert a digg firstPostAge token (e.g. '5d', '17d', '5h', '1w', '1m')
|
||||
into a YYYY-MM-DD string. Returns None when the value is outside the
|
||||
last-30-day window or cannot be parsed.
|
||||
|
||||
Digg uses minutes-symbol-collision for 'months' (per agent-context:
|
||||
'Nh, Nd, Nw, Nm (e.g. 30d, 1w, 12h, 1m)'), so 'Nm' is months ~30 days.
|
||||
"""
|
||||
if not age or not isinstance(age, str):
|
||||
return None
|
||||
age = age.strip().lower()
|
||||
if len(age) < 2:
|
||||
return None
|
||||
unit = age[-1]
|
||||
try:
|
||||
amount = int(age[:-1])
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if amount < 0:
|
||||
return None
|
||||
|
||||
base = today or _today()
|
||||
|
||||
if unit == "h":
|
||||
delta = timedelta(hours=amount)
|
||||
elif unit == "d":
|
||||
delta = timedelta(days=amount)
|
||||
elif unit == "w":
|
||||
delta = timedelta(weeks=amount)
|
||||
elif unit == "m":
|
||||
delta = timedelta(days=amount * 30)
|
||||
else:
|
||||
return None
|
||||
|
||||
if delta > timedelta(days=30):
|
||||
return None
|
||||
|
||||
point = base - delta
|
||||
return point.date().isoformat()
|
||||
|
||||
|
||||
def _build_search_args(query: str, limit: int) -> List[str]:
|
||||
return [
|
||||
CLI_BIN,
|
||||
"search",
|
||||
query,
|
||||
"--since",
|
||||
"30d",
|
||||
"--agent",
|
||||
"--limit",
|
||||
str(limit),
|
||||
]
|
||||
|
||||
|
||||
def _build_posts_args(cluster_url_id: str, posts_per: int) -> List[str]:
|
||||
return [
|
||||
CLI_BIN,
|
||||
"posts",
|
||||
cluster_url_id,
|
||||
"--agent",
|
||||
"--by",
|
||||
"rank",
|
||||
"--limit",
|
||||
str(posts_per),
|
||||
]
|
||||
|
||||
|
||||
def _run_cli(cmd: List[str], timeout: int) -> Dict[str, Any]:
|
||||
"""Invoke digg-pp-cli and parse the JSON envelope.
|
||||
|
||||
Returns ``{"results": [...]}`` on success, ``{"results": [], "error": "..."}``
|
||||
on failure. Never raises; the pipeline relies on shape consistency.
|
||||
"""
|
||||
if not _is_available():
|
||||
return {"results": [], "error": f"{CLI_BIN} not on PATH"}
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=timeout)
|
||||
except subproc.SubprocTimeout as exc:
|
||||
_log(f"Timeout: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
except FileNotFoundError as exc:
|
||||
_log(f"Binary missing: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
except OSError as exc:
|
||||
_log(f"Spawn failed: {exc}")
|
||||
return {"results": [], "error": str(exc)}
|
||||
|
||||
if result.returncode != 0:
|
||||
snippet = (result.stderr or "").strip().splitlines()[:1]
|
||||
first = snippet[0] if snippet else f"exit {result.returncode}"
|
||||
_log(f"CLI exit {result.returncode}: {first}")
|
||||
return {"results": [], "error": first}
|
||||
|
||||
stdout = result.stdout or ""
|
||||
if not stdout.strip():
|
||||
return {"results": []}
|
||||
try:
|
||||
data = json.loads(stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
_log(f"JSON decode failed: {exc}")
|
||||
return {"results": [], "error": f"json decode: {exc}"}
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return {"results": []}
|
||||
results = data.get("results")
|
||||
if not isinstance(results, list):
|
||||
return {"results": []}
|
||||
return data
|
||||
|
||||
|
||||
def search_digg(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search Digg AI 1000 clusters via digg-pp-cli.
|
||||
|
||||
Args:
|
||||
topic: search query.
|
||||
from_date: YYYY-MM-DD start (advisory; --since 30d is the actual filter).
|
||||
to_date: YYYY-MM-DD end (advisory; same).
|
||||
depth: 'quick' | 'default' | 'deep'.
|
||||
|
||||
Returns:
|
||||
Dict with ``results`` list. On failure, ``results`` is empty and an
|
||||
``error`` key carries a one-line description.
|
||||
"""
|
||||
limit = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
if not topic or not topic.strip():
|
||||
return {"results": []}
|
||||
cmd = _build_search_args(topic, limit)
|
||||
_log(f"search '{topic}' (limit={limit}, since=30d)")
|
||||
response = _run_cli(cmd, timeout=SEARCH_TIMEOUT)
|
||||
n = len(response.get("results") or [])
|
||||
_log(f"found {n} clusters")
|
||||
return response
|
||||
|
||||
|
||||
def _build_url(cluster_url_id: str) -> str:
|
||||
return f"https://di.gg/ai/{cluster_url_id}"
|
||||
|
||||
|
||||
def _rank_score(rank: Optional[int]) -> float:
|
||||
"""Convert Digg rank (lower is better, top 50 are notable) into a
|
||||
positive engagement-style signal in [0, 50]. Anything off the top-50
|
||||
leaderboard contributes 0.
|
||||
"""
|
||||
if rank is None:
|
||||
return 0.0
|
||||
try:
|
||||
r = int(rank)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
if r < 1 or r > 50:
|
||||
return 0.0
|
||||
return float(51 - r)
|
||||
|
||||
|
||||
def parse_digg_response(
|
||||
response: Dict[str, Any],
|
||||
query: str = "",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Parse a digg search envelope into normalized item dicts.
|
||||
|
||||
Args:
|
||||
response: payload from ``search_digg``.
|
||||
query: original search query, used for token-overlap relevance.
|
||||
|
||||
Returns:
|
||||
List of dicts ready for ``normalize._normalize_digg``.
|
||||
"""
|
||||
raw = response.get("results") if isinstance(response, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
|
||||
items: List[Dict[str, Any]] = []
|
||||
for i, cluster in enumerate(raw):
|
||||
if not isinstance(cluster, dict):
|
||||
continue
|
||||
cluster_url_id = cluster.get("clusterUrlId")
|
||||
if not cluster_url_id:
|
||||
continue
|
||||
|
||||
title = str(cluster.get("title") or "").strip()
|
||||
tldr = str(cluster.get("tldr") or "").strip()
|
||||
rank = cluster.get("rank")
|
||||
post_count = cluster.get("postCount") or 0
|
||||
unique_authors = cluster.get("uniqueAuthors") or 0
|
||||
first_post_age = cluster.get("firstPostAge")
|
||||
date_str = _parse_first_post_age(first_post_age)
|
||||
if date_str is None and first_post_age:
|
||||
# firstPostAge present but outside 30d -> drop; last30days contract.
|
||||
continue
|
||||
|
||||
rank_decay = max(0.3, 1.0 - (i * 0.02))
|
||||
if query:
|
||||
content_score = token_overlap_relevance(query, f"{title} {tldr}".strip())
|
||||
else:
|
||||
content_score = 0.5
|
||||
rank_boost = min(0.2, _rank_score(rank) / 250.0)
|
||||
relevance = min(1.0, 0.55 * rank_decay + 0.35 * content_score + rank_boost)
|
||||
|
||||
items.append(
|
||||
{
|
||||
"id": str(cluster_url_id),
|
||||
"title": title or f"Digg cluster {i + 1}",
|
||||
"url": _build_url(str(cluster_url_id)),
|
||||
"tldr": tldr,
|
||||
"author": "",
|
||||
"date": date_str,
|
||||
"engagement": {
|
||||
"postCount": int(post_count) if isinstance(post_count, (int, float)) else 0,
|
||||
"uniqueAuthors": int(unique_authors) if isinstance(unique_authors, (int, float)) else 0,
|
||||
"rank": int(rank) if isinstance(rank, (int, float)) else None,
|
||||
"rank_score": _rank_score(rank),
|
||||
},
|
||||
"first_post_age": first_post_age,
|
||||
"posts": [],
|
||||
"relevance": round(relevance, 2),
|
||||
"why_relevant": (
|
||||
f"Digg AI 1000 cluster (rank {rank}, {post_count} posts, {unique_authors} authors)"
|
||||
if rank is not None
|
||||
else f"Digg AI 1000 cluster ({post_count} posts, {unique_authors} authors)"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _parse_post(raw_post: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Reduce a digg post payload into the small dict render uses.
|
||||
|
||||
We deliberately keep this minimal: an inline quote needs the author
|
||||
handle, the body, the post type, and the X URL.
|
||||
"""
|
||||
if not isinstance(raw_post, dict):
|
||||
return None
|
||||
body = str(raw_post.get("body") or "").strip()
|
||||
if not body:
|
||||
return None
|
||||
author = raw_post.get("author") or {}
|
||||
if not isinstance(author, dict):
|
||||
author = {}
|
||||
username = str(author.get("username") or "").strip()
|
||||
if not username:
|
||||
return None
|
||||
x_url = str(raw_post.get("xUrl") or "").strip()
|
||||
if not x_url:
|
||||
return None
|
||||
return {
|
||||
"username": username,
|
||||
"display_name": str(author.get("display_name") or "").strip() or username,
|
||||
"category": str(author.get("category") or "").strip(),
|
||||
"rank": author.get("rank"),
|
||||
"body": body,
|
||||
"post_type": str(raw_post.get("post_type") or "tweet").strip(),
|
||||
"x_url": x_url,
|
||||
"posted_at": raw_post.get("posted_at"),
|
||||
}
|
||||
|
||||
|
||||
def fetch_top_posts(cluster_url_id: str, posts_per: int = POSTS_PER_CLUSTER) -> List[Dict[str, Any]]:
|
||||
"""Fetch top-ranked X posts attached to a cluster.
|
||||
|
||||
Returns an empty list on any failure (timeout, missing cluster, JSON
|
||||
error). Never raises.
|
||||
"""
|
||||
if posts_per <= 0:
|
||||
return []
|
||||
cmd = _build_posts_args(cluster_url_id, posts_per)
|
||||
response = _run_cli(cmd, timeout=POSTS_TIMEOUT)
|
||||
raw = response.get("results") or []
|
||||
out: List[Dict[str, Any]] = []
|
||||
for entry in raw:
|
||||
post = _parse_post(entry)
|
||||
if post is not None:
|
||||
out.append(post)
|
||||
return out
|
||||
|
||||
|
||||
def enrich_with_top_posts(
|
||||
items: List[Dict[str, Any]],
|
||||
top_k: int = 3,
|
||||
posts_per: int = POSTS_PER_CLUSTER,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Attach top X posts to the first ``top_k`` clusters by Digg rank order.
|
||||
|
||||
Mutates and returns the same list. Items that already have posts, or
|
||||
whose ``postCount`` is 0, are skipped.
|
||||
"""
|
||||
if top_k <= 0 or posts_per <= 0:
|
||||
return items
|
||||
enriched = 0
|
||||
for item in items:
|
||||
if enriched >= top_k:
|
||||
break
|
||||
if item.get("posts"):
|
||||
continue
|
||||
engagement = item.get("engagement") or {}
|
||||
if not engagement.get("postCount"):
|
||||
continue
|
||||
cluster_url_id = item.get("id")
|
||||
if not cluster_url_id:
|
||||
continue
|
||||
posts = fetch_top_posts(str(cluster_url_id), posts_per=posts_per)
|
||||
item["posts"] = posts
|
||||
enriched += 1
|
||||
if enriched:
|
||||
_log(f"enriched {enriched} clusters with X posts")
|
||||
return items
|
||||
|
||||
|
||||
def enrich_source_items(items: list, top_k: int = 3, posts_per: int = POSTS_PER_CLUSTER) -> list:
|
||||
"""Attach top X posts to the first ``top_k`` SourceItems that survived dedupe.
|
||||
|
||||
Reads ``metadata['clusterUrlId']`` and writes ``metadata['posts']`` in
|
||||
place. Skips items that already carry a non-empty ``metadata['posts']``,
|
||||
items whose engagement ``postCount`` is 0, and items whose source is not
|
||||
'digg'. Designed to run from `_finalize_items_by_source` so enrichment
|
||||
is spent on the items the brief actually shows.
|
||||
"""
|
||||
if top_k <= 0 or posts_per <= 0:
|
||||
return items
|
||||
enriched = 0
|
||||
for item in items:
|
||||
if enriched >= top_k:
|
||||
break
|
||||
if getattr(item, "source", None) != "digg":
|
||||
continue
|
||||
metadata = getattr(item, "metadata", None) or {}
|
||||
if metadata.get("posts"):
|
||||
continue
|
||||
engagement = getattr(item, "engagement", None) or {}
|
||||
if not engagement.get("postCount"):
|
||||
continue
|
||||
cluster_url_id = metadata.get("clusterUrlId") or item.item_id
|
||||
if not cluster_url_id:
|
||||
continue
|
||||
posts = fetch_top_posts(str(cluster_url_id), posts_per=posts_per)
|
||||
if posts:
|
||||
metadata["posts"] = posts
|
||||
enriched += 1
|
||||
if enriched:
|
||||
_log(f"post-dedupe enriched {enriched} clusters with X posts")
|
||||
return items
|
||||
@@ -372,14 +372,6 @@ def config_exists() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def is_reddit_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Reddit search is available.
|
||||
|
||||
v3 uses ScrapeCreators only.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
|
||||
|
||||
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
||||
"""Determine which Reddit backend to use.
|
||||
|
||||
@@ -116,6 +116,8 @@ def weighted_rrf(
|
||||
"""Fuse ranked lists into a single candidate pool."""
|
||||
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
||||
candidates: dict[str, schema.Candidate] = {}
|
||||
# Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
|
||||
seen_source_items: dict[str, set[tuple[str, str]]] = {}
|
||||
|
||||
for (label, source), items in streams.items():
|
||||
subquery = subqueries[label]
|
||||
@@ -154,6 +156,7 @@ def weighted_rrf(
|
||||
]
|
||||
},
|
||||
)
|
||||
seen_source_items[key] = {(item.source, item.item_id)}
|
||||
continue
|
||||
|
||||
candidate = candidates[key]
|
||||
@@ -179,7 +182,9 @@ def weighted_rrf(
|
||||
candidate.subquery_labels.append(label)
|
||||
if item.source not in candidate.sources:
|
||||
candidate.sources.append(item.source)
|
||||
if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
|
||||
source_item_key = (item.source, item.item_id)
|
||||
if source_item_key not in seen_source_items[key]:
|
||||
seen_source_items[key].add(source_item_key)
|
||||
candidate.source_items.append(item)
|
||||
candidate.metadata.setdefault("provenance", []).append(
|
||||
{
|
||||
@@ -0,0 +1,674 @@
|
||||
"""HTML rendering for shareable last30days reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
from . import render, schema
|
||||
|
||||
|
||||
PROSE_LABELS = [
|
||||
("What I learned:", "What I learned"),
|
||||
("KEY PATTERNS from the research:", "Key patterns from the research"),
|
||||
]
|
||||
|
||||
INVITATION_PATTERN = re.compile(r"^---\nI'm now an expert.*?Just ask\.$", re.MULTILINE | re.DOTALL)
|
||||
EVIDENCE_BLOCK_PATTERN = re.compile(r"<!-- EVIDENCE FOR SYNTHESIS.*?<!-- END EVIDENCE FOR SYNTHESIS -->", re.DOTALL)
|
||||
PASS_THROUGH_FOOTER_PATTERN = re.compile(r"<!-- PASS-THROUGH FOOTER.*?-->\n(.*?)<!-- END PASS-THROUGH FOOTER -->", re.DOTALL)
|
||||
CANONICAL_BOUNDARY_PATTERN = re.compile(r"\n?---\n# END OF last30days CANONICAL OUTPUT.*$", re.DOTALL)
|
||||
# render_for_html emits metadata as <!-- META: ... --> so it survives the
|
||||
# markdown converter (which escapes raw HTML inside paragraphs). Promoted to
|
||||
# a styled <div class="meta"> after conversion.
|
||||
META_MARKER_PATTERN = re.compile(r"<!--\s*META:\s*(.*?)\s*-->")
|
||||
|
||||
CSS = """
|
||||
:root {
|
||||
--bg: #0e0e10;
|
||||
--bg-elev: #18181b;
|
||||
--fg: #fafafa;
|
||||
--fg-muted: #a1a1aa;
|
||||
--fg-subtle: #71717a;
|
||||
--accent: #a855f7;
|
||||
--accent-soft: #c4b5fd;
|
||||
--border: #27272a;
|
||||
--code-bg: #1a1a1d;
|
||||
--max-w: 720px;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root {
|
||||
--bg: #ffffff;
|
||||
--bg-elev: #fafafa;
|
||||
--fg: #18181b;
|
||||
--fg-muted: #52525b;
|
||||
--fg-subtle: #71717a;
|
||||
--accent: #7c3aed;
|
||||
--accent-soft: #6d28d9;
|
||||
--border: #e4e4e7;
|
||||
--code-bg: #f4f4f5;
|
||||
}
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: var(--bg);
|
||||
color: var(--fg);
|
||||
font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif;
|
||||
font-size: 17px;
|
||||
line-height: 1.65;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
text-rendering: optimizeLegibility;
|
||||
}
|
||||
|
||||
body {
|
||||
max-width: var(--max-w);
|
||||
margin: 0 auto;
|
||||
padding: 4rem 1.5rem 6rem;
|
||||
}
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 0.4rem 0.85rem;
|
||||
margin-bottom: 2.5rem;
|
||||
background: var(--bg-elev);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
font-family: 'JetBrains Mono', ui-monospace, 'SF Mono', 'Cascadia Code', Menlo, Consolas, monospace;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: var(--fg-muted);
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
.badge .accent { color: var(--accent); }
|
||||
|
||||
.meta {
|
||||
margin: -1.5rem 0 2.5rem;
|
||||
color: var(--fg-subtle);
|
||||
font-family: 'JetBrains Mono', ui-monospace, 'SF Mono', 'Cascadia Code', Menlo, Consolas, monospace;
|
||||
font-size: 13px;
|
||||
letter-spacing: 0.01em;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0 0 1.5rem;
|
||||
color: var(--fg);
|
||||
font-size: 30px;
|
||||
font-weight: 700;
|
||||
line-height: 1.2;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
h2,
|
||||
.prose-label {
|
||||
margin: 2.75rem 0 1.25rem;
|
||||
color: var(--fg);
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
line-height: 1.35;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
.badge + h2,
|
||||
.badge + .prose-label { margin-top: 0.5rem; }
|
||||
|
||||
h3 {
|
||||
margin: 2rem 0 0.85rem;
|
||||
color: var(--fg);
|
||||
font-size: 17px;
|
||||
font-weight: 600;
|
||||
line-height: 1.4;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0 0 1.4rem;
|
||||
color: var(--fg-muted);
|
||||
}
|
||||
|
||||
p strong,
|
||||
li strong,
|
||||
td strong {
|
||||
color: var(--fg);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
a {
|
||||
color: var(--accent);
|
||||
text-decoration: none;
|
||||
border-bottom: 1px solid transparent;
|
||||
transition: border-color 0.15s ease;
|
||||
}
|
||||
|
||||
a:hover { border-bottom-color: var(--accent); }
|
||||
|
||||
ul,
|
||||
ol {
|
||||
margin: 0 0 1.6rem;
|
||||
padding-left: 1.5rem;
|
||||
color: var(--fg-muted);
|
||||
}
|
||||
|
||||
li {
|
||||
margin: 0.6rem 0;
|
||||
padding-left: 0.4rem;
|
||||
}
|
||||
|
||||
li::marker {
|
||||
color: var(--accent);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
blockquote {
|
||||
margin: 1.5rem 0;
|
||||
padding-left: 1rem;
|
||||
border-left: 3px solid var(--accent);
|
||||
color: var(--fg-muted);
|
||||
}
|
||||
|
||||
hr {
|
||||
margin: 2.5rem 0;
|
||||
border: 0;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
code {
|
||||
font-family: 'JetBrains Mono', ui-monospace, 'SF Mono', 'Cascadia Code', Menlo, Consolas, monospace;
|
||||
font-size: 0.92em;
|
||||
background: var(--code-bg);
|
||||
padding: 0.15rem 0.4rem;
|
||||
border-radius: 4px;
|
||||
color: var(--accent-soft);
|
||||
}
|
||||
|
||||
pre {
|
||||
margin: 1.4rem 0;
|
||||
background: var(--code-bg);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 1rem 1.25rem;
|
||||
overflow-x: auto;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
pre code {
|
||||
background: none;
|
||||
padding: 0;
|
||||
color: var(--fg);
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin: 1.5rem 0;
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
text-align: left;
|
||||
padding: 0.75rem 1rem;
|
||||
border-bottom: 1px solid var(--border);
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
th {
|
||||
color: var(--fg-muted);
|
||||
font-weight: 600;
|
||||
font-size: 13px;
|
||||
letter-spacing: 0;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
td { color: var(--fg-muted); }
|
||||
td:first-child { color: var(--fg); font-weight: 500; }
|
||||
|
||||
.engine-footer {
|
||||
margin: 3rem 0 2.5rem;
|
||||
padding: 1.25rem 1.5rem;
|
||||
background: var(--bg-elev);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
color: var(--fg-muted);
|
||||
}
|
||||
|
||||
.engine-footer pre {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
border: 0;
|
||||
border-radius: 0;
|
||||
font-family: 'JetBrains Mono', ui-monospace, 'SF Mono', 'Cascadia Code', Menlo, Consolas, monospace;
|
||||
font-size: 13.5px;
|
||||
font-weight: 400;
|
||||
line-height: 1.75;
|
||||
color: inherit;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.colophon {
|
||||
margin-top: 4rem;
|
||||
padding-top: 2rem;
|
||||
border-top: 1px solid var(--border);
|
||||
color: var(--fg-subtle);
|
||||
font-size: 13px;
|
||||
font-family: 'JetBrains Mono', ui-monospace, 'SF Mono', 'Cascadia Code', Menlo, Consolas, monospace;
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
.colophon .rerun {
|
||||
display: inline-block;
|
||||
padding: 0.15rem 0.5rem;
|
||||
margin-left: 0.25rem;
|
||||
background: var(--code-bg);
|
||||
border-radius: 4px;
|
||||
color: var(--accent-soft);
|
||||
font-size: 0.95em;
|
||||
}
|
||||
|
||||
@media print {
|
||||
:root {
|
||||
--bg: #ffffff;
|
||||
--bg-elev: #f5f5f5;
|
||||
--fg: #000000;
|
||||
--fg-muted: #1f2937;
|
||||
--fg-subtle: #4b5563;
|
||||
--accent: #6d28d9;
|
||||
--accent-soft: #6d28d9;
|
||||
--border: #d4d4d8;
|
||||
--code-bg: #f4f4f5;
|
||||
}
|
||||
|
||||
@page { size: A4; margin: 1.5cm 2cm; }
|
||||
|
||||
body {
|
||||
max-width: none;
|
||||
padding: 0;
|
||||
font-size: 11pt;
|
||||
}
|
||||
|
||||
a {
|
||||
color: inherit;
|
||||
border-bottom: 0;
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
a[href]::after {
|
||||
content: " (" attr(href) ")";
|
||||
font-size: 0.85em;
|
||||
color: var(--fg-subtle);
|
||||
}
|
||||
|
||||
.engine-footer { page-break-inside: avoid; }
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
body {
|
||||
padding: 2.5rem 1.25rem 4rem;
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
h1 { font-size: 25px; }
|
||||
.badge { font-size: 12px; }
|
||||
th, td { padding: 0.65rem 0.5rem; }
|
||||
}
|
||||
""".strip()
|
||||
|
||||
HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>last30days · __TITLE__</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
__CSS__
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
__BODY__
|
||||
__COLOPHON__
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def render_html(
|
||||
report: schema.Report,
|
||||
*,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
_ = fun_level
|
||||
md = render.render_for_html(report, synthesis_md=synthesis_md, save_path=save_path)
|
||||
md = _strip_evidence_block(md)
|
||||
md = _strip_invitation(md)
|
||||
md = _strip_canonical_boundary(md)
|
||||
md = _promote_prose_labels(md)
|
||||
body = _markdown_to_html(md)
|
||||
body = _wrap_engine_footer(body)
|
||||
body = _promote_meta_marker(body)
|
||||
colophon = _build_colophon(report)
|
||||
return _wrap_in_template(body, colophon, report.topic)
|
||||
|
||||
|
||||
def render_html_comparison(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
*,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
_ = fun_level
|
||||
md = render.render_for_html_comparison(
|
||||
entity_reports, synthesis_md=synthesis_md, save_path=save_path,
|
||||
)
|
||||
md = _strip_evidence_block(md)
|
||||
md = _strip_invitation(md)
|
||||
md = _strip_canonical_boundary(md)
|
||||
md = _promote_prose_labels(md)
|
||||
body = _markdown_to_html(md)
|
||||
body = _wrap_engine_footer(body)
|
||||
body = _promote_meta_marker(body)
|
||||
topic = " vs ".join(label for label, _ in entity_reports)
|
||||
colophon = _build_colophon(entity_reports[0][1], topic=topic)
|
||||
return _wrap_in_template(body, colophon, topic)
|
||||
|
||||
|
||||
def _strip_evidence_block(md: str) -> str:
|
||||
return EVIDENCE_BLOCK_PATTERN.sub("", md)
|
||||
|
||||
|
||||
def _strip_invitation(md: str) -> str:
|
||||
return INVITATION_PATTERN.sub("", md)
|
||||
|
||||
|
||||
def _strip_canonical_boundary(md: str) -> str:
|
||||
return CANONICAL_BOUNDARY_PATTERN.sub("", md)
|
||||
|
||||
|
||||
def _promote_prose_labels(md: str) -> str:
|
||||
for source, normalized in PROSE_LABELS:
|
||||
md = re.sub(
|
||||
rf"^{re.escape(source)}$",
|
||||
f"## {normalized}",
|
||||
md,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
return md
|
||||
|
||||
|
||||
def _markdown_to_html(md: str) -> str:
|
||||
md, footers = _protect_engine_footers(md)
|
||||
global _ENGINE_FOOTER_STORE
|
||||
_ENGINE_FOOTER_STORE = footers
|
||||
# Strip HTML comments EXCEPT preserved markers used for post-processing
|
||||
# (META is promoted to <div class="meta"> after markdown conversion).
|
||||
md = re.sub(r"<!--(?!\s*META:).*?-->", "", md, flags=re.DOTALL)
|
||||
lines = md.splitlines()
|
||||
out: list[str] = []
|
||||
paragraph: list[str] = []
|
||||
list_type: str | None = None
|
||||
in_code = False
|
||||
code_lines: list[str] = []
|
||||
index = 0
|
||||
|
||||
def flush_paragraph() -> None:
|
||||
nonlocal paragraph
|
||||
if paragraph:
|
||||
text = " ".join(part.strip() for part in paragraph).strip()
|
||||
if text:
|
||||
out.append(f"<p>{_inline_markdown(text)}</p>")
|
||||
paragraph = []
|
||||
|
||||
def close_list() -> None:
|
||||
nonlocal list_type
|
||||
if list_type:
|
||||
out.append(f"</{list_type}>")
|
||||
list_type = None
|
||||
|
||||
while index < len(lines):
|
||||
line = lines[index]
|
||||
stripped = line.strip()
|
||||
|
||||
if in_code:
|
||||
if stripped.startswith("```"):
|
||||
out.append(f"<pre><code>{html.escape(chr(10).join(code_lines))}</code></pre>")
|
||||
code_lines = []
|
||||
in_code = False
|
||||
else:
|
||||
code_lines.append(line)
|
||||
index += 1
|
||||
continue
|
||||
|
||||
if stripped.startswith("```"):
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
in_code = True
|
||||
code_lines = []
|
||||
index += 1
|
||||
continue
|
||||
|
||||
if stripped in footers:
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
out.append(stripped)
|
||||
index += 1
|
||||
continue
|
||||
|
||||
if not stripped:
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
index += 1
|
||||
continue
|
||||
|
||||
if stripped == "---":
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
out.append("<hr>")
|
||||
index += 1
|
||||
continue
|
||||
|
||||
if index + 1 < len(lines) and _is_table_row(stripped) and _is_table_separator(lines[index + 1].strip()):
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
table_lines = [stripped]
|
||||
index += 2
|
||||
while index < len(lines) and _is_table_row(lines[index].strip()):
|
||||
table_lines.append(lines[index].strip())
|
||||
index += 1
|
||||
out.append(_render_table(table_lines))
|
||||
continue
|
||||
|
||||
heading = re.match(r"^(#{1,4})\s+(.+)$", stripped)
|
||||
if heading:
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
level = min(len(heading.group(1)), 3)
|
||||
out.append(f"<h{level}>{_inline_markdown(heading.group(2))}</h{level}>")
|
||||
index += 1
|
||||
continue
|
||||
|
||||
if stripped.startswith(">"):
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
quote_lines = []
|
||||
while index < len(lines) and lines[index].strip().startswith(">"):
|
||||
quote_lines.append(lines[index].strip().lstrip(">").strip())
|
||||
index += 1
|
||||
out.append(f"<blockquote>{_inline_markdown(' '.join(quote_lines))}</blockquote>")
|
||||
continue
|
||||
|
||||
unordered = re.match(r"^[-*]\s+(.+)$", stripped)
|
||||
ordered = re.match(r"^\d+[.)]\s+(.+)$", stripped)
|
||||
if unordered or ordered:
|
||||
flush_paragraph()
|
||||
next_type = "ul" if unordered else "ol"
|
||||
if list_type != next_type:
|
||||
close_list()
|
||||
out.append(f"<{next_type}>")
|
||||
list_type = next_type
|
||||
item = unordered.group(1) if unordered else ordered.group(1)
|
||||
out.append(f"<li>{_inline_markdown(item)}</li>")
|
||||
index += 1
|
||||
continue
|
||||
|
||||
if stripped.startswith("🌐 last30days"):
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
badge_text = _inline_markdown(stripped.removeprefix("🌐").strip())
|
||||
out.append(f'<div class="badge"><span class="accent">🌐</span> {badge_text}</div>')
|
||||
index += 1
|
||||
continue
|
||||
|
||||
paragraph.append(line)
|
||||
index += 1
|
||||
|
||||
if in_code:
|
||||
out.append(f"<pre><code>{html.escape(chr(10).join(code_lines))}</code></pre>")
|
||||
flush_paragraph()
|
||||
close_list()
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
def _protect_engine_footers(md: str) -> tuple[str, dict[str, str]]:
|
||||
footers: dict[str, str] = {}
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
token = f"__LAST30DAYS_ENGINE_FOOTER_{len(footers)}__"
|
||||
footers[token] = match.group(1).strip("\n")
|
||||
return f"\n{token}\n"
|
||||
|
||||
return PASS_THROUGH_FOOTER_PATTERN.sub(replace, md), footers
|
||||
|
||||
|
||||
def _wrap_engine_footer(body: str) -> str:
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
footer = html.escape(_ENGINE_FOOTER_STORE.get(match.group(0), ""), quote=False)
|
||||
return f'<div class="engine-footer"><pre>{footer}</pre></div>'
|
||||
|
||||
return re.sub(
|
||||
r"__LAST30DAYS_ENGINE_FOOTER_\d+__",
|
||||
replace,
|
||||
body,
|
||||
)
|
||||
|
||||
|
||||
def _promote_meta_marker(body: str) -> str:
|
||||
"""Promote ``<!-- META: ... -->`` markers into a styled ``<div class="meta">``.
|
||||
|
||||
The marker is preserved through the comment-strip pass (see
|
||||
_markdown_to_html exemption) but the markdown converter wraps it in
|
||||
``<p>`` and HTML-escapes the angle brackets. After conversion the body
|
||||
contains shapes like:
|
||||
<p><!-- META: TEXT --></p>
|
||||
<p><!-- META: TEXT --></p> (when not escaped)
|
||||
Both collapse to ``<div class="meta">TEXT</div>``.
|
||||
"""
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
text = match.group(1).strip()
|
||||
return f'<div class="meta">{text}</div>'
|
||||
|
||||
# Escaped form (most common after markdown conversion)
|
||||
body = re.sub(
|
||||
r"<p>\s*<!--\s*META:\s*(.*?)\s*-->\s*</p>",
|
||||
replace,
|
||||
body,
|
||||
)
|
||||
body = re.sub(r"<!--\s*META:\s*(.*?)\s*-->", replace, body)
|
||||
# Unescaped form (paranoid fallback)
|
||||
body = re.sub(r"<p>\s*<!--\s*META:\s*(.*?)\s*-->\s*</p>", replace, body)
|
||||
body = re.sub(r"<!--\s*META:\s*(.*?)\s*-->", replace, body)
|
||||
return body
|
||||
|
||||
|
||||
_ENGINE_FOOTER_STORE: dict[str, str] = {}
|
||||
|
||||
|
||||
def _inline_markdown(text: str) -> str:
|
||||
escaped = html.escape(text, quote=True)
|
||||
code_tokens: dict[str, str] = {}
|
||||
|
||||
def code_replace(match: re.Match[str]) -> str:
|
||||
token = f"__CODE_{len(code_tokens)}__"
|
||||
code_tokens[token] = f"<code>{match.group(1)}</code>"
|
||||
return token
|
||||
|
||||
escaped = re.sub(r"`([^`]+)`", code_replace, escaped)
|
||||
escaped = re.sub(r"\*\*([^*]+)\*\*", r"<strong>\1</strong>", escaped)
|
||||
escaped = re.sub(
|
||||
r"\[([^\]]+)\]\(([^)\s]+)\)",
|
||||
r'<a href="\2">\1</a>',
|
||||
escaped,
|
||||
)
|
||||
for token, value in code_tokens.items():
|
||||
escaped = escaped.replace(token, value)
|
||||
return escaped
|
||||
|
||||
|
||||
def _is_table_row(line: str) -> bool:
|
||||
return "|" in line and len(_split_table_cells(line)) >= 2
|
||||
|
||||
|
||||
def _is_table_separator(line: str) -> bool:
|
||||
cells = _split_table_cells(line)
|
||||
return bool(cells) and all(re.fullmatch(r":?-{3,}:?", cell.strip()) for cell in cells)
|
||||
|
||||
|
||||
def _split_table_cells(line: str) -> list[str]:
|
||||
return [cell.strip() for cell in line.strip().strip("|").split("|")]
|
||||
|
||||
|
||||
def _render_table(rows: list[str]) -> str:
|
||||
header = _split_table_cells(rows[0])
|
||||
body_rows = [_split_table_cells(row) for row in rows[1:]]
|
||||
out = ["<table>", "<thead>", "<tr>"]
|
||||
out.extend(f"<th>{_inline_markdown(cell)}</th>" for cell in header)
|
||||
out.extend(["</tr>", "</thead>", "<tbody>"])
|
||||
for row in body_rows:
|
||||
out.append("<tr>")
|
||||
out.extend(f"<td>{_inline_markdown(cell)}</td>" for cell in row)
|
||||
out.append("</tr>")
|
||||
out.extend(["</tbody>", "</table>"])
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _build_colophon(report: schema.Report, *, topic: str | None = None) -> str:
|
||||
display_topic = topic or report.topic
|
||||
generated = _generated_date(report)
|
||||
version = render._skill_version()
|
||||
escaped_topic = html.escape(display_topic)
|
||||
rerun = html.escape(f"/last30days {display_topic}")
|
||||
return (
|
||||
'<div class="colophon">\n'
|
||||
f" Generated {generated} by /last30days v{html.escape(version)} · topic: {escaped_topic}<br>\n"
|
||||
f' Re-run for fresh data: <span class="rerun">{rerun}</span>\n'
|
||||
"</div>"
|
||||
)
|
||||
|
||||
|
||||
def _generated_date(report: schema.Report) -> str:
|
||||
if report.generated_at:
|
||||
return report.generated_at[:10]
|
||||
return date.today().strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _wrap_in_template(body: str, colophon: str, title: str) -> str:
|
||||
return (
|
||||
HTML_TEMPLATE
|
||||
.replace("__TITLE__", html.escape(title))
|
||||
.replace("__CSS__", CSS)
|
||||
.replace("__BODY__", body)
|
||||
.replace("__COLOPHON__", colophon)
|
||||
)
|
||||
@@ -49,6 +49,7 @@ def normalize_source_items(
|
||||
"xquik": _normalize_x,
|
||||
"pinterest": _normalize_pinterest,
|
||||
"polymarket": _normalize_polymarket,
|
||||
"digg": _normalize_digg,
|
||||
"grounding": _normalize_grounding,
|
||||
"xiaohongshu": _normalize_grounding,
|
||||
"github": _normalize_github,
|
||||
@@ -110,6 +111,19 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
|
||||
return default
|
||||
|
||||
|
||||
def _join_comment_excerpts(
|
||||
top_comments: list[Any],
|
||||
key: str,
|
||||
limit: int = 3,
|
||||
) -> str:
|
||||
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
|
||||
return " ".join(
|
||||
str(comment.get(key) or "").strip()
|
||||
for comment in top_comments[:limit]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
|
||||
|
||||
def _domain_from_url(url: str) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
@@ -169,11 +183,7 @@ def _normalize_reddit(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
body = "\n".join(
|
||||
part
|
||||
for part in [
|
||||
@@ -338,11 +348,7 @@ def _normalize_hackernews(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = " ".join(
|
||||
str(comment.get("text") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
comment_text = _join_comment_excerpts(top_comments, "text")
|
||||
title = str(item.get("title") or "").strip()
|
||||
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
||||
return _source_item(
|
||||
@@ -394,6 +400,53 @@ def _normalize_microblog(
|
||||
)
|
||||
|
||||
|
||||
def _normalize_digg(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
index: int,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
"""Normalizer for Digg AI 1000 clusters.
|
||||
|
||||
Each cluster is one item. The TLDR carries the most useful body for
|
||||
rerank and synthesis. Top-ranked X posts attached at search time are
|
||||
passed through under metadata['posts'] so render can emit them as
|
||||
inline 'via Digg AI 1000' quotes.
|
||||
"""
|
||||
title = str(item.get("title") or "").strip()
|
||||
tldr = str(item.get("tldr") or "").strip()
|
||||
body = "\n\n".join(part for part in [title, tldr] if part)
|
||||
posts = item.get("posts") or []
|
||||
if not isinstance(posts, list):
|
||||
posts = []
|
||||
cluster_url_id = str(item.get("id") or f"DG{index + 1}")
|
||||
return _source_item(
|
||||
item_id=cluster_url_id,
|
||||
source=source,
|
||||
title=title or f"Digg cluster {index + 1}",
|
||||
body=body,
|
||||
url=str(item.get("url") or f"https://di.gg/ai/{cluster_url_id}"),
|
||||
author="",
|
||||
container="Digg AI 1000",
|
||||
published_at=item.get("date"),
|
||||
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
|
||||
engagement=item.get("engagement") or {},
|
||||
relevance_hint=item.get("relevance", 0.5),
|
||||
why_relevant=str(item.get("why_relevant") or ""),
|
||||
snippet=tldr[:400],
|
||||
metadata={
|
||||
"clusterUrlId": cluster_url_id,
|
||||
"tldr": tldr,
|
||||
"rank": (item.get("engagement") or {}).get("rank"),
|
||||
"uniqueAuthors": (item.get("engagement") or {}).get("uniqueAuthors"),
|
||||
"postCount": (item.get("engagement") or {}).get("postCount"),
|
||||
"firstPostAge": item.get("first_post_age"),
|
||||
"posts": posts,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalize_polymarket(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
@@ -441,11 +494,7 @@ def _normalize_github(
|
||||
title = str(item.get("title") or "").strip()
|
||||
snippet_text = str(item.get("snippet") or "").strip()
|
||||
top_comments = item.get("metadata", {}).get("top_comments") or []
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
||||
metadata = item.get("metadata") or {}
|
||||
return _source_item(
|
||||
@@ -15,6 +15,7 @@ from . import (
|
||||
bluesky,
|
||||
dates,
|
||||
dedupe,
|
||||
digg,
|
||||
entity_extract,
|
||||
env,
|
||||
github,
|
||||
@@ -30,6 +31,7 @@ from . import (
|
||||
query,
|
||||
reddit,
|
||||
reddit_public,
|
||||
relevance,
|
||||
rerank,
|
||||
schema,
|
||||
signals,
|
||||
@@ -78,6 +80,7 @@ MOCK_AVAILABLE_SOURCES = [
|
||||
"github",
|
||||
"perplexity",
|
||||
"xquik",
|
||||
"digg",
|
||||
]
|
||||
|
||||
|
||||
@@ -105,6 +108,8 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.extend(["hackernews", "polymarket"])
|
||||
if config.get("GITHUB_TOKEN") or which("gh"):
|
||||
available.append("github")
|
||||
if which("digg-pp-cli"):
|
||||
available.append("digg")
|
||||
if env.is_bluesky_available(config):
|
||||
available.append("bluesky")
|
||||
if env.is_truthsocial_available(config):
|
||||
@@ -500,11 +505,12 @@ def _normalize_score_dedupe(
|
||||
source, raw_items, from_date, to_date,
|
||||
freshness_mode=freshness_mode,
|
||||
)
|
||||
normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
|
||||
prepared_query = relevance.PreparedQuery(ranking_query)
|
||||
normalized = signals.annotate_stream(normalized, prepared_query, freshness_mode)
|
||||
normalized = signals.prune_low_relevance(normalized)
|
||||
normalized = dedupe.dedupe_items(normalized)
|
||||
for item in normalized:
|
||||
item.snippet = snippet.extract_best_snippet(item, ranking_query)
|
||||
item.snippet = snippet.extract_best_snippet(item, prepared_query)
|
||||
return normalized
|
||||
|
||||
|
||||
@@ -529,6 +535,12 @@ def _finalize_items_by_source(
|
||||
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
|
||||
if keywords:
|
||||
items = polymarket.filter_items_against_keywords(items, keywords)
|
||||
if source == "digg" and items:
|
||||
# Pull top-ranked X posts only for the survivors that will appear
|
||||
# in the brief. Spending the enrichment budget here (rather than
|
||||
# at retrieval time) keeps the inline 'via Digg AI 1000' quotes
|
||||
# paired with the clusters dedupe actually kept.
|
||||
digg.enrich_source_items(items, top_k=3)
|
||||
finalized[source] = items
|
||||
return finalized
|
||||
|
||||
@@ -964,6 +976,13 @@ def _retrieve_stream(
|
||||
if source == "hackernews":
|
||||
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
||||
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
||||
if source == "digg":
|
||||
result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
|
||||
items = digg.parse_digg_response(result, query=subquery.search_query)
|
||||
# Enrichment with attached X posts is deferred to
|
||||
# _finalize_items_by_source so it runs on the items that actually
|
||||
# survive dedupe rather than on top-K of the raw fanout.
|
||||
return items, {}
|
||||
if source == "bluesky":
|
||||
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
||||
return bluesky.parse_bluesky_response(result), {}
|
||||
@@ -1056,6 +1075,45 @@ def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[d
|
||||
"why_relevant": "Brave web search",
|
||||
}
|
||||
],
|
||||
"digg": [
|
||||
{
|
||||
"id": "mock1abc",
|
||||
"title": f"Digg AI 1000 cluster about {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock1abc",
|
||||
"tldr": f"Curated cluster summarizing recent {subquery.search_query} discussion across the AI 1000.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(3)[0],
|
||||
"engagement": {"postCount": 8, "uniqueAuthors": 5, "rank": 2, "rank_score": 49.0},
|
||||
"first_post_age": "3d",
|
||||
"posts": [
|
||||
{
|
||||
"username": "exampledev",
|
||||
"display_name": "Example Dev",
|
||||
"category": "Engineer",
|
||||
"rank": 142,
|
||||
"body": f"Quote from the AI 1000 about {subquery.search_query}.",
|
||||
"post_type": "tweet",
|
||||
"x_url": "https://x.com/exampledev/status/1",
|
||||
"posted_at": dates.get_date_range(3)[0],
|
||||
},
|
||||
],
|
||||
"relevance": 0.84,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
{
|
||||
"id": "mock2def",
|
||||
"title": f"Second Digg cluster on {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock2def",
|
||||
"tldr": f"Another angle on {subquery.search_query}.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(8)[0],
|
||||
"engagement": {"postCount": 3, "uniqueAuthors": 2, "rank": 18, "rank_score": 33.0},
|
||||
"first_post_age": "8d",
|
||||
"posts": [],
|
||||
"relevance": 0.71,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
],
|
||||
}
|
||||
if source == "grounding":
|
||||
return payloads.get(source, []), {
|
||||
@@ -67,6 +67,7 @@ SOURCE_CAPABILITIES = {
|
||||
"bluesky": {"discussion", "social"},
|
||||
"truthsocial": {"discussion", "social"},
|
||||
"polymarket": {"market"},
|
||||
"digg": {"discussion", "social", "link"},
|
||||
"xiaohongshu": {"video", "video_shortform", "social"},
|
||||
"github": {"discussion", "link"},
|
||||
"grounding": {"web", "reference", "link"},
|
||||
@@ -93,13 +93,6 @@ class GeminiClient(ReasoningClient):
|
||||
)
|
||||
return extract_gemini_text(payload)
|
||||
|
||||
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
|
||||
|
||||
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
|
||||
|
||||
|
||||
class OpenAIClient(ReasoningClient):
|
||||
name = "openai"
|
||||
|
||||
@@ -71,8 +71,29 @@ def _normalize_phrase(text: str) -> str:
|
||||
return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split())
|
||||
|
||||
|
||||
class PreparedQuery:
|
||||
"""Precomputed query shape reused across items in a stream.
|
||||
|
||||
Built once per ranking_query; reused by token_overlap_relevance so the
|
||||
per-item normalize/score loops don't re-tokenize the same query N times.
|
||||
"""
|
||||
|
||||
__slots__ = ("raw", "q_tokens", "informative_q_tokens", "normalized_phrase")
|
||||
|
||||
def __init__(self, query: str) -> None:
|
||||
self.raw = query
|
||||
self.q_tokens = tokenize(query)
|
||||
informative = {t for t in self.q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
||||
self.informative_q_tokens = informative or self.q_tokens
|
||||
self.normalized_phrase = _normalize_phrase(query)
|
||||
|
||||
|
||||
def _as_prepared(query: "str | PreparedQuery") -> PreparedQuery:
|
||||
return query if isinstance(query, PreparedQuery) else PreparedQuery(query)
|
||||
|
||||
|
||||
def token_overlap_relevance(
|
||||
query: str,
|
||||
query: "str | PreparedQuery",
|
||||
text: str,
|
||||
hashtags: Optional[List[str]] = None,
|
||||
) -> float:
|
||||
@@ -95,7 +116,8 @@ def token_overlap_relevance(
|
||||
Returns:
|
||||
Float between 0.0 and 1.0 (0.5 for empty queries)
|
||||
"""
|
||||
q_tokens = tokenize(query)
|
||||
prepared = _as_prepared(query)
|
||||
q_tokens = prepared.q_tokens
|
||||
|
||||
# Combine text and hashtags for matching
|
||||
combined = text
|
||||
@@ -119,9 +141,7 @@ def token_overlap_relevance(
|
||||
if overlap == 0:
|
||||
return 0.0
|
||||
|
||||
informative_q_tokens = {t for t in q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
||||
if not informative_q_tokens:
|
||||
informative_q_tokens = q_tokens
|
||||
informative_q_tokens = prepared.informative_q_tokens
|
||||
|
||||
coverage = overlap / len(q_tokens)
|
||||
informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens)
|
||||
@@ -129,7 +149,7 @@ def token_overlap_relevance(
|
||||
precision = overlap / precision_denominator
|
||||
|
||||
phrase_bonus = 0.0
|
||||
normalized_query = _normalize_phrase(query)
|
||||
normalized_query = prepared.normalized_phrase
|
||||
normalized_text = _normalize_phrase(combined)
|
||||
if normalized_query and normalized_query in normalized_text:
|
||||
phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
|
||||
@@ -12,7 +12,7 @@ from . import dates, schema
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
"""Read plugin version from .claude-plugin/plugin.json if available.
|
||||
"""Read plugin version from a plugin manifest if available.
|
||||
|
||||
Tries nearest plugin.json by walking up from render.py's own location.
|
||||
Falls back to "?" if not found. This keeps the badge emission from
|
||||
@@ -20,12 +20,13 @@ def _skill_version() -> str:
|
||||
"""
|
||||
here = pathlib.Path(__file__).resolve()
|
||||
for parent in [here.parent, *here.parents]:
|
||||
candidate = parent / ".claude-plugin" / "plugin.json"
|
||||
if candidate.is_file():
|
||||
try:
|
||||
return json.loads(candidate.read_text()).get("version", "?")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return "?"
|
||||
for manifest_dir in (".codex-plugin", ".claude-plugin"):
|
||||
candidate = parent / manifest_dir / "plugin.json"
|
||||
if candidate.is_file():
|
||||
try:
|
||||
return json.loads(candidate.read_text()).get("version", "?")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return "?"
|
||||
return "?"
|
||||
|
||||
|
||||
@@ -52,6 +53,7 @@ SOURCE_LABELS = {
|
||||
"xiaohongshu": "Xiaohongshu",
|
||||
"x": "X",
|
||||
"github": "GitHub",
|
||||
"digg": "Digg AI 1000",
|
||||
"perplexity": "Perplexity",
|
||||
}
|
||||
|
||||
@@ -170,6 +172,168 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_for_html(
|
||||
report: schema.Report,
|
||||
synthesis_md: str | None = None,
|
||||
*,
|
||||
save_path: str | None = None,
|
||||
) -> str:
|
||||
"""Render markdown intended for shareable HTML conversion.
|
||||
|
||||
This output keeps the public badge, compact source/date metadata, an
|
||||
optional one-line data quality note, optional synthesized brief markdown,
|
||||
and the engine footer. It deliberately omits the debug file header,
|
||||
model-facing safety note, and evidence scratchpad emitted by
|
||||
render_compact().
|
||||
|
||||
When synthesis_md is None, the body is intentionally sparse: badge,
|
||||
metadata, optional data quality note, and engine footer only.
|
||||
"""
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
*_render_html_metadata(report),
|
||||
]
|
||||
if synthesis_md:
|
||||
lines.extend(["", synthesis_md.strip()])
|
||||
# Data quality warnings are NOT rendered into the HTML artifact. The HTML
|
||||
# is meant to be shared (Slack, email, Notion); recipients haven't asked
|
||||
# for technical commentary about how the run was produced. Generators see
|
||||
# the same warnings via collect_html_warnings() routed to stderr by the
|
||||
# CLI, so they can fix quality issues before sharing.
|
||||
_append_html_footer(lines, report, save_path)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_for_html_comparison(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
synthesis_md: str | None = None,
|
||||
*,
|
||||
save_path: str | None = None,
|
||||
) -> str:
|
||||
"""Render comparison markdown intended for shareable HTML conversion.
|
||||
|
||||
Same semantics as render_for_html(), but metadata and data quality notes
|
||||
are aggregated across the compared entities.
|
||||
"""
|
||||
if not entity_reports:
|
||||
raise ValueError("render_for_html_comparison requires at least one report")
|
||||
|
||||
entities = [label for label, _ in entity_reports]
|
||||
main_report = entity_reports[0][1]
|
||||
meta = (
|
||||
f"<!-- META: {main_report.range_from} to {main_report.range_to} "
|
||||
f"· comparing {len(entities)}: {', '.join(entities)} -->"
|
||||
)
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
meta,
|
||||
]
|
||||
if synthesis_md:
|
||||
lines.extend(["", synthesis_md.strip()])
|
||||
# Comparison data quality notes also go to stderr, not into the artifact.
|
||||
_append_html_footer(lines, main_report, save_path)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def collect_html_warnings(report: schema.Report) -> list[str]:
|
||||
"""Collect data quality warnings for stderr output (NOT for the HTML artifact).
|
||||
|
||||
Returns a list of human-readable warning strings. Empty list if the run
|
||||
was clean. Used by the CLI to emit diagnostics to stderr after writing
|
||||
the HTML to stdout/file.
|
||||
"""
|
||||
notes: list[str] = []
|
||||
if _render_degraded_run_warning(report):
|
||||
notes.append("Run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
||||
elif _render_pre_research_warning(report):
|
||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
||||
freshness_warning = _assess_data_freshness(report)
|
||||
if freshness_warning:
|
||||
notes.append(freshness_warning)
|
||||
notes.extend(report.warnings)
|
||||
return _dedupe_notes(notes)
|
||||
|
||||
|
||||
def collect_html_warnings_comparison(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
) -> list[str]:
|
||||
"""Collect comparison-mode warnings, prefixed by entity label."""
|
||||
notes: list[str] = []
|
||||
for label, report in entity_reports:
|
||||
for w in collect_html_warnings(report):
|
||||
notes.append(f"{label}: {w}")
|
||||
return notes
|
||||
|
||||
|
||||
def _render_html_metadata(report: schema.Report) -> list[str]:
|
||||
"""Inline metadata as an HTML comment marker.
|
||||
|
||||
html_render.py post-processes ``<!-- META: ... -->`` markers into a
|
||||
``<div class="meta">`` after markdown conversion, so the metadata escapes
|
||||
the markdown converter's HTML-escaping pass cleanly. Same pattern as the
|
||||
PASS_THROUGH_FOOTER marker used for the engine tree.
|
||||
"""
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
if non_empty:
|
||||
sources = ", ".join(_source_label(s) for s in non_empty)
|
||||
else:
|
||||
sources = "no active sources"
|
||||
return [
|
||||
f"<!-- META: {report.range_from} to {report.range_to} · {sources} -->",
|
||||
]
|
||||
|
||||
|
||||
def _render_html_data_quality_note(report: schema.Report) -> str | None:
|
||||
notes: list[str] = []
|
||||
degraded_warning = _render_degraded_run_warning(report)
|
||||
if degraded_warning:
|
||||
notes.append("This run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
||||
pre_research_warning = _render_pre_research_warning(report)
|
||||
if pre_research_warning and not degraded_warning:
|
||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
||||
freshness_warning = _assess_data_freshness(report)
|
||||
if freshness_warning:
|
||||
notes.append(freshness_warning)
|
||||
notes.extend(report.warnings)
|
||||
if not notes:
|
||||
return None
|
||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
||||
|
||||
|
||||
def _render_html_comparison_data_quality_note(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
) -> str | None:
|
||||
notes: list[str] = []
|
||||
for label, report in entity_reports:
|
||||
note = _render_html_data_quality_note(report)
|
||||
if note:
|
||||
clean = note.removeprefix("> **Data quality note:** ").strip()
|
||||
notes.append(f"{label}: {clean}")
|
||||
if not notes:
|
||||
return None
|
||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
||||
|
||||
|
||||
def _dedupe_notes(notes: list[str]) -> list[str]:
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for note in notes:
|
||||
normalized = " ".join(str(note).split())
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
out.append(normalized)
|
||||
return out
|
||||
|
||||
|
||||
def _append_html_footer(lines: list[str], report: schema.Report, save_path: str | None) -> None:
|
||||
footer = _render_emoji_footer(report, save_path)
|
||||
lines.append("")
|
||||
lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")
|
||||
lines.extend(footer)
|
||||
lines.append("<!-- END PASS-THROUGH FOOTER -->")
|
||||
|
||||
|
||||
def _render_canonical_boundary() -> list[str]:
|
||||
"""Emit the explicit END-OF-CANONICAL-OUTPUT boundary.
|
||||
|
||||
@@ -663,7 +827,7 @@ def render_full(report: schema.Report) -> str:
|
||||
lines.append("## All Items by Source")
|
||||
lines.append("")
|
||||
source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest",
|
||||
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "perplexity"]
|
||||
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "digg", "perplexity"]
|
||||
for source in source_order:
|
||||
items = report.items_by_source.get(source, [])
|
||||
if not items:
|
||||
@@ -689,6 +853,9 @@ def render_full(report: schema.Report) -> str:
|
||||
tc_score = tc.get("score", "")
|
||||
attribution = _comment_attribution(item.source, tc.get("author"))
|
||||
lines.append(f" Top comment {attribution} ({tc_score} {vote_label}): {excerpt}")
|
||||
# Digg AI 1000: inline X-post quotes attached to the cluster.
|
||||
for post in _digg_posts_for(item, limit=3):
|
||||
lines.append(f" > {_format_digg_quote(post)}")
|
||||
# Comment insights for Reddit
|
||||
insights = item.metadata.get("comment_insights", [])
|
||||
if insights:
|
||||
@@ -810,6 +977,8 @@ def _render_candidate(candidate: schema.Candidate, prefix: str) -> list[str]:
|
||||
source = primary.source if primary else None
|
||||
attribution = _comment_attribution(source, tc.get("author"))
|
||||
lines.append(f" - {attribution} ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
|
||||
for post in _digg_posts_for(primary):
|
||||
lines.append(f" - {_format_digg_quote(post)}")
|
||||
insight = _comment_insight(primary)
|
||||
if insight:
|
||||
lines.append(f" - Insight: {_truncate(insight, 220)}")
|
||||
@@ -1060,6 +1229,7 @@ _FOOTER_SOURCES: list[tuple[str, str, str, str, list[tuple[str, str]]]] = [
|
||||
("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||
("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||
("github", "🐙", "GitHub", "item", [("reactions", "reactions"), ("comments", "comments")]),
|
||||
("digg", "⛏️", "Digg AI 1000", "cluster", [("postCount", "posts"), ("uniqueAuthors", "authors")]),
|
||||
]
|
||||
|
||||
|
||||
@@ -1317,6 +1487,7 @@ ENGAGEMENT_DISPLAY: dict[str, list[tuple[str, str]]] = {
|
||||
"polymarket": [],
|
||||
"github": [("reactions", "react"), ("comments", "cmt")],
|
||||
"perplexity": [("citations", "cite")],
|
||||
"digg": [("postCount", "posts"), ("uniqueAuthors", "auth")],
|
||||
}
|
||||
|
||||
|
||||
@@ -1504,16 +1675,6 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score
|
||||
return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]
|
||||
|
||||
|
||||
def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None:
|
||||
if not item:
|
||||
return None
|
||||
comments = item.metadata.get("top_comments") or []
|
||||
if not comments or not isinstance(comments[0], dict):
|
||||
return None
|
||||
top = comments[0]
|
||||
return str(top.get("excerpt") or top.get("text") or "").strip() or None
|
||||
|
||||
|
||||
def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
||||
if not item:
|
||||
return None
|
||||
@@ -1523,6 +1684,39 @@ def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
||||
return str(insights[0]).strip() or None
|
||||
|
||||
|
||||
def _digg_posts_for(item: schema.SourceItem | None, limit: int = 2) -> list[dict]:
|
||||
"""Return up to `limit` parsed Digg posts attached as enrichment to a cluster.
|
||||
|
||||
Returns an empty list for non-digg sources or clusters without enrichment.
|
||||
"""
|
||||
if not item or item.source != "digg":
|
||||
return []
|
||||
posts = item.metadata.get("posts") or []
|
||||
if not isinstance(posts, list):
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for entry in posts:
|
||||
if isinstance(entry, dict) and entry.get("body") and entry.get("username"):
|
||||
out.append(entry)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _format_digg_quote(post: dict, body_limit: int = 200) -> str:
|
||||
"""Format a Digg-attached X post as an inline 'via Digg AI 1000' quote line."""
|
||||
handle = post.get("username") or ""
|
||||
x_url = post.get("x_url") or ""
|
||||
body = (post.get("body") or "").replace("\n", " ").strip()
|
||||
if len(body) > body_limit:
|
||||
body = body[: body_limit - 1].rstrip() + "…"
|
||||
if x_url and handle:
|
||||
return f"[@{handle}]({x_url}) via Digg AI 1000: {body}"
|
||||
if handle:
|
||||
return f"@{handle} via Digg AI 1000: {body}"
|
||||
return f"via Digg AI 1000: {body}"
|
||||
|
||||
|
||||
def _transcript_highlights(item: schema.SourceItem | None) -> list[str]:
|
||||
if not item or item.source != "youtube":
|
||||
return []
|
||||
@@ -12,6 +12,7 @@ SOURCE_QUALITY = {
|
||||
"xiaohongshu": 0.7,
|
||||
"hackernews": 0.8,
|
||||
"youtube": 0.85,
|
||||
"digg": 0.85,
|
||||
"reddit": 0.6,
|
||||
"x": 0.68,
|
||||
"bluesky": 0.66,
|
||||
@@ -26,7 +27,10 @@ def source_quality(source: str) -> float:
|
||||
return SOURCE_QUALITY.get(source, 0.6)
|
||||
|
||||
|
||||
def local_relevance(item: schema.SourceItem, ranking_query: str) -> float:
|
||||
def local_relevance(
|
||||
item: schema.SourceItem,
|
||||
ranking_query: "str | relevance.PreparedQuery",
|
||||
) -> float:
|
||||
text = "\n".join(
|
||||
part
|
||||
for part in [item.title, item.body, item.snippet]
|
||||
@@ -92,6 +96,7 @@ ENGAGEMENT_WEIGHTS: dict[str, list[tuple[str, float]]] = {
|
||||
"bluesky": [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)],
|
||||
"truthsocial": [("likes", 0.45), ("reposts", 0.30), ("replies", 0.25)],
|
||||
"polymarket": [("volume", 0.60), ("liquidity", 0.40)],
|
||||
"digg": [("postCount", 0.40), ("uniqueAuthors", 0.30), ("rank_score", 0.30)],
|
||||
}
|
||||
|
||||
|
||||
@@ -175,13 +180,14 @@ def normalize(values: list[float | None]) -> list[int | None]:
|
||||
|
||||
def annotate_stream(
|
||||
items: list[schema.SourceItem],
|
||||
ranking_query: str,
|
||||
ranking_query: "str | relevance.PreparedQuery",
|
||||
freshness_mode: str,
|
||||
) -> list[schema.SourceItem]:
|
||||
"""Attach local scoring metadata and return items sorted by local_rank_score."""
|
||||
prepared_query = ranking_query if isinstance(ranking_query, relevance.PreparedQuery) else relevance.PreparedQuery(ranking_query)
|
||||
engagement_scores = normalize([engagement_raw(item) for item in items])
|
||||
for item, eng_score in zip(items, engagement_scores, strict=True):
|
||||
item.local_relevance = local_relevance(item, ranking_query)
|
||||
item.local_relevance = local_relevance(item, prepared_query)
|
||||
item.freshness = freshness(item, freshness_mode)
|
||||
item.engagement_score = eng_score
|
||||
item.source_quality = source_quality(item.source)
|
||||
@@ -26,7 +26,7 @@ def _windows(words: list[str], size: int, overlap: int) -> list[str]:
|
||||
|
||||
def extract_best_snippet(
|
||||
item: schema.SourceItem,
|
||||
ranking_query: str,
|
||||
ranking_query: "str | relevance.PreparedQuery",
|
||||
max_words: int = 120,
|
||||
) -> str:
|
||||
"""Prefer existing snippets, else extract the best matching evidence window."""
|
||||
@@ -43,8 +43,9 @@ def extract_best_snippet(
|
||||
if not candidates:
|
||||
return _truncate_words(body, max_words)
|
||||
|
||||
prepared_query = ranking_query if isinstance(ranking_query, relevance.PreparedQuery) else relevance.PreparedQuery(ranking_query)
|
||||
best = max(
|
||||
candidates,
|
||||
key=lambda candidate: relevance.token_overlap_relevance(ranking_query, candidate),
|
||||
key=lambda candidate: relevance.token_overlap_relevance(prepared_query, candidate),
|
||||
)
|
||||
return _truncate_words(best, max_words)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Subprocess helpers: safe timeout + process-group cleanup.
|
||||
|
||||
Used by bird_x.py (Node.js Bird search) and youtube_yt.py (yt-dlp search
|
||||
and transcript download). Both need the same os.setsid/killpg cleanup
|
||||
dance on timeout to avoid orphaning child processes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Sequence
|
||||
|
||||
|
||||
class SubprocTimeout(Exception):
|
||||
"""Raised when a subprocess exceeds its timeout and is killed."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SubprocResult:
|
||||
"""Result of a subprocess run that captured stdout and stderr."""
|
||||
|
||||
returncode: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
def run_with_timeout(
|
||||
cmd: Sequence[str],
|
||||
*,
|
||||
timeout: int,
|
||||
env: Optional[dict] = None,
|
||||
on_pid: Optional[callable] = None,
|
||||
) -> SubprocResult:
|
||||
"""Run a subprocess with process-group cleanup on timeout.
|
||||
|
||||
Spawns ``cmd`` inside its own process group via ``os.setsid`` where
|
||||
available. If ``communicate(timeout=...)`` raises ``TimeoutExpired``,
|
||||
signals ``SIGTERM`` to the entire group, falls back to ``proc.kill()``
|
||||
if the signal fails, then waits up to 5 seconds for cleanup, and
|
||||
raises ``SubprocTimeout``.
|
||||
|
||||
Args:
|
||||
cmd: Command and arguments to spawn.
|
||||
timeout: Timeout in seconds passed to ``communicate()``.
|
||||
env: Optional environment dict. If None, inherits parent env.
|
||||
on_pid: Optional callable invoked with the child PID right after
|
||||
spawn. Used by bird_x.py to register child PIDs for cleanup
|
||||
tracking. Exceptions raised by the callback are suppressed.
|
||||
|
||||
Returns:
|
||||
SubprocResult with returncode, stdout, and stderr as strings.
|
||||
|
||||
Raises:
|
||||
SubprocTimeout: If the process exceeded ``timeout``.
|
||||
FileNotFoundError: If the executable is not found.
|
||||
OSError: For other spawn failures.
|
||||
"""
|
||||
preexec = os.setsid if hasattr(os, "setsid") else None
|
||||
|
||||
proc = subprocess.Popen(
|
||||
list(cmd),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=env,
|
||||
)
|
||||
|
||||
if on_pid is not None:
|
||||
try:
|
||||
on_pid(proc.pid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
raise SubprocTimeout(f"Command {cmd[0]} timed out after {timeout}s")
|
||||
|
||||
return SubprocResult(
|
||||
returncode=proc.returncode,
|
||||
stdout=stdout or "",
|
||||
stderr=stderr or "",
|
||||
)
|
||||
@@ -124,6 +124,7 @@ SOURCE_COMPLETION_ORDER = [
|
||||
"polymarket",
|
||||
"grounding",
|
||||
"xiaohongshu",
|
||||
"digg",
|
||||
]
|
||||
|
||||
SOURCE_COMPLETION_META = {
|
||||
@@ -138,6 +139,7 @@ SOURCE_COMPLETION_META = {
|
||||
"polymarket": ("Polymarket", "market", "markets", Colors.GREEN),
|
||||
"grounding": ("Web", "result", "results", Colors.GREEN),
|
||||
"xiaohongshu": ("Xiaohongshu", "post", "posts", Colors.RED),
|
||||
"digg": ("Digg", "cluster", "clusters", Colors.YELLOW),
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user