Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Matt Van Horn
|
b31ec05c74 |
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Research"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,17 +1,16 @@
|
||||
{
|
||||
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "last30days-skill",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"owner": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.2.4",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"version": "3.0.9",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.2.4",
|
||||
"version": "3.0.14",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
@@ -10,5 +10,6 @@
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"],
|
||||
"hooks": {}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"name": "last30days"
|
||||
}
|
||||
+4
-4
@@ -1,6 +1,5 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by skills/last30days/scripts/build-skill.sh to produce a
|
||||
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
|
||||
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
@@ -33,8 +32,9 @@ release-notes.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
uv.lock export-ignore
|
||||
|
||||
# Platform adapters are kept in git archives because Claude Code and Codex
|
||||
# plugin installs use the same repository archive as their source payload.
|
||||
# Platform adapters - skill-upload path is platform-agnostic
|
||||
.agents/ export-ignore
|
||||
.codex-plugin/ export-ignore
|
||||
.hermes-plugin/ export-ignore
|
||||
|
||||
# CI workflows - repo-only, not needed at skill runtime
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
name: Bug Report
|
||||
description: Report a bug or unexpected behavior
|
||||
labels: [bug]
|
||||
body:
|
||||
- type: textarea
|
||||
id: summary
|
||||
attributes:
|
||||
label: Summary
|
||||
description: What happened?
|
||||
placeholder: Describe the bug in 1-2 sentences.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: How can we reproduce this?
|
||||
placeholder: |
|
||||
1. Run `python3 scripts/last30days.py "topic" --emit compact`
|
||||
2. ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected Behavior
|
||||
description: What should have happened?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: traceback
|
||||
attributes:
|
||||
label: Error / Traceback
|
||||
description: Paste the full traceback or error output.
|
||||
render: text
|
||||
- type: dropdown
|
||||
id: install
|
||||
attributes:
|
||||
label: Install Method
|
||||
options:
|
||||
- Claude Code plugin
|
||||
- Gemini CLI extension
|
||||
- Codex plugin
|
||||
- Hermes skill
|
||||
- Manual (git clone)
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: OS
|
||||
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
|
||||
@@ -1,24 +0,0 @@
|
||||
name: Feature Request
|
||||
description: Suggest a new feature or improvement
|
||||
labels: [enhancement]
|
||||
body:
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem
|
||||
description: What problem does this solve?
|
||||
placeholder: When I try to ..., I can't ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: How should this work?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
description: Other approaches you thought of (optional).
|
||||
@@ -1,19 +0,0 @@
|
||||
## Summary
|
||||
|
||||
<!-- What does this PR do? 1-3 sentences. -->
|
||||
|
||||
## Changes
|
||||
|
||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
||||
|
||||
-
|
||||
|
||||
## Testing
|
||||
|
||||
<!-- How did you verify this works? -->
|
||||
|
||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
||||
|
||||
## Related Issues
|
||||
|
||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
bash scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
name: Security
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
name: Dependency audit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Export locked dependency set
|
||||
run: |
|
||||
uv export \
|
||||
--locked \
|
||||
--all-groups \
|
||||
--no-hashes \
|
||||
--format requirements.txt \
|
||||
--output-file /tmp/last30days-requirements.txt
|
||||
|
||||
# Advisory-first: visibility before enforcement. This repo handles API keys,
|
||||
# cookies, browser tokens, and local env files, so dependency CVEs should be
|
||||
# visible in CI logs even before the project has a clean blocking baseline.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
- name: Run pip-audit against locked dependencies
|
||||
continue-on-error: true
|
||||
run: uvx --python 3.12 pip-audit -r /tmp/last30days-requirements.txt --progress-spinner=off
|
||||
|
||||
secret-scan:
|
||||
name: Secret scan
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout full history for diff-aware scanning
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Advisory-first: this reports verified secrets in pull requests and pushes to
|
||||
# main, but does not block merges until maintainers confirm a clean baseline.
|
||||
# The TruffleHog action automatically scans the PR range for pull_request
|
||||
# events and the pushed commit range for push events.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
# Contributor policy: never commit real secrets in fixtures, tests, docs, or
|
||||
# examples; use obvious dummy values and env-based auth patterns instead.
|
||||
- name: Run TruffleHog OSS secret scan
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
uses: trufflesecurity/trufflehog@v3.95.2
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: ./
|
||||
version: v3.95.2
|
||||
extra_args: --only-verified
|
||||
@@ -1,26 +0,0 @@
|
||||
name: Validate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run test suite
|
||||
run: uv run pytest
|
||||
@@ -28,3 +28,9 @@ htmlcov/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
|
||||
# Marketing video build artifacts
|
||||
marketing/v3.1-launch/node_modules/
|
||||
marketing/v3.1-launch/out/
|
||||
marketing/*/node_modules/
|
||||
marketing/*/out/
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
# last30days Skill
|
||||
|
||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
||||
|
||||
## Orientation
|
||||
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
||||
- Feature design starts from the slash-command UX. A new engine flag with no SKILL.md integration is incomplete — the model invoking the skill won't know the flag exists.
|
||||
- README and PR examples show `/last30days <topic>` first. Direct CLI invocation (`python3 scripts/last30days.py ...`) is a fallback for scripting, cron, and dev-time engine testing; label it as such, never as the primary path.
|
||||
- Slash commands don't pass shell mechanics through. `/last30days OpenClaw --emit=html | pbcopy` is invalid in any harness — either use the slash form (no flags or pipes; let the model translate user intent into engine flags) or use the direct CLI form (full `python3 ...` with explicit flags and a real shell).
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Security hygiene
|
||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
||||
- Use the env-based auth patterns in `skills/last30days/scripts/lib/env.py`; tests and fixtures must use obvious dummy values only.
|
||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
@@ -5,57 +5,6 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- `LAST30DAYS_YOUTUBE_SSH_HOST` env var: when set, yt-dlp YouTube search invocations are routed through `ssh <host>` for residential-IP egress. Bypasses YouTube's bot-wall on datacenter IPs (Hetzner/DigitalOcean/AWS) where `ytsearch:` returns 0 results regardless of cookies (the IP fingerprint is checked first). The named host must be configured in `~/.ssh/config` and have yt-dlp installed. Host value is validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection (e.g. a leading `-` masquerading as a flag). The transcript path is unchanged (uses the existing HTTP fallback when SSH-routing is on, since the timedtext API isn't bot-walled).
|
||||
|
||||
### Changed
|
||||
|
||||
- Replace the SKILL_ROOT resolver loops in Step 1 and comparison-mode with a single `SKILL_DIR` substitution pattern. The model templates the absolute path of the SKILL.md's own directory (which it always knows from the Read tool result); the bash block just validates that `scripts/last30days.py` lives there. Removes ~80 lines of bash across the two locations. Fixes a real bug: the previous resolver could pick a different install than the SKILL.md the model loaded from (spec-vs-engine divergence) and didn't enumerate harnesses like Hermes at all. The simplification works for any harness without enumeration because it just uses wherever SKILL.md was loaded from. STEP 0's marketplaces-stale-clone hop is unchanged.
|
||||
- Rename "Digg AI 1000" to just "Digg" in user-facing output (footer line, source label, inline-quote suffix, why_relevant, container attribution). Internal references to the upstream Digg AI 1000 product remain in code comments and docstrings.
|
||||
- Bump `POSTS_PER_CLUSTER` from 3 to 5 and the render-side display limit from 2 to 3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous 3/2 caps routinely truncated cluster context (e.g. dropped a Jason Calacanis quote tweet on a `cli-printing-press` run).
|
||||
- Rewrite SKILL.md path resolution. STEP 0 narrows from a global canonical-path enforcement to a Claude-Code-marketplaces-only stale-clone guard. Step 1 SKILL_ROOT resolver walks a single precedence list (Claude plugin cache, then `~/.codex/skills/`, `~/.agents/skills/`, repo checkout, `./.skills/last30days` for `npx skills add`, CWD, Gemini). Adds SKILL.md frontmatter fallback to `render.py::_skill_version` so the badge no longer prints `v?` on installs that don't include `.claude-plugin/plugin.json`.
|
||||
|
||||
- Switch SKILL.md's `--plan` and `--competitors-plan` invocation templates from inline single-quoted JSON to heredoc-written tmpfiles. Apostrophes in resolved context strings ("McDonald's", "people's choice", "developer's") previously closed the outer single-quote and broke shell parsing before the engine started — observed in a Codex run during PR #400 testing. The engine's `parse_plan()` / `parse_competitors_plan()` already supported file paths (via `os.path.isfile()` probe); only the template prose changed. Fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403).
|
||||
|
||||
### Removed
|
||||
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1. Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`.** The maintainer dev-deploy script is gone. Every job it did has a better replacement: `npx skills add . -g -y` symlinks the working tree into every detected harness's skill dir (better than sync.sh's copy model — edits propagate live), `hermes skills install mvanhorn/last30days-skill --force` handles Hermes, `clawhub install last30days-official` handles OpenClaw, and the Claude marketplace cache target was a "test against the official install path" hack we shouldn't have been recommending in the first place. The `test_sync_cache_path_uses_skill_version` test was dropped along with it. CLAUDE.md, HERMES_SETUP.md, the PR template, and a render.py docstring were updated to drop references; CHANGELOG and historical docs (release notes, plan files) keep their existing mentions as accurate history.
|
||||
|
||||
## [3.2.0] - 2026-05-09
|
||||
|
||||
### Added
|
||||
|
||||
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
|
||||
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
|
||||
|
||||
## [3.1.1] - 2026-04-24
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
|
||||
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
|
||||
|
||||
## [3.1.0] - 2026-04-22
|
||||
|
||||
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
|
||||
|
||||
### Added
|
||||
|
||||
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
|
||||
|
||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||
|
||||
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
|
||||
|
||||
## [3.0.14] - 2026-04-22
|
||||
|
||||
### Changed
|
||||
@@ -112,7 +61,6 @@ Individual changelog entries for 3.0.10 through 3.0.14 below document the increm
|
||||
### Fixed
|
||||
|
||||
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
||||
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
|
||||
|
||||
### Behavior fallback
|
||||
|
||||
|
||||
@@ -1 +1,25 @@
|
||||
@AGENTS.md
|
||||
# last30days Skill
|
||||
|
||||
Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||
Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `scripts/last30days.py` — main research engine
|
||||
- `scripts/lib/` — search, enrichment, rendering modules
|
||||
- `scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `SKILL.md` — skill definition (deployed to ~/.claude/skills/last30days/)
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
python3 scripts/last30days.py "test query" --emit=compact # Run research
|
||||
bash scripts/sync.sh # Deploy to ~/.claude, ~/.agents, ~/.codex
|
||||
```
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- After edits: run `bash scripts/sync.sh` to deploy
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
# Concepts
|
||||
|
||||
Shared vocabulary for `last30days-skill`. Terms here have a precise project-specific meaning — distinct enough from their general technical sense that a new contributor would need them defined to follow conversations, PR descriptions, or the SKILL.md contract.
|
||||
|
||||
## The package
|
||||
|
||||
### Skill
|
||||
|
||||
A self-contained agent-instructions package consisting of a `SKILL.md` prose contract plus a sibling `scripts/` directory containing the executable code the SKILL.md invokes. The package conforms to the [Agent Skills](https://agentskills.io) open format and installs across every major harness (Claude Code, Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ others) via `npx skills add`, harness-native plugin installers, or per-harness skill directories. A Skill is the unit of distribution; the Skill is the product.
|
||||
|
||||
### Engine
|
||||
|
||||
The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do the actual research work. The Engine and SKILL.md have a contract: SKILL.md tells the model which flags to pass (`--plan`, `--competitors-plan`, `--x-handle`, `--subreddits`, `--emit=compact`, etc.), and the Engine produces a specific output shape (badge line, ranked evidence clusters, emoji-tree footer) that the model is contractually required to pass through. The Engine is implementation; the SKILL.md prose is the agent-facing surface.
|
||||
|
||||
### Harness
|
||||
|
||||
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
||||
|
||||
## Distribution
|
||||
|
||||
### Beta channel
|
||||
|
||||
A parallel install of the Skill, sourced from the private `mvanhorn/last30days-skill-private` repo and installed as `/last30days-beta` rather than `/last30days`. The Beta channel exists so experimental changes can be tested by real users before they ship to the public `/last30days`. Promotion from Beta to public happens via a review PR against this (public) repo — Beta-only changes never ship to public without that PR. The Beta channel workflow guide lives in `BETA.md` in the private repo.
|
||||
+1
-1
@@ -23,7 +23,7 @@ v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos
|
||||
### @thinkun
|
||||
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
||||
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
||||
> Thinker, technologist, AI expert, music-tinkerer. Founder of [Thinkun](https://thinkun.com). [@thinkun on GitHub](https://github.com/thinkun) · [@unthink on X](https://x.com/unthink)
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @thomasmktong
|
||||
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
||||
|
||||
+23
-13
@@ -10,20 +10,28 @@ This guide covers installing last30days on Hermes AI Agent.
|
||||
|
||||
## Installation
|
||||
|
||||
### Option 1: Via sync.sh (Recommended)
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
# Clone the repo
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
cd last30days-skill
|
||||
|
||||
# Run the sync script
|
||||
bash scripts/sync.sh
|
||||
```
|
||||
|
||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
||||
This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days/`
|
||||
|
||||
### Developer / live-edit alternative
|
||||
|
||||
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
|
||||
### Option 2: Manual Copy
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
mkdir -p ~/.hermes/skills/research
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
|
||||
# Create directory
|
||||
mkdir -p ~/.hermes/skills/research/last30days
|
||||
|
||||
# Copy files
|
||||
cp -r scripts ~/.hermes/skills/research/last30days/
|
||||
cp .hermes-plugin/SKILL.md ~/.hermes/skills/research/last30days/
|
||||
```
|
||||
|
||||
## Usage
|
||||
@@ -51,7 +59,7 @@ On first run, the skill will guide you through setup:
|
||||
|
||||
2. **Optional: ScrapeCreators**
|
||||
- Adds TikTok, Instagram, Reddit backup
|
||||
- 100 free credits (no expiration)
|
||||
- 10,000 free API calls
|
||||
- Sign up at scrapecreators.com
|
||||
|
||||
3. **Optional: API Keys**
|
||||
@@ -98,11 +106,13 @@ python3.12 scripts/last30days.py --diagnose
|
||||
|
||||
## Updating
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
```
|
||||
To update to the latest version:
|
||||
|
||||
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step.
|
||||
```bash
|
||||
cd last30days-skill
|
||||
git pull
|
||||
bash scripts/sync.sh
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
|
||||
@@ -12,20 +12,23 @@
|
||||
|
||||
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
||||
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [SKILL.md](SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
|
||||
**Claude Code (recommended — auto-updates via marketplace):**
|
||||
Claude Code:
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
||||
OpenClaw:
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
clawhub install last30days-official
|
||||
```
|
||||
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
|
||||
|
||||
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
|
||||
Hermes:
|
||||
```
|
||||
# The skill auto-deploys when you run sync.sh
|
||||
# Or manually copy to ~/.hermes/skills/research/last30days/
|
||||
```
|
||||
|
||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
@@ -65,7 +68,6 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||
@@ -94,28 +96,6 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
## What v3 Changed
|
||||
|
||||
### Shareable HTML briefs
|
||||
|
||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
||||
|
||||
```
|
||||
/last30days OpenClaw --emit=html
|
||||
```
|
||||
|
||||
or just ask in plain language:
|
||||
|
||||
```
|
||||
/last30days OpenClaw, give me a shareable HTML brief
|
||||
/last30days Cursor IDE for slack
|
||||
/last30days Anthropic earnings export as html
|
||||
```
|
||||
|
||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
||||
|
||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
||||
|
||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
||||
|
||||
### Intelligent search: the killer feature
|
||||
|
||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
@@ -152,10 +132,8 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
|
||||
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
||||
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
||||
- **TikTok, Instagram, Threads.** All three activate automatically once `SCRAPECREATORS_API_KEY` is set — same key, same per-call cost. Suppress any of them with `EXCLUDE_SOURCES=tiktok,instagram,threads` (any comma-separated subset).
|
||||
- **Pinterest.** Per-query opt-in (visual pins, narrow utility): the model passes `--search=pinterest` for the runs that need it. Requires `SCRAPECREATORS_API_KEY`.
|
||||
- **YouTube + TikTok comments.** Persistent opt-in via `INCLUDE_SOURCES=youtube_comments,tiktok_comments` because each video pulls N extra ScrapeCreators calls on top of the base search. Surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` and `INCLUDE_SOURCES=perplexity` (it's a separate paid API — opt-in keeps you from being surprise-billed).
|
||||
- **Threads, Pinterest, YouTube + TikTok comments.** Opt-in sources via ScrapeCreators. Set `INCLUDE_SOURCES=tiktok,instagram` and add threads, pinterest, youtube_comments, tiktok_comments for more. `youtube_comments` and `tiktok_comments` surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` to unlock.
|
||||
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
||||
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
||||
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
||||
@@ -167,61 +145,12 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install | Updates |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recommended)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Recommended because the Claude Code marketplace handles updates for you — the plugin cache is versioned and auto-refreshes when a new release publishes. Run `claude plugin update last30days@last30days-skill` to force a check.
|
||||
|
||||
If you'd rather use the agent-skills install path on Claude Code, that's also supported:
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
The native plugin and the `npx skills` install can coexist; Claude Code dedupes the slash command.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
||||
|
||||
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
|
||||
|
||||
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
Update later with:
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
Or update everything you've installed globally via `npx skills`:
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + |
|
||||
| **Claude Code** | `/plugin marketplace add mvanhorn/last30days-skill` |
|
||||
| **OpenClaw** | `clawhub install last30days-official` |
|
||||
| **Gemini CLI** | Clone then `gemini extensions install ./last30days-skill` (see below) |
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
@@ -229,7 +158,15 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
||||
3. Click the `+` button in the Skills panel and drop the file in
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
||||
Enable "Code execution and file creation" under Capabilities first - skills won't run without it.
|
||||
|
||||
### Claude Code
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Update later with `claude plugin update last30days@last30days-skill`.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
@@ -237,14 +174,22 @@ Enable "Code execution and file creation" under Capabilities first — skills wo
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
### Gemini CLI
|
||||
|
||||
Gemini CLI v0.9.0 has an upstream installer bug that can fail with `Configuration file not found at /tmp/gemini-extensionXXXXXX/gemini-extension.json` ([upstream issue](https://github.com/google-gemini/gemini-cli/issues/11452)). Workaround:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill
|
||||
gemini extensions install ./last30days-skill
|
||||
```
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
Or build the claude.ai `.skill` file from source: `bash scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||
|
||||
@@ -258,28 +203,10 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
| X / Twitter | Log into x.com in any browser | Free |
|
||||
| YouTube | `brew install yt-dlp` | Free |
|
||||
| Bluesky | App password from bsky.app | Free |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls |
|
||||
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
||||
| Web search | Brave Search key | 2,000 free queries/month |
|
||||
|
||||
### macOS Keychain (optional)
|
||||
|
||||
On macOS you can store keys in the system Keychain instead of a `.env` file. The skill picks them up automatically as the lowest-priority source — `.env` files and process environment still win on collision.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: last30days
|
||||
version: "3.2.4"
|
||||
version: "3.0.1"
|
||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
@@ -13,9 +13,9 @@ metadata:
|
||||
openclaw:
|
||||
emoji: "📰"
|
||||
requires:
|
||||
env: []
|
||||
optionalEnv:
|
||||
env:
|
||||
- SCRAPECREATORS_API_KEY
|
||||
optionalEnv:
|
||||
- OPENAI_API_KEY
|
||||
- XAI_API_KEY
|
||||
- OPENROUTER_API_KEY
|
||||
@@ -45,7 +45,6 @@ metadata:
|
||||
- instagram
|
||||
- hackernews
|
||||
- polymarket
|
||||
- digg
|
||||
- bluesky
|
||||
- truthsocial
|
||||
- trends
|
||||
@@ -60,32 +59,23 @@ metadata:
|
||||
- clawhub
|
||||
---
|
||||
|
||||
# STEP 0: STALE-CLONE SELF-CHECK — RUN BEFORE READING BELOW
|
||||
# STEP 0: CANONICAL PATH SELF-CHECK — RUN BEFORE READING BELOW
|
||||
|
||||
Before reading anything else in this file, check whether you loaded SKILL.md from the one known stale-clone location: Claude Code's marketplaces directory.
|
||||
|
||||
**Run this check:**
|
||||
Before reading anything else in this file, verify that the SKILL.md you just loaded is the versioned cache copy, not a stale marketplace clone:
|
||||
|
||||
```bash
|
||||
CLAUDE_CACHE_LATEST=$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)
|
||||
# Two cache layouts ship in the wild — nested ({version}/skills/last30days/SKILL.md)
|
||||
# and flat ({version}/SKILL.md). Resolve to whichever shape actually exists.
|
||||
CLAUDE_CACHE_SKILL_MD=""
|
||||
if [ -n "$CLAUDE_CACHE_LATEST" ]; then
|
||||
if [ -f "$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md" ]; then
|
||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md"
|
||||
elif [ -f "$CLAUDE_CACHE_LATEST/SKILL.md" ]; then
|
||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/SKILL.md"
|
||||
fi
|
||||
fi
|
||||
echo "CLAUDE_CACHE_SKILL_MD=$CLAUDE_CACHE_SKILL_MD"
|
||||
CANONICAL_ROOT="$HOME/.claude/plugins/cache/last30days-skill/last30days"
|
||||
CANONICAL_LATEST=$(ls -d "$CANONICAL_ROOT"/*/ 2>/dev/null | sort -V | tail -1)
|
||||
CANONICAL_LATEST="${CANONICAL_LATEST%/}"
|
||||
```
|
||||
|
||||
If the SKILL.md path you just Read contains `/.claude/plugins/marketplaces/` AND `$CLAUDE_CACHE_SKILL_MD` is non-empty, STOP and re-read `$CLAUDE_CACHE_SKILL_MD` before proceeding. Otherwise the SKILL.md you have is fine — continue.
|
||||
If the SKILL.md file you just read is NOT under `$CANONICAL_LATEST/`, STOP and re-read `$CANONICAL_LATEST/SKILL.md` before proceeding. Do not run `--help`, do not plan, do not invoke the engine against any other path.
|
||||
|
||||
**Why this specific check:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test. STEP 0 defends against that one Claude Code-specific bug.
|
||||
**Why:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test.
|
||||
|
||||
**Other install paths are fine:** `~/.codex/skills/`, `~/.agents/skills/`, an `npx skills add` install dir, or a repo checkout are all valid load points - the resolver in Step 1 picks them up. Do NOT abort or hop on those paths.
|
||||
**How to self-check:** the file path you used in your last Read tool call should match `$CANONICAL_LATEST/SKILL.md`. If it contains `marketplaces/` or any other prefix, that is the stale-path failure mode. Re-read from `$CANONICAL_LATEST/SKILL.md` and restart this contract from the top.
|
||||
|
||||
The same pinned resolver appears later in Step 1 for the engine Bash invocation. That guard is necessary but insufficient — by the time you reach Step 1, you may have already internalized an out-of-date flag list from the stale SKILL.md above it. This STEP 0 runs first so the CONTRACT itself is read from the right file.
|
||||
|
||||
---
|
||||
|
||||
@@ -97,7 +87,7 @@ You are inside the `/last30days` SKILL. This is a specific research tool with a
|
||||
|
||||
**How v3.0.7 fixes it:** three structural anchors.
|
||||
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
||||
2. **The SKILL_DIR substitution** in the engine Bash calls uses the directory of the SKILL.md the model just Read — no resolver list, no precedence walk. Whichever install the harness loaded SKILL.md from is the install whose engine runs. Aligns spec-with-code and works for any harness without enumerating its install path.
|
||||
2. **The pinned SKILL_ROOT resolution** in the engine Bash calls always points to the public plugin cache, never `~/.openclaw/` or other stale copies.
|
||||
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
||||
|
||||
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
||||
@@ -114,7 +104,7 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
```
|
||||
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_DIR/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_DIR/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
|
||||
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
||||
|
||||
@@ -186,13 +176,13 @@ The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses
|
||||
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
|
||||
```
|
||||
|
||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan "$QUERY_PLAN_FILE"` (the path to a tmpfile you wrote via heredoc — see Step 1 for the pattern; never inline `--plan '$JSON'`, apostrophes in search/ranking strings break shell parsing).
|
||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan '$JSON'`.
|
||||
|
||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan "$QUERY_PLAN_FILE"` (or any path the engine can read). A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan '$JSON'`. A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||
|
||||
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
|
||||
|
||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan "$QUERY_PLAN_FILE"` (or another path the engine can read)? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan, then write it to a tmpfile per the Step 1 pattern. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan '$JSON'`? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||
|
||||
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
|
||||
|
||||
@@ -243,7 +233,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
||||
|
||||
---
|
||||
|
||||
# last30days v3.2.4: Research Any Topic from the Last 30 Days
|
||||
# last30days v3.0.1: Research Any Topic from the Last 30 Days
|
||||
|
||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||
|
||||
@@ -327,14 +317,14 @@ Common patterns:
|
||||
|
||||
- Always active: Reddit, Hacker News, Polymarket
|
||||
- If gh CLI is installed (check `which gh`): add GitHub
|
||||
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg
|
||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
||||
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
||||
- If SCRAPECREATORS_API_KEY is set: add TikTok, Instagram, Threads (suppress any of these via EXCLUDE_SOURCES)
|
||||
- If SCRAPECREATORS_API_KEY is set and the user explicitly requested pinterest for this query (e.g. via `--search=pinterest`): add Pinterest
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains threads: add Threads
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest
|
||||
- If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky
|
||||
- If OPENROUTER_API_KEY is set and INCLUDE_SOURCES contains perplexity: add Perplexity
|
||||
- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying
|
||||
- If OPENROUTER_API_KEY is set: add Perplexity
|
||||
|
||||
Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma):
|
||||
|
||||
@@ -591,50 +581,18 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe
|
||||
|
||||
**Invocation:**
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Write the per-entity plan to a tmpfile and pass the path to the engine.
|
||||
# The engine's parse_competitors_plan() reads file paths transparently. This
|
||||
# avoids the inline-single-quoted-JSON apostrophe trap (resolved context
|
||||
# strings like "people's choice" or "McDonald's" otherwise close the outer
|
||||
# single-quote and break shell parsing before the engine is even invoked).
|
||||
# Trailing XXXXXX (no .json suffix) so BSD/macOS mktemp works the same as
|
||||
# GNU; BSD only substitutes X's at the end of the template.
|
||||
COMPETITORS_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-competitors.XXXXXX")
|
||||
trap 'rm -f "$COMPETITORS_PLAN_FILE"' EXIT
|
||||
cat > "$COMPETITORS_PLAN_FILE" <<'PLAN_EOF'
|
||||
{
|
||||
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
||||
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
||||
}
|
||||
PLAN_EOF
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||
--emit=compact \
|
||||
--save-dir="${LAST30DAYS_MEMORY_DIR}" \
|
||||
--save-suffix=v3 \
|
||||
--x-handle={TOPIC_A_HANDLE} \
|
||||
--subreddits={TOPIC_A_SUBS} \
|
||||
--competitors-plan "$COMPETITORS_PLAN_FILE"
|
||||
--competitors-plan '{
|
||||
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
||||
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
||||
}'
|
||||
```
|
||||
|
||||
**The quoted heredoc marker `'PLAN_EOF'` is load-bearing** — quoting suppresses shell interpolation so apostrophes, `$`, backticks, etc. pass through verbatim. If you ever switch to an unquoted `<<PLAN_EOF`, every variable reference and apostrophe inside the JSON becomes a parse hazard.
|
||||
|
||||
Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive).
|
||||
|
||||
**Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan.
|
||||
@@ -866,7 +824,7 @@ Only show lines for platforms where something was resolved. Skip empty lines. On
|
||||
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
||||
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
||||
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key), digg (Digg clusters - only if `digg-pp-cli` is on PATH)
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key)
|
||||
|
||||
**Intent → freshness_mode mapping:**
|
||||
- breaking_news, prediction → `strict_recent`
|
||||
@@ -909,45 +867,34 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
||||
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
# PIN SKILL_ROOT to the public plugin cache (highest-version dir wins on upgrade).
|
||||
# DO NOT write your own path-discovery loop. The 2026-04-18 Peter Steinberger run 1
|
||||
# regression was caused by a custom discovery loop landing on ~/.openclaw/skills/last30days/
|
||||
# (a stale copy from a private-repo sync pattern). That path contains a pre-plan-007
|
||||
# engine and produces non-canonical output. This pinned resolution ignores every stale
|
||||
# copy (~/.openclaw/, ~/.agents/, ~/.codex/) and picks the plugin cache exclusively.
|
||||
SKILL_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
|
||||
SKILL_ROOT="${SKILL_ROOT%/}"
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
# Fallback for repo checkout / Gemini / Codex hosts where the plugin cache does not exist.
|
||||
# Only runs if the public plugin cache is missing entirely.
|
||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
echo "ERROR: Could not find scripts/last30days.py in public plugin cache or repo checkout" >&2
|
||||
echo "Expected: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/scripts/last30days.py" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||
```
|
||||
|
||||
**If you ran Steps 0.55 and 0.75 (agent planning), pass the plan via a tmpfile and add the targeting flags:**
|
||||
|
||||
```bash
|
||||
# Write QUERY_PLAN_JSON to a tmpfile before the engine invocation above.
|
||||
# parse_plan() reads file paths transparently; this avoids inline-JSON
|
||||
# shell-quoting hazards (apostrophes in search_query / ranking_query
|
||||
# strings break single-quoted command-line JSON). Trailing XXXXXX (no
|
||||
# .json suffix) for BSD/macOS portability — BSD mktemp only substitutes
|
||||
# X's at the end of the template.
|
||||
QUERY_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-plan.XXXXXX")
|
||||
trap 'rm -f "$QUERY_PLAN_FILE"' EXIT
|
||||
cat > "$QUERY_PLAN_FILE" <<'PLAN_EOF'
|
||||
{QUERY_PLAN_JSON_FROM_STEP_0.75}
|
||||
PLAN_EOF
|
||||
```
|
||||
|
||||
Then add to the engine command:
|
||||
|
||||
- `--plan "$QUERY_PLAN_FILE"` (path to the file you just wrote)
|
||||
**If you ran Steps 0.55 and 0.75 (agent planning), add these flags:**
|
||||
- `--plan 'QUERY_PLAN_JSON'` (replace with actual JSON from Step 0.75)
|
||||
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
|
||||
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
|
||||
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
|
||||
@@ -1550,33 +1497,6 @@ Close with `I have all the links to the {N} {source list} I pulled from. Just as
|
||||
|
||||
---
|
||||
|
||||
## SHAREABLE HTML BRIEF (when the user asked for one)
|
||||
|
||||
**This section fires if EITHER trigger is true:**
|
||||
|
||||
- `$ARGUMENTS` contains `--emit=html`, `--emit:html`, or `--html` as a flag
|
||||
- The user's natural-language request asks for an HTML brief, shareable doc, or file for sharing (Slack, email, Notion, "export as HTML", etc). Use your judgment for phrasing variants.
|
||||
|
||||
**If neither trigger fires, skip this entire section and proceed to WAIT FOR USER'S RESPONSE.** No HTML save flow, no reference read needed.
|
||||
|
||||
**When triggered, you MUST:**
|
||||
|
||||
- Read `references/save-html-brief.md` BEFORE proceeding to WAIT FOR USER'S RESPONSE
|
||||
- Follow that file's instructions exactly - it is the canonical source for the save flow
|
||||
- Append the confirmation line (`📎 Shareable brief saved to <path>`) to your already-emitted chat response
|
||||
|
||||
**You MUST NOT:**
|
||||
|
||||
- Improvise the HTML save flow from memory or from instructions you've seen before
|
||||
- Skip the reference read because the steps "look familiar"
|
||||
- Save to a different path than the reference specifies
|
||||
- Add data quality warnings, debug headers, or safety notes to the saved HTML
|
||||
- Re-research the topic for the HTML render - the engine cache covers the second invocation
|
||||
|
||||
**Why the directive is forceful:** the reference file is the only source of truth for the save flow. Skipping it produces broken artifacts - wrong path conventions, missing synthesis content, leaked engine debug output, or warnings that don't belong in shareable docs.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S RESPONSE
|
||||
|
||||
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.
|
||||
@@ -1688,7 +1608,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 100 free credits)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 10,000 free API calls)
|
||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||
- Stores research findings in local SQLite database (watchlist mode only)
|
||||
@@ -1701,7 +1621,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Does not log, cache, or write API keys to output files
|
||||
- Does not send data to any endpoint not listed above
|
||||
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (100 free credits one-time, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (10,000 free API calls, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
||||
|
||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||
@@ -0,0 +1,77 @@
|
||||
# last30days Skill Specification
|
||||
|
||||
## Overview
|
||||
|
||||
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. OpenAI auth can come from `OPENAI_API_KEY` or Codex login credentials.
|
||||
|
||||
The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability.
|
||||
|
||||
## Architecture
|
||||
|
||||
The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`:
|
||||
|
||||
- **env.py**: Load API keys from `~/.config/last30days/.env` and Codex auth from `~/.codex/auth.json`
|
||||
- **dates.py**: Date range calculation and confidence scoring
|
||||
- **cache.py**: 24-hour TTL caching keyed by topic + date range
|
||||
- **http.py**: stdlib-only HTTP client with retry logic
|
||||
- **models.py**: Auto-selection of OpenAI/xAI models with 7-day caching
|
||||
- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit
|
||||
- **xai_x.py**: xAI Responses API + x_search for X
|
||||
- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics
|
||||
- **hackernews.py**: Hacker News search via Algolia API (free, no auth)
|
||||
- **polymarket.py**: Polymarket prediction market search via Gamma API (free, no auth)
|
||||
- **normalize.py**: Convert raw API responses to canonical schema
|
||||
- **score.py**: Compute popularity-aware scores (relevance + recency + engagement)
|
||||
- **dedupe.py**: Near-duplicate detection via text similarity
|
||||
- **render.py**: Generate markdown and JSON outputs
|
||||
- **schema.py**: Type definitions and validation
|
||||
|
||||
## Embedding in Other Skills
|
||||
|
||||
Other skills can import the research context in several ways:
|
||||
|
||||
### Inline Context Injection
|
||||
```markdown
|
||||
## Recent Research Context
|
||||
!python3 ~/.claude/skills/last30days/scripts/last30days.py "your topic" --emit=context
|
||||
```
|
||||
|
||||
### Read from File
|
||||
```markdown
|
||||
## Research Context
|
||||
!cat ~/.local/share/last30days/out/last30days.context.md
|
||||
```
|
||||
|
||||
### Get Path for Dynamic Loading
|
||||
```bash
|
||||
CONTEXT_PATH=$(python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=path)
|
||||
cat "$CONTEXT_PATH"
|
||||
```
|
||||
|
||||
### JSON for Programmatic Use
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=json > research.json
|
||||
```
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py <topic> [options]
|
||||
|
||||
Options:
|
||||
--refresh Bypass cache and fetch fresh data
|
||||
--mock Use fixtures instead of real API calls
|
||||
--emit=MODE Output mode: compact|json|md|context|path (default: compact)
|
||||
--sources=MODE Source selection: auto|reddit|x|both (default: auto)
|
||||
```
|
||||
|
||||
## Output Files
|
||||
|
||||
All outputs are written to `~/.local/share/last30days/out/`:
|
||||
|
||||
- `report.md` - Human-readable full report
|
||||
- `report.json` - Normalized data with scores
|
||||
- `last30days.context.md` - Compact reusable snippet for other skills
|
||||
- `raw_openai.json` - Raw OpenAI API response
|
||||
- `raw_xai.json` - Raw xAI API response
|
||||
- `raw_reddit_threads_enriched.json` - Enriched Reddit thread data
|
||||
@@ -0,0 +1,47 @@
|
||||
# last30days Implementation Tasks
|
||||
|
||||
## Setup & Configuration
|
||||
- [x] Create directory structure
|
||||
- [x] Write SPEC.md
|
||||
- [x] Write TASKS.md
|
||||
- [x] Write SKILL.md with proper frontmatter
|
||||
|
||||
## Core Library Modules
|
||||
- [x] scripts/lib/env.py - Environment and API key loading
|
||||
- [x] scripts/lib/dates.py - Date range and confidence utilities
|
||||
- [x] scripts/lib/cache.py - TTL-based caching
|
||||
- [x] scripts/lib/http.py - HTTP client with retry
|
||||
- [x] scripts/lib/models.py - Auto model selection
|
||||
- [x] scripts/lib/schema.py - Data structures
|
||||
- [x] scripts/lib/openai_reddit.py - OpenAI Responses API
|
||||
- [x] scripts/lib/xai_x.py - xAI Responses API
|
||||
- [x] scripts/lib/reddit_enrich.py - Reddit thread JSON fetcher
|
||||
- [x] scripts/lib/normalize.py - Schema normalization
|
||||
- [x] scripts/lib/score.py - Popularity scoring
|
||||
- [x] scripts/lib/dedupe.py - Near-duplicate detection
|
||||
- [x] scripts/lib/render.py - Output rendering
|
||||
|
||||
## Main Script
|
||||
- [x] scripts/last30days.py - CLI orchestrator
|
||||
|
||||
## Fixtures
|
||||
- [x] fixtures/openai_sample.json
|
||||
- [x] fixtures/xai_sample.json
|
||||
- [x] fixtures/reddit_thread_sample.json
|
||||
- [x] fixtures/models_openai_sample.json
|
||||
- [x] fixtures/models_xai_sample.json
|
||||
|
||||
## Tests
|
||||
- [x] tests/test_dates.py
|
||||
- [x] tests/test_cache.py
|
||||
- [x] tests/test_models.py
|
||||
- [x] tests/test_score.py
|
||||
- [x] tests/test_dedupe.py
|
||||
- [x] tests/test_normalize.py
|
||||
- [x] tests/test_render.py
|
||||
|
||||
## Validation
|
||||
- [x] Run tests in mock mode
|
||||
- [x] Demo --emit=compact
|
||||
- [x] Demo --emit=context
|
||||
- [x] Verify file tree
|
||||
|
Before Width: | Height: | Size: 2.7 MiB After Width: | Height: | Size: 2.7 MiB |
|
Before Width: | Height: | Size: 2.3 MiB After Width: | Height: | Size: 2.3 MiB |
|
Before Width: | Height: | Size: 3.8 MiB After Width: | Height: | Size: 3.8 MiB |
|
Before Width: | Height: | Size: 2.6 MiB After Width: | Height: | Size: 2.6 MiB |
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
||||
type: feat
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
||||
type: fix
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
||||
type: feat
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
||||
type: fix
|
||||
status: active
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
---
|
||||
title: "marketing: v3.1 launch video — 30s Remotion piece for X"
|
||||
type: feat
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
---
|
||||
|
||||
# marketing: v3.1 launch video — 30s Remotion piece for X
|
||||
|
||||
## Overview
|
||||
|
||||
Build a 30-second Remotion-rendered MP4 marketing video announcing **v3.1: Competitors mode** for `/last30days`. Posts to X. Frames the new vs-mode-with-auto-discovery as the headline — three full passes, three save files, one comparison — without burying it in CLI minutiae.
|
||||
|
||||
Marketing release tag is **v3.1** (rebrands the 3.0.11-3.0.14 bundle into one shippable narrative for the launch tweet). Engine version stays 3.0.14 — `3.1` is the marketing version, not a code version bump.
|
||||
|
||||
## Script (60 frames per second × 30 seconds = 900 frames; this version assumes 30fps × 30s = 900 frames at 30fps)
|
||||
|
||||
Total runtime: 30.0 seconds @ 30fps = 900 frames. Six scenes, on-screen text only (silent autoplay-friendly).
|
||||
|
||||
### Scene 1 — Hook (0:00 – 0:03 | frames 0-90)
|
||||
|
||||
**Visual:** Black background. The `/last30days` badge animates in (the literal `🌐 last30days v3.1 · synced 2026-04-22` line) with the spring scale-in used in slick devtool intros.
|
||||
|
||||
**Caption (overlay, large):**
|
||||
```
|
||||
What if one search
|
||||
ran 3 at once?
|
||||
```
|
||||
|
||||
### Scene 2 — Set the world (0:03 – 0:08 | frames 90-240)
|
||||
|
||||
**Visual:** Single mac terminal window center-stage. Type-on animation:
|
||||
```
|
||||
$ /last30days OpenAI
|
||||
```
|
||||
Below it, a simple result card stub appears (Reddit upvote count + X likes), then static.
|
||||
|
||||
**Caption (small bottom-left):**
|
||||
```
|
||||
The old way: one topic.
|
||||
```
|
||||
|
||||
### Scene 3 — The reveal (0:08 – 0:14 | frames 240-420)
|
||||
|
||||
**Visual:** The terminal types one more flag:
|
||||
```
|
||||
$ /last30days OpenAI --competitors
|
||||
```
|
||||
Hard cut → the single terminal **splits into 3 panes** side by side. Each pane shows a different topic header animating in, in this order:
|
||||
- Left: `OpenAI`
|
||||
- Middle: `vs Anthropic`
|
||||
- Right: `vs xAI`
|
||||
|
||||
Pane content scrolls fake "search progress" lines (Reddit, X, YouTube indicators) in parallel, like a live fan-out.
|
||||
|
||||
**Caption (top center):**
|
||||
```
|
||||
Now it discovers competitors
|
||||
and runs all 3.
|
||||
```
|
||||
|
||||
### Scene 4 — Result reveal (0:14 – 0:21 | frames 420-630)
|
||||
|
||||
**Visual:** The 3 panes collapse into a single comparison surface — the `## Head-to-Head` table from the actual engine output, with rows fading in one by one (What it is, Streams, Best for, Trajectory). Each entity column lights up as its row populates.
|
||||
|
||||
**Caption (bottom):**
|
||||
```
|
||||
3 full passes. 3 save files.
|
||||
1 comparison.
|
||||
```
|
||||
|
||||
### Scene 5 — How it's special (0:21 – 0:26 | frames 630-780)
|
||||
|
||||
**Visual:** Cut to a clean text card, large mono font:
|
||||
```
|
||||
You pick the topic.
|
||||
The agent picks the peers.
|
||||
The engine fans out.
|
||||
```
|
||||
Each line fades in 1.5s apart.
|
||||
|
||||
### Scene 6 — CTA (0:26 – 0:30 | frames 780-900)
|
||||
|
||||
**Visual:** Black background. Centered:
|
||||
- Top: `🌐 last30days v3.1`
|
||||
- Middle: `/last30days {topic} --competitors`
|
||||
- Bottom: `github.com/mvanhorn/last30days-skill`
|
||||
|
||||
Subtle pulse on the install line.
|
||||
|
||||
**End frame holds for ~0.5s.**
|
||||
|
||||
## Problem Frame
|
||||
|
||||
The 3.0.11-3.0.14 release bundle ships a transformative feature (per-entity vs-mode fanout + `--competitors` shortcut + per-entity save files), but the value lands flat in a tweet thread or screenshot. A 30s video does what static text cannot: shows the fan-out happening in real time and the 3 → 1 collapse into a comparison. Higher tweet engagement, easier to RT/QT.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. Final artifact: a single MP4 file, 1920×1080, 30fps, ~30 seconds (±0.5s), under 30MB, suitable for direct X upload.
|
||||
- R2. Six-scene script as defined above, with text/visuals/timing matching to within 5 frames.
|
||||
- R3. Branded look: matches the `🌐 last30days v3.1` badge style (terminal aesthetic, mono font, dark background).
|
||||
- R4. Silent — no voiceover, no music in v1. Captions baked in. Designed for autoplay-muted feeds.
|
||||
- R5. Reproducible: another contributor (or a future-me) can re-render with one command. Project lives in-repo so the source is versioned.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No voiceover. Text-on-screen only. (Voice can be a v1.1 if engagement is high.)
|
||||
- No background music in the rendered MP4. (Music can be added in post via QuickTime/iMovie if desired before posting.)
|
||||
- No localization. English captions only.
|
||||
- No A/B test variants. One video.
|
||||
- No 9:16 vertical version. 16:9 only. (Vertical can be a separate render after launch validates the format.)
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Voiceover variant: defer to follow-up if v1 lands well.
|
||||
- 9:16 mobile cut: defer; same source compositions can re-render at 1080×1920 in a follow-up.
|
||||
- Animated GIF for embedding in README.md: defer; can be ffmpeg-extracted from the MP4.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `SKILL.md` — the comparison render scaffold (`## Head-to-Head` table) is the visual reference for Scene 4's table look.
|
||||
- `scripts/lib/render.py` `_render_comparison_scaffold` — emits the 9-axis table whose visual style we're recreating in a more polished form.
|
||||
- `CHANGELOG.md` 3.0.11-3.0.14 entries — the prose source for the script's beats.
|
||||
- `.claude-plugin/plugin.json` version 3.0.14 — current code version (marketing version is 3.1 for the launch).
|
||||
|
||||
### External References
|
||||
|
||||
- Remotion 4.x docs (https://www.remotion.dev/docs/) — current API for compositions, sequences, springs, and render CLI.
|
||||
- X video specs 2026: max 2 min 20 s, ≤512MB, MP4 with H.264 + AAC, recommended 1920×1080 for landscape autoplay.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- No prior `marketing/` dir or video plans in `docs/plans/`. This is a greenfield asset directory.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Remotion, not ffmpeg-only.** Remotion's React-based compositions handle the typed-on terminal effect, spring-animated badges, and scene transitions far more cleanly than raw ffmpeg filtergraphs. Render output is still MP4 via Remotion's bundled ffmpeg.
|
||||
- **In-repo asset directory at `marketing/v3.1-launch/`.** Lives with the product so future versions can fork the project. Adds `marketing/` to `.gitignore` exceptions only for source files; rendered MP4 stays out of git (uploaded separately).
|
||||
- **16:9 1920×1080 @ 30fps.** Best fit for X landscape autoplay on desktop and mobile feed. 30fps is plenty for typed-text + UI animation; 60fps doubles render time without obvious quality gain.
|
||||
- **Silent + captions.** X autoplay defaults to muted. Sound-off is the realistic viewing condition. Captions baked into the visual.
|
||||
- **Six scenes, one composition.** Single Remotion composition with sequenced child compositions per scene. Easier to re-time than scene-files. Frame-numbered timing in the script enables precise edits.
|
||||
- **Mono font (JetBrains Mono or Geist Mono).** Matches the terminal aesthetic of the actual `/last30days` output. Available via Google Fonts or @remotion/google-fonts.
|
||||
- **Marketing version 3.1 ≠ engine version 3.0.14.** `3.1` is the launch label. Engine stays 3.0.14. Avoids confusion in CHANGELOG.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **Aspect ratio?** 16:9 1080p. Best X autoplay format; vertical can be a follow-up.
|
||||
- **Voiceover or silent?** Silent + on-screen captions. Autoplay-muted is the realistic condition.
|
||||
- **In-repo or separate repo?** In-repo at `marketing/v3.1-launch/`. Rendered MP4 not committed; source compositions are.
|
||||
- **Length?** Exactly 30s (900 frames @ 30fps). No flex.
|
||||
- **Marketing version label?** `v3.1`. Engine code version stays 3.0.14.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact animation easing curves per scene — pick during build via Remotion preview iteration.
|
||||
- Whether the comparison-table mock in Scene 4 uses canned text or pulls from the actual saved `*-raw.md` files. Probably canned for visual control.
|
||||
- Whether Scene 3's "search progress" lines are typed individually or use a marquee scroll. Pick during preview.
|
||||
- Exact accent color palette beyond "terminal dark." Iterate against preview.
|
||||
|
||||
## Output Structure
|
||||
|
||||
marketing/
|
||||
v3.1-launch/
|
||||
package.json # Remotion dependency manifest
|
||||
tsconfig.json # TypeScript config
|
||||
remotion.config.ts # Remotion render config (codec, fps, resolution)
|
||||
src/
|
||||
index.ts # Remotion entry — registers compositions
|
||||
Root.tsx # Root composition definition
|
||||
LaunchVideo.tsx # Main 30s composition that sequences scenes
|
||||
scenes/
|
||||
Scene1Hook.tsx
|
||||
Scene2OldWay.tsx
|
||||
Scene3FanOut.tsx
|
||||
Scene4Comparison.tsx
|
||||
Scene5HowItWorks.tsx
|
||||
Scene6CTA.tsx
|
||||
components/
|
||||
TerminalWindow.tsx # Reusable mac-style terminal frame
|
||||
TypedLine.tsx # Type-on animation primitive
|
||||
ComparisonTable.tsx # The Head-to-Head table mock
|
||||
BadgeBar.tsx # The 🌐 last30days v3.1 badge
|
||||
lib/
|
||||
timing.ts # Frame ranges per scene (single source of truth)
|
||||
colors.ts # Brand palette
|
||||
public/ # Static assets (logo, fonts if local)
|
||||
out/ # Rendered MP4 lives here (gitignored)
|
||||
README.md # How to preview/render
|
||||
|
||||
## High-Level Technical Design
|
||||
|
||||
> *Directional guidance for review — not implementation specification.*
|
||||
|
||||
```
|
||||
LaunchVideo (durationInFrames = 900)
|
||||
├── <Sequence from={0} durationInFrames={90}> <Scene1Hook />
|
||||
├── <Sequence from={90} durationInFrames={150}> <Scene2OldWay />
|
||||
├── <Sequence from={240} durationInFrames={180}> <Scene3FanOut />
|
||||
├── <Sequence from={420} durationInFrames={210}> <Scene4Comparison />
|
||||
├── <Sequence from={630} durationInFrames={150}> <Scene5HowItWorks />
|
||||
└── <Sequence from={780} durationInFrames={120}> <Scene6CTA />
|
||||
```
|
||||
|
||||
Per-scene components use `useCurrentFrame()` + `interpolate()` + `spring()` for timing. `TerminalWindow` is the dominant motif across scenes 2-4.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: Remotion project scaffold**
|
||||
|
||||
**Goal:** Spin up a working Remotion project at `marketing/v3.1-launch/` that previews a blank composition and renders to MP4.
|
||||
|
||||
**Requirements:** R1, R5
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/package.json`
|
||||
- Create: `marketing/v3.1-launch/tsconfig.json`
|
||||
- Create: `marketing/v3.1-launch/remotion.config.ts`
|
||||
- Create: `marketing/v3.1-launch/src/index.ts`
|
||||
- Create: `marketing/v3.1-launch/src/Root.tsx`
|
||||
- Create: `marketing/v3.1-launch/README.md`
|
||||
- Modify: `.gitignore` (add `marketing/v3.1-launch/out/`, `marketing/v3.1-launch/node_modules/`)
|
||||
|
||||
**Approach:**
|
||||
- Use `npx create-video@latest --blank` (Remotion 4.x scaffolding) targeting `marketing/v3.1-launch/`. Strip the demo composition.
|
||||
- Configure: 1920×1080, 30fps, H.264, AAC (audio codec needed even for silent — empty track).
|
||||
- README documents `npm install`, `npm run preview` (Remotion Studio), `npm run render` (one-command MP4).
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none for scaffold, but verification = `npm run preview` opens Remotion Studio with a blank 30s composition; `npm run render` produces a black MP4 at the right resolution.
|
||||
|
||||
**Verification:**
|
||||
- Studio loads at localhost:3000 with the empty `LaunchVideo` composition listed.
|
||||
- A render produces `out/launch-video.mp4` at 1920×1080, 30s, valid MP4.
|
||||
|
||||
- [ ] **Unit 2: Reusable components (Terminal, TypedLine, BadgeBar)**
|
||||
|
||||
**Goal:** Build the three primitive components scenes 2-6 will compose. Each is independently previewable.
|
||||
|
||||
**Requirements:** R3, R5
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/components/TerminalWindow.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/TypedLine.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/BadgeBar.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/lib/colors.ts`
|
||||
- Create: `marketing/v3.1-launch/src/lib/timing.ts`
|
||||
|
||||
**Approach:**
|
||||
- `TerminalWindow`: mac-style traffic-light header, dark gradient background, mono content area. Accepts children.
|
||||
- `TypedLine`: takes a string and a `startFrame`, renders character-by-character at ~30 chars/sec. Reuses Remotion's `interpolate(useCurrentFrame() - startFrame, [0, lengthFrames], [0, text.length])` clamped.
|
||||
- `BadgeBar`: renders the literal `🌐 last30days v3.1 · synced 2026-04-22` line in mono with the same gradient color treatment as the engine's compact emit.
|
||||
- `colors.ts`: 5-7 brand colors (terminal-bg, terminal-fg, accent-cyan, accent-magenta, muted, success-green, warning-amber).
|
||||
- `timing.ts`: exports the scene frame ranges as named constants. Single source of truth for any retiming.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — visual components verified in Remotion Studio.
|
||||
|
||||
**Verification:**
|
||||
- Each component renders standalone in Studio when wrapped in a temporary preview composition.
|
||||
- TypedLine animates character-by-character without flicker.
|
||||
|
||||
- [ ] **Unit 3: Scenes 1-3 (Hook, Old way, Fan-out reveal)**
|
||||
|
||||
**Goal:** Build the first half of the video (frames 0-420). The narrative arc up through the visual fan-out.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 2
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene1Hook.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene2OldWay.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene3FanOut.tsx`
|
||||
- Modify: `marketing/v3.1-launch/src/Root.tsx` (register sequences)
|
||||
|
||||
**Approach:**
|
||||
- Scene 1: spring-in BadgeBar, large overlay caption, 3-second hold.
|
||||
- Scene 2: TerminalWindow with TypedLine (`$ /last30days OpenAI`), then a single result-card mock fading in.
|
||||
- Scene 3: typing animation appends `--competitors`, hard cut, three TerminalWindow components arranged in a row with staggered fade-in. Each pane shows a different entity header + scrolling progress lines.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verified visually in Studio.
|
||||
|
||||
**Verification:**
|
||||
- Scrub the 0-14s range in Studio; visuals match the script timing within 5 frames.
|
||||
- The fan-out moment (frame 240) lands cleanly; no jank in the transition from 1 → 3 panes.
|
||||
|
||||
- [ ] **Unit 4: Scenes 4-6 (Comparison reveal, How it works, CTA)**
|
||||
|
||||
**Goal:** Build the back half of the video (frames 420-900). Resolution + payoff + call to action.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 2
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene4Comparison.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene5HowItWorks.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene6CTA.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/ComparisonTable.tsx`
|
||||
|
||||
**Approach:**
|
||||
- Scene 4: ComparisonTable component renders 3-column markdown-style table; rows fade in one by one (stagger 15-20 frames). Uses canned data — OpenAI / Anthropic / xAI with believable cell content drawn from real 3.0.13 outputs.
|
||||
- Scene 5: three-line text card; lines fade in 45 frames apart.
|
||||
- Scene 6: three centered text blocks; install line gets a 1Hz subtle opacity pulse for emphasis.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verified visually.
|
||||
|
||||
**Verification:**
|
||||
- Scrub 14-30s; table reveal feels paced (not too slow, not strobed); CTA holds long enough to read (~3-4s).
|
||||
- ComparisonTable cells are legible at 1920×1080 (mono font ≥ 28px).
|
||||
|
||||
- [ ] **Unit 5: Final composition wiring + render**
|
||||
|
||||
**Goal:** Wire the six scenes into the master `LaunchVideo` composition, render to MP4, verify against X upload constraints.
|
||||
|
||||
**Requirements:** R1, R2, R5
|
||||
|
||||
**Dependencies:** Units 3, 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `marketing/v3.1-launch/src/LaunchVideo.tsx` (sequence all 6 scenes)
|
||||
- Modify: `marketing/v3.1-launch/README.md` (add render command + verification checklist)
|
||||
|
||||
**Approach:**
|
||||
- `LaunchVideo` is a single Composition that imports `Scene1Hook` … `Scene6CTA` and wraps each in `<Sequence from=… durationInFrames=…>` matching `lib/timing.ts`.
|
||||
- Run `npx remotion render LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18`.
|
||||
- Verify output: 30.0s ±0.1s, 1920×1080, file size <30MB, opens in QuickTime, plays without dropped frames.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verification is the rendered MP4 itself.
|
||||
|
||||
**Verification:**
|
||||
- `ffprobe out/last30days-v3.1-launch.mp4` reports 1920×1080, 30fps, ~30.0s, h264, faststart-friendly.
|
||||
- Manual play-through end-to-end in QuickTime feels coherent and on-pace.
|
||||
- File <30MB so X upload is instant.
|
||||
|
||||
- [ ] **Unit 6: Polish pass + ship**
|
||||
|
||||
**Goal:** Watch the full render, fix obvious jank, do a second render, and stage for X posting.
|
||||
|
||||
**Requirements:** R1, R2, R3
|
||||
|
||||
**Dependencies:** Unit 5
|
||||
|
||||
**Files:**
|
||||
- Possibly modify: any scene file based on watch-through findings.
|
||||
|
||||
**Approach:**
|
||||
- Watch the rendered MP4 at full size. Note: timing felt off, transitions too fast, captions overflow, color clash, anything visibly broken.
|
||||
- Iterate: edit scene component → re-preview in Studio → re-render full MP4.
|
||||
- Cap at 2 polish passes; ship the better of the two renders.
|
||||
- Final MP4 sits at `marketing/v3.1-launch/out/last30days-v3.1-launch.mp4` ready for X upload.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — pure subjective polish.
|
||||
|
||||
**Verification:**
|
||||
- User watches the final render and approves.
|
||||
- No glaring visual bugs (overflowing text, frozen frames, color clashes).
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** None — this is a standalone marketing artifact. Doesn't touch the Python engine, doesn't change any user-facing behavior.
|
||||
- **State lifecycle risks:** None.
|
||||
- **API surface parity:** N/A.
|
||||
- **Unchanged invariants:** The shipped 3.0.14 engine is untouched.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Remotion install pulls 200MB+ of node_modules. | `marketing/v3.1-launch/node_modules/` in `.gitignore`; checked-in source stays small. |
|
||||
| Render time blows past patience (>5 min for 30s @ 1080p30). | Bun-based render or `--concurrency` flag. Default Remotion is fast enough on M-series Macs. If slow, lower preview to 720p, render final at 1080p. |
|
||||
| Captions overflow the 1920px width on certain fonts. | Use a known mono font with a measured per-character width; cap caption lines at 36 chars. |
|
||||
| The "fan-out" visual in Scene 3 looks confusing instead of magical. | Polish pass (Unit 6) is the safety net; if still bad, fall back to a simpler "1 → 3 panes wipe" instead of typed split. |
|
||||
| File size >30MB hits X upload friction. | Use `--crf=18` (high quality, reasonable size); fall back to `--crf=23` if over. 30s @ 1080p30 H.264 is normally 5-15MB. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- README at `marketing/v3.1-launch/README.md` documents preview / render commands.
|
||||
- After render, the MP4 is uploaded directly to X. Tweet copy is the user's call (this plan stops at the rendered file).
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Related code: `scripts/lib/render.py` (`_render_comparison_scaffold` is the visual model for Scene 4); `SKILL.md` Competitor mode section (the narrative source).
|
||||
- Related PRs: #308, #311, #312 (the 3.0.11 → 3.0.14 release bundle this video markets as "v3.1").
|
||||
- External docs: https://www.remotion.dev/docs/ (Remotion 4.x API).
|
||||
- X video specs: https://help.x.com/en/using-x/twitter-videos.
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
title: Search-quality eval is manual by default, not a CI gate on every PR
|
||||
date: 2026-05-10
|
||||
category: docs/solutions/architecture
|
||||
module: skills/last30days/scripts/evaluate_search_quality.py
|
||||
problem_type: design_decision
|
||||
component: ci_policy
|
||||
severity: low
|
||||
applies_when:
|
||||
- a contributor proposes wiring search-quality eval into PR CI
|
||||
- a change affects retrieval, ranking, grounding, or synthesis quality and a reviewer asks "why aren't we testing this in CI?"
|
||||
- someone is deciding whether a new evaluator-style script belongs in the default CI workflow
|
||||
related_components:
|
||||
- search_quality_evaluation
|
||||
- ci_workflow
|
||||
- llm_judging
|
||||
tags:
|
||||
- ci-policy
|
||||
- eval
|
||||
- design-decision
|
||||
- cost-vs-signal
|
||||
- non-determinism
|
||||
- manual-gates
|
||||
---
|
||||
|
||||
# Search-quality eval is manual by default, not a CI gate on every PR
|
||||
|
||||
## Context
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` compares a baseline revision against a candidate revision across a fixed pool of reviewer topics. It produces two flavors of metrics: deterministic overlap (Jaccard, retention) and LLM-judged quality scores. The natural impulse on seeing an evaluator script is to wire it into CI on every PR — "regression catcher, run it automatically." We deliberately don't.
|
||||
|
||||
Three properties of this particular evaluator make CI-on-every-PR the wrong default:
|
||||
|
||||
1. **Live API access.** The candidate revision typically needs the engine to actually run, which means real ScrapeCreators calls, real reddit fetches, real YouTube searches. CI runs would either need production credentials or a record/replay fixture set that drifts almost immediately as external APIs change shape.
|
||||
|
||||
2. **Cost and latency.** A full eval pass runs the pipeline N times across reviewer topics. Multiplied by every PR (including doc-only PRs), the spend is meaningful and the wall-clock pushes CI from ~30s to many minutes.
|
||||
|
||||
3. **Non-determinism in the judging path.** The LLM-judged metrics are valuable for review but depend on judge-model behavior on a given day. A flaky eval that fails 1 PR in 20 because the judge re-scored an item differently is a worse CI signal than no eval at all — it teaches contributors to retry rather than read the result.
|
||||
|
||||
The deterministic overlap metrics are useful regression signals but they are not the same as user-facing correctness. A change that improves overlap can degrade synthesis quality; a change that drops overlap can be a deliberate improvement. So even the deterministic side isn't safe to auto-fail on.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Keep search-quality eval available, just not automatic
|
||||
|
||||
The script stays runnable by maintainers and contributors. The pattern is:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_PYTHON=python3.13 \
|
||||
python3 skills/last30days/scripts/evaluate_search_quality.py \
|
||||
--baseline main --candidate HEAD
|
||||
```
|
||||
|
||||
Reviewers can request a manual eval run when a PR is in the retrieval/ranking/synthesis path and the risk warrants it. Contributors can run it locally before submitting if they want signal upfront.
|
||||
|
||||
### 2. Standard PR CI gates remain deterministic and contract-shaped
|
||||
|
||||
`pytest` (offline-safe), plugin-contract checks, version-consistency contracts, ruff/lint. Anything that returns the same answer twice for the same input. Quality-of-output assessment lives outside that loop.
|
||||
|
||||
### 3. The middle ground is `workflow_dispatch`, not auto-PR-gating
|
||||
|
||||
If maintainers want a GitHub-triggered eval that doesn't make every PR pay the live-API cost, the right shape is a manually-dispatched workflow (or a label-triggered one) — not a `pull_request:` workflow that runs unconditionally. That keeps the cost knob in human hands.
|
||||
|
||||
### 4. Revisit if the eval can ever be made offline-deterministic
|
||||
|
||||
The blocker is the live-API + non-determinism combination. If a future iteration of the script can compute meaningful Jaccard/retention metrics against static fixtures (no live API calls, no LLM judging), the decision flips and it becomes a candidate for default CI. The decision below tracks that condition; revisit when it's met.
|
||||
|
||||
## What this means in practice
|
||||
|
||||
- Don't merge PRs that wire `evaluate_search_quality.py` into the default `validate.yml` workflow.
|
||||
- Do merge PRs that add `workflow_dispatch` triggers or label-gated runs.
|
||||
- When reviewing a retrieval/ranking change, request a manual eval if the diff suggests it could regress quality — don't expect CI to catch it.
|
||||
|
||||
## Links
|
||||
|
||||
- `skills/last30days/scripts/evaluate_search_quality.py` — the evaluator script
|
||||
- `docs/search-quality-eval.md` — user-facing usage documentation
|
||||
- `.github/workflows/validate.yml` — the default CI workflow (deterministic gates only)
|
||||
|
||||
---
|
||||
|
||||
*Adapted from a draft ADR proposed by @hnshah in [#374](https://github.com/mvanhorn/last30days-skill/pull/374), restructured into the `docs/solutions/` convention. The original ADR text correctly identified the constraint; this version adds the "why workflow_dispatch is the middle ground" framing and the revisit-condition.*
|
||||
@@ -1,219 +0,0 @@
|
||||
---
|
||||
title: Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
date: 2026-05-16
|
||||
category: docs/solutions/workflow-issues
|
||||
module: ci-release-engineering
|
||||
problem_type: workflow_issue
|
||||
component: testing_framework
|
||||
severity: high
|
||||
applies_when:
|
||||
- a test asserts consistency between two release-time artifacts (e.g., SKILL.md version and a hardcoded pin in a shell script)
|
||||
- one artifact is updated as part of a version bump and the other requires a manual lockstep update
|
||||
- multiple long-lived PRs are open simultaneously against the same base branch
|
||||
symptoms:
|
||||
- every open PR's CI fails after a version bump even though the PRs are unrelated to versioning
|
||||
- the failing test references a stale hardcoded value that was not updated alongside the bumped version
|
||||
- PR authors must rebase and manually fix an artifact they did not touch
|
||||
root_cause: missing_workflow_step
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- development_workflow
|
||||
- documentation
|
||||
tags:
|
||||
- ci
|
||||
- release-engineering
|
||||
- consistency-test
|
||||
- version-pin
|
||||
- cascade-failure
|
||||
- test-design
|
||||
- workflow
|
||||
---
|
||||
|
||||
# Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
|
||||
## Context
|
||||
|
||||
A `tests/test_version_consistency.py::test_sync_cache_path_uses_skill_version` test was added to enforce that the version string embedded in `skills/last30days/scripts/sync.sh` (a hardcoded plugin-cache path segment) matched the version frontmatter in `skills/last30days/SKILL.md`. The intention was sound: the cache path had to stay in lockstep with the skill version or the sync would silently pull stale files.
|
||||
|
||||
The test worked as designed until a release shipped. At that point it turned into a cascade-failure machine:
|
||||
|
||||
1. A release PR bumps `SKILL.md` version (e.g., 3.2.0 → 3.2.1) **and** bumps the `sync.sh` pin. That PR's CI is green.
|
||||
2. The release PR merges to `main`.
|
||||
3. Every PR that was open at merge time was branched from pre-release `main`. Those PRs have `SKILL.md` 3.2.1 (inherited via merge-base with `main`) but their branch never touched `sync.sh`.
|
||||
4. CI for those PRs runs the consistency test against the new `main` — `SKILL.md` says 3.2.1, `sync.sh` still says 3.2.0 — and fails.
|
||||
5. All open PRs are now red simultaneously, with a failure that has nothing to do with their changes.
|
||||
|
||||
This affected at least five PRs during the 2026-05-13 to 2026-05-15 window: PR #400 (caught during rebase, required a manual pin bump), PRs #390 and #392 (OpenClaw `SCRAPECREATORS_API_KEY` fix, both stalled for the same stale-pin reason), and at least two others. A follow-up hotfix PR (#397 — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`) was required just to unblock the queue.
|
||||
|
||||
The permanent fix was PR #405: delete `sync.sh` entirely (the install workflow made it redundant) and drop `test_sync_cache_path_uses_skill_version`. Once both were gone, no version-consistency cascade was possible.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Don't write consistency tests that read two files and assert one matches a substring derived from the other
|
||||
|
||||
This pattern looks safe but is not:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # reads SKILL.md
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # asserts sync.sh contains that string
|
||||
)
|
||||
```
|
||||
|
||||
It encodes the assumption that both files are always updated together, in the same commit, on the same branch. That assumption breaks the moment two files have independent lifecycle owners — a versioned manifest and a deployment script are archetypal examples.
|
||||
|
||||
### 2. If the values genuinely need to stay in sync, derive one from the other at runtime
|
||||
|
||||
Remove the hardcoded pin from `sync.sh` and compute it:
|
||||
|
||||
```bash
|
||||
# sync.sh — derive version from SKILL.md at runtime, no pin to maintain
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "$(dirname "$0")/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
CACHE_PATH="last30days-skill/last30days/${SKILL_VERSION}"
|
||||
```
|
||||
|
||||
Now there is only one source of truth (`SKILL.md`). The test that asserted they matched becomes vacuous and should be deleted. If `SKILL.md` is wrong, the sync itself will fail loudly — which is better feedback than a CI gate on a different PR.
|
||||
|
||||
### 3. If two values must stay independent for legitimate reasons, update them together and make the test self-skip if either source is missing
|
||||
|
||||
If separate versioning is genuinely required (e.g., SKILL.md versions for harness consumers, sync.sh versions a private artifact store with its own cadence), update both in the same PR — never staggered — and write the test to self-skip rather than error when either file is absent:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_sh = SKILL_ROOT / "scripts" / "sync.sh"
|
||||
if not sync_sh.exists():
|
||||
self.skipTest("sync.sh not present; skipping pin consistency check")
|
||||
sync_text = sync_sh.read_text(encoding="utf-8")
|
||||
version = _skill_version()
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text,
|
||||
)
|
||||
```
|
||||
|
||||
Self-skipping means deleting the file is a non-event in CI — no cascading red, no hotfix PR to the queue.
|
||||
|
||||
### 4. Run consistency tests against the merge-base diff, not main
|
||||
|
||||
If you keep a two-file consistency test, scope it so it only fails when the PR itself modifies one of the two files but not the other. A GitHub Actions step can do this:
|
||||
|
||||
```yaml
|
||||
- name: Check sync.sh version pin consistency
|
||||
run: |
|
||||
BASE=$(git merge-base HEAD origin/main)
|
||||
SKILL_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'SKILL\.md' || true)
|
||||
SYNC_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'sync\.sh' || true)
|
||||
if [ "$SKILL_CHANGED" -gt 0 ] && [ "$SYNC_CHANGED" -eq 0 ]; then
|
||||
echo "SKILL.md version bumped but sync.sh pin was not updated"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
This only fires when your PR touched `SKILL.md` and left `sync.sh` alone — never because a release merged to `main` after you branched.
|
||||
|
||||
### 5. Ask whether you actually need this test
|
||||
|
||||
If the values are wrong, downstream tooling will fail loudly: the sync will fetch the wrong artifact, installs will break, or the harness will reject the version. A test that exists only to catch a human-bookkeeping error at release time adds cascade-fail risk without offering a meaningfully earlier signal. Weigh that cost before adding any two-file consistency gate.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The damage from a stale-pin consistency test is asymmetric. It:
|
||||
|
||||
- Fails on every open PR simultaneously the moment a release lands on `main` — not just the PR that forgot to update the pin.
|
||||
- Produces a failure message that points at a line in a test file with no obvious relationship to the PR's actual changes.
|
||||
- Requires either a hotfix PR (touching a file the failing PRs have no business touching) or a manual rebase of every affected branch.
|
||||
- Blocks work that has already been reviewed and approved.
|
||||
|
||||
In this repo the effect was measurable: at least five PRs stalled across a two-day window, one hotfix PR was shipped just to unblock the queue, and multiple authors spent time debugging a failure completely unrelated to their changes.
|
||||
|
||||
The broader principle is that tests which gate on *bookkeeping consistency between files* impose their maintenance cost on every contributor, every time, even when those contributors did nothing wrong. That cost compounds with team size and release cadence.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Apply this guidance whenever you find yourself:
|
||||
|
||||
- Writing a test that reads two files and asserts that a string in one matches a value derived from the other.
|
||||
- Adding a CI step labeled "consistency check," "sync check," or "pin check" where the check compares a hardcoded value against a computed one from a separate file.
|
||||
- Working in a repo where a versioned manifest (e.g., `SKILL.md`, `package.json`, `pyproject.toml`) and a deployment artifact (e.g., a shell script, a Dockerfile, a Helm values file) are both maintained by hand.
|
||||
- Reviewing a PR that touches only one of two "paired" files and fails a consistency test for the other.
|
||||
|
||||
It does *not* apply to tests that read a single source of truth and validate its internal structure (e.g., asserting that `SKILL.md`'s frontmatter version is double-quoted, or that `package.json`'s `version` field is a valid semver string). Those tests have one file and one assertion; they cannot cascade across branches.
|
||||
|
||||
## Examples
|
||||
|
||||
### Before — the pattern that caused the cascade
|
||||
|
||||
Original `tests/test_version_consistency.py` (deleted in commit `9fb19ea`):
|
||||
|
||||
```python
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_ROOT = ROOT / "skills" / "last30days"
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
|
||||
if not match:
|
||||
raise AssertionError("SKILL.md version frontmatter not found")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
class TestVersionConsistency(unittest.TestCase):
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # source 1: SKILL.md frontmatter
|
||||
self.assertIn( # assertion: sync.sh must contain
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # source 2: hardcoded string in sync.sh
|
||||
)
|
||||
```
|
||||
|
||||
`sync.sh` contained a line like:
|
||||
|
||||
```bash
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/3.2.0"
|
||||
```
|
||||
|
||||
When SKILL.md bumped to `3.2.1` in a release PR, `sync.sh` was updated in the same PR and CI stayed green. But every PR branched before that release still had `sync.sh` at `3.2.0`. Their CI failed immediately, with an assertion error pointing at the test, not at the release PR.
|
||||
|
||||
### After — what we did: delete both
|
||||
|
||||
PR #405 deleted `sync.sh` (the install workflow replaced it) and dropped `test_sync_cache_path_uses_skill_version` in the same change. No consistency gate, no pin to maintain, no cascade possible.
|
||||
|
||||
### After — what we could have done instead: derive at runtime
|
||||
|
||||
If `sync.sh` had still been needed, the right fix would have been to remove the hardcoded version from the script and derive it from `SKILL.md`:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# sync.sh — no hardcoded version; reads SKILL.md as single source of truth
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "${SCRIPT_DIR}/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
|
||||
if [ -z "$SKILL_VERSION" ]; then
|
||||
echo "error: could not parse version from SKILL.md" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/${SKILL_VERSION}"
|
||||
# ... rest of sync logic
|
||||
```
|
||||
|
||||
With this in place, `test_sync_cache_path_uses_skill_version` has no reason to exist — there is nothing to assert. Delete it. If the version parsing breaks, `sync.sh` itself exits non-zero with a clear message.
|
||||
|
||||
## Related
|
||||
|
||||
- **PR #397** (merged) — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`. The hotfix that unblocked the cascade temporarily by bumping the pin.
|
||||
- **PR #400** (merged) — caught the same cascade during rebase; had to bump the pin to clear CI.
|
||||
- **PR #390** (closed) and **PR #392** (rebased + merged) — OpenClaw `SCRAPECREATORS_API_KEY` fix; both blocked by the cascade until rebased onto post-#405 main.
|
||||
- **PR #405** (merged) — the permanent fix: deleted `sync.sh` + `test_sync_cache_path_uses_skill_version` together.
|
||||
- **PR #412** (merged) — adjacent work that consolidated SKILL.md version parsing into `lib/skill_meta.py`, reducing future drift risk by giving the version field one canonical reader.
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"triggerOnUpdates": true,
|
||||
"statusCheck": true
|
||||
}
|
||||
@@ -97,20 +97,7 @@ if [[ -n "$HAS_BSKY" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
|
||||
# Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer
|
||||
# whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing.
|
||||
SC_ADD=3
|
||||
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
|
||||
EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \
|
||||
| sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//')
|
||||
if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||
fi
|
||||
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
@@ -120,6 +107,6 @@ else
|
||||
# Setup done but missing ScrapeCreators — recommend it
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
||||
echo " 10,000 free API calls, no credit card — scrapecreators.com"
|
||||
echo " last30days has no affiliation with any API provider."
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# /last30days v3.1 Launch Video
|
||||
|
||||
30-second Remotion-rendered MP4 announcing **v3.1: Competitors mode** for posting on X.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
cd marketing/v3.1-launch
|
||||
npm install # one-time, ~200MB of node_modules
|
||||
npm run preview # opens Remotion Studio at localhost:3000 to scrub frames
|
||||
npm run render # writes out/last30days-v3.1-launch.mp4 (high quality, CRF 18)
|
||||
npm run render:fast # writes a CRF 23 preview for fast iteration
|
||||
```
|
||||
|
||||
## Specs
|
||||
|
||||
- 1920×1080, 30fps, 30 seconds (900 frames)
|
||||
- H.264 / MP4
|
||||
- Silent (autoplay-muted-friendly; captions baked in)
|
||||
- Marketing label `v3.1` (engine code version stays 3.0.14)
|
||||
|
||||
## Scene timing (single source of truth: `src/lib/timing.ts`)
|
||||
|
||||
| Scene | Frames | Time | What |
|
||||
|-------|--------|------|------|
|
||||
| 1. Hook | 0-89 | 0.0-3.0s | Badge animates in + "What if one search ran 3 at once?" |
|
||||
| 2. Old way | 90-239 | 3.0-8.0s | Single terminal: `/last30days OpenAI` |
|
||||
| 3. Fan-out | 240-419 | 8.0-14.0s | `--competitors` types in → splits into 3 panes |
|
||||
| 4. Comparison | 420-629 | 14.0-21.0s | 3 panes collapse into Head-to-Head table |
|
||||
| 5. How | 630-779 | 21.0-26.0s | 3-line text card |
|
||||
| 6. CTA | 780-899 | 26.0-30.0s | Install command + repo URL |
|
||||
|
||||
Edit `src/lib/timing.ts` to retime scenes; the `LaunchVideo` composition reads from there.
|
||||
|
||||
## Output
|
||||
|
||||
Rendered MP4 lives at `out/last30days-v3.1-launch.mp4` (gitignored). Upload directly to X.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "last30days-v3-1-launch-video",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "30s Remotion launch video for /last30days v3.1 (competitors mode).",
|
||||
"scripts": {
|
||||
"preview": "remotion studio src/index.ts",
|
||||
"render": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18",
|
||||
"render:fast": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch-preview.mp4 --codec=h264 --crf=23"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "19.0.0",
|
||||
"react-dom": "19.0.0",
|
||||
"remotion": "4.0.250",
|
||||
"@remotion/cli": "4.0.250",
|
||||
"@remotion/google-fonts": "4.0.250"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "19.0.0",
|
||||
"@types/node": "22.10.0",
|
||||
"typescript": "5.6.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { Config } from "@remotion/cli/config";
|
||||
|
||||
Config.setVideoImageFormat("jpeg");
|
||||
Config.setOverwriteOutput(true);
|
||||
Config.setConcurrency(null);
|
||||
Config.setCodec("h264");
|
||||
@@ -0,0 +1,35 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, Sequence } from "remotion";
|
||||
import { SCENES } from "./lib/timing";
|
||||
import { Scene1Hook } from "./scenes/Scene1Hook";
|
||||
import { Scene2OldWay } from "./scenes/Scene2OldWay";
|
||||
import { Scene3FanOut } from "./scenes/Scene3FanOut";
|
||||
import { Scene4Comparison } from "./scenes/Scene4Comparison";
|
||||
import { Scene5HowItWorks } from "./scenes/Scene5HowItWorks";
|
||||
import { Scene6CTA } from "./scenes/Scene6CTA";
|
||||
import { COLORS } from "./lib/colors";
|
||||
|
||||
export const LaunchVideo: React.FC = () => {
|
||||
return (
|
||||
<AbsoluteFill style={{ background: COLORS.bgDeep }}>
|
||||
<Sequence from={SCENES.hook.from} durationInFrames={SCENES.hook.durationInFrames}>
|
||||
<Scene1Hook />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.oldWay.from} durationInFrames={SCENES.oldWay.durationInFrames}>
|
||||
<Scene2OldWay />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.fanOut.from} durationInFrames={SCENES.fanOut.durationInFrames}>
|
||||
<Scene3FanOut />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.comparison.from} durationInFrames={SCENES.comparison.durationInFrames}>
|
||||
<Scene4Comparison />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.howItWorks.from} durationInFrames={SCENES.howItWorks.durationInFrames}>
|
||||
<Scene5HowItWorks />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.cta.from} durationInFrames={SCENES.cta.durationInFrames}>
|
||||
<Scene6CTA />
|
||||
</Sequence>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
import React from "react";
|
||||
import { Composition } from "remotion";
|
||||
import { LaunchVideo } from "./LaunchVideo";
|
||||
import { FPS, TOTAL_FRAMES } from "./lib/timing";
|
||||
|
||||
export const RemotionRoot: React.FC = () => {
|
||||
return (
|
||||
<>
|
||||
<Composition
|
||||
id="LaunchVideo"
|
||||
component={LaunchVideo}
|
||||
durationInFrames={TOTAL_FRAMES}
|
||||
fps={FPS}
|
||||
width={1920}
|
||||
height={1080}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,68 @@
|
||||
import React from "react";
|
||||
import { spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
startFrame?: number;
|
||||
size?: "small" | "large";
|
||||
};
|
||||
|
||||
export const BadgeBar: React.FC<Props> = ({ startFrame = 0, size = "large" }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
const elapsed = Math.max(0, frame - startFrame);
|
||||
|
||||
const scale = spring({
|
||||
frame: elapsed,
|
||||
fps,
|
||||
config: { damping: 12, stiffness: 90 },
|
||||
from: 0.85,
|
||||
to: 1,
|
||||
});
|
||||
const opacity = spring({
|
||||
frame: elapsed,
|
||||
fps,
|
||||
config: { damping: 20 },
|
||||
from: 0,
|
||||
to: 1,
|
||||
});
|
||||
|
||||
const fontSize = size === "large" ? 56 : 28;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
transform: `scale(${scale})`,
|
||||
opacity,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize,
|
||||
color: COLORS.fgPrimary,
|
||||
letterSpacing: 0.5,
|
||||
}}
|
||||
>
|
||||
<span style={{ fontSize: fontSize * 1.1, marginRight: 16 }}>🌐</span>
|
||||
<span>last30days</span>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 14,
|
||||
color: COLORS.accentCyan,
|
||||
fontWeight: 600,
|
||||
}}
|
||||
>
|
||||
v3.1
|
||||
</span>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 16,
|
||||
color: COLORS.fgDim,
|
||||
fontSize: fontSize * 0.55,
|
||||
}}
|
||||
>
|
||||
· synced 2026-04-22
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,135 @@
|
||||
import React from "react";
|
||||
import { interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Row = {
|
||||
dimension: string;
|
||||
cells: [string, string, string];
|
||||
};
|
||||
|
||||
type Props = {
|
||||
startFrame: number;
|
||||
entities: [string, string, string];
|
||||
rows: Row[];
|
||||
rowStaggerFrames?: number;
|
||||
};
|
||||
|
||||
export const ComparisonTable: React.FC<Props> = ({
|
||||
startFrame,
|
||||
entities,
|
||||
rows,
|
||||
rowStaggerFrames = 18,
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const headerOpacity = interpolate(
|
||||
frame - startFrame,
|
||||
[0, 12],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
|
||||
const colTemplate = "1.4fr 1fr 1fr 1fr";
|
||||
const cellPad = "16px 22px";
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
background: COLORS.bgPanel,
|
||||
borderRadius: 16,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
overflow: "hidden",
|
||||
fontFamily: FONT_MONO,
|
||||
boxShadow: "0 24px 60px rgba(0,0,0,0.6)",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: colTemplate,
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderBottom: `1px solid ${COLORS.border}`,
|
||||
opacity: headerOpacity,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 22,
|
||||
fontWeight: 500,
|
||||
}}
|
||||
>
|
||||
Dimension
|
||||
</div>
|
||||
{entities.map((entity, idx) => (
|
||||
<div
|
||||
key={entity}
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: idx === 0 ? COLORS.accentCyan : COLORS.fgPrimary,
|
||||
fontSize: 26,
|
||||
fontWeight: 600,
|
||||
borderLeft: `1px solid ${COLORS.border}`,
|
||||
}}
|
||||
>
|
||||
{entity}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{rows.map((row, idx) => {
|
||||
const rowStart = startFrame + 12 + idx * rowStaggerFrames;
|
||||
const rowOpacity = interpolate(
|
||||
frame - rowStart,
|
||||
[0, 14],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
const rowSlide = interpolate(
|
||||
frame - rowStart,
|
||||
[0, 14],
|
||||
[12, 0],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
return (
|
||||
<div
|
||||
key={row.dimension}
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: colTemplate,
|
||||
borderBottom:
|
||||
idx === rows.length - 1 ? "none" : `1px solid ${COLORS.border}`,
|
||||
opacity: rowOpacity,
|
||||
transform: `translateY(${rowSlide}px)`,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 22,
|
||||
}}
|
||||
>
|
||||
{row.dimension}
|
||||
</div>
|
||||
{row.cells.map((cell, cellIdx) => (
|
||||
<div
|
||||
key={cellIdx}
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgPrimary,
|
||||
fontSize: 22,
|
||||
borderLeft: `1px solid ${COLORS.border}`,
|
||||
lineHeight: 1.35,
|
||||
}}
|
||||
>
|
||||
{cell}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,95 @@
|
||||
import React from "react";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
title?: string;
|
||||
width?: number | string;
|
||||
height?: number | string;
|
||||
children?: React.ReactNode;
|
||||
glow?: boolean;
|
||||
};
|
||||
|
||||
export const TerminalWindow: React.FC<Props> = ({
|
||||
title = "/last30days",
|
||||
width = "100%",
|
||||
height = "100%",
|
||||
children,
|
||||
glow = false,
|
||||
}) => {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width,
|
||||
height,
|
||||
background: COLORS.bgPanel,
|
||||
borderRadius: 16,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
boxShadow: glow
|
||||
? `0 0 60px ${COLORS.accentCyan}33, 0 24px 60px rgba(0,0,0,0.6)`
|
||||
: "0 24px 60px rgba(0,0,0,0.6)",
|
||||
overflow: "hidden",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
fontFamily: FONT_MONO,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
height: 36,
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderBottom: `1px solid ${COLORS.border}`,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
padding: "0 16px",
|
||||
gap: 8,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficRed,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficYellow,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficGreen,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 16,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 14,
|
||||
fontFamily: FONT_MONO,
|
||||
letterSpacing: 0.5,
|
||||
}}
|
||||
>
|
||||
{title}
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
padding: "20px 28px",
|
||||
color: COLORS.fgPrimary,
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
import React from "react";
|
||||
import { interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
text: string;
|
||||
startFrame: number;
|
||||
charsPerSec?: number;
|
||||
fontSize?: number;
|
||||
color?: string;
|
||||
prefix?: string;
|
||||
prefixColor?: string;
|
||||
showCursor?: boolean;
|
||||
fps?: number;
|
||||
};
|
||||
|
||||
export const TypedLine: React.FC<Props> = ({
|
||||
text,
|
||||
startFrame,
|
||||
charsPerSec = 28,
|
||||
fontSize = 32,
|
||||
color = COLORS.fgPrimary,
|
||||
prefix,
|
||||
prefixColor = COLORS.accentGreen,
|
||||
showCursor = true,
|
||||
fps = 30,
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
const elapsed = Math.max(0, frame - startFrame);
|
||||
const totalChars = text.length;
|
||||
const lengthFrames = Math.ceil((totalChars / charsPerSec) * fps);
|
||||
const visibleChars = Math.round(
|
||||
interpolate(elapsed, [0, lengthFrames], [0, totalChars], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
}),
|
||||
);
|
||||
const visible = text.slice(0, visibleChars);
|
||||
const done = visibleChars >= totalChars;
|
||||
const cursorOn = showCursor && Math.floor(frame / 15) % 2 === 0;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize,
|
||||
color,
|
||||
whiteSpace: "pre",
|
||||
lineHeight: 1.4,
|
||||
}}
|
||||
>
|
||||
{prefix ? (
|
||||
<span style={{ color: prefixColor, marginRight: 12 }}>{prefix}</span>
|
||||
) : null}
|
||||
<span>{visible}</span>
|
||||
{(!done || cursorOn) && (
|
||||
<span
|
||||
style={{
|
||||
display: "inline-block",
|
||||
width: fontSize * 0.55,
|
||||
height: fontSize * 0.95,
|
||||
background: color,
|
||||
verticalAlign: "text-bottom",
|
||||
marginLeft: 2,
|
||||
opacity: cursorOn ? 0.85 : 0,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
import { registerRoot } from "remotion";
|
||||
import { RemotionRoot } from "./Root";
|
||||
|
||||
registerRoot(RemotionRoot);
|
||||
@@ -0,0 +1,20 @@
|
||||
export const COLORS = {
|
||||
bgDeep: "#0a0e14",
|
||||
bgPanel: "#11161d",
|
||||
bgPanelSoft: "#161c25",
|
||||
fgPrimary: "#e6e6e6",
|
||||
fgMuted: "#8a93a3",
|
||||
fgDim: "#5b6573",
|
||||
border: "#2a3340",
|
||||
accentCyan: "#36d6f7",
|
||||
accentMagenta: "#ff55a3",
|
||||
accentGreen: "#5fff9f",
|
||||
accentAmber: "#ffc857",
|
||||
trafficRed: "#ff5f57",
|
||||
trafficYellow: "#febc2e",
|
||||
trafficGreen: "#28c840",
|
||||
} as const;
|
||||
|
||||
export const FONT_MONO = '"JetBrains Mono", "SF Mono", "Menlo", monospace';
|
||||
export const FONT_SANS =
|
||||
'"Inter", "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif';
|
||||
@@ -0,0 +1,13 @@
|
||||
// Single source of truth for scene frame ranges.
|
||||
// 30fps × 30s = 900 frames total.
|
||||
export const FPS = 30;
|
||||
export const TOTAL_FRAMES = 900;
|
||||
|
||||
export const SCENES = {
|
||||
hook: { from: 0, durationInFrames: 90 },
|
||||
oldWay: { from: 90, durationInFrames: 150 },
|
||||
fanOut: { from: 240, durationInFrames: 180 },
|
||||
comparison: { from: 420, durationInFrames: 210 },
|
||||
howItWorks: { from: 630, durationInFrames: 150 },
|
||||
cta: { from: 780, durationInFrames: 120 },
|
||||
} as const;
|
||||
@@ -0,0 +1,61 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { BadgeBar } from "../components/BadgeBar";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene1Hook: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const captionOpacity = interpolate(frame, [20, 35, 75, 90], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const captionLift = spring({
|
||||
frame: frame - 20,
|
||||
fps,
|
||||
config: { damping: 15, stiffness: 70 },
|
||||
from: 16,
|
||||
to: 0,
|
||||
});
|
||||
|
||||
const badgeFadeOut = interpolate(frame, [70, 90], [1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 40%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 60%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 64,
|
||||
}}
|
||||
>
|
||||
<div style={{ opacity: badgeFadeOut }}>
|
||||
<BadgeBar />
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionOpacity,
|
||||
transform: `translateY(${captionLift}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 96,
|
||||
fontWeight: 600,
|
||||
color: COLORS.fgPrimary,
|
||||
textAlign: "center",
|
||||
letterSpacing: -1.5,
|
||||
lineHeight: 1.1,
|
||||
maxWidth: 1400,
|
||||
}}
|
||||
>
|
||||
What if one search
|
||||
<br />
|
||||
ran <span style={{ color: COLORS.accentCyan }}>3 at once?</span>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,100 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { TerminalWindow } from "../components/TerminalWindow";
|
||||
import { TypedLine } from "../components/TypedLine";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene2OldWay: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const enter = interpolate(frame, [0, 20], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const slide = interpolate(frame, [0, 20], [40, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Result card fades in after type completes (~70 frames)
|
||||
const resultFade = interpolate(frame, [70, 95], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Caption appears late
|
||||
const captionFade = interpolate(frame, [110, 130, 150], [0, 1, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: 80,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 1400,
|
||||
height: 520,
|
||||
opacity: enter,
|
||||
transform: `translateY(${slide}px)`,
|
||||
}}
|
||||
>
|
||||
<TerminalWindow title="bash">
|
||||
<TypedLine
|
||||
text="/last30days OpenAI"
|
||||
startFrame={20}
|
||||
prefix="$"
|
||||
fontSize={42}
|
||||
/>
|
||||
<div
|
||||
style={{
|
||||
opacity: resultFade,
|
||||
marginTop: 36,
|
||||
padding: "20px 24px",
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderRadius: 12,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 26,
|
||||
color: COLORS.fgPrimary,
|
||||
lineHeight: 1.6,
|
||||
}}
|
||||
>
|
||||
<div style={{ color: COLORS.accentGreen }}>
|
||||
✅ All agents reported back!
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted, marginTop: 6 }}>
|
||||
├─ 🟠 Reddit: 14 threads
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted }}>
|
||||
├─ 🔵 X: 22 posts
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted }}>
|
||||
└─ 🟡 HN: 1 story
|
||||
</div>
|
||||
</div>
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
marginTop: 60,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 38,
|
||||
color: COLORS.fgMuted,
|
||||
}}
|
||||
>
|
||||
The old way: <span style={{ color: COLORS.fgPrimary }}>one topic.</span>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,221 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { TerminalWindow } from "../components/TerminalWindow";
|
||||
import { TypedLine } from "../components/TypedLine";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const PROGRESS_LINES = [
|
||||
{ source: "Reddit", color: "#ff6a3d" },
|
||||
{ source: "X", color: "#36d6f7" },
|
||||
{ source: "YouTube", color: "#ff5757" },
|
||||
{ source: "TikTok", color: "#5fff9f" },
|
||||
{ source: "Instagram", color: "#ff55a3" },
|
||||
];
|
||||
|
||||
const ENTITIES: { label: string; tag: string; accent: string }[] = [
|
||||
{ label: "OpenAI", tag: "$ /last30days OpenAI", accent: COLORS.accentCyan },
|
||||
{ label: "Anthropic", tag: "$ /last30days Anthropic", accent: COLORS.accentMagenta },
|
||||
{ label: "xAI", tag: "$ /last30days xAI", accent: COLORS.accentAmber },
|
||||
];
|
||||
|
||||
const FanPane: React.FC<{
|
||||
label: string;
|
||||
tag: string;
|
||||
accent: string;
|
||||
panelStart: number;
|
||||
}> = ({ label, tag, accent, panelStart }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
const localFrame = Math.max(0, frame - panelStart);
|
||||
|
||||
const enter = spring({
|
||||
frame: localFrame,
|
||||
fps,
|
||||
config: { damping: 18, stiffness: 80 },
|
||||
from: 0,
|
||||
to: 1,
|
||||
});
|
||||
const slide = interpolate(localFrame, [0, 20], [40, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
opacity: enter,
|
||||
transform: `translateY(${slide}px)`,
|
||||
height: 480,
|
||||
}}
|
||||
>
|
||||
<TerminalWindow title={label} glow>
|
||||
<div
|
||||
style={{
|
||||
color: accent,
|
||||
fontSize: 18,
|
||||
fontFamily: FONT_MONO,
|
||||
marginBottom: 14,
|
||||
}}
|
||||
>
|
||||
{tag}
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
fontSize: 22,
|
||||
color: COLORS.accentGreen,
|
||||
fontFamily: FONT_MONO,
|
||||
marginBottom: 12,
|
||||
}}
|
||||
>
|
||||
[Competitors] running...
|
||||
</div>
|
||||
{PROGRESS_LINES.map((line, idx) => {
|
||||
const lineStart = panelStart + 16 + idx * 6;
|
||||
const lineFade = interpolate(
|
||||
frame - lineStart,
|
||||
[0, 8],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
// pulse the in-progress dot
|
||||
const dotOn = Math.floor((frame - lineStart) / 6) % 2 === 0;
|
||||
return (
|
||||
<div
|
||||
key={line.source}
|
||||
style={{
|
||||
opacity: lineFade,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 20,
|
||||
color: COLORS.fgMuted,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
marginBottom: 6,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
display: "inline-block",
|
||||
width: 10,
|
||||
height: 10,
|
||||
borderRadius: 10,
|
||||
background: dotOn ? line.color : COLORS.bgPanelSoft,
|
||||
marginRight: 12,
|
||||
boxShadow: dotOn ? `0 0 10px ${line.color}` : "none",
|
||||
}}
|
||||
/>
|
||||
<span style={{ color: line.color, marginRight: 8 }}>
|
||||
►
|
||||
</span>
|
||||
<span>{line.source}</span>
|
||||
<span style={{ marginLeft: "auto", color: COLORS.fgDim }}>
|
||||
{dotOn ? "..." : "·"}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const Scene3FanOut: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
// Phase 1 (0-30 frames): single terminal types --competitors flag
|
||||
// Phase 2 (30+): split into 3 panes
|
||||
const splitProgress = interpolate(frame, [30, 50], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const singleOpacity = interpolate(frame, [0, 8, 30, 45], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Caption fades in shortly after panes settle so it has time to read.
|
||||
const captionFade = interpolate(frame, [60, 80], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: 60,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
textAlign: "center",
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 42,
|
||||
color: COLORS.fgPrimary,
|
||||
marginBottom: 32,
|
||||
}}
|
||||
>
|
||||
Now it discovers competitors
|
||||
<br />
|
||||
<span style={{ color: COLORS.accentCyan }}>and runs all 3.</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
position: "relative",
|
||||
}}
|
||||
>
|
||||
{/* Single terminal during phase 1, fades out as panes appear */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: 0,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
opacity: singleOpacity,
|
||||
}}
|
||||
>
|
||||
<div style={{ width: 1200, height: 380 }}>
|
||||
<TerminalWindow title="bash">
|
||||
<TypedLine
|
||||
text="/last30days OpenAI --competitors"
|
||||
startFrame={0}
|
||||
prefix="$"
|
||||
fontSize={42}
|
||||
/>
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Three panes fade in starting frame ~30 */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: 0,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 24,
|
||||
opacity: splitProgress,
|
||||
}}
|
||||
>
|
||||
{ENTITIES.map((entity, idx) => (
|
||||
<FanPane
|
||||
key={entity.label}
|
||||
label={entity.label}
|
||||
tag={entity.tag}
|
||||
accent={entity.accent}
|
||||
panelStart={45 + idx * 8}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,105 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { ComparisonTable } from "../components/ComparisonTable";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const ROWS = [
|
||||
{
|
||||
dimension: "What it is",
|
||||
cells: [
|
||||
"GPT-5 leader, Plus + API",
|
||||
"Claude 4, safety-first",
|
||||
"Grok, X-native, fast",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "30-day momentum",
|
||||
cells: [
|
||||
"GPT-5 launch wave",
|
||||
"Claude 4.7 1M context",
|
||||
"Grok 5 reveal",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "Community vibe",
|
||||
cells: [
|
||||
"Defensive but deep",
|
||||
"Quiet, devs-only",
|
||||
"Loud, meme-rich",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "Best for",
|
||||
cells: [
|
||||
"Mainstream + tools",
|
||||
"Long-context coding",
|
||||
"Live X intel",
|
||||
] as [string, string, string],
|
||||
},
|
||||
];
|
||||
|
||||
export const Scene4Comparison: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const captionFade = interpolate(frame, [0, 12, 180, 210], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const captionLift = interpolate(frame, [0, 14], [16, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const tableFade = interpolate(frame, [16, 32], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: "48px 80px",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
transform: `translateY(${captionLift}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 38,
|
||||
color: COLORS.fgMuted,
|
||||
textAlign: "center",
|
||||
marginBottom: 36,
|
||||
}}
|
||||
>
|
||||
<span style={{ color: COLORS.accentCyan, fontWeight: 600 }}>
|
||||
3 full passes.
|
||||
</span>
|
||||
<span style={{ marginLeft: 18, color: COLORS.accentMagenta, fontWeight: 600 }}>
|
||||
3 save files.
|
||||
</span>
|
||||
<span style={{ marginLeft: 18, color: COLORS.fgPrimary, fontWeight: 600 }}>
|
||||
1 comparison.
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
maxWidth: 1640,
|
||||
opacity: tableFade,
|
||||
}}
|
||||
>
|
||||
<ComparisonTable
|
||||
startFrame={20}
|
||||
entities={["OpenAI", "Anthropic", "xAI"]}
|
||||
rows={ROWS}
|
||||
rowStaggerFrames={22}
|
||||
/>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const LINES = [
|
||||
{ text: "You pick the topic.", color: COLORS.fgPrimary },
|
||||
{ text: "The agent picks the peers.", color: COLORS.accentCyan },
|
||||
{ text: "The engine fans out.", color: COLORS.accentMagenta },
|
||||
];
|
||||
|
||||
export const Scene5HowItWorks: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 60%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 48,
|
||||
}}
|
||||
>
|
||||
{LINES.map((line, idx) => {
|
||||
const start = 10 + idx * 28;
|
||||
const fade = interpolate(frame, [start, start + 14], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const slide = interpolate(frame, [start, start + 18], [24, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
return (
|
||||
<div
|
||||
key={line.text}
|
||||
style={{
|
||||
opacity: fade,
|
||||
transform: `translateY(${slide}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 78,
|
||||
fontWeight: 600,
|
||||
color: line.color,
|
||||
letterSpacing: -1,
|
||||
}}
|
||||
>
|
||||
{line.text}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,73 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { BadgeBar } from "../components/BadgeBar";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene6CTA: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const installFade = interpolate(frame, [20, 40], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
// Stronger 1Hz pulse on the install line for end-of-video emphasis
|
||||
const pulse = 0.8 + 0.2 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||
const glowPulse = 0.4 + 0.4 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||
|
||||
const repoFade = interpolate(frame, [50, 70], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const enterScale = spring({
|
||||
frame,
|
||||
fps,
|
||||
config: { damping: 18, stiffness: 90 },
|
||||
from: 0.95,
|
||||
to: 1,
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 50%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 56,
|
||||
transform: `scale(${enterScale})`,
|
||||
}}
|
||||
>
|
||||
<BadgeBar />
|
||||
<div
|
||||
style={{
|
||||
opacity: installFade * pulse,
|
||||
padding: "18px 36px",
|
||||
background: COLORS.bgPanel,
|
||||
border: `1px solid ${COLORS.accentCyan}`,
|
||||
borderRadius: 14,
|
||||
boxShadow: `0 0 ${40 + glowPulse * 60}px ${COLORS.accentCyan}${Math.round(40 + glowPulse * 80).toString(16)}`,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 44,
|
||||
color: COLORS.fgPrimary,
|
||||
}}
|
||||
>
|
||||
<span style={{ color: COLORS.accentGreen, marginRight: 18 }}>$</span>
|
||||
/last30days <span style={{ color: COLORS.fgMuted }}>{"{topic}"}</span>{" "}
|
||||
<span style={{ color: COLORS.accentCyan }}>--competitors</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
opacity: repoFade,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 28,
|
||||
color: COLORS.fgMuted,
|
||||
}}
|
||||
>
|
||||
github.com/mvanhorn/last30days-skill
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.4 MiB |
+9
-6
@@ -1,14 +1,16 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.2.4"
|
||||
version = "3.0.0"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
dependencies = [
|
||||
"requests>=2.32,<3",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.0.3,<10",
|
||||
"pytest>=9,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
]
|
||||
|
||||
@@ -22,9 +24,9 @@ addopts = [
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["skills/last30days/scripts", "tests"]
|
||||
source = ["scripts", "tests"]
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
@@ -32,6 +34,7 @@ omit = [
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ OpenClaw:
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`.
|
||||
OpenAI Codex CLI: run `codex` from a checkout of this repo and v3's skill at `.agents/skills/last30days/SKILL.md` will be discovered automatically. Or copy `SKILL.md` to `~/.agents/skills/last30days/SKILL.md` for a global install.
|
||||
|
||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
||||
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root)
|
||||
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||
#
|
||||
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
||||
# directory containing SKILL.md and the scripts/ runtime from skills/last30days.
|
||||
# See
|
||||
# directory containing SKILL.md and the scripts/ runtime. See
|
||||
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
@@ -18,7 +17,14 @@ fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/last30days.skill"
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD
|
||||
|
||||
# claude.ai's .skill bundle only needs the root SKILL.md + scripts/ runtime.
|
||||
# Claude Code needs skills/ and .claude-plugin/ in the git archive
|
||||
# (that's why they're NOT in .gitattributes export-ignore), but the .skill
|
||||
# bundle must strip them to keep a single canonical SKILL.md and stay under
|
||||
# the 200-file cap.
|
||||
zip -d "$OUT" "last30days/skills/*" "last30days/.claude-plugin/*" > /dev/null 2>&1 || true
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# A/B test runner: public release vs private beta
|
||||
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West"
|
||||
# Usage: bash scripts/compare.sh "Kanye West"
|
||||
#
|
||||
# Runs /last30days (public release) and /last30days-beta (private beta)
|
||||
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
||||
@@ -9,8 +9,8 @@
|
||||
set -e
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>"
|
||||
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose"
|
||||
echo "Usage: bash scripts/compare.sh <topic>"
|
||||
echo " Example: bash scripts/compare.sh Kevin Rose"
|
||||
exit 1
|
||||
fi
|
||||
TOPIC="$*"
|
||||
+3
-8
@@ -20,11 +20,9 @@ sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from lib import env as envlib
|
||||
from lib import schema
|
||||
from lib.providers import GEMINI_FLASH_LITE
|
||||
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
||||
|
||||
|
||||
@@ -44,7 +42,7 @@ def _load_default_topics() -> list[tuple[str, str]]:
|
||||
|
||||
DEFAULT_TOPICS = _load_default_topics()
|
||||
DEFAULT_SEARCH = ""
|
||||
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
|
||||
DEFAULT_JUDGE_MODEL = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||
|
||||
|
||||
@@ -309,10 +307,7 @@ def create_eval_env() -> dict[str, str]:
|
||||
|
||||
|
||||
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
||||
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py"
|
||||
if not engine.exists():
|
||||
engine = repo_dir / "scripts" / "last30days.py"
|
||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
||||
cmd = [sys.executable, "scripts/last30days.py", topic, "--emit=json"]
|
||||
if search:
|
||||
cmd.extend(["--search", search])
|
||||
if quick:
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
# ruff: noqa: E402
|
||||
"""last30days CLI."""
|
||||
"""last30days v3.0.0 CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -41,7 +41,7 @@ if os.name == "nt":
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from lib import env, html_render, pipeline, render, schema, ui
|
||||
from lib import env, pipeline, render, schema, ui
|
||||
|
||||
_child_pids: set[int] = set()
|
||||
_child_pids_lock = threading.Lock()
|
||||
@@ -91,50 +91,30 @@ def slugify(value: str) -> str:
|
||||
return slug or "last30days"
|
||||
|
||||
|
||||
def save_output(
|
||||
report: schema.Report,
|
||||
emit: str,
|
||||
save_dir: str,
|
||||
suffix: str = "",
|
||||
synthesis_md: str | None = None,
|
||||
topic_override: str | None = None,
|
||||
rendered_content: str | None = None,
|
||||
) -> Path:
|
||||
def save_output(report: schema.Report, emit: str, save_dir: str, suffix: str = "") -> Path:
|
||||
from datetime import datetime
|
||||
path = Path(save_dir).expanduser().resolve()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
slug = slugify(topic_override or report.topic)
|
||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
||||
raw_label = "raw-html" if emit == "html" else "raw"
|
||||
slug = slugify(report.topic)
|
||||
extension = "json" if emit == "json" else "md"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
out_path = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
||||
out_path = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
if out_path.exists():
|
||||
out_path = path / f"{slug}-{raw_label}{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Markdown saves keep the complete debug artifact. JSON and HTML preserve
|
||||
# their requested wire format so file extensions match their content.
|
||||
if rendered_content is not None:
|
||||
content = rendered_content
|
||||
elif emit in {"json", "html"}:
|
||||
content = emit_output(report, emit, synthesis_md=synthesis_md)
|
||||
out_path = path / f"{slug}-raw{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Always save the FULL dump to disk (all items, all sources, transcripts).
|
||||
# Claude sees compact clusters via --emit=compact on stdout.
|
||||
# The saved file is the complete debug artifact.
|
||||
if emit == "json":
|
||||
content = emit_output(report, emit)
|
||||
else:
|
||||
content = render.render_full(report)
|
||||
out_path.write_text(content, encoding="utf-8")
|
||||
return out_path
|
||||
|
||||
|
||||
def emit_output(
|
||||
report: schema.Report,
|
||||
emit: str,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
def emit_output(report: schema.Report, emit: str, fun_level: str = "medium", save_path: str | None = None) -> str:
|
||||
if emit == "json":
|
||||
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
|
||||
if emit == "html":
|
||||
return html_render.render_html(
|
||||
report, fun_level=fun_level, save_path=save_path, synthesis_md=synthesis_md,
|
||||
)
|
||||
if emit in {"compact", "md"}:
|
||||
return render.render_compact(report, fun_level=fun_level, save_path=save_path)
|
||||
if emit == "context":
|
||||
@@ -147,7 +127,6 @@ def emit_comparison_output(
|
||||
emit: str,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
if emit == "json":
|
||||
payload = {
|
||||
@@ -159,13 +138,6 @@ def emit_comparison_output(
|
||||
],
|
||||
}
|
||||
return json.dumps(payload, indent=2, sort_keys=True)
|
||||
if emit == "html":
|
||||
return html_render.render_html_comparison(
|
||||
entity_reports,
|
||||
fun_level=fun_level,
|
||||
save_path=save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
if emit in {"compact", "md"}:
|
||||
return render.render_comparison_multi(
|
||||
entity_reports, fun_level=fun_level, save_path=save_path,
|
||||
@@ -175,10 +147,6 @@ def emit_comparison_output(
|
||||
raise SystemExit(f"Unsupported emit mode: {emit}")
|
||||
|
||||
|
||||
def comparison_topic(entity_reports: list[tuple[str, schema.Report]]) -> str:
|
||||
return " vs ".join(label for label, _ in entity_reports)
|
||||
|
||||
|
||||
def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str) -> str:
|
||||
"""Compute the user-friendly save path string that will be shown in the footer.
|
||||
|
||||
@@ -188,10 +156,9 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
||||
from pathlib import Path as _Path
|
||||
path = _Path(save_dir).expanduser().resolve()
|
||||
slug = slugify(topic)
|
||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
||||
raw_label = "raw-html" if emit == "html" else "raw"
|
||||
extension = "json" if emit == "json" else "md"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
raw = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
||||
raw = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
try:
|
||||
home = _Path.home().resolve()
|
||||
relative = raw.relative_to(home)
|
||||
@@ -200,14 +167,6 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
||||
return str(raw)
|
||||
|
||||
|
||||
def read_synthesis_file(path: str) -> str:
|
||||
try:
|
||||
return Path(path).expanduser().read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
sys.stderr.write(f"[last30days] Cannot read --synthesis-file: {exc}\n")
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
def persist_report(report: schema.Report) -> dict[str, int]:
|
||||
import store
|
||||
|
||||
@@ -234,7 +193,7 @@ def persist_report(report: schema.Report) -> dict[str, int]:
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
|
||||
parser.add_argument("topic", nargs="*", help="Research topic")
|
||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md", "html"])
|
||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md"])
|
||||
parser.add_argument("--search", help="Comma-separated source list")
|
||||
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
|
||||
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
|
||||
@@ -242,7 +201,6 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
||||
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
||||
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
||||
parser.add_argument("--synthesis-file", help="Markdown synthesis to embed in --emit=html output")
|
||||
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
|
||||
parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
|
||||
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
|
||||
@@ -541,13 +499,6 @@ def main() -> int:
|
||||
|
||||
config = env.get_config()
|
||||
|
||||
# Surface SSH-routing config as an env var so library modules (e.g.
|
||||
# youtube_yt) can read it without taking a config dependency. This
|
||||
# routes yt-dlp through `ssh <host>` to bypass YouTube's bot-wall on
|
||||
# datacenter IPs (see lib/youtube_yt.py for details).
|
||||
if config.get("LAST30DAYS_YOUTUBE_SSH_HOST") and "LAST30DAYS_YOUTUBE_SSH_HOST" not in os.environ:
|
||||
os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"]
|
||||
|
||||
# Handle setup subcommand
|
||||
topic = " ".join(args.topic).strip()
|
||||
if topic.lower() == "setup":
|
||||
@@ -586,13 +537,6 @@ def main() -> int:
|
||||
parser.print_usage(sys.stderr)
|
||||
return 2
|
||||
|
||||
synthesis_md = None
|
||||
if args.synthesis_file:
|
||||
if args.emit == "html":
|
||||
synthesis_md = read_synthesis_file(args.synthesis_file)
|
||||
else:
|
||||
sys.stderr.write("[last30days] Warning: --synthesis-file is only used with --emit=html; ignoring.\n")
|
||||
|
||||
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
|
||||
from lib import preflight
|
||||
refuse_msg = preflight.check_class_1_trap(topic)
|
||||
@@ -888,15 +832,10 @@ def main() -> int:
|
||||
pass
|
||||
|
||||
fun_level = config.get("FUN_LEVEL", "medium").lower()
|
||||
# Comparison HTML is the one case where the saved file's title and content
|
||||
# have to be overridden away from the leading entity's report. Compute the
|
||||
# gate once so the footer-display and save-output paths can't disagree.
|
||||
is_comparison_html = bool(entity_reports) and args.emit == "html"
|
||||
footer_save_path = None
|
||||
if args.save_dir:
|
||||
save_topic_for_display = comparison_topic(entity_reports) if is_comparison_html else report.topic
|
||||
footer_save_path = compute_save_path_display(
|
||||
args.save_dir, save_topic_for_display, args.save_suffix or "", args.emit
|
||||
args.save_dir, report.topic, args.save_suffix or "", args.emit
|
||||
)
|
||||
|
||||
# Signal to render_compact whether pre-research flags were supplied.
|
||||
@@ -915,31 +854,15 @@ def main() -> int:
|
||||
|
||||
if entity_reports:
|
||||
rendered = emit_comparison_output(
|
||||
entity_reports,
|
||||
args.emit,
|
||||
fun_level=fun_level,
|
||||
save_path=footer_save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
entity_reports, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||
)
|
||||
else:
|
||||
rendered = emit_output(
|
||||
report,
|
||||
args.emit,
|
||||
fun_level=fun_level,
|
||||
save_path=footer_save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
report, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||
)
|
||||
if args.save_dir:
|
||||
# Save the main topic's raw file (single-entity or comparison main).
|
||||
save_path = save_output(
|
||||
report,
|
||||
args.emit,
|
||||
args.save_dir,
|
||||
suffix=args.save_suffix or "",
|
||||
synthesis_md=synthesis_md,
|
||||
topic_override=comparison_topic(entity_reports) if is_comparison_html else None,
|
||||
rendered_content=rendered if is_comparison_html else None,
|
||||
)
|
||||
save_path = save_output(report, args.emit, args.save_dir, suffix=args.save_suffix or "")
|
||||
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
||||
# Competitor / vs-mode: also save a per-entity raw file for each peer.
|
||||
# Matches historical vs-mode behavior (N passes → N save files).
|
||||
@@ -948,7 +871,6 @@ def main() -> int:
|
||||
peer_path = save_output(
|
||||
entity_report, args.emit, args.save_dir,
|
||||
suffix=args.save_suffix or "",
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
sys.stderr.write(f"[last30days] Saved output to {peer_path}\n")
|
||||
sys.stderr.flush()
|
||||
@@ -7,22 +7,18 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log, subproc
|
||||
from . import http, log
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
# How many times to retry the bird-search subprocess when stdout is non-JSON
|
||||
# (typically an HTML anti-bot interstitial from Twitter's edge).
|
||||
MAX_JSON_DECODE_RETRIES = 2
|
||||
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
|
||||
|
||||
|
||||
def _first_of(*values):
|
||||
"""Return first value that is not None."""
|
||||
@@ -154,14 +150,16 @@ def get_bird_status() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"""Invoke the vendored bird-search.mjs subprocess once.
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
Returns (result, error_dict). If error_dict is non-None, treat it as the
|
||||
final result and do not retry — those errors are terminal (timeout,
|
||||
spawn failure). If error_dict is None, the subprocess ran to completion
|
||||
and `result` is the SubprocResult; the caller decides whether to retry
|
||||
based on the result.stdout content.
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
@@ -170,109 +168,62 @@ def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"--json",
|
||||
]
|
||||
|
||||
pid_holder: list[int] = []
|
||||
# Use process groups for clean cleanup on timeout/kill
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
|
||||
def _register(pid: int) -> None:
|
||||
pid_holder.append(pid)
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
# Register for cleanup tracking (if available)
|
||||
try:
|
||||
from last30days import register_child_pid
|
||||
register_child_pid(pid)
|
||||
from last30days import register_child_pid, unregister_child_pid
|
||||
register_child_pid(proc.pid)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(
|
||||
cmd,
|
||||
timeout=timeout,
|
||||
env=_subprocess_env(),
|
||||
on_pid=_register,
|
||||
)
|
||||
except subproc.SubprocTimeout:
|
||||
return None, {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
except Exception as e:
|
||||
return None, {"error": str(e), "items": []}
|
||||
finally:
|
||||
if pid_holder:
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
# Kill the entire process group
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
finally:
|
||||
try:
|
||||
from last30days import unregister_child_pid
|
||||
unregister_child_pid(pid_holder[0])
|
||||
unregister_child_pid(proc.pid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result, None
|
||||
|
||||
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
Retries the subprocess on JSON-decode failure (typically a Twitter
|
||||
anti-bot HTML interstitial in stdout) up to MAX_JSON_DECODE_RETRIES
|
||||
times with JSON_DECODE_RETRY_DELAY seconds between attempts. Terminal
|
||||
errors (subprocess timeout, non-zero return code) are returned
|
||||
immediately without retry.
|
||||
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds (per attempt)
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
last_decode_error: Optional[str] = None
|
||||
|
||||
for attempt in range(MAX_JSON_DECODE_RETRIES):
|
||||
result, terminal_error = _invoke_bird_subprocess(query, count, timeout)
|
||||
if terminal_error is not None:
|
||||
return terminal_error
|
||||
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
if proc.returncode != 0:
|
||||
error = stderr.strip() if stderr else "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
output = result.stdout.strip()
|
||||
output = stdout.strip() if stdout else ""
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError as e:
|
||||
# Twitter's edge sometimes serves an HTML anti-bot interstitial
|
||||
# in place of JSON. Tag the failure shape so it's distinguishable
|
||||
# from "no results" in logs, then retry the subprocess.
|
||||
looks_html = output.lstrip().lower().startswith(("<!doctype", "<html", "<"))
|
||||
attempt_num = attempt + 1
|
||||
log_msg = (
|
||||
f"Bird search returned non-JSON stdout "
|
||||
f"(looks_html={looks_html}, attempt {attempt_num}/{MAX_JSON_DECODE_RETRIES}, "
|
||||
f"first 80 chars: {output[:80]!r})"
|
||||
)
|
||||
last_decode_error = str(e)
|
||||
if attempt_num < MAX_JSON_DECODE_RETRIES:
|
||||
log.source_log(
|
||||
"X/bird",
|
||||
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
|
||||
)
|
||||
time.sleep(JSON_DECODE_RETRY_DELAY)
|
||||
continue
|
||||
log.source_log("X/bird", log_msg)
|
||||
return {
|
||||
"error": (
|
||||
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
|
||||
f"(likely Twitter anti-bot interstitial): {e}"
|
||||
),
|
||||
"items": [],
|
||||
}
|
||||
|
||||
parsed = json.loads(output)
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
# Defensive fallthrough — loop should always return above.
|
||||
return {
|
||||
"error": f"Bird search exhausted retries: {last_decode_error}",
|
||||
"items": [],
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "items": []}
|
||||
|
||||
|
||||
def search_x(
|
||||
@@ -379,29 +330,47 @@ def search_handles(
|
||||
"--json",
|
||||
]
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
||||
except subproc.SubprocTimeout:
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
except OSError as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
if result.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return []
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
|
||||
if proc.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||
return []
|
||||
|
||||
output = (stdout or "").strip()
|
||||
if not output:
|
||||
return []
|
||||
|
||||
response = json.loads(output)
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
return []
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
@@ -39,14 +39,11 @@ def normalize_text(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
|
||||
if len(norm) < n:
|
||||
return {norm} if norm else set()
|
||||
return {norm[index:index + n] for index in range(len(norm) - n + 1)}
|
||||
|
||||
|
||||
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
||||
return _ngrams_of_normalized(normalize_text(text), n)
|
||||
text = normalize_text(text)
|
||||
if len(text) < n:
|
||||
return {text} if text else set()
|
||||
return {text[index:index + n] for index in range(len(text) - n + 1)}
|
||||
|
||||
|
||||
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
||||
@@ -93,7 +90,7 @@ class _PreparedText:
|
||||
|
||||
def __init__(self, raw: str) -> None:
|
||||
norm = normalize_text(raw)
|
||||
self.ngrams = _ngrams_of_normalized(norm)
|
||||
self.ngrams = get_ngrams(norm) if norm else set()
|
||||
self.tokens = _tokenize(norm)
|
||||
|
||||
|
||||
@@ -29,23 +29,6 @@ else:
|
||||
|
||||
CODEX_AUTH_FILE = Path(os.environ.get("CODEX_AUTH_FILE", str(Path.home() / ".codex" / "auth.json")))
|
||||
|
||||
# macOS Keychain integration: items stored with this service prefix are picked
|
||||
# up automatically on Darwin as the lowest-priority credential source.
|
||||
# Example: `security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."`.
|
||||
KEYCHAIN_SERVICE_PREFIX = "last30days-"
|
||||
|
||||
# Single source of truth for which credentials the Keychain loader looks up.
|
||||
# The setup-keychain.sh helper mirrors this list and is held in sync via
|
||||
# tests/test_env_keychain.py::test_keychain_keys_match_setup_script.
|
||||
KEYCHAIN_KEYS = (
|
||||
"OPENAI_API_KEY", "XAI_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY", "SCRAPECREATORS_API_KEY", "APIFY_API_TOKEN",
|
||||
"AUTH_TOKEN", "CT0", "BSKY_HANDLE", "BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN", "BRAVE_API_KEY", "EXA_API_KEY", "SERPER_API_KEY",
|
||||
"OPENROUTER_API_KEY", "PARALLEL_API_KEY", "XQUIK_API_KEY",
|
||||
"XIAOHONGSHU_API_BASE",
|
||||
)
|
||||
|
||||
AuthSource = Literal["api_key", "codex", "none"]
|
||||
AuthStatus = Literal["ok", "missing", "expired", "missing_account_id"]
|
||||
|
||||
@@ -70,10 +53,6 @@ class OpenAIAuth:
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file has overly permissive permissions."""
|
||||
if os.name == "nt":
|
||||
# Windows reports synthesized POSIX mode bits that do not reflect NTFS ACLs.
|
||||
return
|
||||
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
# Check if group or other can read (bits 0o044)
|
||||
@@ -112,46 +91,6 @@ def load_env_file(path: Path) -> dict[str, str]:
|
||||
return env
|
||||
|
||||
|
||||
def _load_keychain(keys: list[str]) -> dict[str, str]:
|
||||
"""Load credentials from macOS Keychain (no-op on other platforms).
|
||||
|
||||
Each key is looked up as a generic password with service name
|
||||
``f"{KEYCHAIN_SERVICE_PREFIX}{key}"`` for the current user. Missing items
|
||||
and lookup failures are silent — Keychain is the lowest-priority source
|
||||
and is meant to be additive over `.env` files and process environment.
|
||||
"""
|
||||
import platform
|
||||
if platform.system() != "Darwin":
|
||||
return {}
|
||||
|
||||
import shutil
|
||||
security = shutil.which("security")
|
||||
if not security:
|
||||
return {}
|
||||
|
||||
import subprocess
|
||||
import pwd
|
||||
# USER can be unset under sudo, in Docker without --env USER, or in some CI
|
||||
# runners; fall back to the OS user record so lookups still match items
|
||||
# stored by setup-keychain.sh (which uses $USER).
|
||||
user = os.environ.get("USER") or pwd.getpwuid(os.getuid()).pw_name
|
||||
env: dict[str, str] = {}
|
||||
for key in keys:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[security, "find-generic-password",
|
||||
"-a", user,
|
||||
"-s", f"{KEYCHAIN_SERVICE_PREFIX}{key}",
|
||||
"-w"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
continue
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
env[key] = result.stdout.strip()
|
||||
return env
|
||||
|
||||
|
||||
def _decode_jwt_payload(token: str) -> dict[str, Any] | None:
|
||||
"""Decode JWT payload without verification."""
|
||||
try:
|
||||
@@ -275,7 +214,6 @@ def get_config() -> dict[str, Any]:
|
||||
1. Environment variables (os.environ)
|
||||
2. .claude/last30days.env (per-project config)
|
||||
3. ~/.config/last30days/.env (global config)
|
||||
4. macOS Keychain items prefixed ``last30days-`` (Darwin only)
|
||||
"""
|
||||
# Load from global config file
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||
@@ -284,14 +222,9 @@ def get_config() -> dict[str, Any]:
|
||||
project_env_path = _find_project_env()
|
||||
project_env = load_env_file(project_env_path) if project_env_path else {}
|
||||
|
||||
# Merge file sources: project > global
|
||||
# Merge: project overrides global
|
||||
merged_env = {**file_env, **project_env}
|
||||
|
||||
# Keychain is the lowest-priority source (Darwin only; no-op elsewhere).
|
||||
# Loaded before openai_auth so OPENAI_API_KEY can come from Keychain too.
|
||||
keychain_env = _load_keychain(list(KEYCHAIN_KEYS))
|
||||
merged_env = {**keychain_env, **merged_env}
|
||||
|
||||
openai_auth = get_openai_auth(merged_env)
|
||||
|
||||
# Build config: Codex/OpenAI auth + process.env > project .env > global .env
|
||||
@@ -332,21 +265,16 @@ def get_config() -> dict[str, Any]:
|
||||
('FROM_BROWSER', None),
|
||||
('SETUP_COMPLETE', None),
|
||||
('INCLUDE_SOURCES', ''),
|
||||
('EXCLUDE_SOURCES', ''),
|
||||
('LAST30DAYS_YOUTUBE_SSH_HOST', None),
|
||||
]
|
||||
|
||||
for key, default in keys:
|
||||
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
||||
|
||||
# Track which config source was used (highest-priority file source wins
|
||||
# the label; keychain is only reported when nothing else is configured).
|
||||
# Track which config source was used
|
||||
if project_env_path:
|
||||
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
||||
elif CONFIG_FILE and CONFIG_FILE.exists():
|
||||
config['_CONFIG_SOURCE'] = f'global:{CONFIG_FILE}'
|
||||
elif keychain_env:
|
||||
config['_CONFIG_SOURCE'] = 'keychain'
|
||||
else:
|
||||
config['_CONFIG_SOURCE'] = 'env_only'
|
||||
|
||||
@@ -444,6 +372,14 @@ def config_exists() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def is_reddit_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Reddit search is available.
|
||||
|
||||
v3 uses ScrapeCreators only.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
|
||||
|
||||
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
||||
"""Determine which Reddit backend to use.
|
||||
|
||||
@@ -589,12 +525,12 @@ def _parse_include_sources(config: dict[str, Any]) -> set[str]:
|
||||
def is_threads_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Threads source is available.
|
||||
|
||||
Returns True when SCRAPECREATORS_API_KEY is set. Threads runs alongside
|
||||
TikTok and Instagram as part of the SC family — same key, same per-call
|
||||
cost shape, so the same default-on rule applies. Suppress via
|
||||
EXCLUDE_SOURCES=threads.
|
||||
Requires SCRAPECREATORS_API_KEY AND 'threads' in INCLUDE_SOURCES.
|
||||
Threads is an opt-in source - it is not activated by default.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||
return False
|
||||
return 'threads' in _parse_include_sources(config)
|
||||
|
||||
|
||||
def is_instagram_available(config: dict[str, Any]) -> bool:
|
||||
@@ -116,8 +116,6 @@ def weighted_rrf(
|
||||
"""Fuse ranked lists into a single candidate pool."""
|
||||
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
||||
candidates: dict[str, schema.Candidate] = {}
|
||||
# Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
|
||||
seen_source_items: dict[str, set[tuple[str, str]]] = {}
|
||||
|
||||
for (label, source), items in streams.items():
|
||||
subquery = subqueries[label]
|
||||
@@ -156,7 +154,6 @@ def weighted_rrf(
|
||||
]
|
||||
},
|
||||
)
|
||||
seen_source_items[key] = {(item.source, item.item_id)}
|
||||
continue
|
||||
|
||||
candidate = candidates[key]
|
||||
@@ -182,9 +179,7 @@ def weighted_rrf(
|
||||
candidate.subquery_labels.append(label)
|
||||
if item.source not in candidate.sources:
|
||||
candidate.sources.append(item.source)
|
||||
source_item_key = (item.source, item.item_id)
|
||||
if source_item_key not in seen_source_items[key]:
|
||||
seen_source_items[key].add(source_item_key)
|
||||
if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
|
||||
candidate.source_items.append(item)
|
||||
candidate.metadata.setdefault("provenance", []).append(
|
||||
{
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
@@ -206,90 +205,29 @@ def web_search(
|
||||
backend = "parallel"
|
||||
else:
|
||||
return [], {}
|
||||
items: list[dict] = []
|
||||
artifact: dict = {}
|
||||
if backend == "brave":
|
||||
key = config.get("BRAVE_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
|
||||
items, artifact = brave_search(query, date_range, key)
|
||||
elif backend == "exa":
|
||||
return brave_search(query, date_range, key)
|
||||
if backend == "exa":
|
||||
key = config.get("EXA_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
|
||||
items, artifact = exa_search(query, date_range, key)
|
||||
elif backend == "serper":
|
||||
return exa_search(query, date_range, key)
|
||||
if backend == "serper":
|
||||
key = config.get("SERPER_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
|
||||
items, artifact = serper_search(query, date_range, key)
|
||||
elif backend == "parallel":
|
||||
return serper_search(query, date_range, key)
|
||||
if backend == "parallel":
|
||||
key = config.get("PARALLEL_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
|
||||
items, artifact = parallel_search(query, date_range, key)
|
||||
elif backend != "none":
|
||||
return parallel_search(query, date_range, key)
|
||||
if backend != "none":
|
||||
raise ValueError(f"Unsupported web backend: {backend!r}")
|
||||
else:
|
||||
return [], {}
|
||||
if items and not _reddit_excluded(config):
|
||||
items = _enrich_reddit_items(items)
|
||||
return items, artifact
|
||||
|
||||
|
||||
def _reddit_excluded(config: dict) -> bool:
|
||||
"""Return True when EXCLUDE_SOURCES contains 'reddit'.
|
||||
|
||||
Respects the same suppression knob the pipeline uses for source gating,
|
||||
so a user who set EXCLUDE_SOURCES=reddit doesn't get Reddit content
|
||||
smuggled back in via web-search URLs.
|
||||
"""
|
||||
raw = (config.get("EXCLUDE_SOURCES") or "").split(",")
|
||||
return any(s.strip().lower() == "reddit" for s in raw)
|
||||
|
||||
|
||||
def _enrich_reddit_items(items: list[dict]) -> list[dict]:
|
||||
"""Enrich web search results that are Reddit URLs with thread body and comments.
|
||||
|
||||
Claude Code's WebFetch blocks reddit.com, so the model can't retrieve
|
||||
Reddit content from web search results. This fetches it via the public
|
||||
JSON API (reddit.com/.../.json) which bypasses that restriction.
|
||||
|
||||
Callers should gate this with EXCLUDE_SOURCES=reddit handling (see
|
||||
`_reddit_excluded`) so a user who explicitly excluded Reddit doesn't
|
||||
get Reddit content via web-search URLs.
|
||||
"""
|
||||
from . import reddit_enrich
|
||||
from .reddit_enrich import RedditRateLimitError
|
||||
|
||||
for item in items:
|
||||
url = item.get("url", "")
|
||||
if "reddit.com" not in url or "/comments/" not in url:
|
||||
continue
|
||||
try:
|
||||
thread_data = reddit_enrich.fetch_thread_data(url, timeout=8)
|
||||
if not thread_data:
|
||||
continue
|
||||
parsed = reddit_enrich.parse_thread_data(thread_data)
|
||||
# selftext lives under parsed["submission"], not at the top level
|
||||
selftext = (parsed.get("submission") or {}).get("selftext", "")
|
||||
if selftext:
|
||||
item["snippet"] = selftext[:2000]
|
||||
comments = parsed.get("comments", [])
|
||||
top = reddit_enrich.get_top_comments(comments)
|
||||
if top:
|
||||
item["top_comments"] = [
|
||||
{"score": c.get("score", 0), "excerpt": (c.get("body") or "")[:200]}
|
||||
for c in top[:5]
|
||||
]
|
||||
item["enriched_via"] = "reddit_json_api"
|
||||
except RedditRateLimitError as exc:
|
||||
# Stop iterating to avoid flooding more 429s
|
||||
sys.stderr.write(f"[Web] Reddit rate-limited, halting enrichment: {exc}\n")
|
||||
break
|
||||
except Exception as exc:
|
||||
sys.stderr.write(f"[Web] Reddit enrichment failed for {url}: {exc}\n")
|
||||
return items
|
||||
return [], {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -88,26 +88,17 @@ def search_hackernews(
|
||||
|
||||
# Use extracted core subject instead of raw topic for cleaner Algolia matching
|
||||
core = extract_core_subject(topic)
|
||||
# Hyphens and commas tokenize awkwardly in Algolia; flatten them so themed
|
||||
# queries like "ts-bun-node" or "claude, personal agents" become plain words.
|
||||
core_flat = _flatten_query_for_algolia(core)
|
||||
_log(f"Searching for '{core_flat}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
|
||||
# Use relevance-sorted search with minimum engagement filter.
|
||||
# NOTE: restrictSearchableAttributes=title omitted intentionally — it would
|
||||
# miss Ask HN/Show HN threads where the topic appears in the body.
|
||||
params = {
|
||||
"query": core_flat,
|
||||
"query": core,
|
||||
"tags": "story",
|
||||
"numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
|
||||
"hitsPerPage": str(count),
|
||||
}
|
||||
# Algolia defaults to AND across query tokens, so a 4-5 word theme query
|
||||
# matches no stories. Mark all-but-the-first token as optional so Algolia
|
||||
# ranks by how many tokens match instead of requiring every one.
|
||||
tokens = core_flat.split()
|
||||
if len(tokens) > 1:
|
||||
params["optionalWords"] = " ".join(tokens[1:])
|
||||
|
||||
from urllib.parse import urlencode
|
||||
url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
|
||||
@@ -126,56 +117,28 @@ def search_hackernews(
|
||||
return response
|
||||
|
||||
|
||||
_WORD_BOUNDARY_RE_CACHE: Dict[str, "re.Pattern[str]"] = {}
|
||||
|
||||
|
||||
def _flatten_query_for_algolia(text: str) -> str:
|
||||
"""Normalise query for Algolia + post-filter comparison.
|
||||
|
||||
Multi-keyword theme queries frequently contain commas (delimiters) or
|
||||
hyphens (compound terms like ``ts-bun-node``); both tokenize awkwardly.
|
||||
Flatten them to spaces and collapse runs of whitespace so the search
|
||||
parameter and the post-filter operate on the same shape.
|
||||
"""
|
||||
return " ".join(text.replace(",", " ").replace("-", " ").split())
|
||||
|
||||
|
||||
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
|
||||
"""Check if any query token appears as a whole word in the title.
|
||||
"""Check if the query term appears in the title content, not just an HN prefix or author.
|
||||
|
||||
Returns True when the query is empty (no filter), or when at least one
|
||||
query token matches as a whole word in the title after stripping
|
||||
"Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes.
|
||||
|
||||
We previously required *every* token to appear (all-words), which killed
|
||||
every Algolia hit on multi-keyword themes like "claude, personal agents,
|
||||
agentic infra" because real HN titles never contain all five tokens
|
||||
verbatim. Relaxing to any-word matches Algolia's `optionalWords` behaviour
|
||||
in `search_hackernews`. Token-overlap relevance scoring at parse time
|
||||
demotes hits where only one weak token matched, so the loosened gate
|
||||
won't surface noise to the top of the ranking.
|
||||
|
||||
Word-boundary matching (rather than naive substring) prevents short
|
||||
tokens like ``ai`` or ``ts`` from matching unrelated words like
|
||||
``email`` or ``artists``.
|
||||
Returns True if the query (or any multi-word token) appears in the title
|
||||
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
|
||||
and ignoring the author name. Returns True when query is empty (no filter).
|
||||
"""
|
||||
if not query:
|
||||
return True
|
||||
stripped = _HN_PREFIXES.sub("", title).strip()
|
||||
# Also check that the match isn't solely in the author's username
|
||||
check_text = stripped.lower()
|
||||
# Normalise the query the same way search_hackernews does so post-filter
|
||||
# tokens line up with what Algolia actually saw.
|
||||
query_words = [w for w in _flatten_query_for_algolia(query.lower()).split() if w]
|
||||
if not query_words:
|
||||
return True
|
||||
query_lower = query.lower()
|
||||
# Check each word of the query independently; all must appear somewhere
|
||||
# in the stripped title (not just the prefix).
|
||||
query_words = query_lower.split()
|
||||
for word in query_words:
|
||||
pattern = _WORD_BOUNDARY_RE_CACHE.get(word)
|
||||
if pattern is None:
|
||||
pattern = re.compile(rf"\b{re.escape(word)}\b")
|
||||
_WORD_BOUNDARY_RE_CACHE[word] = pattern
|
||||
if pattern.search(check_text):
|
||||
return True
|
||||
return False
|
||||
if word in check_text:
|
||||
continue
|
||||
# Word not found in stripped title — reject
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -23,19 +22,9 @@ def log(msg: str):
|
||||
MAX_RETRIES = 5
|
||||
MAX_429_RETRIES = 2
|
||||
RETRY_DELAY = 2.0
|
||||
# DNS resolution failures (gaierror) are transient — typically resolved by a
|
||||
# brief backoff and retry. Use a dedicated minimum attempt count + exponential
|
||||
# delays (1s, 2s, 4s) so callers that pass a small `retries` value still get a
|
||||
# meaningful chance to recover from a transient resolution failure.
|
||||
MIN_DNS_RETRIES = 3
|
||||
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
||||
|
||||
|
||||
def _is_dns_failure(err: urllib.error.URLError) -> bool:
|
||||
"""Return True if a URLError was caused by DNS resolution (gaierror)."""
|
||||
return isinstance(getattr(err, "reason", None), socket.gaierror)
|
||||
|
||||
|
||||
class HTTPError(Exception):
|
||||
"""HTTP request error with status code."""
|
||||
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
||||
@@ -96,13 +85,7 @@ def request(
|
||||
|
||||
last_error = None
|
||||
rate_limit_count = 0
|
||||
# DNS failures get a dedicated minimum attempt count + exponential backoff.
|
||||
# `effective_retries` is the actual loop bound; we expand it on the first
|
||||
# gaierror if the caller passed a smaller `retries` value than MIN_DNS_RETRIES.
|
||||
effective_retries = retries
|
||||
dns_attempts = 0
|
||||
attempt = 0
|
||||
while attempt < effective_retries:
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
body = response.read().decode('utf-8')
|
||||
@@ -132,8 +115,6 @@ def request(
|
||||
if rate_limit_count >= max_429_retries:
|
||||
raise last_error
|
||||
|
||||
# HTTP errors respect the caller's original `retries`; only DNS
|
||||
# failures get the widened `effective_retries` budget.
|
||||
if attempt < retries - 1:
|
||||
if e.code == 429:
|
||||
# Respect Retry-After header, fall back to exponential backoff
|
||||
@@ -149,43 +130,11 @@ def request(
|
||||
else:
|
||||
delay = RETRY_DELAY * (2 ** attempt)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure may have widened `effective_retries`, but that
|
||||
# widening is DNS-only — don't grant extra HTTP attempts.
|
||||
break
|
||||
except urllib.error.URLError as e:
|
||||
log(f"URL Error: {e.reason}")
|
||||
last_error = HTTPError(f"URL Error: {e.reason}")
|
||||
if _is_dns_failure(e):
|
||||
# DNS resolution failures are transient; expand the retry budget
|
||||
# to MIN_DNS_RETRIES if the caller passed fewer, and use
|
||||
# exponential backoff (1s, 2s, 4s, ...) instead of the linear
|
||||
# default. Counts DNS attempts separately so other URLError
|
||||
# causes don't bypass the regular retry budget.
|
||||
dns_attempts += 1
|
||||
if effective_retries < MIN_DNS_RETRIES:
|
||||
log(
|
||||
f"DNS resolution failed; expanding retry budget from "
|
||||
f"{effective_retries} to {MIN_DNS_RETRIES}"
|
||||
)
|
||||
effective_retries = MIN_DNS_RETRIES
|
||||
if attempt < effective_retries - 1:
|
||||
delay = 2 ** (dns_attempts - 1) # 1s, 2s, 4s, 8s, ...
|
||||
log(
|
||||
f"DNS resolution failure (attempt {dns_attempts}); "
|
||||
f"retrying in {delay:.1f}s"
|
||||
)
|
||||
time.sleep(delay)
|
||||
elif attempt < retries - 1:
|
||||
# Non-DNS URLError (e.g. ConnectionRefused) respects the
|
||||
# caller's original retry budget, not the DNS-widened bound.
|
||||
if attempt < retries - 1:
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure widening `effective_retries` does not carry over
|
||||
# to non-DNS error paths.
|
||||
break
|
||||
except json.JSONDecodeError as e:
|
||||
log(f"JSON decode error: {e}")
|
||||
last_error = HTTPError(f"Invalid JSON response: {e}")
|
||||
@@ -195,13 +144,7 @@ def request(
|
||||
log(f"Connection error: {type(e).__name__}: {e}")
|
||||
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
||||
if attempt < retries - 1:
|
||||
# Socket errors respect the caller's original retry budget.
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Original budget exhausted; DNS widening doesn't apply here.
|
||||
break
|
||||
|
||||
attempt += 1
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
@@ -12,6 +12,11 @@ import sys
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import dates, http, log
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||
@@ -231,17 +236,30 @@ def _user_reels(
|
||||
"""
|
||||
_log(f"User reels: @{handle}")
|
||||
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
|
||||
try:
|
||||
data = http.get(
|
||||
reels_url,
|
||||
params={"handle": handle},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"User reels error for @{handle}: {e}")
|
||||
return []
|
||||
if not _requests:
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"handle": handle})
|
||||
url = f"{reels_url}?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"User reels error (urllib) for @{handle}: {e}")
|
||||
return []
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
reels_url,
|
||||
params={"handle": handle},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"User reels error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
|
||||
_log(f" -> {len(raw_items)} reels from @{handle}")
|
||||
@@ -275,17 +293,31 @@ def search_instagram(
|
||||
|
||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"query": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
||||
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
||||
@@ -335,7 +367,7 @@ def fetch_captions(
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = config["max_captions"]
|
||||
|
||||
if not video_items or not token:
|
||||
if not video_items or not token or not _requests:
|
||||
return {}
|
||||
|
||||
top_items = video_items[:max_captions]
|
||||
@@ -360,24 +392,26 @@ def fetch_captions(
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
data = http.get(
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||
params={"url": url},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=15,
|
||||
retries=1,
|
||||
)
|
||||
transcripts = data.get("transcripts") or []
|
||||
if transcripts and isinstance(transcripts, list):
|
||||
transcript_text = " ".join(
|
||||
t.get("text", "") for t in transcripts
|
||||
if isinstance(t, dict) and t.get("text")
|
||||
)
|
||||
if transcript_text:
|
||||
words = transcript_text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = transcript_text
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
transcripts = data.get("transcripts") or []
|
||||
if transcripts and isinstance(transcripts, list):
|
||||
# Combine all transcript segments
|
||||
transcript_text = " ".join(
|
||||
t.get("text", "") for t in transcripts
|
||||
if isinstance(t, dict) and t.get("text")
|
||||
)
|
||||
if transcript_text:
|
||||
words = transcript_text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = transcript_text
|
||||
except Exception as e:
|
||||
_log(f"Transcript fetch failed for {vid}: {e}")
|
||||
|
||||
@@ -49,7 +49,6 @@ def normalize_source_items(
|
||||
"xquik": _normalize_x,
|
||||
"pinterest": _normalize_pinterest,
|
||||
"polymarket": _normalize_polymarket,
|
||||
"digg": _normalize_digg,
|
||||
"grounding": _normalize_grounding,
|
||||
"xiaohongshu": _normalize_grounding,
|
||||
"github": _normalize_github,
|
||||
@@ -111,19 +110,6 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
|
||||
return default
|
||||
|
||||
|
||||
def _join_comment_excerpts(
|
||||
top_comments: list[Any],
|
||||
key: str,
|
||||
limit: int = 3,
|
||||
) -> str:
|
||||
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
|
||||
return " ".join(
|
||||
str(comment.get(key) or "").strip()
|
||||
for comment in top_comments[:limit]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
|
||||
|
||||
def _domain_from_url(url: str) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
@@ -183,7 +169,11 @@ def _normalize_reddit(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
body = "\n".join(
|
||||
part
|
||||
for part in [
|
||||
@@ -348,7 +338,11 @@ def _normalize_hackernews(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "text")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("text") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
title = str(item.get("title") or "").strip()
|
||||
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
||||
return _source_item(
|
||||
@@ -400,53 +394,6 @@ def _normalize_microblog(
|
||||
)
|
||||
|
||||
|
||||
def _normalize_digg(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
index: int,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
"""Normalizer for Digg AI 1000 clusters.
|
||||
|
||||
Each cluster is one item. The TLDR carries the most useful body for
|
||||
rerank and synthesis. Top-ranked X posts attached at search time are
|
||||
passed through under metadata['posts'] so render can emit them as
|
||||
inline 'via Digg' quotes.
|
||||
"""
|
||||
title = str(item.get("title") or "").strip()
|
||||
tldr = str(item.get("tldr") or "").strip()
|
||||
body = "\n\n".join(part for part in [title, tldr] if part)
|
||||
posts = item.get("posts") or []
|
||||
if not isinstance(posts, list):
|
||||
posts = []
|
||||
cluster_url_id = str(item.get("id") or f"DG{index + 1}")
|
||||
return _source_item(
|
||||
item_id=cluster_url_id,
|
||||
source=source,
|
||||
title=title or f"Digg cluster {index + 1}",
|
||||
body=body,
|
||||
url=str(item.get("url") or f"https://di.gg/ai/{cluster_url_id}"),
|
||||
author="",
|
||||
container="Digg",
|
||||
published_at=item.get("date"),
|
||||
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
|
||||
engagement=item.get("engagement") or {},
|
||||
relevance_hint=item.get("relevance", 0.5),
|
||||
why_relevant=str(item.get("why_relevant") or ""),
|
||||
snippet=tldr[:400],
|
||||
metadata={
|
||||
"clusterUrlId": cluster_url_id,
|
||||
"tldr": tldr,
|
||||
"rank": (item.get("engagement") or {}).get("rank"),
|
||||
"uniqueAuthors": (item.get("engagement") or {}).get("uniqueAuthors"),
|
||||
"postCount": (item.get("engagement") or {}).get("postCount"),
|
||||
"firstPostAge": item.get("first_post_age"),
|
||||
"posts": posts,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalize_polymarket(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
@@ -494,7 +441,11 @@ def _normalize_github(
|
||||
title = str(item.get("title") or "").strip()
|
||||
snippet_text = str(item.get("snippet") or "").strip()
|
||||
top_comments = item.get("metadata", {}).get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
||||
metadata = item.get("metadata") or {}
|
||||
return _source_item(
|
||||
@@ -11,6 +11,11 @@ import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import dates, http, log
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
|
||||
@@ -135,17 +140,31 @@ def search_pinterest(
|
||||
|
||||
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"keyword": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/search?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Extract items from response - try common SC response shapes
|
||||
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
|
||||
@@ -15,7 +15,6 @@ from . import (
|
||||
bluesky,
|
||||
dates,
|
||||
dedupe,
|
||||
digg,
|
||||
entity_extract,
|
||||
env,
|
||||
github,
|
||||
@@ -31,7 +30,6 @@ from . import (
|
||||
query,
|
||||
reddit,
|
||||
reddit_public,
|
||||
relevance,
|
||||
rerank,
|
||||
schema,
|
||||
signals,
|
||||
@@ -80,7 +78,6 @@ MOCK_AVAILABLE_SOURCES = [
|
||||
"github",
|
||||
"perplexity",
|
||||
"xquik",
|
||||
"digg",
|
||||
]
|
||||
|
||||
|
||||
@@ -108,8 +105,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.extend(["hackernews", "polymarket"])
|
||||
if config.get("GITHUB_TOKEN") or which("gh"):
|
||||
available.append("github")
|
||||
if which("digg-pp-cli"):
|
||||
available.append("digg")
|
||||
if env.is_bluesky_available(config):
|
||||
available.append("bluesky")
|
||||
if env.is_truthsocial_available(config):
|
||||
@@ -128,9 +123,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("pinterest")
|
||||
if env.is_xquik_available(config):
|
||||
available.append("xquik")
|
||||
exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()}
|
||||
if exclude:
|
||||
available = [s for s in available if s not in exclude]
|
||||
return available
|
||||
|
||||
|
||||
@@ -508,12 +500,11 @@ def _normalize_score_dedupe(
|
||||
source, raw_items, from_date, to_date,
|
||||
freshness_mode=freshness_mode,
|
||||
)
|
||||
prepared_query = relevance.PreparedQuery(ranking_query)
|
||||
normalized = signals.annotate_stream(normalized, prepared_query, freshness_mode)
|
||||
normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
|
||||
normalized = signals.prune_low_relevance(normalized)
|
||||
normalized = dedupe.dedupe_items(normalized)
|
||||
for item in normalized:
|
||||
item.snippet = snippet.extract_best_snippet(item, prepared_query)
|
||||
item.snippet = snippet.extract_best_snippet(item, ranking_query)
|
||||
return normalized
|
||||
|
||||
|
||||
@@ -538,12 +529,6 @@ def _finalize_items_by_source(
|
||||
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
|
||||
if keywords:
|
||||
items = polymarket.filter_items_against_keywords(items, keywords)
|
||||
if source == "digg" and items:
|
||||
# Pull top-ranked X posts only for the survivors that will appear
|
||||
# in the brief. Spending the enrichment budget here (rather than
|
||||
# at retrieval time) keeps the inline 'via Digg' quotes
|
||||
# paired with the clusters dedupe actually kept.
|
||||
digg.enrich_source_items(items, top_k=3)
|
||||
finalized[source] = items
|
||||
return finalized
|
||||
|
||||
@@ -979,13 +964,6 @@ def _retrieve_stream(
|
||||
if source == "hackernews":
|
||||
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
||||
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
||||
if source == "digg":
|
||||
result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
|
||||
items = digg.parse_digg_response(result, query=subquery.search_query)
|
||||
# Enrichment with attached X posts is deferred to
|
||||
# _finalize_items_by_source so it runs on the items that actually
|
||||
# survive dedupe rather than on top-K of the raw fanout.
|
||||
return items, {}
|
||||
if source == "bluesky":
|
||||
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
||||
return bluesky.parse_bluesky_response(result), {}
|
||||
@@ -1078,45 +1056,6 @@ def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[d
|
||||
"why_relevant": "Brave web search",
|
||||
}
|
||||
],
|
||||
"digg": [
|
||||
{
|
||||
"id": "mock1abc",
|
||||
"title": f"Digg cluster about {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock1abc",
|
||||
"tldr": f"Curated cluster summarizing recent {subquery.search_query} discussion across the AI 1000.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(3)[0],
|
||||
"engagement": {"postCount": 8, "uniqueAuthors": 5, "rank": 2, "rank_score": 49.0},
|
||||
"first_post_age": "3d",
|
||||
"posts": [
|
||||
{
|
||||
"username": "exampledev",
|
||||
"display_name": "Example Dev",
|
||||
"category": "Engineer",
|
||||
"rank": 142,
|
||||
"body": f"Quote from the AI 1000 about {subquery.search_query}.",
|
||||
"post_type": "tweet",
|
||||
"x_url": "https://x.com/exampledev/status/1",
|
||||
"posted_at": dates.get_date_range(3)[0],
|
||||
},
|
||||
],
|
||||
"relevance": 0.84,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
{
|
||||
"id": "mock2def",
|
||||
"title": f"Second Digg cluster on {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock2def",
|
||||
"tldr": f"Another angle on {subquery.search_query}.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(8)[0],
|
||||
"engagement": {"postCount": 3, "uniqueAuthors": 2, "rank": 18, "rank_score": 33.0},
|
||||
"first_post_age": "8d",
|
||||
"posts": [],
|
||||
"relevance": 0.71,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
],
|
||||
}
|
||||
if source == "grounding":
|
||||
return payloads.get(source, []), {
|
||||
@@ -67,7 +67,6 @@ SOURCE_CAPABILITIES = {
|
||||
"bluesky": {"discussion", "social"},
|
||||
"truthsocial": {"discussion", "social"},
|
||||
"polymarket": {"market"},
|
||||
"digg": {"discussion", "social", "link"},
|
||||
"xiaohongshu": {"video", "video_shortform", "social"},
|
||||
"github": {"discussion", "link"},
|
||||
"grounding": {"web", "reference", "link"},
|
||||
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
from . import env, http, schema
|
||||
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite"
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_PRO = "gemini-3.1-pro-preview"
|
||||
OPENAI_DEFAULT = "gpt-5.4-nano"
|
||||
XAI_DEFAULT = "grok-4-1-fast"
|
||||
@@ -93,6 +93,13 @@ class GeminiClient(ReasoningClient):
|
||||
)
|
||||
return extract_gemini_text(payload)
|
||||
|
||||
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
|
||||
|
||||
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
|
||||
|
||||
|
||||
class OpenAIClient(ReasoningClient):
|
||||
name = "openai"
|
||||
|
||||
@@ -232,8 +239,8 @@ def _resolve_model_pins(config: dict[str, Any], depth: str, provider_name: str)
|
||||
rerank_model = config.get("LAST30DAYS_RERANK_MODEL") or default_rerank
|
||||
|
||||
if provider_name == "gemini":
|
||||
_require_gemini_31(planner_model, role="planner")
|
||||
_require_gemini_31(rerank_model, role="rerank")
|
||||
_require_gemini_31_preview(planner_model, role="planner")
|
||||
_require_gemini_31_preview(rerank_model, role="rerank")
|
||||
|
||||
return planner_model, rerank_model
|
||||
|
||||
@@ -344,11 +351,11 @@ def _resolve_x_backend(config: dict[str, Any]) -> str | None:
|
||||
return env.get_x_source(config)
|
||||
|
||||
|
||||
def _require_gemini_31(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-"):
|
||||
def _require_gemini_31_preview(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-") and model.endswith("-preview"):
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"{role} must use a Gemini 3.1 model. Got: {model}"
|
||||
f"{role} must use a Gemini 3.1 preview model. Got: {model}"
|
||||
)
|
||||
|
||||
|
||||
@@ -71,29 +71,8 @@ def _normalize_phrase(text: str) -> str:
|
||||
return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split())
|
||||
|
||||
|
||||
class PreparedQuery:
|
||||
"""Precomputed query shape reused across items in a stream.
|
||||
|
||||
Built once per ranking_query; reused by token_overlap_relevance so the
|
||||
per-item normalize/score loops don't re-tokenize the same query N times.
|
||||
"""
|
||||
|
||||
__slots__ = ("raw", "q_tokens", "informative_q_tokens", "normalized_phrase")
|
||||
|
||||
def __init__(self, query: str) -> None:
|
||||
self.raw = query
|
||||
self.q_tokens = tokenize(query)
|
||||
informative = {t for t in self.q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
||||
self.informative_q_tokens = informative or self.q_tokens
|
||||
self.normalized_phrase = _normalize_phrase(query)
|
||||
|
||||
|
||||
def _as_prepared(query: "str | PreparedQuery") -> PreparedQuery:
|
||||
return query if isinstance(query, PreparedQuery) else PreparedQuery(query)
|
||||
|
||||
|
||||
def token_overlap_relevance(
|
||||
query: "str | PreparedQuery",
|
||||
query: str,
|
||||
text: str,
|
||||
hashtags: Optional[List[str]] = None,
|
||||
) -> float:
|
||||
@@ -116,8 +95,7 @@ def token_overlap_relevance(
|
||||
Returns:
|
||||
Float between 0.0 and 1.0 (0.5 for empty queries)
|
||||
"""
|
||||
prepared = _as_prepared(query)
|
||||
q_tokens = prepared.q_tokens
|
||||
q_tokens = tokenize(query)
|
||||
|
||||
# Combine text and hashtags for matching
|
||||
combined = text
|
||||
@@ -141,7 +119,9 @@ def token_overlap_relevance(
|
||||
if overlap == 0:
|
||||
return 0.0
|
||||
|
||||
informative_q_tokens = prepared.informative_q_tokens
|
||||
informative_q_tokens = {t for t in q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
||||
if not informative_q_tokens:
|
||||
informative_q_tokens = q_tokens
|
||||
|
||||
coverage = overlap / len(q_tokens)
|
||||
informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens)
|
||||
@@ -149,7 +129,7 @@ def token_overlap_relevance(
|
||||
precision = overlap / precision_denominator
|
||||
|
||||
phrase_bonus = 0.0
|
||||
normalized_query = prepared.normalized_phrase
|
||||
normalized_query = _normalize_phrase(query)
|
||||
normalized_text = _normalize_phrase(combined)
|
||||
if normalized_query and normalized_query in normalized_text:
|
||||
phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
|
||||
@@ -8,40 +8,24 @@ from collections import Counter
|
||||
from datetime import date
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import dates, schema, skill_meta
|
||||
from . import dates, schema
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
"""Read plugin version from .claude-plugin/plugin.json, falling back to SKILL.md frontmatter.
|
||||
"""Read plugin version from .claude-plugin/plugin.json if available.
|
||||
|
||||
Per-harness skill install dirs (`~/.claude/skills`, `~/.codex/skills`, `~/.agents/skills`,
|
||||
Hermes, etc.) do not always carry `.claude-plugin/plugin.json` — that file ships with
|
||||
plugin-cache installs but not with per-harness skill installs. SKILL.md frontmatter is
|
||||
the fallback that keeps the badge from emitting v? on those installs. Returns "?" only
|
||||
if no usable version string is found from either source (missing files, corrupt JSON,
|
||||
or SKILL.md without a version line).
|
||||
|
||||
A corrupt manifest at one ancestor does not shadow a valid manifest at a deeper one
|
||||
(continue, not break). SKILL.md parsing accepts double-quoted, single-quoted, or
|
||||
unquoted YAML version scalars (delegated to skill_meta.read_skill_version).
|
||||
Tries nearest plugin.json by walking up from render.py's own location.
|
||||
Falls back to "?" if not found. This keeps the badge emission from
|
||||
crashing on non-plugin-cache installs (repo checkout, Gemini, Codex).
|
||||
"""
|
||||
here = pathlib.Path(__file__).resolve()
|
||||
for parent in here.parents:
|
||||
manifest = parent / ".claude-plugin" / "plugin.json"
|
||||
if manifest.is_file():
|
||||
for parent in [here.parent, *here.parents]:
|
||||
candidate = parent / ".claude-plugin" / "plugin.json"
|
||||
if candidate.is_file():
|
||||
try:
|
||||
version = json.loads(manifest.read_text()).get("version")
|
||||
return json.loads(candidate.read_text()).get("version", "?")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
if version:
|
||||
return version
|
||||
|
||||
# No usable manifest found at any ancestor — fall back to SKILL.md frontmatter.
|
||||
# First SKILL.md found in the walk is THIS skill's; never traverse past it.
|
||||
for parent in here.parents:
|
||||
skill_md = parent / "SKILL.md"
|
||||
if skill_md.is_file():
|
||||
return skill_meta.read_skill_version(skill_md) or "?"
|
||||
return "?"
|
||||
return "?"
|
||||
|
||||
|
||||
@@ -68,7 +52,6 @@ SOURCE_LABELS = {
|
||||
"xiaohongshu": "Xiaohongshu",
|
||||
"x": "X",
|
||||
"github": "GitHub",
|
||||
"digg": "Digg",
|
||||
"perplexity": "Perplexity",
|
||||
}
|
||||
|
||||
@@ -96,7 +79,7 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
f"# last30days v{_skill_version()}: {report.topic}",
|
||||
f"# last30days v3.0.0: {report.topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Date range: {report.range_from} to {report.range_to}",
|
||||
@@ -187,168 +170,6 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_for_html(
|
||||
report: schema.Report,
|
||||
synthesis_md: str | None = None,
|
||||
*,
|
||||
save_path: str | None = None,
|
||||
) -> str:
|
||||
"""Render markdown intended for shareable HTML conversion.
|
||||
|
||||
This output keeps the public badge, compact source/date metadata, an
|
||||
optional one-line data quality note, optional synthesized brief markdown,
|
||||
and the engine footer. It deliberately omits the debug file header,
|
||||
model-facing safety note, and evidence scratchpad emitted by
|
||||
render_compact().
|
||||
|
||||
When synthesis_md is None, the body is intentionally sparse: badge,
|
||||
metadata, optional data quality note, and engine footer only.
|
||||
"""
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
*_render_html_metadata(report),
|
||||
]
|
||||
if synthesis_md:
|
||||
lines.extend(["", synthesis_md.strip()])
|
||||
# Data quality warnings are NOT rendered into the HTML artifact. The HTML
|
||||
# is meant to be shared (Slack, email, Notion); recipients haven't asked
|
||||
# for technical commentary about how the run was produced. Generators see
|
||||
# the same warnings via collect_html_warnings() routed to stderr by the
|
||||
# CLI, so they can fix quality issues before sharing.
|
||||
_append_html_footer(lines, report, save_path)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_for_html_comparison(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
synthesis_md: str | None = None,
|
||||
*,
|
||||
save_path: str | None = None,
|
||||
) -> str:
|
||||
"""Render comparison markdown intended for shareable HTML conversion.
|
||||
|
||||
Same semantics as render_for_html(), but metadata and data quality notes
|
||||
are aggregated across the compared entities.
|
||||
"""
|
||||
if not entity_reports:
|
||||
raise ValueError("render_for_html_comparison requires at least one report")
|
||||
|
||||
entities = [label for label, _ in entity_reports]
|
||||
main_report = entity_reports[0][1]
|
||||
meta = (
|
||||
f"<!-- META: {main_report.range_from} to {main_report.range_to} "
|
||||
f"· comparing {len(entities)}: {', '.join(entities)} -->"
|
||||
)
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
meta,
|
||||
]
|
||||
if synthesis_md:
|
||||
lines.extend(["", synthesis_md.strip()])
|
||||
# Comparison data quality notes also go to stderr, not into the artifact.
|
||||
_append_html_footer(lines, main_report, save_path)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def collect_html_warnings(report: schema.Report) -> list[str]:
|
||||
"""Collect data quality warnings for stderr output (NOT for the HTML artifact).
|
||||
|
||||
Returns a list of human-readable warning strings. Empty list if the run
|
||||
was clean. Used by the CLI to emit diagnostics to stderr after writing
|
||||
the HTML to stdout/file.
|
||||
"""
|
||||
notes: list[str] = []
|
||||
if _render_degraded_run_warning(report):
|
||||
notes.append("Run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
||||
elif _render_pre_research_warning(report):
|
||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
||||
freshness_warning = _assess_data_freshness(report)
|
||||
if freshness_warning:
|
||||
notes.append(freshness_warning)
|
||||
notes.extend(report.warnings)
|
||||
return _dedupe_notes(notes)
|
||||
|
||||
|
||||
def collect_html_warnings_comparison(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
) -> list[str]:
|
||||
"""Collect comparison-mode warnings, prefixed by entity label."""
|
||||
notes: list[str] = []
|
||||
for label, report in entity_reports:
|
||||
for w in collect_html_warnings(report):
|
||||
notes.append(f"{label}: {w}")
|
||||
return notes
|
||||
|
||||
|
||||
def _render_html_metadata(report: schema.Report) -> list[str]:
|
||||
"""Inline metadata as an HTML comment marker.
|
||||
|
||||
html_render.py post-processes ``<!-- META: ... -->`` markers into a
|
||||
``<div class="meta">`` after markdown conversion, so the metadata escapes
|
||||
the markdown converter's HTML-escaping pass cleanly. Same pattern as the
|
||||
PASS_THROUGH_FOOTER marker used for the engine tree.
|
||||
"""
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
if non_empty:
|
||||
sources = ", ".join(_source_label(s) for s in non_empty)
|
||||
else:
|
||||
sources = "no active sources"
|
||||
return [
|
||||
f"<!-- META: {report.range_from} to {report.range_to} · {sources} -->",
|
||||
]
|
||||
|
||||
|
||||
def _render_html_data_quality_note(report: schema.Report) -> str | None:
|
||||
notes: list[str] = []
|
||||
degraded_warning = _render_degraded_run_warning(report)
|
||||
if degraded_warning:
|
||||
notes.append("This run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
||||
pre_research_warning = _render_pre_research_warning(report)
|
||||
if pre_research_warning and not degraded_warning:
|
||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
||||
freshness_warning = _assess_data_freshness(report)
|
||||
if freshness_warning:
|
||||
notes.append(freshness_warning)
|
||||
notes.extend(report.warnings)
|
||||
if not notes:
|
||||
return None
|
||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
||||
|
||||
|
||||
def _render_html_comparison_data_quality_note(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
) -> str | None:
|
||||
notes: list[str] = []
|
||||
for label, report in entity_reports:
|
||||
note = _render_html_data_quality_note(report)
|
||||
if note:
|
||||
clean = note.removeprefix("> **Data quality note:** ").strip()
|
||||
notes.append(f"{label}: {clean}")
|
||||
if not notes:
|
||||
return None
|
||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
||||
|
||||
|
||||
def _dedupe_notes(notes: list[str]) -> list[str]:
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for note in notes:
|
||||
normalized = " ".join(str(note).split())
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
out.append(normalized)
|
||||
return out
|
||||
|
||||
|
||||
def _append_html_footer(lines: list[str], report: schema.Report, save_path: str | None) -> None:
|
||||
footer = _render_emoji_footer(report, save_path)
|
||||
lines.append("")
|
||||
lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")
|
||||
lines.extend(footer)
|
||||
lines.append("<!-- END PASS-THROUGH FOOTER -->")
|
||||
|
||||
|
||||
def _render_canonical_boundary() -> list[str]:
|
||||
"""Emit the explicit END-OF-CANONICAL-OUTPUT boundary.
|
||||
|
||||
@@ -602,7 +423,7 @@ def render_comparison_multi(
|
||||
|
||||
lines: list[str] = [
|
||||
*_render_badge(),
|
||||
f"# last30days v{_skill_version()}: {synthesized_topic}",
|
||||
f"# last30days v3.0.0: {synthesized_topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Comparison mode: {len(entities)} entities ({', '.join(entities)})",
|
||||
@@ -790,7 +611,7 @@ def render_full(report: schema.Report) -> str:
|
||||
# Start with the same header as compact
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
lines = [
|
||||
f"# last30days v{_skill_version()}: {report.topic}",
|
||||
f"# last30days v3.0.0: {report.topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Date range: {report.range_from} to {report.range_to}",
|
||||
@@ -842,7 +663,7 @@ def render_full(report: schema.Report) -> str:
|
||||
lines.append("## All Items by Source")
|
||||
lines.append("")
|
||||
source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest",
|
||||
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "digg", "perplexity"]
|
||||
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "perplexity"]
|
||||
for source in source_order:
|
||||
items = report.items_by_source.get(source, [])
|
||||
if not items:
|
||||
@@ -868,9 +689,6 @@ def render_full(report: schema.Report) -> str:
|
||||
tc_score = tc.get("score", "")
|
||||
attribution = _comment_attribution(item.source, tc.get("author"))
|
||||
lines.append(f" Top comment {attribution} ({tc_score} {vote_label}): {excerpt}")
|
||||
# Digg: inline X-post quotes attached to the cluster.
|
||||
for post in _digg_posts_for(item, limit=3):
|
||||
lines.append(f" > {_format_digg_quote(post)}")
|
||||
# Comment insights for Reddit
|
||||
insights = item.metadata.get("comment_insights", [])
|
||||
if insights:
|
||||
@@ -992,8 +810,6 @@ def _render_candidate(candidate: schema.Candidate, prefix: str) -> list[str]:
|
||||
source = primary.source if primary else None
|
||||
attribution = _comment_attribution(source, tc.get("author"))
|
||||
lines.append(f" - {attribution} ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
|
||||
for post in _digg_posts_for(primary):
|
||||
lines.append(f" - {_format_digg_quote(post)}")
|
||||
insight = _comment_insight(primary)
|
||||
if insight:
|
||||
lines.append(f" - Insight: {_truncate(insight, 220)}")
|
||||
@@ -1244,7 +1060,6 @@ _FOOTER_SOURCES: list[tuple[str, str, str, str, list[tuple[str, str]]]] = [
|
||||
("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||
("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||
("github", "🐙", "GitHub", "item", [("reactions", "reactions"), ("comments", "comments")]),
|
||||
("digg", "⛏️", "Digg", "cluster", [("postCount", "posts"), ("uniqueAuthors", "authors")]),
|
||||
]
|
||||
|
||||
|
||||
@@ -1502,7 +1317,6 @@ ENGAGEMENT_DISPLAY: dict[str, list[tuple[str, str]]] = {
|
||||
"polymarket": [],
|
||||
"github": [("reactions", "react"), ("comments", "cmt")],
|
||||
"perplexity": [("citations", "cite")],
|
||||
"digg": [("postCount", "posts"), ("uniqueAuthors", "auth")],
|
||||
}
|
||||
|
||||
|
||||
@@ -1690,6 +1504,16 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score
|
||||
return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]
|
||||
|
||||
|
||||
def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None:
|
||||
if not item:
|
||||
return None
|
||||
comments = item.metadata.get("top_comments") or []
|
||||
if not comments or not isinstance(comments[0], dict):
|
||||
return None
|
||||
top = comments[0]
|
||||
return str(top.get("excerpt") or top.get("text") or "").strip() or None
|
||||
|
||||
|
||||
def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
||||
if not item:
|
||||
return None
|
||||
@@ -1699,39 +1523,6 @@ def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
||||
return str(insights[0]).strip() or None
|
||||
|
||||
|
||||
def _digg_posts_for(item: schema.SourceItem | None, limit: int = 3) -> list[dict]:
|
||||
"""Return up to `limit` parsed Digg posts attached as enrichment to a cluster.
|
||||
|
||||
Returns an empty list for non-digg sources or clusters without enrichment.
|
||||
"""
|
||||
if not item or item.source != "digg":
|
||||
return []
|
||||
posts = item.metadata.get("posts") or []
|
||||
if not isinstance(posts, list):
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for entry in posts:
|
||||
if isinstance(entry, dict) and entry.get("body") and entry.get("username"):
|
||||
out.append(entry)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _format_digg_quote(post: dict, body_limit: int = 200) -> str:
|
||||
"""Format a Digg-attached X post as an inline 'via Digg' quote line."""
|
||||
handle = post.get("username") or ""
|
||||
x_url = post.get("x_url") or ""
|
||||
body = (post.get("body") or "").replace("\n", " ").strip()
|
||||
if len(body) > body_limit:
|
||||
body = body[: body_limit - 1].rstrip() + "…"
|
||||
if x_url and handle:
|
||||
return f"[@{handle}]({x_url}) via Digg: {body}"
|
||||
if handle:
|
||||
return f"@{handle} via Digg: {body}"
|
||||
return f"via Digg: {body}"
|
||||
|
||||
|
||||
def _transcript_highlights(item: schema.SourceItem | None) -> list[str]:
|
||||
if not item or item.source != "youtube":
|
||||
return []
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user