Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Matt Van Horn
|
b31ec05c74 |
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Research"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,17 +1,16 @@
|
||||
{
|
||||
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "last30days-skill",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"owner": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.3.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"version": "3.0.9",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.3.0",
|
||||
"version": "3.0.14",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
@@ -10,5 +10,6 @@
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"],
|
||||
"hooks": {}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"name": "last30days"
|
||||
}
|
||||
+4
-4
@@ -1,6 +1,5 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by skills/last30days/scripts/build-skill.sh to produce a
|
||||
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
|
||||
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
@@ -33,8 +32,9 @@ release-notes.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
uv.lock export-ignore
|
||||
|
||||
# Platform adapters are kept in git archives because Claude Code and Codex
|
||||
# plugin installs use the same repository archive as their source payload.
|
||||
# Platform adapters - skill-upload path is platform-agnostic
|
||||
.agents/ export-ignore
|
||||
.codex-plugin/ export-ignore
|
||||
.hermes-plugin/ export-ignore
|
||||
|
||||
# CI workflows - repo-only, not needed at skill runtime
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
name: Bug Report
|
||||
description: Report a bug or unexpected behavior
|
||||
labels: [bug]
|
||||
body:
|
||||
- type: textarea
|
||||
id: summary
|
||||
attributes:
|
||||
label: Summary
|
||||
description: What happened?
|
||||
placeholder: Describe the bug in 1-2 sentences.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: How can we reproduce this?
|
||||
placeholder: |
|
||||
1. Run `python3 skills/last30days/scripts/last30days.py "topic" --emit=compact`
|
||||
2. ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected Behavior
|
||||
description: What should have happened?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: traceback
|
||||
attributes:
|
||||
label: Error / Traceback
|
||||
description: Paste the full traceback or error output.
|
||||
render: text
|
||||
- type: dropdown
|
||||
id: install
|
||||
attributes:
|
||||
label: Install Method
|
||||
options:
|
||||
- Claude Code plugin
|
||||
- Gemini CLI extension
|
||||
- Codex plugin
|
||||
- Hermes skill
|
||||
- Manual (git clone)
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: OS
|
||||
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
|
||||
@@ -1,24 +0,0 @@
|
||||
name: Feature Request
|
||||
description: Suggest a new feature or improvement
|
||||
labels: [enhancement]
|
||||
body:
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem
|
||||
description: What problem does this solve?
|
||||
placeholder: When I try to ..., I can't ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: How should this work?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
description: Other approaches you thought of (optional).
|
||||
@@ -1,19 +0,0 @@
|
||||
## Summary
|
||||
|
||||
<!-- What does this PR do? 1-3 sentences. -->
|
||||
|
||||
## Changes
|
||||
|
||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
||||
|
||||
-
|
||||
|
||||
## Testing
|
||||
|
||||
<!-- How did you verify this works? -->
|
||||
|
||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
||||
|
||||
## Related Issues
|
||||
|
||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
bash scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
name: Security
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
name: Dependency audit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Export locked dependency set
|
||||
run: |
|
||||
uv export \
|
||||
--locked \
|
||||
--all-groups \
|
||||
--no-hashes \
|
||||
--format requirements.txt \
|
||||
--output-file /tmp/last30days-requirements.txt
|
||||
|
||||
# Advisory-first: visibility before enforcement. This repo handles API keys,
|
||||
# cookies, browser tokens, and local env files, so dependency CVEs should be
|
||||
# visible in CI logs even before the project has a clean blocking baseline.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
- name: Run pip-audit against locked dependencies
|
||||
continue-on-error: true
|
||||
run: uvx --python 3.12 pip-audit -r /tmp/last30days-requirements.txt --progress-spinner=off
|
||||
|
||||
secret-scan:
|
||||
name: Secret scan
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout full history for diff-aware scanning
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Advisory-first: this reports verified secrets in pull requests and pushes to
|
||||
# main, but does not block merges until maintainers confirm a clean baseline.
|
||||
# The TruffleHog action automatically scans the PR range for pull_request
|
||||
# events and the pushed commit range for push events.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
# Contributor policy: never commit real secrets in fixtures, tests, docs, or
|
||||
# examples; use obvious dummy values and env-based auth patterns instead.
|
||||
- name: Run TruffleHog OSS secret scan
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
uses: trufflesecurity/trufflehog@v3.95.2
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: ./
|
||||
version: v3.95.2
|
||||
extra_args: --only-verified
|
||||
@@ -1,26 +0,0 @@
|
||||
name: Validate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run test suite
|
||||
run: uv run pytest
|
||||
+5
-3
@@ -28,7 +28,9 @@ htmlcov/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
.context/
|
||||
|
||||
/work
|
||||
/print
|
||||
# Marketing video build artifacts
|
||||
marketing/v3.1-launch/node_modules/
|
||||
marketing/v3.1-launch/out/
|
||||
marketing/*/node_modules/
|
||||
marketing/*/out/
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
# last30days Skill
|
||||
|
||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
||||
|
||||
## Orientation
|
||||
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
||||
- Feature design starts from the slash-command UX. A new engine flag with no SKILL.md integration is incomplete — the model invoking the skill won't know the flag exists.
|
||||
- README and PR examples show `/last30days <topic>` first. Direct CLI invocation (`python3 scripts/last30days.py ...`) is a fallback for scripting, cron, and dev-time engine testing; label it as such, never as the primary path.
|
||||
- Slash commands don't pass shell mechanics through. `/last30days OpenClaw --emit=html | pbcopy` is invalid in any harness — either use the slash form (no flags or pipes; let the model translate user intent into engine flags) or use the direct CLI form (full `python3 ...` with explicit flags and a real shell).
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Security hygiene
|
||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
||||
- Use the env-based auth patterns in `skills/last30days/scripts/lib/env.py`; tests and fixtures must use obvious dummy values only.
|
||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
||||
|
||||
## Maintaining CONFIGURATION.md
|
||||
|
||||
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
|
||||
|
||||
Update `CONFIGURATION.md` when:
|
||||
|
||||
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
|
||||
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
|
||||
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Hermes…)
|
||||
- adding a new optional source that requires its own credential
|
||||
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
|
||||
|
||||
Keep the existing structure organized by how often each layer is touched: per-run flags → env vars / `.env` → optional trend-monitoring stack → per-client patterns. Add new content into the right section rather than appending at the end.
|
||||
|
||||
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
-181
@@ -5,186 +5,6 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [3.3.0] - 2026-05-17
|
||||
|
||||
A week-long shipping cycle: ~75 PRs merged plus 7 community fixes salvaged through PR triage. Big themes: install story modernized for the multi-harness world (Claude Code, Codex, Cursor, Gemini CLI, Copilot, Windsurf, and 50+ Agent Skills hosts), new emit and source modes, and a substantial reliability sweep across Reddit, X, Windows, YouTube, and the planner.
|
||||
|
||||
### Added
|
||||
|
||||
**Emit modes and sources**
|
||||
|
||||
- `--emit=html` for shareable, print-friendly HTML research briefs ([#332](https://github.com/mvanhorn/last30days-skill/pull/332)).
|
||||
- **Digg AI 1000 source**, auto-enabled when `digg-pp-cli` is on PATH ([#370](https://github.com/mvanhorn/last30days-skill/pull/370)). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief.
|
||||
|
||||
**Configuration knobs**
|
||||
|
||||
- `EXCLUDE_SOURCES` env var — the inverse of `INCLUDE_SOURCES`, honored in source count and pipeline filter ([#399](https://github.com/mvanhorn/last30days-skill/pull/399)).
|
||||
- `LAST30DAYS_YOUTUBE_SSH_HOST` — opt-in SSH routing for `yt-dlp` through a residential-IP host, for users on datacenter VPS hit by YouTube's bot-wall ([#376](https://github.com/mvanhorn/last30days-skill/pull/376)). Host validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection. Transcript path unchanged (uses HTTP fallback).
|
||||
- macOS Keychain as a credential source — reads from the system keychain when env vars and config files aren't set ([#407](https://github.com/mvanhorn/last30days-skill/pull/407)).
|
||||
- Configuration enablement: env-var defaults and source-resilience patterns across the config layer ([#344](https://github.com/mvanhorn/last30days-skill/pull/344)).
|
||||
|
||||
**Pipeline and storage**
|
||||
|
||||
- Reddit URL auto-enrichment from web search via the public JSON API ([#366](https://github.com/mvanhorn/last30days-skill/pull/366)).
|
||||
- Per-run finding sightings recorded in the SQLite store ([#373](https://github.com/mvanhorn/last30days-skill/pull/373)).
|
||||
- Brave browser support for X/Twitter cookie extraction ([#320](https://github.com/mvanhorn/last30days-skill/pull/320)).
|
||||
|
||||
**Tests and CI**
|
||||
|
||||
- Full pytest suite restored to CI; 13 rotted tests repaired ([#416](https://github.com/mvanhorn/last30days-skill/pull/416)).
|
||||
- `greptile.json` added with `triggerOnUpdates` + `statusCheck` ([#418](https://github.com/mvanhorn/last30days-skill/pull/418)).
|
||||
- Advisory security workflow ([#368](https://github.com/mvanhorn/last30days-skill/pull/368)).
|
||||
- Parallel grounding backend test coverage ([#355](https://github.com/mvanhorn/last30days-skill/pull/355)).
|
||||
|
||||
**Docs**
|
||||
|
||||
- New `CONFIGURATION.md` with README pointers ([#339](https://github.com/mvanhorn/last30days-skill/pull/339)).
|
||||
- `docs/solutions/` learning capture for release-time consistency-test cascades ([#413](https://github.com/mvanhorn/last30days-skill/pull/413)) and the eval-not-in-CI design decision ([#417](https://github.com/mvanhorn/last30days-skill/pull/417)).
|
||||
|
||||
### Changed
|
||||
|
||||
**Install story modernized**
|
||||
|
||||
- `npx skills add` is now the canonical install path for every harness ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). README and SKILL.md flipped to recommend `npx skills add . -g -y` over per-harness manual instructions. Surfaces Gemini CLI, Copilot, Windsurf, and 50+ other Agent Skills hosts that the install pattern reaches.
|
||||
- README dropped the Gemini CLI native-extension install path (now covered by `npx skills add`).
|
||||
- `hooks.json` made polyglot for Gemini CLI + Claude Code compatibility ([#318](https://github.com/mvanhorn/last30days-skill/pull/318)).
|
||||
|
||||
**Skill semantics and multi-harness reframe**
|
||||
|
||||
- `AGENTS.md` is now canonical; `CLAUDE.md` points at it ([#410](https://github.com/mvanhorn/last30days-skill/pull/410)). Reframes the project as a multi-harness Agent Skills package rather than a Claude-Code-specific tool.
|
||||
- SKILL.md path resolution rewritten: STEP 0 narrows to a Claude-Code-marketplaces-only stale-clone guard; Step 1 walks a single `SKILL_DIR` substitution pattern ([#400](https://github.com/mvanhorn/last30days-skill/pull/400), [#409](https://github.com/mvanhorn/last30days-skill/pull/409)). Removes ~80 lines of bash and fixes a real spec-vs-engine divergence where the previous resolver could pick a different install than the SKILL.md the model loaded from.
|
||||
- SKILL.md version regex consolidated into `lib/skill_meta.py` ([#412](https://github.com/mvanhorn/last30days-skill/pull/412)).
|
||||
- `--plan` / `--competitors-plan` invocation templates switched from inline single-quoted JSON to heredoc-written tmpfiles ([#404](https://github.com/mvanhorn/last30days-skill/pull/404), fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403)). Apostrophes in resolved context strings ("McDonald's", "people's choice") no longer break shell parsing.
|
||||
- `POSTS_PER_CLUSTER` raised 3→5 and render-side display limit 2→3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous caps routinely truncated cluster context.
|
||||
- Digg AI 1000 renamed to "Digg" in user-facing output ([#372](https://github.com/mvanhorn/last30days-skill/pull/372)) — footer line, source label, inline-quote suffix, why_relevant, container attribution. Internal references retain the upstream product name.
|
||||
- GitHub repo resolution canonicalized for ambiguous product comparisons ([#302](https://github.com/mvanhorn/last30days-skill/pull/302)).
|
||||
|
||||
**Dependencies and tooling**
|
||||
|
||||
- Dropped `requests` runtime dependency. All providers route through stdlib `urllib` via the `lib/http` wrapper ([#393](https://github.com/mvanhorn/last30days-skill/pull/393)).
|
||||
- Migrated to `gemini-3.1-flash-lite` GA model ([#378](https://github.com/mvanhorn/last30days-skill/pull/378)).
|
||||
- Aligned Codex/Claude plugin manifests + added Codex `AGENTS.md` ([#321](https://github.com/mvanhorn/last30days-skill/pull/321)).
|
||||
- pytest dev dep bumped 9.0.2 → 9.0.3 ([#414](https://github.com/mvanhorn/last30days-skill/pull/414)).
|
||||
|
||||
### Removed
|
||||
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1 ([#400](https://github.com/mvanhorn/last30days-skill/pull/400)). Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`** — maintainer dev-deploy script ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). Replaced by `npx skills add . -g -y` (live-symlink into every detected harness's skill dir — better than sync.sh's copy model since edits propagate live). Hermes uses `hermes skills install mvanhorn/last30days-skill --force`; OpenClaw uses `clawhub install last30days-official`.
|
||||
- Orphaned `SPEC.md` and `TASKS.md` ([#419](https://github.com/mvanhorn/last30days-skill/pull/419)).
|
||||
|
||||
### Fixed
|
||||
|
||||
**Reddit**
|
||||
|
||||
- `lstrip("r/")` mangled subreddits starting with `r` (`r/robotics` → `obotics`, `r/ruby` → `uby`); replaced with `removeprefix("r/")` at 4 sites (Alex Key, salvaged from #288).
|
||||
- Browser-like User-Agent + `Accept-Language`/`Accept-Encoding`/`Connection` headers + gzip decompression to fix `urllib` 403s on Reddit's public JSON endpoint (Franco Carballar, salvaged from #199).
|
||||
- HTTP 402 re-raised across all three ScrapeCreators paths (`_global_search`, `_subreddit_search`, `fetch_post_comments`) so the OpenAI/public-JSON fallback chain triggers when credits are exhausted (Jonathan Oppenheim, salvaged from #170).
|
||||
|
||||
**Authentication and credentials**
|
||||
|
||||
- Restored multi-key rotation for `SCRAPECREATORS_API_KEY` accidentally dropped in v3.0.6 (Eric Oberhofer, salvaged from #287). Comma-separated keys round-robin via `random.choice` per run.
|
||||
|
||||
**Windows compatibility**
|
||||
|
||||
- `os.killpg` in `_cleanup_children()` guarded with `hasattr(os, "killpg")`, falls back to `os.kill(SIGTERM)` (gujishh, salvaged from #226).
|
||||
- POSIX-style secret-permission warning skipped on Windows ([#357](https://github.com/mvanhorn/last30days-skill/pull/357)).
|
||||
- Render uses forward slashes in save-path footer for Windows ([#338](https://github.com/mvanhorn/last30days-skill/pull/338)).
|
||||
|
||||
**xAI / X / xurl**
|
||||
|
||||
- `parse_x_response` now raises `http.HTTPError` on empty output, missing JSON, or decode failure — surfaces in `errors_by_source` instead of silently returning an empty result list (Kaustav Mishra, salvaged from #155).
|
||||
- `xurl` treats `PermissionError` from PATH lookup as unavailable ([#322](https://github.com/mvanhorn/last30days-skill/pull/322)).
|
||||
|
||||
**YouTube**
|
||||
|
||||
- SC YouTube + multi-token HN searches unblocked ([#388](https://github.com/mvanhorn/last30days-skill/pull/388)).
|
||||
- Transcript-fetch ratio surfaced + degraded-run nudge for stale `yt-dlp` ([#340](https://github.com/mvanhorn/last30days-skill/pull/340)).
|
||||
|
||||
**bird_x / HTTP**
|
||||
|
||||
- Subprocess retry on non-JSON stdout to handle X anti-bot HTML interstitials ([#383](https://github.com/mvanhorn/last30days-skill/pull/383)).
|
||||
- HTTP retry budget expanded + exponential backoff on DNS resolution failure ([#382](https://github.com/mvanhorn/last30days-skill/pull/382)).
|
||||
- Parallel AI search aligned with current API schema ([#341](https://github.com/mvanhorn/last30days-skill/pull/341)).
|
||||
- Parallel web backend routed through grounding ([#354](https://github.com/mvanhorn/last30days-skill/pull/354)).
|
||||
|
||||
**Planner and sources**
|
||||
|
||||
- `xquik` registered in `SOURCE_CAPABILITIES` ([#336](https://github.com/mvanhorn/last30days-skill/pull/336), fixes [#319](https://github.com/mvanhorn/last30days-skill/issues/319)).
|
||||
- Honor explicit optional source requests ([#356](https://github.com/mvanhorn/last30days-skill/pull/356)).
|
||||
- ScrapeCreators source-gating aligned between code and docs ([#415](https://github.com/mvanhorn/last30days-skill/pull/415)).
|
||||
- OpenClaw works without ScrapeCreators key ([#392](https://github.com/mvanhorn/last30days-skill/pull/392), by @thinkun).
|
||||
|
||||
**Render, version display, hosting paths**
|
||||
|
||||
- Hardcoded `v3.0.0` in render replaced with dynamic `_skill_version()` ([#365](https://github.com/mvanhorn/last30days-skill/pull/365)).
|
||||
- Comparison HTML artifacts saved correctly ([#389](https://github.com/mvanhorn/last30days-skill/pull/389)).
|
||||
- `OPENROUTER_DEFAULT` model ID corrected ([#323](https://github.com/mvanhorn/last30days-skill/pull/323)).
|
||||
- OpenClaw poll-timing initialized once ([#358](https://github.com/mvanhorn/last30days-skill/pull/358)).
|
||||
- Prefer sandboxed Safari cookie path ([#343](https://github.com/mvanhorn/last30days-skill/pull/343)).
|
||||
- Preserve clean mode for last-run state ([#334](https://github.com/mvanhorn/last30days-skill/pull/334)).
|
||||
- Replaced hardcoded `/Users/mvanhorn/...` paths in `test-v1-vs-v2.sh` with portable env-var overrides (Dave Morin, salvaged from #297).
|
||||
|
||||
**Hooks**
|
||||
|
||||
- `check-config.sh` path-quoting fix for paths with spaces ([#337](https://github.com/mvanhorn/last30days-skill/pull/337)).
|
||||
- Replaced unsafe `eval` with `declare` in `check-config.sh` ([#364](https://github.com/mvanhorn/last30days-skill/pull/364)).
|
||||
|
||||
**Sync and version metadata**
|
||||
|
||||
- `sync.sh` pointed at this repo's plugin cache, not the private repo's ([#402](https://github.com/mvanhorn/last30days-skill/pull/402)).
|
||||
- Sync cache target bumped to 3.2.1 to match SKILL.md ([#397](https://github.com/mvanhorn/last30days-skill/pull/397)).
|
||||
- ScrapeCreators free-tier credit count corrected to 100 in docs ([#369](https://github.com/mvanhorn/last30days-skill/pull/369), fixes [#367](https://github.com/mvanhorn/last30days-skill/issues/367)).
|
||||
- Gemini extension version synced ([#349](https://github.com/mvanhorn/last30days-skill/pull/349)).
|
||||
- Various stale path/link fixes ([#345](https://github.com/mvanhorn/last30days-skill/pull/345), [#346](https://github.com/mvanhorn/last30days-skill/pull/346), [#347](https://github.com/mvanhorn/last30days-skill/pull/347), [#348](https://github.com/mvanhorn/last30days-skill/pull/348), [#351](https://github.com/mvanhorn/last30days-skill/pull/351)).
|
||||
|
||||
### Contributors
|
||||
|
||||
First-time contributors whose fixes shipped in this release (most via PR triage salvage — fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
||||
|
||||
- Dave Morin — portable test-harness paths
|
||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
||||
- Eric Oberhofer — multi-key rotation restored
|
||||
- gujishh — Windows process cleanup
|
||||
- Franco Carballar — Reddit browser-like headers
|
||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
||||
- Kaustav Mishra — xAI error surfacing
|
||||
- [@thinkun](https://github.com/thinkun) ([#363](https://github.com/mvanhorn/last30days-skill/pull/363)) — OpenClaw ScrapeCreators-key-optional fix
|
||||
|
||||
Full PR list at [github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0](https://github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0).
|
||||
|
||||
## [3.2.0] - 2026-05-09
|
||||
|
||||
### Added
|
||||
|
||||
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
|
||||
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
|
||||
|
||||
## [3.1.1] - 2026-04-24
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
|
||||
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
|
||||
|
||||
## [3.1.0] - 2026-04-22
|
||||
|
||||
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
|
||||
|
||||
### Added
|
||||
|
||||
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||
- **Broken README link.** The README's "source of truth" link pointed at root `SKILL.md`, which is no longer maintained after the plugin-layout restructure. Fixed to point at `skills/last30days/SKILL.md`.
|
||||
|
||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||
|
||||
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
|
||||
|
||||
## [3.0.14] - 2026-04-22
|
||||
|
||||
### Changed
|
||||
@@ -241,7 +61,6 @@ Individual changelog entries for 3.0.10 through 3.0.14 below document the increm
|
||||
### Fixed
|
||||
|
||||
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
||||
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
|
||||
|
||||
### Behavior fallback
|
||||
|
||||
|
||||
@@ -1 +1,25 @@
|
||||
@AGENTS.md
|
||||
# last30days Skill
|
||||
|
||||
Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||
Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `scripts/last30days.py` — main research engine
|
||||
- `scripts/lib/` — search, enrichment, rendering modules
|
||||
- `scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `SKILL.md` — skill definition (deployed to ~/.claude/skills/last30days/)
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
python3 scripts/last30days.py "test query" --emit=compact # Run research
|
||||
bash scripts/sync.sh # Deploy to ~/.claude, ~/.agents, ~/.codex
|
||||
```
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- After edits: run `bash scripts/sync.sh` to deploy
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
# Concepts
|
||||
|
||||
Shared vocabulary for `last30days-skill`. Terms here have a precise project-specific meaning — distinct enough from their general technical sense that a new contributor would need them defined to follow conversations, PR descriptions, or the SKILL.md contract.
|
||||
|
||||
## The package
|
||||
|
||||
### Skill
|
||||
|
||||
A self-contained agent-instructions package consisting of a `SKILL.md` prose contract plus a sibling `scripts/` directory containing the executable code the SKILL.md invokes. The package conforms to the [Agent Skills](https://agentskills.io) open format and installs across every major harness (Claude Code, Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ others) via `npx skills add`, harness-native plugin installers, or per-harness skill directories. A Skill is the unit of distribution; the Skill is the product.
|
||||
|
||||
### Engine
|
||||
|
||||
The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do the actual research work. The Engine and SKILL.md have a contract: SKILL.md tells the model which flags to pass (`--plan`, `--competitors-plan`, `--x-handle`, `--subreddits`, `--emit=compact`, etc.), and the Engine produces a specific output shape (badge line, ranked evidence clusters, emoji-tree footer) that the model is contractually required to pass through. The Engine is implementation; the SKILL.md prose is the agent-facing surface.
|
||||
|
||||
### Harness
|
||||
|
||||
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
||||
|
||||
## Distribution
|
||||
|
||||
### Beta channel
|
||||
|
||||
A parallel install of the Skill, sourced from the private `mvanhorn/last30days-skill-private` repo and installed as `/last30days-beta` rather than `/last30days`. The Beta channel exists so experimental changes can be tested by real users before they ship to the public `/last30days`. Promotion from Beta to public happens via a review PR against this (public) repo — Beta-only changes never ship to public without that PR. The Beta channel workflow guide lives in `BETA.md` in the private repo.
|
||||
@@ -1,268 +0,0 @@
|
||||
# Configuration
|
||||
|
||||
Everything you can tune in `/last30days` without editing the engine source.
|
||||
Three layers, in order of how often you'll touch them:
|
||||
|
||||
1. **Per-run flags** - what you pass on the command line.
|
||||
2. **Environment variables and `.env`** - what's enabled across all runs.
|
||||
3. **Optional trend-monitoring stack** - SQLite store, watchlist, briefings.
|
||||
|
||||
Per-client patterns and the experimental beta channel are at the bottom.
|
||||
|
||||
> Skip ahead: [Where output is saved](#where-output-is-saved) - [API keys](#api-keys-env) - [Reasoning provider](#reasoning-provider-priority) - [Web search backend](#web-search-backend-priority) - [Trend monitoring](#trend-monitoring-store--watchlist--briefings) - [Per-client patterns](#per-client-patterns) - [Beta channel](#beta-channel)
|
||||
|
||||
## Why this document exists
|
||||
|
||||
This is a focused **configuration reference** maintained alongside the engine. The runtime contract (the voice rules, the planner protocol, the LAWs the synthesizing model follows) lives in [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md) - that file is authoritative when the two ever differ. This file's job is narrower: surface every knob a user or operator can turn, in one place, kept current with the code so client-facing setups stay reliable. New configuration knobs added to the engine should be reflected here in the same PR.
|
||||
|
||||
---
|
||||
|
||||
## Where output is saved
|
||||
|
||||
| Platform | Default path | Override |
|
||||
|---|---|---|
|
||||
| Linux / macOS | `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` | set `LAST30DAYS_MEMORY_DIR=/path` |
|
||||
| Windows | `LAST30DAYS_MEMORY_DIR` defaults to `C:\Users\<you>\Documents\Last30Days\` | set `LAST30DAYS_MEMORY_DIR=C:\path` |
|
||||
|
||||
Each run produces one file per topic, slug-named:
|
||||
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
|
||||
|
||||
**Per-run overrides:**
|
||||
- `--save-dir <path>` - one-off output location.
|
||||
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
||||
|
||||
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
||||
|
||||
---
|
||||
|
||||
## API keys (`.env`)
|
||||
|
||||
The skill reads keys from a `.env` file. Two locations are supported, in priority order:
|
||||
|
||||
1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present.
|
||||
2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback.
|
||||
|
||||
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
||||
|
||||
The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case.
|
||||
|
||||
**Source-by-source** - what each key unlocks:
|
||||
|
||||
| Source | Key(s) | Required for | Free tier |
|
||||
|---|---|---|---|
|
||||
| Reddit (public) | none | always on | yes |
|
||||
| Hacker News | none | always on | yes |
|
||||
| Polymarket | none | always on | yes |
|
||||
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
|
||||
| YouTube | `yt-dlp` CLI installed | always on if `yt-dlp` present | yes |
|
||||
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; xAI / ScrapeCreators = paid |
|
||||
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
|
||||
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
|
||||
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
|
||||
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
|
||||
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
|
||||
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
|
||||
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
|
||||
| Perplexity Deep Research | `OPENROUTER_API_KEY` | `--deep-research` flag (~$0.90/query) | no |
|
||||
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
|
||||
|
||||
**Example `.env` skeleton** (placeholders only - replace with your own values):
|
||||
|
||||
```bash
|
||||
# Reasoning + planning (one provider; see priority below)
|
||||
GOOGLE_API_KEY=<your-gemini-key>
|
||||
|
||||
# Web search backend (one is enough; Brave is the cheapest)
|
||||
BRAVE_API_KEY=<your-brave-key>
|
||||
|
||||
# Optional sources
|
||||
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
|
||||
# X authentication (one option only)
|
||||
XAI_API_KEY=<your-xai-key>
|
||||
# OR cookie-jar (no key needed; logs in via your browser session)
|
||||
# FROM_BROWSER=firefox
|
||||
|
||||
# Bluesky
|
||||
BSKY_HANDLE=<your-handle>.bsky.social
|
||||
BSKY_APP_PASSWORD=<your-app-password>
|
||||
```
|
||||
|
||||
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
||||
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search.
|
||||
|
||||
### Bluesky app-password format and search host
|
||||
|
||||
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
|
||||
|
||||
The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`:
|
||||
|
||||
```bash
|
||||
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Reasoning provider priority
|
||||
|
||||
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
||||
|
||||
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
||||
2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`)
|
||||
3. **xAI** - `XAI_API_KEY`
|
||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (also unlocks `--deep-research`)
|
||||
5. **Local / deterministic** - always available, lowest quality
|
||||
|
||||
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
|
||||
|
||||
---
|
||||
|
||||
## Web search backend priority
|
||||
|
||||
Used by `--auto-resolve` (when WebSearch isn't available from the host) and Step 2 supplements. Auto-detect priority (override per-run with `--web-backend=<name>`):
|
||||
|
||||
1. **Brave** - `BRAVE_API_KEY`
|
||||
2. **Exa** - `EXA_API_KEY`
|
||||
3. **Serper** - `SERPER_API_KEY`
|
||||
4. **Parallel** - `PARALLEL_API_KEY`
|
||||
5. **Host's native WebSearch** - Claude Code, Codex, Gemini all have one built in
|
||||
|
||||
Visible quality difference between hosts with vs without a configured backend. If your client setup produces thinner results than yours, this is usually why.
|
||||
|
||||
---
|
||||
|
||||
## Trend monitoring (`--store` + watchlist + briefings)
|
||||
|
||||
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
|
||||
|
||||
### `--store` flag
|
||||
|
||||
Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate.
|
||||
|
||||
**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`.
|
||||
|
||||
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
|
||||
|
||||
### `watchlist.py` - recurring topics
|
||||
|
||||
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
|
||||
|
||||
Two-step flow (the watchlist holds the topic; an external scheduler invokes the run):
|
||||
|
||||
```bash
|
||||
# 1. Add the topic to the watchlist
|
||||
# Default schedule daily 8am; --weekly switches to Mondays 8am
|
||||
python3 scripts/watchlist.py add "british airways middle east" --weekly
|
||||
|
||||
# 2. Configure delivery and budget (optional)
|
||||
python3 scripts/watchlist.py config delivery "https://hooks.slack.com/services/..."
|
||||
python3 scripts/watchlist.py config budget 5.00
|
||||
|
||||
# 3. Trigger via cron / Task Scheduler / GitHub Actions
|
||||
python3 scripts/watchlist.py run-one "british airways middle east"
|
||||
# or run every enabled topic, gated by daily_budget
|
||||
python3 scripts/watchlist.py run-all
|
||||
```
|
||||
|
||||
The schedule field stored on each topic is metadata - the actual cron / Task Scheduler invocation is your responsibility. Watchlist runs hardcode `--quick` and `--lookback-days 90` when spawning the underlying engine.
|
||||
|
||||
### `briefing.py` - daily / weekly digests
|
||||
|
||||
[`scripts/briefing.py`](skills/last30days/scripts/briefing.py) reads the SQLite store and emits structured data the agent then synthesizes into prose. Modes: `generate` (daily), `generate --weekly`, `show [--date DATE]` (display a saved briefing). Briefs save to `~/.local/share/last30days/briefs/`.
|
||||
|
||||
### Recommended cadence pattern
|
||||
|
||||
| Step | Cadence | Command |
|
||||
|---|---|---|
|
||||
| Baseline | one-time per topic | `/last30days "<topic>" --days=30 --store` |
|
||||
| Add to watchlist | one-time per topic | `python3 scripts/watchlist.py add "<topic>" --weekly` |
|
||||
| Recurring run | daily or weekly (external scheduler) | `python3 scripts/watchlist.py run-all` |
|
||||
| Digest | weekly | `python3 scripts/briefing.py generate --weekly` |
|
||||
|
||||
---
|
||||
|
||||
## Per-client patterns
|
||||
|
||||
The skill is built to flex around different client environments. Four patterns that compose well:
|
||||
|
||||
### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders)
|
||||
|
||||
The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
||||
SCRAPECREATORS_API_KEY=<acme-scoped-key-or-shared>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
||||
```
|
||||
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no flags or wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
|
||||
### 2. Per-client save dir + suffix wrapper
|
||||
|
||||
For workflows where you don't `cd` into a client folder (running from anywhere, scripted batches), a tiny shell function isolates each client's research without engine changes.
|
||||
|
||||
PowerShell example:
|
||||
|
||||
```powershell
|
||||
function Run-L30D-Client {
|
||||
param([string]$ClientSlug, [Parameter(ValueFromRemainingArguments=$true)]$Args)
|
||||
$env:LAST30DAYS_MEMORY_DIR = "C:\Users\$env:USERNAME\Clients\$ClientSlug\Research\Last30Days"
|
||||
/last30days @Args --save-suffix=$ClientSlug
|
||||
}
|
||||
# Usage: Run-L30D-Client acme "british airways middle east"
|
||||
```
|
||||
|
||||
Bash example:
|
||||
|
||||
```bash
|
||||
l30d-client() {
|
||||
local client=$1; shift
|
||||
LAST30DAYS_MEMORY_DIR="$HOME/Clients/$client/Research/Last30Days" \
|
||||
/last30days "$@" --save-suffix="$client"
|
||||
}
|
||||
# Usage: l30d-client acme "british airways middle east"
|
||||
```
|
||||
|
||||
### 3. Custom category-peer subreddits
|
||||
|
||||
[`scripts/lib/categories.py`](skills/last30days/scripts/lib/categories.py) holds a table of `(category_id, trigger_keywords, peer_subreddits)`. If a client lives in a vertical that isn't covered (legal-tech, real-estate-tech, B2B HR SaaS), add a row. Pure data, no logic.
|
||||
|
||||
Section 2a of `SKILL.md` documents the merging rule the skill applies when your topic matches a category.
|
||||
|
||||
### 4. Pre-built `--competitors-plan` JSON
|
||||
|
||||
For competitor-vs-comparisons that recur, a pre-written JSON skeleton per client industry saves real time:
|
||||
|
||||
```json
|
||||
{
|
||||
"Competitor B": {
|
||||
"x_handle": "competitor_b_handle",
|
||||
"subreddits": ["sub1", "sub2"],
|
||||
"github_user": "competitor-b-org",
|
||||
"context": "Founded 2019, focused on ..."
|
||||
},
|
||||
"Competitor C": { ... }
|
||||
}
|
||||
```
|
||||
|
||||
Pass as `--competitors-plan @client/competitors-plan.json` (or as a string). See `SKILL.md` section "If QUERY_TYPE = COMPARISON" for the full schema.
|
||||
|
||||
---
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental customizations live on a private companion repo (`mvanhorn/last30days-skill-private`) installed as `/last30days-beta`. Never ship beta-only changes to the public marketplace without a review PR against the public repo. Workflow guide: `BETA.md` in the private repo.
|
||||
|
||||
This is the right home for client-specific changes you don't intend to upstream - custom category rows, internal subreddit lists, per-vertical plan templates.
|
||||
|
||||
---
|
||||
|
||||
## Cross-references
|
||||
|
||||
- The CLI flag surface: `python3 scripts/last30days.py --help`
|
||||
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
|
||||
- Engine spec (some sections stale; SKILL.md wins on conflicts): [`SPEC.md`](SPEC.md)
|
||||
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
|
||||
+1
-1
@@ -23,7 +23,7 @@ v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos
|
||||
### @thinkun
|
||||
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
||||
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
||||
> Thinker, technologist, AI expert, music-tinkerer. Founder of [Thinkun](https://thinkun.com). [@thinkun on GitHub](https://github.com/thinkun) · [@unthink on X](https://x.com/unthink)
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @thomasmktong
|
||||
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
||||
|
||||
+23
-13
@@ -10,20 +10,28 @@ This guide covers installing last30days on Hermes AI Agent.
|
||||
|
||||
## Installation
|
||||
|
||||
### Option 1: Via sync.sh (Recommended)
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
# Clone the repo
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
cd last30days-skill
|
||||
|
||||
# Run the sync script
|
||||
bash scripts/sync.sh
|
||||
```
|
||||
|
||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
||||
This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days/`
|
||||
|
||||
### Developer / live-edit alternative
|
||||
|
||||
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
|
||||
### Option 2: Manual Copy
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
mkdir -p ~/.hermes/skills/research
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
|
||||
# Create directory
|
||||
mkdir -p ~/.hermes/skills/research/last30days
|
||||
|
||||
# Copy files
|
||||
cp -r scripts ~/.hermes/skills/research/last30days/
|
||||
cp .hermes-plugin/SKILL.md ~/.hermes/skills/research/last30days/
|
||||
```
|
||||
|
||||
## Usage
|
||||
@@ -51,7 +59,7 @@ On first run, the skill will guide you through setup:
|
||||
|
||||
2. **Optional: ScrapeCreators**
|
||||
- Adds TikTok, Instagram, Reddit backup
|
||||
- 100 free credits (no expiration)
|
||||
- 10,000 free API calls
|
||||
- Sign up at scrapecreators.com
|
||||
|
||||
3. **Optional: API Keys**
|
||||
@@ -98,11 +106,13 @@ python3.12 scripts/last30days.py --diagnose
|
||||
|
||||
## Updating
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
```
|
||||
To update to the latest version:
|
||||
|
||||
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step.
|
||||
```bash
|
||||
cd last30days-skill
|
||||
git pull
|
||||
bash scripts/sync.sh
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
|
||||
@@ -14,19 +14,21 @@
|
||||
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
|
||||
**Claude Code (recommended — auto-updates via marketplace):**
|
||||
Claude Code:
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
||||
OpenClaw:
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
clawhub install last30days-official
|
||||
```
|
||||
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
|
||||
|
||||
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
|
||||
Hermes:
|
||||
```
|
||||
# The skill auto-deploys when you run sync.sh
|
||||
# Or manually copy to ~/.hermes/skills/research/last30days/
|
||||
```
|
||||
|
||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
@@ -66,7 +68,6 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||
@@ -95,28 +96,6 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
## What v3 Changed
|
||||
|
||||
### Shareable HTML briefs
|
||||
|
||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
||||
|
||||
```
|
||||
/last30days OpenClaw --emit=html
|
||||
```
|
||||
|
||||
or just ask in plain language:
|
||||
|
||||
```
|
||||
/last30days OpenClaw, give me a shareable HTML brief
|
||||
/last30days Cursor IDE for slack
|
||||
/last30days Anthropic earnings export as html
|
||||
```
|
||||
|
||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
||||
|
||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
||||
|
||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
||||
|
||||
### Intelligent search: the killer feature
|
||||
|
||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
@@ -153,10 +132,8 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
|
||||
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
||||
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
||||
- **TikTok, Instagram, Threads.** All three activate automatically once `SCRAPECREATORS_API_KEY` is set — same key, same per-call cost. Suppress any of them with `EXCLUDE_SOURCES=tiktok,instagram,threads` (any comma-separated subset).
|
||||
- **Pinterest.** Per-query opt-in (visual pins, narrow utility): the model passes `--search=pinterest` for the runs that need it. Requires `SCRAPECREATORS_API_KEY`.
|
||||
- **YouTube + TikTok comments.** Persistent opt-in via `INCLUDE_SOURCES=youtube_comments,tiktok_comments` because each video pulls N extra ScrapeCreators calls on top of the base search. Surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` and `INCLUDE_SOURCES=perplexity` (it's a separate paid API — opt-in keeps you from being surprise-billed).
|
||||
- **Threads, Pinterest, YouTube + TikTok comments.** Opt-in sources via ScrapeCreators. Set `INCLUDE_SOURCES=tiktok,instagram` and add threads, pinterest, youtube_comments, tiktok_comments for more. `youtube_comments` and `tiktok_comments` surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` to unlock.
|
||||
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
||||
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
||||
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
||||
@@ -168,61 +145,12 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install | Updates |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recommended)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Recommended because the Claude Code marketplace handles updates for you — the plugin cache is versioned and auto-refreshes when a new release publishes. Run `claude plugin update last30days@last30days-skill` to force a check.
|
||||
|
||||
If you'd rather use the agent-skills install path on Claude Code, that's also supported:
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
The native plugin and the `npx skills` install can coexist; Claude Code dedupes the slash command.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
||||
|
||||
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
|
||||
|
||||
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
Update later with:
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
Or update everything you've installed globally via `npx skills`:
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + |
|
||||
| **Claude Code** | `/plugin marketplace add mvanhorn/last30days-skill` |
|
||||
| **OpenClaw** | `clawhub install last30days-official` |
|
||||
| **Gemini CLI** | Clone then `gemini extensions install ./last30days-skill` (see below) |
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
@@ -230,7 +158,15 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
||||
3. Click the `+` button in the Skills panel and drop the file in
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
||||
Enable "Code execution and file creation" under Capabilities first - skills won't run without it.
|
||||
|
||||
### Claude Code
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Update later with `claude plugin update last30days@last30days-skill`.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
@@ -238,14 +174,22 @@ Enable "Code execution and file creation" under Capabilities first — skills wo
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
### Gemini CLI
|
||||
|
||||
Gemini CLI v0.9.0 has an upstream installer bug that can fail with `Configuration file not found at /tmp/gemini-extensionXXXXXX/gemini-extension.json` ([upstream issue](https://github.com/google-gemini/gemini-cli/issues/11452)). Workaround:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill
|
||||
gemini extensions install ./last30days-skill
|
||||
```
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
Or build the claude.ai `.skill` file from source: `bash scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||
|
||||
@@ -259,40 +203,10 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
| X / Twitter | Log into x.com in any browser | Free |
|
||||
| YouTube | `brew install yt-dlp` | Free |
|
||||
| Bluesky | App password from bsky.app | Free |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls |
|
||||
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
||||
| Web search | Brave Search key | 2,000 free queries/month |
|
||||
|
||||
### macOS Keychain (optional)
|
||||
|
||||
On macOS you can store keys in the system Keychain instead of a `.env` file. The skill picks them up automatically as the lowest-priority source — `.env` files and process environment still win on collision.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
||||
|
||||
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
|
||||
|
||||
## Configuration
|
||||
|
||||
Two things you'll likely want to know on day one:
|
||||
|
||||
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each run produces `<slug>-raw[-suffix].md`.
|
||||
|
||||
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: last30days
|
||||
version: "3.3.0"
|
||||
version: "3.0.1"
|
||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
@@ -13,9 +13,9 @@ metadata:
|
||||
openclaw:
|
||||
emoji: "📰"
|
||||
requires:
|
||||
env: []
|
||||
optionalEnv:
|
||||
env:
|
||||
- SCRAPECREATORS_API_KEY
|
||||
optionalEnv:
|
||||
- OPENAI_API_KEY
|
||||
- XAI_API_KEY
|
||||
- OPENROUTER_API_KEY
|
||||
@@ -45,7 +45,6 @@ metadata:
|
||||
- instagram
|
||||
- hackernews
|
||||
- polymarket
|
||||
- digg
|
||||
- bluesky
|
||||
- truthsocial
|
||||
- trends
|
||||
@@ -60,32 +59,23 @@ metadata:
|
||||
- clawhub
|
||||
---
|
||||
|
||||
# STEP 0: STALE-CLONE SELF-CHECK — RUN BEFORE READING BELOW
|
||||
# STEP 0: CANONICAL PATH SELF-CHECK — RUN BEFORE READING BELOW
|
||||
|
||||
Before reading anything else in this file, check whether you loaded SKILL.md from the one known stale-clone location: Claude Code's marketplaces directory.
|
||||
|
||||
**Run this check:**
|
||||
Before reading anything else in this file, verify that the SKILL.md you just loaded is the versioned cache copy, not a stale marketplace clone:
|
||||
|
||||
```bash
|
||||
CLAUDE_CACHE_LATEST=$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)
|
||||
# Two cache layouts ship in the wild — nested ({version}/skills/last30days/SKILL.md)
|
||||
# and flat ({version}/SKILL.md). Resolve to whichever shape actually exists.
|
||||
CLAUDE_CACHE_SKILL_MD=""
|
||||
if [ -n "$CLAUDE_CACHE_LATEST" ]; then
|
||||
if [ -f "$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md" ]; then
|
||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md"
|
||||
elif [ -f "$CLAUDE_CACHE_LATEST/SKILL.md" ]; then
|
||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/SKILL.md"
|
||||
fi
|
||||
fi
|
||||
echo "CLAUDE_CACHE_SKILL_MD=$CLAUDE_CACHE_SKILL_MD"
|
||||
CANONICAL_ROOT="$HOME/.claude/plugins/cache/last30days-skill/last30days"
|
||||
CANONICAL_LATEST=$(ls -d "$CANONICAL_ROOT"/*/ 2>/dev/null | sort -V | tail -1)
|
||||
CANONICAL_LATEST="${CANONICAL_LATEST%/}"
|
||||
```
|
||||
|
||||
If the SKILL.md path you just Read contains `/.claude/plugins/marketplaces/` AND `$CLAUDE_CACHE_SKILL_MD` is non-empty, STOP and re-read `$CLAUDE_CACHE_SKILL_MD` before proceeding. Otherwise the SKILL.md you have is fine — continue.
|
||||
If the SKILL.md file you just read is NOT under `$CANONICAL_LATEST/`, STOP and re-read `$CANONICAL_LATEST/SKILL.md` before proceeding. Do not run `--help`, do not plan, do not invoke the engine against any other path.
|
||||
|
||||
**Why this specific check:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test. STEP 0 defends against that one Claude Code-specific bug.
|
||||
**Why:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test.
|
||||
|
||||
**Other install paths are fine:** `~/.codex/skills/`, `~/.agents/skills/`, an `npx skills add` install dir, or a repo checkout are all valid load points - the resolver in Step 1 picks them up. Do NOT abort or hop on those paths.
|
||||
**How to self-check:** the file path you used in your last Read tool call should match `$CANONICAL_LATEST/SKILL.md`. If it contains `marketplaces/` or any other prefix, that is the stale-path failure mode. Re-read from `$CANONICAL_LATEST/SKILL.md` and restart this contract from the top.
|
||||
|
||||
The same pinned resolver appears later in Step 1 for the engine Bash invocation. That guard is necessary but insufficient — by the time you reach Step 1, you may have already internalized an out-of-date flag list from the stale SKILL.md above it. This STEP 0 runs first so the CONTRACT itself is read from the right file.
|
||||
|
||||
---
|
||||
|
||||
@@ -97,7 +87,7 @@ You are inside the `/last30days` SKILL. This is a specific research tool with a
|
||||
|
||||
**How v3.0.7 fixes it:** three structural anchors.
|
||||
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
||||
2. **The SKILL_DIR substitution** in the engine Bash calls uses the directory of the SKILL.md the model just Read — no resolver list, no precedence walk. Whichever install the harness loaded SKILL.md from is the install whose engine runs. Aligns spec-with-code and works for any harness without enumerating its install path.
|
||||
2. **The pinned SKILL_ROOT resolution** in the engine Bash calls always points to the public plugin cache, never `~/.openclaw/` or other stale copies.
|
||||
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
||||
|
||||
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
||||
@@ -114,7 +104,7 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
```
|
||||
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_DIR/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_DIR/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
|
||||
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
||||
|
||||
@@ -186,13 +176,13 @@ The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses
|
||||
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
|
||||
```
|
||||
|
||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan "$QUERY_PLAN_FILE"` (the path to a tmpfile you wrote via heredoc — see Step 1 for the pattern; never inline `--plan '$JSON'`, apostrophes in search/ranking strings break shell parsing).
|
||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan '$JSON'`.
|
||||
|
||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan "$QUERY_PLAN_FILE"` (or any path the engine can read). A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan '$JSON'`. A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||
|
||||
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
|
||||
|
||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan "$QUERY_PLAN_FILE"` (or another path the engine can read)? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan, then write it to a tmpfile per the Step 1 pattern. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan '$JSON'`? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||
|
||||
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
|
||||
|
||||
@@ -243,7 +233,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
||||
|
||||
---
|
||||
|
||||
# last30days v3.3.0: Research Any Topic from the Last 30 Days
|
||||
# last30days v3.0.1: Research Any Topic from the Last 30 Days
|
||||
|
||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||
|
||||
@@ -327,14 +317,14 @@ Common patterns:
|
||||
|
||||
- Always active: Reddit, Hacker News, Polymarket
|
||||
- If gh CLI is installed (check `which gh`): add GitHub
|
||||
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg
|
||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
||||
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
||||
- If SCRAPECREATORS_API_KEY is set: add TikTok, Instagram, Threads (suppress any of these via EXCLUDE_SOURCES)
|
||||
- If SCRAPECREATORS_API_KEY is set and the user explicitly requested pinterest for this query (e.g. via `--search=pinterest`): add Pinterest
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains threads: add Threads
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest
|
||||
- If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky
|
||||
- If OPENROUTER_API_KEY is set and INCLUDE_SOURCES contains perplexity: add Perplexity
|
||||
- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying
|
||||
- If OPENROUTER_API_KEY is set: add Perplexity
|
||||
|
||||
Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma):
|
||||
|
||||
@@ -591,50 +581,18 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe
|
||||
|
||||
**Invocation:**
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Write the per-entity plan to a tmpfile and pass the path to the engine.
|
||||
# The engine's parse_competitors_plan() reads file paths transparently. This
|
||||
# avoids the inline-single-quoted-JSON apostrophe trap (resolved context
|
||||
# strings like "people's choice" or "McDonald's" otherwise close the outer
|
||||
# single-quote and break shell parsing before the engine is even invoked).
|
||||
# Trailing XXXXXX (no .json suffix) so BSD/macOS mktemp works the same as
|
||||
# GNU; BSD only substitutes X's at the end of the template.
|
||||
COMPETITORS_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-competitors.XXXXXX")
|
||||
trap 'rm -f "$COMPETITORS_PLAN_FILE"' EXIT
|
||||
cat > "$COMPETITORS_PLAN_FILE" <<'PLAN_EOF'
|
||||
{
|
||||
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
||||
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
||||
}
|
||||
PLAN_EOF
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||
--emit=compact \
|
||||
--save-dir="${LAST30DAYS_MEMORY_DIR}" \
|
||||
--save-suffix=v3 \
|
||||
--x-handle={TOPIC_A_HANDLE} \
|
||||
--subreddits={TOPIC_A_SUBS} \
|
||||
--competitors-plan "$COMPETITORS_PLAN_FILE"
|
||||
--competitors-plan '{
|
||||
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
||||
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
||||
}'
|
||||
```
|
||||
|
||||
**The quoted heredoc marker `'PLAN_EOF'` is load-bearing** — quoting suppresses shell interpolation so apostrophes, `$`, backticks, etc. pass through verbatim. If you ever switch to an unquoted `<<PLAN_EOF`, every variable reference and apostrophe inside the JSON becomes a parse hazard.
|
||||
|
||||
Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive).
|
||||
|
||||
**Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan.
|
||||
@@ -866,7 +824,7 @@ Only show lines for platforms where something was resolved. Skip empty lines. On
|
||||
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
||||
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
||||
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key), digg (Digg clusters - only if `digg-pp-cli` is on PATH)
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key)
|
||||
|
||||
**Intent → freshness_mode mapping:**
|
||||
- breaking_news, prediction → `strict_recent`
|
||||
@@ -909,45 +867,34 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
||||
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
# PIN SKILL_ROOT to the public plugin cache (highest-version dir wins on upgrade).
|
||||
# DO NOT write your own path-discovery loop. The 2026-04-18 Peter Steinberger run 1
|
||||
# regression was caused by a custom discovery loop landing on ~/.openclaw/skills/last30days/
|
||||
# (a stale copy from a private-repo sync pattern). That path contains a pre-plan-007
|
||||
# engine and produces non-canonical output. This pinned resolution ignores every stale
|
||||
# copy (~/.openclaw/, ~/.agents/, ~/.codex/) and picks the plugin cache exclusively.
|
||||
SKILL_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
|
||||
SKILL_ROOT="${SKILL_ROOT%/}"
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
# Fallback for repo checkout / Gemini / Codex hosts where the plugin cache does not exist.
|
||||
# Only runs if the public plugin cache is missing entirely.
|
||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
echo "ERROR: Could not find scripts/last30days.py in public plugin cache or repo checkout" >&2
|
||||
echo "Expected: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/scripts/last30days.py" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||
```
|
||||
|
||||
**If you ran Steps 0.55 and 0.75 (agent planning), pass the plan via a tmpfile and add the targeting flags:**
|
||||
|
||||
```bash
|
||||
# Write QUERY_PLAN_JSON to a tmpfile before the engine invocation above.
|
||||
# parse_plan() reads file paths transparently; this avoids inline-JSON
|
||||
# shell-quoting hazards (apostrophes in search_query / ranking_query
|
||||
# strings break single-quoted command-line JSON). Trailing XXXXXX (no
|
||||
# .json suffix) for BSD/macOS portability — BSD mktemp only substitutes
|
||||
# X's at the end of the template.
|
||||
QUERY_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-plan.XXXXXX")
|
||||
trap 'rm -f "$QUERY_PLAN_FILE"' EXIT
|
||||
cat > "$QUERY_PLAN_FILE" <<'PLAN_EOF'
|
||||
{QUERY_PLAN_JSON_FROM_STEP_0.75}
|
||||
PLAN_EOF
|
||||
```
|
||||
|
||||
Then add to the engine command:
|
||||
|
||||
- `--plan "$QUERY_PLAN_FILE"` (path to the file you just wrote)
|
||||
**If you ran Steps 0.55 and 0.75 (agent planning), add these flags:**
|
||||
- `--plan 'QUERY_PLAN_JSON'` (replace with actual JSON from Step 0.75)
|
||||
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
|
||||
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
|
||||
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
|
||||
@@ -1550,33 +1497,6 @@ Close with `I have all the links to the {N} {source list} I pulled from. Just as
|
||||
|
||||
---
|
||||
|
||||
## SHAREABLE HTML BRIEF (when the user asked for one)
|
||||
|
||||
**This section fires if EITHER trigger is true:**
|
||||
|
||||
- `$ARGUMENTS` contains `--emit=html`, `--emit:html`, or `--html` as a flag
|
||||
- The user's natural-language request asks for an HTML brief, shareable doc, or file for sharing (Slack, email, Notion, "export as HTML", etc). Use your judgment for phrasing variants.
|
||||
|
||||
**If neither trigger fires, skip this entire section and proceed to WAIT FOR USER'S RESPONSE.** No HTML save flow, no reference read needed.
|
||||
|
||||
**When triggered, you MUST:**
|
||||
|
||||
- Read `references/save-html-brief.md` BEFORE proceeding to WAIT FOR USER'S RESPONSE
|
||||
- Follow that file's instructions exactly - it is the canonical source for the save flow
|
||||
- Append the confirmation line (`📎 Shareable brief saved to <path>`) to your already-emitted chat response
|
||||
|
||||
**You MUST NOT:**
|
||||
|
||||
- Improvise the HTML save flow from memory or from instructions you've seen before
|
||||
- Skip the reference read because the steps "look familiar"
|
||||
- Save to a different path than the reference specifies
|
||||
- Add data quality warnings, debug headers, or safety notes to the saved HTML
|
||||
- Re-research the topic for the HTML render - the engine cache covers the second invocation
|
||||
|
||||
**Why the directive is forceful:** the reference file is the only source of truth for the save flow. Skipping it produces broken artifacts - wrong path conventions, missing synthesis content, leaked engine debug output, or warnings that don't belong in shareable docs.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S RESPONSE
|
||||
|
||||
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.
|
||||
@@ -1688,7 +1608,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 100 free credits)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 10,000 free API calls)
|
||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||
- Stores research findings in local SQLite database (watchlist mode only)
|
||||
@@ -1701,7 +1621,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Does not log, cache, or write API keys to output files
|
||||
- Does not send data to any endpoint not listed above
|
||||
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (100 free credits one-time, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (10,000 free API calls, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
||||
|
||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||
@@ -0,0 +1,77 @@
|
||||
# last30days Skill Specification
|
||||
|
||||
## Overview
|
||||
|
||||
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. OpenAI auth can come from `OPENAI_API_KEY` or Codex login credentials.
|
||||
|
||||
The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability.
|
||||
|
||||
## Architecture
|
||||
|
||||
The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`:
|
||||
|
||||
- **env.py**: Load API keys from `~/.config/last30days/.env` and Codex auth from `~/.codex/auth.json`
|
||||
- **dates.py**: Date range calculation and confidence scoring
|
||||
- **cache.py**: 24-hour TTL caching keyed by topic + date range
|
||||
- **http.py**: stdlib-only HTTP client with retry logic
|
||||
- **models.py**: Auto-selection of OpenAI/xAI models with 7-day caching
|
||||
- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit
|
||||
- **xai_x.py**: xAI Responses API + x_search for X
|
||||
- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics
|
||||
- **hackernews.py**: Hacker News search via Algolia API (free, no auth)
|
||||
- **polymarket.py**: Polymarket prediction market search via Gamma API (free, no auth)
|
||||
- **normalize.py**: Convert raw API responses to canonical schema
|
||||
- **score.py**: Compute popularity-aware scores (relevance + recency + engagement)
|
||||
- **dedupe.py**: Near-duplicate detection via text similarity
|
||||
- **render.py**: Generate markdown and JSON outputs
|
||||
- **schema.py**: Type definitions and validation
|
||||
|
||||
## Embedding in Other Skills
|
||||
|
||||
Other skills can import the research context in several ways:
|
||||
|
||||
### Inline Context Injection
|
||||
```markdown
|
||||
## Recent Research Context
|
||||
!python3 ~/.claude/skills/last30days/scripts/last30days.py "your topic" --emit=context
|
||||
```
|
||||
|
||||
### Read from File
|
||||
```markdown
|
||||
## Research Context
|
||||
!cat ~/.local/share/last30days/out/last30days.context.md
|
||||
```
|
||||
|
||||
### Get Path for Dynamic Loading
|
||||
```bash
|
||||
CONTEXT_PATH=$(python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=path)
|
||||
cat "$CONTEXT_PATH"
|
||||
```
|
||||
|
||||
### JSON for Programmatic Use
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=json > research.json
|
||||
```
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py <topic> [options]
|
||||
|
||||
Options:
|
||||
--refresh Bypass cache and fetch fresh data
|
||||
--mock Use fixtures instead of real API calls
|
||||
--emit=MODE Output mode: compact|json|md|context|path (default: compact)
|
||||
--sources=MODE Source selection: auto|reddit|x|both (default: auto)
|
||||
```
|
||||
|
||||
## Output Files
|
||||
|
||||
All outputs are written to `~/.local/share/last30days/out/`:
|
||||
|
||||
- `report.md` - Human-readable full report
|
||||
- `report.json` - Normalized data with scores
|
||||
- `last30days.context.md` - Compact reusable snippet for other skills
|
||||
- `raw_openai.json` - Raw OpenAI API response
|
||||
- `raw_xai.json` - Raw xAI API response
|
||||
- `raw_reddit_threads_enriched.json` - Enriched Reddit thread data
|
||||
@@ -0,0 +1,47 @@
|
||||
# last30days Implementation Tasks
|
||||
|
||||
## Setup & Configuration
|
||||
- [x] Create directory structure
|
||||
- [x] Write SPEC.md
|
||||
- [x] Write TASKS.md
|
||||
- [x] Write SKILL.md with proper frontmatter
|
||||
|
||||
## Core Library Modules
|
||||
- [x] scripts/lib/env.py - Environment and API key loading
|
||||
- [x] scripts/lib/dates.py - Date range and confidence utilities
|
||||
- [x] scripts/lib/cache.py - TTL-based caching
|
||||
- [x] scripts/lib/http.py - HTTP client with retry
|
||||
- [x] scripts/lib/models.py - Auto model selection
|
||||
- [x] scripts/lib/schema.py - Data structures
|
||||
- [x] scripts/lib/openai_reddit.py - OpenAI Responses API
|
||||
- [x] scripts/lib/xai_x.py - xAI Responses API
|
||||
- [x] scripts/lib/reddit_enrich.py - Reddit thread JSON fetcher
|
||||
- [x] scripts/lib/normalize.py - Schema normalization
|
||||
- [x] scripts/lib/score.py - Popularity scoring
|
||||
- [x] scripts/lib/dedupe.py - Near-duplicate detection
|
||||
- [x] scripts/lib/render.py - Output rendering
|
||||
|
||||
## Main Script
|
||||
- [x] scripts/last30days.py - CLI orchestrator
|
||||
|
||||
## Fixtures
|
||||
- [x] fixtures/openai_sample.json
|
||||
- [x] fixtures/xai_sample.json
|
||||
- [x] fixtures/reddit_thread_sample.json
|
||||
- [x] fixtures/models_openai_sample.json
|
||||
- [x] fixtures/models_xai_sample.json
|
||||
|
||||
## Tests
|
||||
- [x] tests/test_dates.py
|
||||
- [x] tests/test_cache.py
|
||||
- [x] tests/test_models.py
|
||||
- [x] tests/test_score.py
|
||||
- [x] tests/test_dedupe.py
|
||||
- [x] tests/test_normalize.py
|
||||
- [x] tests/test_render.py
|
||||
|
||||
## Validation
|
||||
- [x] Run tests in mock mode
|
||||
- [x] Demo --emit=compact
|
||||
- [x] Demo --emit=context
|
||||
- [x] Verify file tree
|
||||
|
Before Width: | Height: | Size: 2.7 MiB After Width: | Height: | Size: 2.7 MiB |
|
Before Width: | Height: | Size: 2.3 MiB After Width: | Height: | Size: 2.3 MiB |
|
Before Width: | Height: | Size: 3.8 MiB After Width: | Height: | Size: 3.8 MiB |
|
Before Width: | Height: | Size: 2.6 MiB After Width: | Height: | Size: 2.6 MiB |
+11
-12
@@ -142,7 +142,7 @@ The repo vendors a search-only subset of Bird's Twitter GraphQL client and shell
|
||||
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
||||
| Replies/quotes | Real | Real |
|
||||
| Author handle | Real | Real |
|
||||
| Relevance score | Default 0.7 (re-ranked by relevance.py) | AI-assessed 0.0-1.0 |
|
||||
| Relevance score | Default 0.7 (re-ranked by score.py) | AI-assessed 0.0-1.0 |
|
||||
|
||||
### Depth settings
|
||||
|
||||
@@ -183,14 +183,13 @@ After both searches complete:
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `skills/last30days/scripts/last30days.py` | Main CLI entry point |
|
||||
| `skills/last30days/scripts/lib/pipeline.py` | Multi-source retrieval orchestration |
|
||||
| `skills/last30days/scripts/lib/reddit_public.py` | Reddit public JSON search |
|
||||
| `skills/last30days/scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `skills/last30days/scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `skills/last30days/scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `skills/last30days/scripts/lib/providers.py` | Reasoning provider and model selection |
|
||||
| `skills/last30days/scripts/lib/env.py` | API key loading, source detection |
|
||||
| `skills/last30days/scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `skills/last30days/scripts/lib/relevance.py` | Query matching and relevance scoring |
|
||||
| `skills/last30days/scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
| `scripts/last30days.py` | Main orchestrator, concurrent execution |
|
||||
| `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API |
|
||||
| `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `scripts/lib/models.py` | Auto-select best available model |
|
||||
| `scripts/lib/env.py` | API key loading, source detection |
|
||||
| `scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `scripts/lib/score.py` | Relevance scoring |
|
||||
| `scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
||||
type: feat
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
||||
type: fix
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
||||
type: feat
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
||||
type: fix
|
||||
status: active
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
---
|
||||
title: "marketing: v3.1 launch video — 30s Remotion piece for X"
|
||||
type: feat
|
||||
status: active
|
||||
date: 2026-04-22
|
||||
---
|
||||
|
||||
# marketing: v3.1 launch video — 30s Remotion piece for X
|
||||
|
||||
## Overview
|
||||
|
||||
Build a 30-second Remotion-rendered MP4 marketing video announcing **v3.1: Competitors mode** for `/last30days`. Posts to X. Frames the new vs-mode-with-auto-discovery as the headline — three full passes, three save files, one comparison — without burying it in CLI minutiae.
|
||||
|
||||
Marketing release tag is **v3.1** (rebrands the 3.0.11-3.0.14 bundle into one shippable narrative for the launch tweet). Engine version stays 3.0.14 — `3.1` is the marketing version, not a code version bump.
|
||||
|
||||
## Script (60 frames per second × 30 seconds = 900 frames; this version assumes 30fps × 30s = 900 frames at 30fps)
|
||||
|
||||
Total runtime: 30.0 seconds @ 30fps = 900 frames. Six scenes, on-screen text only (silent autoplay-friendly).
|
||||
|
||||
### Scene 1 — Hook (0:00 – 0:03 | frames 0-90)
|
||||
|
||||
**Visual:** Black background. The `/last30days` badge animates in (the literal `🌐 last30days v3.1 · synced 2026-04-22` line) with the spring scale-in used in slick devtool intros.
|
||||
|
||||
**Caption (overlay, large):**
|
||||
```
|
||||
What if one search
|
||||
ran 3 at once?
|
||||
```
|
||||
|
||||
### Scene 2 — Set the world (0:03 – 0:08 | frames 90-240)
|
||||
|
||||
**Visual:** Single mac terminal window center-stage. Type-on animation:
|
||||
```
|
||||
$ /last30days OpenAI
|
||||
```
|
||||
Below it, a simple result card stub appears (Reddit upvote count + X likes), then static.
|
||||
|
||||
**Caption (small bottom-left):**
|
||||
```
|
||||
The old way: one topic.
|
||||
```
|
||||
|
||||
### Scene 3 — The reveal (0:08 – 0:14 | frames 240-420)
|
||||
|
||||
**Visual:** The terminal types one more flag:
|
||||
```
|
||||
$ /last30days OpenAI --competitors
|
||||
```
|
||||
Hard cut → the single terminal **splits into 3 panes** side by side. Each pane shows a different topic header animating in, in this order:
|
||||
- Left: `OpenAI`
|
||||
- Middle: `vs Anthropic`
|
||||
- Right: `vs xAI`
|
||||
|
||||
Pane content scrolls fake "search progress" lines (Reddit, X, YouTube indicators) in parallel, like a live fan-out.
|
||||
|
||||
**Caption (top center):**
|
||||
```
|
||||
Now it discovers competitors
|
||||
and runs all 3.
|
||||
```
|
||||
|
||||
### Scene 4 — Result reveal (0:14 – 0:21 | frames 420-630)
|
||||
|
||||
**Visual:** The 3 panes collapse into a single comparison surface — the `## Head-to-Head` table from the actual engine output, with rows fading in one by one (What it is, Streams, Best for, Trajectory). Each entity column lights up as its row populates.
|
||||
|
||||
**Caption (bottom):**
|
||||
```
|
||||
3 full passes. 3 save files.
|
||||
1 comparison.
|
||||
```
|
||||
|
||||
### Scene 5 — How it's special (0:21 – 0:26 | frames 630-780)
|
||||
|
||||
**Visual:** Cut to a clean text card, large mono font:
|
||||
```
|
||||
You pick the topic.
|
||||
The agent picks the peers.
|
||||
The engine fans out.
|
||||
```
|
||||
Each line fades in 1.5s apart.
|
||||
|
||||
### Scene 6 — CTA (0:26 – 0:30 | frames 780-900)
|
||||
|
||||
**Visual:** Black background. Centered:
|
||||
- Top: `🌐 last30days v3.1`
|
||||
- Middle: `/last30days {topic} --competitors`
|
||||
- Bottom: `github.com/mvanhorn/last30days-skill`
|
||||
|
||||
Subtle pulse on the install line.
|
||||
|
||||
**End frame holds for ~0.5s.**
|
||||
|
||||
## Problem Frame
|
||||
|
||||
The 3.0.11-3.0.14 release bundle ships a transformative feature (per-entity vs-mode fanout + `--competitors` shortcut + per-entity save files), but the value lands flat in a tweet thread or screenshot. A 30s video does what static text cannot: shows the fan-out happening in real time and the 3 → 1 collapse into a comparison. Higher tweet engagement, easier to RT/QT.
|
||||
|
||||
## Requirements Trace
|
||||
|
||||
- R1. Final artifact: a single MP4 file, 1920×1080, 30fps, ~30 seconds (±0.5s), under 30MB, suitable for direct X upload.
|
||||
- R2. Six-scene script as defined above, with text/visuals/timing matching to within 5 frames.
|
||||
- R3. Branded look: matches the `🌐 last30days v3.1` badge style (terminal aesthetic, mono font, dark background).
|
||||
- R4. Silent — no voiceover, no music in v1. Captions baked in. Designed for autoplay-muted feeds.
|
||||
- R5. Reproducible: another contributor (or a future-me) can re-render with one command. Project lives in-repo so the source is versioned.
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
- No voiceover. Text-on-screen only. (Voice can be a v1.1 if engagement is high.)
|
||||
- No background music in the rendered MP4. (Music can be added in post via QuickTime/iMovie if desired before posting.)
|
||||
- No localization. English captions only.
|
||||
- No A/B test variants. One video.
|
||||
- No 9:16 vertical version. 16:9 only. (Vertical can be a separate render after launch validates the format.)
|
||||
|
||||
### Deferred to Separate Tasks
|
||||
|
||||
- Voiceover variant: defer to follow-up if v1 lands well.
|
||||
- 9:16 mobile cut: defer; same source compositions can re-render at 1080×1920 in a follow-up.
|
||||
- Animated GIF for embedding in README.md: defer; can be ffmpeg-extracted from the MP4.
|
||||
|
||||
## Context & Research
|
||||
|
||||
### Relevant Code and Patterns
|
||||
|
||||
- `SKILL.md` — the comparison render scaffold (`## Head-to-Head` table) is the visual reference for Scene 4's table look.
|
||||
- `scripts/lib/render.py` `_render_comparison_scaffold` — emits the 9-axis table whose visual style we're recreating in a more polished form.
|
||||
- `CHANGELOG.md` 3.0.11-3.0.14 entries — the prose source for the script's beats.
|
||||
- `.claude-plugin/plugin.json` version 3.0.14 — current code version (marketing version is 3.1 for the launch).
|
||||
|
||||
### External References
|
||||
|
||||
- Remotion 4.x docs (https://www.remotion.dev/docs/) — current API for compositions, sequences, springs, and render CLI.
|
||||
- X video specs 2026: max 2 min 20 s, ≤512MB, MP4 with H.264 + AAC, recommended 1920×1080 for landscape autoplay.
|
||||
|
||||
### Institutional Learnings
|
||||
|
||||
- No prior `marketing/` dir or video plans in `docs/plans/`. This is a greenfield asset directory.
|
||||
|
||||
## Key Technical Decisions
|
||||
|
||||
- **Remotion, not ffmpeg-only.** Remotion's React-based compositions handle the typed-on terminal effect, spring-animated badges, and scene transitions far more cleanly than raw ffmpeg filtergraphs. Render output is still MP4 via Remotion's bundled ffmpeg.
|
||||
- **In-repo asset directory at `marketing/v3.1-launch/`.** Lives with the product so future versions can fork the project. Adds `marketing/` to `.gitignore` exceptions only for source files; rendered MP4 stays out of git (uploaded separately).
|
||||
- **16:9 1920×1080 @ 30fps.** Best fit for X landscape autoplay on desktop and mobile feed. 30fps is plenty for typed-text + UI animation; 60fps doubles render time without obvious quality gain.
|
||||
- **Silent + captions.** X autoplay defaults to muted. Sound-off is the realistic viewing condition. Captions baked into the visual.
|
||||
- **Six scenes, one composition.** Single Remotion composition with sequenced child compositions per scene. Easier to re-time than scene-files. Frame-numbered timing in the script enables precise edits.
|
||||
- **Mono font (JetBrains Mono or Geist Mono).** Matches the terminal aesthetic of the actual `/last30days` output. Available via Google Fonts or @remotion/google-fonts.
|
||||
- **Marketing version 3.1 ≠ engine version 3.0.14.** `3.1` is the launch label. Engine stays 3.0.14. Avoids confusion in CHANGELOG.
|
||||
|
||||
## Open Questions
|
||||
|
||||
### Resolved During Planning
|
||||
|
||||
- **Aspect ratio?** 16:9 1080p. Best X autoplay format; vertical can be a follow-up.
|
||||
- **Voiceover or silent?** Silent + on-screen captions. Autoplay-muted is the realistic condition.
|
||||
- **In-repo or separate repo?** In-repo at `marketing/v3.1-launch/`. Rendered MP4 not committed; source compositions are.
|
||||
- **Length?** Exactly 30s (900 frames @ 30fps). No flex.
|
||||
- **Marketing version label?** `v3.1`. Engine code version stays 3.0.14.
|
||||
|
||||
### Deferred to Implementation
|
||||
|
||||
- Exact animation easing curves per scene — pick during build via Remotion preview iteration.
|
||||
- Whether the comparison-table mock in Scene 4 uses canned text or pulls from the actual saved `*-raw.md` files. Probably canned for visual control.
|
||||
- Whether Scene 3's "search progress" lines are typed individually or use a marquee scroll. Pick during preview.
|
||||
- Exact accent color palette beyond "terminal dark." Iterate against preview.
|
||||
|
||||
## Output Structure
|
||||
|
||||
marketing/
|
||||
v3.1-launch/
|
||||
package.json # Remotion dependency manifest
|
||||
tsconfig.json # TypeScript config
|
||||
remotion.config.ts # Remotion render config (codec, fps, resolution)
|
||||
src/
|
||||
index.ts # Remotion entry — registers compositions
|
||||
Root.tsx # Root composition definition
|
||||
LaunchVideo.tsx # Main 30s composition that sequences scenes
|
||||
scenes/
|
||||
Scene1Hook.tsx
|
||||
Scene2OldWay.tsx
|
||||
Scene3FanOut.tsx
|
||||
Scene4Comparison.tsx
|
||||
Scene5HowItWorks.tsx
|
||||
Scene6CTA.tsx
|
||||
components/
|
||||
TerminalWindow.tsx # Reusable mac-style terminal frame
|
||||
TypedLine.tsx # Type-on animation primitive
|
||||
ComparisonTable.tsx # The Head-to-Head table mock
|
||||
BadgeBar.tsx # The 🌐 last30days v3.1 badge
|
||||
lib/
|
||||
timing.ts # Frame ranges per scene (single source of truth)
|
||||
colors.ts # Brand palette
|
||||
public/ # Static assets (logo, fonts if local)
|
||||
out/ # Rendered MP4 lives here (gitignored)
|
||||
README.md # How to preview/render
|
||||
|
||||
## High-Level Technical Design
|
||||
|
||||
> *Directional guidance for review — not implementation specification.*
|
||||
|
||||
```
|
||||
LaunchVideo (durationInFrames = 900)
|
||||
├── <Sequence from={0} durationInFrames={90}> <Scene1Hook />
|
||||
├── <Sequence from={90} durationInFrames={150}> <Scene2OldWay />
|
||||
├── <Sequence from={240} durationInFrames={180}> <Scene3FanOut />
|
||||
├── <Sequence from={420} durationInFrames={210}> <Scene4Comparison />
|
||||
├── <Sequence from={630} durationInFrames={150}> <Scene5HowItWorks />
|
||||
└── <Sequence from={780} durationInFrames={120}> <Scene6CTA />
|
||||
```
|
||||
|
||||
Per-scene components use `useCurrentFrame()` + `interpolate()` + `spring()` for timing. `TerminalWindow` is the dominant motif across scenes 2-4.
|
||||
|
||||
## Implementation Units
|
||||
|
||||
- [ ] **Unit 1: Remotion project scaffold**
|
||||
|
||||
**Goal:** Spin up a working Remotion project at `marketing/v3.1-launch/` that previews a blank composition and renders to MP4.
|
||||
|
||||
**Requirements:** R1, R5
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/package.json`
|
||||
- Create: `marketing/v3.1-launch/tsconfig.json`
|
||||
- Create: `marketing/v3.1-launch/remotion.config.ts`
|
||||
- Create: `marketing/v3.1-launch/src/index.ts`
|
||||
- Create: `marketing/v3.1-launch/src/Root.tsx`
|
||||
- Create: `marketing/v3.1-launch/README.md`
|
||||
- Modify: `.gitignore` (add `marketing/v3.1-launch/out/`, `marketing/v3.1-launch/node_modules/`)
|
||||
|
||||
**Approach:**
|
||||
- Use `npx create-video@latest --blank` (Remotion 4.x scaffolding) targeting `marketing/v3.1-launch/`. Strip the demo composition.
|
||||
- Configure: 1920×1080, 30fps, H.264, AAC (audio codec needed even for silent — empty track).
|
||||
- README documents `npm install`, `npm run preview` (Remotion Studio), `npm run render` (one-command MP4).
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none for scaffold, but verification = `npm run preview` opens Remotion Studio with a blank 30s composition; `npm run render` produces a black MP4 at the right resolution.
|
||||
|
||||
**Verification:**
|
||||
- Studio loads at localhost:3000 with the empty `LaunchVideo` composition listed.
|
||||
- A render produces `out/launch-video.mp4` at 1920×1080, 30s, valid MP4.
|
||||
|
||||
- [ ] **Unit 2: Reusable components (Terminal, TypedLine, BadgeBar)**
|
||||
|
||||
**Goal:** Build the three primitive components scenes 2-6 will compose. Each is independently previewable.
|
||||
|
||||
**Requirements:** R3, R5
|
||||
|
||||
**Dependencies:** Unit 1
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/components/TerminalWindow.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/TypedLine.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/BadgeBar.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/lib/colors.ts`
|
||||
- Create: `marketing/v3.1-launch/src/lib/timing.ts`
|
||||
|
||||
**Approach:**
|
||||
- `TerminalWindow`: mac-style traffic-light header, dark gradient background, mono content area. Accepts children.
|
||||
- `TypedLine`: takes a string and a `startFrame`, renders character-by-character at ~30 chars/sec. Reuses Remotion's `interpolate(useCurrentFrame() - startFrame, [0, lengthFrames], [0, text.length])` clamped.
|
||||
- `BadgeBar`: renders the literal `🌐 last30days v3.1 · synced 2026-04-22` line in mono with the same gradient color treatment as the engine's compact emit.
|
||||
- `colors.ts`: 5-7 brand colors (terminal-bg, terminal-fg, accent-cyan, accent-magenta, muted, success-green, warning-amber).
|
||||
- `timing.ts`: exports the scene frame ranges as named constants. Single source of truth for any retiming.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — visual components verified in Remotion Studio.
|
||||
|
||||
**Verification:**
|
||||
- Each component renders standalone in Studio when wrapped in a temporary preview composition.
|
||||
- TypedLine animates character-by-character without flicker.
|
||||
|
||||
- [ ] **Unit 3: Scenes 1-3 (Hook, Old way, Fan-out reveal)**
|
||||
|
||||
**Goal:** Build the first half of the video (frames 0-420). The narrative arc up through the visual fan-out.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 2
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene1Hook.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene2OldWay.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene3FanOut.tsx`
|
||||
- Modify: `marketing/v3.1-launch/src/Root.tsx` (register sequences)
|
||||
|
||||
**Approach:**
|
||||
- Scene 1: spring-in BadgeBar, large overlay caption, 3-second hold.
|
||||
- Scene 2: TerminalWindow with TypedLine (`$ /last30days OpenAI`), then a single result-card mock fading in.
|
||||
- Scene 3: typing animation appends `--competitors`, hard cut, three TerminalWindow components arranged in a row with staggered fade-in. Each pane shows a different entity header + scrolling progress lines.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verified visually in Studio.
|
||||
|
||||
**Verification:**
|
||||
- Scrub the 0-14s range in Studio; visuals match the script timing within 5 frames.
|
||||
- The fan-out moment (frame 240) lands cleanly; no jank in the transition from 1 → 3 panes.
|
||||
|
||||
- [ ] **Unit 4: Scenes 4-6 (Comparison reveal, How it works, CTA)**
|
||||
|
||||
**Goal:** Build the back half of the video (frames 420-900). Resolution + payoff + call to action.
|
||||
|
||||
**Requirements:** R2
|
||||
|
||||
**Dependencies:** Unit 2
|
||||
|
||||
**Files:**
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene4Comparison.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene5HowItWorks.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/scenes/Scene6CTA.tsx`
|
||||
- Create: `marketing/v3.1-launch/src/components/ComparisonTable.tsx`
|
||||
|
||||
**Approach:**
|
||||
- Scene 4: ComparisonTable component renders 3-column markdown-style table; rows fade in one by one (stagger 15-20 frames). Uses canned data — OpenAI / Anthropic / xAI with believable cell content drawn from real 3.0.13 outputs.
|
||||
- Scene 5: three-line text card; lines fade in 45 frames apart.
|
||||
- Scene 6: three centered text blocks; install line gets a 1Hz subtle opacity pulse for emphasis.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verified visually.
|
||||
|
||||
**Verification:**
|
||||
- Scrub 14-30s; table reveal feels paced (not too slow, not strobed); CTA holds long enough to read (~3-4s).
|
||||
- ComparisonTable cells are legible at 1920×1080 (mono font ≥ 28px).
|
||||
|
||||
- [ ] **Unit 5: Final composition wiring + render**
|
||||
|
||||
**Goal:** Wire the six scenes into the master `LaunchVideo` composition, render to MP4, verify against X upload constraints.
|
||||
|
||||
**Requirements:** R1, R2, R5
|
||||
|
||||
**Dependencies:** Units 3, 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `marketing/v3.1-launch/src/LaunchVideo.tsx` (sequence all 6 scenes)
|
||||
- Modify: `marketing/v3.1-launch/README.md` (add render command + verification checklist)
|
||||
|
||||
**Approach:**
|
||||
- `LaunchVideo` is a single Composition that imports `Scene1Hook` … `Scene6CTA` and wraps each in `<Sequence from=… durationInFrames=…>` matching `lib/timing.ts`.
|
||||
- Run `npx remotion render LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18`.
|
||||
- Verify output: 30.0s ±0.1s, 1920×1080, file size <30MB, opens in QuickTime, plays without dropped frames.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — verification is the rendered MP4 itself.
|
||||
|
||||
**Verification:**
|
||||
- `ffprobe out/last30days-v3.1-launch.mp4` reports 1920×1080, 30fps, ~30.0s, h264, faststart-friendly.
|
||||
- Manual play-through end-to-end in QuickTime feels coherent and on-pace.
|
||||
- File <30MB so X upload is instant.
|
||||
|
||||
- [ ] **Unit 6: Polish pass + ship**
|
||||
|
||||
**Goal:** Watch the full render, fix obvious jank, do a second render, and stage for X posting.
|
||||
|
||||
**Requirements:** R1, R2, R3
|
||||
|
||||
**Dependencies:** Unit 5
|
||||
|
||||
**Files:**
|
||||
- Possibly modify: any scene file based on watch-through findings.
|
||||
|
||||
**Approach:**
|
||||
- Watch the rendered MP4 at full size. Note: timing felt off, transitions too fast, captions overflow, color clash, anything visibly broken.
|
||||
- Iterate: edit scene component → re-preview in Studio → re-render full MP4.
|
||||
- Cap at 2 polish passes; ship the better of the two renders.
|
||||
- Final MP4 sits at `marketing/v3.1-launch/out/last30days-v3.1-launch.mp4` ready for X upload.
|
||||
|
||||
**Test scenarios:**
|
||||
- Test expectation: none — pure subjective polish.
|
||||
|
||||
**Verification:**
|
||||
- User watches the final render and approves.
|
||||
- No glaring visual bugs (overflowing text, frozen frames, color clashes).
|
||||
|
||||
## System-Wide Impact
|
||||
|
||||
- **Interaction graph:** None — this is a standalone marketing artifact. Doesn't touch the Python engine, doesn't change any user-facing behavior.
|
||||
- **State lifecycle risks:** None.
|
||||
- **API surface parity:** N/A.
|
||||
- **Unchanged invariants:** The shipped 3.0.14 engine is untouched.
|
||||
|
||||
## Risks & Dependencies
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|------------|
|
||||
| Remotion install pulls 200MB+ of node_modules. | `marketing/v3.1-launch/node_modules/` in `.gitignore`; checked-in source stays small. |
|
||||
| Render time blows past patience (>5 min for 30s @ 1080p30). | Bun-based render or `--concurrency` flag. Default Remotion is fast enough on M-series Macs. If slow, lower preview to 720p, render final at 1080p. |
|
||||
| Captions overflow the 1920px width on certain fonts. | Use a known mono font with a measured per-character width; cap caption lines at 36 chars. |
|
||||
| The "fan-out" visual in Scene 3 looks confusing instead of magical. | Polish pass (Unit 6) is the safety net; if still bad, fall back to a simpler "1 → 3 panes wipe" instead of typed split. |
|
||||
| File size >30MB hits X upload friction. | Use `--crf=18` (high quality, reasonable size); fall back to `--crf=23` if over. 30s @ 1080p30 H.264 is normally 5-15MB. |
|
||||
|
||||
## Documentation / Operational Notes
|
||||
|
||||
- README at `marketing/v3.1-launch/README.md` documents preview / render commands.
|
||||
- After render, the MP4 is uploaded directly to X. Tweet copy is the user's call (this plan stops at the rendered file).
|
||||
|
||||
## Sources & References
|
||||
|
||||
- Related code: `scripts/lib/render.py` (`_render_comparison_scaffold` is the visual model for Scene 4); `SKILL.md` Competitor mode section (the narrative source).
|
||||
- Related PRs: #308, #311, #312 (the 3.0.11 → 3.0.14 release bundle this video markets as "v3.1").
|
||||
- External docs: https://www.remotion.dev/docs/ (Remotion 4.x API).
|
||||
- X video specs: https://help.x.com/en/using-x/twitter-videos.
|
||||
@@ -1,6 +1,6 @@
|
||||
# Search Quality Eval
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
|
||||
What it does:
|
||||
|
||||
@@ -18,13 +18,13 @@ What it does:
|
||||
Recommended usage:
|
||||
|
||||
```bash
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py
|
||||
uv run python scripts/evaluate_search_quality.py
|
||||
```
|
||||
|
||||
Useful flags:
|
||||
|
||||
```bash
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py \
|
||||
uv run python scripts/evaluate_search_quality.py \
|
||||
--baseline-rev origin/main \
|
||||
--candidate-rev HEAD \
|
||||
--no-default-topics \
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
title: Search-quality eval is manual by default, not a CI gate on every PR
|
||||
date: 2026-05-10
|
||||
category: docs/solutions/architecture
|
||||
module: skills/last30days/scripts/evaluate_search_quality.py
|
||||
problem_type: design_decision
|
||||
component: ci_policy
|
||||
severity: low
|
||||
applies_when:
|
||||
- a contributor proposes wiring search-quality eval into PR CI
|
||||
- a change affects retrieval, ranking, grounding, or synthesis quality and a reviewer asks "why aren't we testing this in CI?"
|
||||
- someone is deciding whether a new evaluator-style script belongs in the default CI workflow
|
||||
related_components:
|
||||
- search_quality_evaluation
|
||||
- ci_workflow
|
||||
- llm_judging
|
||||
tags:
|
||||
- ci-policy
|
||||
- eval
|
||||
- design-decision
|
||||
- cost-vs-signal
|
||||
- non-determinism
|
||||
- manual-gates
|
||||
---
|
||||
|
||||
# Search-quality eval is manual by default, not a CI gate on every PR
|
||||
|
||||
## Context
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` compares a baseline revision against a candidate revision across a fixed pool of reviewer topics. It produces two flavors of metrics: deterministic overlap (Jaccard, retention) and LLM-judged quality scores. The natural impulse on seeing an evaluator script is to wire it into CI on every PR — "regression catcher, run it automatically." We deliberately don't.
|
||||
|
||||
Three properties of this particular evaluator make CI-on-every-PR the wrong default:
|
||||
|
||||
1. **Live API access.** The candidate revision typically needs the engine to actually run, which means real ScrapeCreators calls, real reddit fetches, real YouTube searches. CI runs would either need production credentials or a record/replay fixture set that drifts almost immediately as external APIs change shape.
|
||||
|
||||
2. **Cost and latency.** A full eval pass runs the pipeline N times across reviewer topics. Multiplied by every PR (including doc-only PRs), the spend is meaningful and the wall-clock pushes CI from ~30s to many minutes.
|
||||
|
||||
3. **Non-determinism in the judging path.** The LLM-judged metrics are valuable for review but depend on judge-model behavior on a given day. A flaky eval that fails 1 PR in 20 because the judge re-scored an item differently is a worse CI signal than no eval at all — it teaches contributors to retry rather than read the result.
|
||||
|
||||
The deterministic overlap metrics are useful regression signals but they are not the same as user-facing correctness. A change that improves overlap can degrade synthesis quality; a change that drops overlap can be a deliberate improvement. So even the deterministic side isn't safe to auto-fail on.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Keep search-quality eval available, just not automatic
|
||||
|
||||
The script stays runnable by maintainers and contributors. The pattern is:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_PYTHON=python3.13 \
|
||||
python3 skills/last30days/scripts/evaluate_search_quality.py \
|
||||
--baseline main --candidate HEAD
|
||||
```
|
||||
|
||||
Reviewers can request a manual eval run when a PR is in the retrieval/ranking/synthesis path and the risk warrants it. Contributors can run it locally before submitting if they want signal upfront.
|
||||
|
||||
### 2. Standard PR CI gates remain deterministic and contract-shaped
|
||||
|
||||
`pytest` (offline-safe), plugin-contract checks, version-consistency contracts, ruff/lint. Anything that returns the same answer twice for the same input. Quality-of-output assessment lives outside that loop.
|
||||
|
||||
### 3. The middle ground is `workflow_dispatch`, not auto-PR-gating
|
||||
|
||||
If maintainers want a GitHub-triggered eval that doesn't make every PR pay the live-API cost, the right shape is a manually-dispatched workflow (or a label-triggered one) — not a `pull_request:` workflow that runs unconditionally. That keeps the cost knob in human hands.
|
||||
|
||||
### 4. Revisit if the eval can ever be made offline-deterministic
|
||||
|
||||
The blocker is the live-API + non-determinism combination. If a future iteration of the script can compute meaningful Jaccard/retention metrics against static fixtures (no live API calls, no LLM judging), the decision flips and it becomes a candidate for default CI. The decision below tracks that condition; revisit when it's met.
|
||||
|
||||
## What this means in practice
|
||||
|
||||
- Don't merge PRs that wire `evaluate_search_quality.py` into the default `validate.yml` workflow.
|
||||
- Do merge PRs that add `workflow_dispatch` triggers or label-gated runs.
|
||||
- When reviewing a retrieval/ranking change, request a manual eval if the diff suggests it could regress quality — don't expect CI to catch it.
|
||||
|
||||
## Links
|
||||
|
||||
- `skills/last30days/scripts/evaluate_search_quality.py` — the evaluator script
|
||||
- `docs/search-quality-eval.md` — user-facing usage documentation
|
||||
- `.github/workflows/validate.yml` — the default CI workflow (deterministic gates only)
|
||||
|
||||
---
|
||||
|
||||
*Adapted from a draft ADR proposed by @hnshah in [#374](https://github.com/mvanhorn/last30days-skill/pull/374), restructured into the `docs/solutions/` convention. The original ADR text correctly identified the constraint; this version adds the "why workflow_dispatch is the middle ground" framing and the revisit-condition.*
|
||||
@@ -1,219 +0,0 @@
|
||||
---
|
||||
title: Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
date: 2026-05-16
|
||||
category: docs/solutions/workflow-issues
|
||||
module: ci-release-engineering
|
||||
problem_type: workflow_issue
|
||||
component: testing_framework
|
||||
severity: high
|
||||
applies_when:
|
||||
- a test asserts consistency between two release-time artifacts (e.g., SKILL.md version and a hardcoded pin in a shell script)
|
||||
- one artifact is updated as part of a version bump and the other requires a manual lockstep update
|
||||
- multiple long-lived PRs are open simultaneously against the same base branch
|
||||
symptoms:
|
||||
- every open PR's CI fails after a version bump even though the PRs are unrelated to versioning
|
||||
- the failing test references a stale hardcoded value that was not updated alongside the bumped version
|
||||
- PR authors must rebase and manually fix an artifact they did not touch
|
||||
root_cause: missing_workflow_step
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- development_workflow
|
||||
- documentation
|
||||
tags:
|
||||
- ci
|
||||
- release-engineering
|
||||
- consistency-test
|
||||
- version-pin
|
||||
- cascade-failure
|
||||
- test-design
|
||||
- workflow
|
||||
---
|
||||
|
||||
# Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
|
||||
## Context
|
||||
|
||||
A `tests/test_version_consistency.py::test_sync_cache_path_uses_skill_version` test was added to enforce that the version string embedded in `skills/last30days/scripts/sync.sh` (a hardcoded plugin-cache path segment) matched the version frontmatter in `skills/last30days/SKILL.md`. The intention was sound: the cache path had to stay in lockstep with the skill version or the sync would silently pull stale files.
|
||||
|
||||
The test worked as designed until a release shipped. At that point it turned into a cascade-failure machine:
|
||||
|
||||
1. A release PR bumps `SKILL.md` version (e.g., 3.2.0 → 3.2.1) **and** bumps the `sync.sh` pin. That PR's CI is green.
|
||||
2. The release PR merges to `main`.
|
||||
3. Every PR that was open at merge time was branched from pre-release `main`. Those PRs have `SKILL.md` 3.2.1 (inherited via merge-base with `main`) but their branch never touched `sync.sh`.
|
||||
4. CI for those PRs runs the consistency test against the new `main` — `SKILL.md` says 3.2.1, `sync.sh` still says 3.2.0 — and fails.
|
||||
5. All open PRs are now red simultaneously, with a failure that has nothing to do with their changes.
|
||||
|
||||
This affected at least five PRs during the 2026-05-13 to 2026-05-15 window: PR #400 (caught during rebase, required a manual pin bump), PRs #390 and #392 (OpenClaw `SCRAPECREATORS_API_KEY` fix, both stalled for the same stale-pin reason), and at least two others. A follow-up hotfix PR (#397 — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`) was required just to unblock the queue.
|
||||
|
||||
The permanent fix was PR #405: delete `sync.sh` entirely (the install workflow made it redundant) and drop `test_sync_cache_path_uses_skill_version`. Once both were gone, no version-consistency cascade was possible.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Don't write consistency tests that read two files and assert one matches a substring derived from the other
|
||||
|
||||
This pattern looks safe but is not:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # reads SKILL.md
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # asserts sync.sh contains that string
|
||||
)
|
||||
```
|
||||
|
||||
It encodes the assumption that both files are always updated together, in the same commit, on the same branch. That assumption breaks the moment two files have independent lifecycle owners — a versioned manifest and a deployment script are archetypal examples.
|
||||
|
||||
### 2. If the values genuinely need to stay in sync, derive one from the other at runtime
|
||||
|
||||
Remove the hardcoded pin from `sync.sh` and compute it:
|
||||
|
||||
```bash
|
||||
# sync.sh — derive version from SKILL.md at runtime, no pin to maintain
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "$(dirname "$0")/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
CACHE_PATH="last30days-skill/last30days/${SKILL_VERSION}"
|
||||
```
|
||||
|
||||
Now there is only one source of truth (`SKILL.md`). The test that asserted they matched becomes vacuous and should be deleted. If `SKILL.md` is wrong, the sync itself will fail loudly — which is better feedback than a CI gate on a different PR.
|
||||
|
||||
### 3. If two values must stay independent for legitimate reasons, update them together and make the test self-skip if either source is missing
|
||||
|
||||
If separate versioning is genuinely required (e.g., SKILL.md versions for harness consumers, sync.sh versions a private artifact store with its own cadence), update both in the same PR — never staggered — and write the test to self-skip rather than error when either file is absent:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_sh = SKILL_ROOT / "scripts" / "sync.sh"
|
||||
if not sync_sh.exists():
|
||||
self.skipTest("sync.sh not present; skipping pin consistency check")
|
||||
sync_text = sync_sh.read_text(encoding="utf-8")
|
||||
version = _skill_version()
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text,
|
||||
)
|
||||
```
|
||||
|
||||
Self-skipping means deleting the file is a non-event in CI — no cascading red, no hotfix PR to the queue.
|
||||
|
||||
### 4. Run consistency tests against the merge-base diff, not main
|
||||
|
||||
If you keep a two-file consistency test, scope it so it only fails when the PR itself modifies one of the two files but not the other. A GitHub Actions step can do this:
|
||||
|
||||
```yaml
|
||||
- name: Check sync.sh version pin consistency
|
||||
run: |
|
||||
BASE=$(git merge-base HEAD origin/main)
|
||||
SKILL_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'SKILL\.md' || true)
|
||||
SYNC_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'sync\.sh' || true)
|
||||
if [ "$SKILL_CHANGED" -gt 0 ] && [ "$SYNC_CHANGED" -eq 0 ]; then
|
||||
echo "SKILL.md version bumped but sync.sh pin was not updated"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
This only fires when your PR touched `SKILL.md` and left `sync.sh` alone — never because a release merged to `main` after you branched.
|
||||
|
||||
### 5. Ask whether you actually need this test
|
||||
|
||||
If the values are wrong, downstream tooling will fail loudly: the sync will fetch the wrong artifact, installs will break, or the harness will reject the version. A test that exists only to catch a human-bookkeeping error at release time adds cascade-fail risk without offering a meaningfully earlier signal. Weigh that cost before adding any two-file consistency gate.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The damage from a stale-pin consistency test is asymmetric. It:
|
||||
|
||||
- Fails on every open PR simultaneously the moment a release lands on `main` — not just the PR that forgot to update the pin.
|
||||
- Produces a failure message that points at a line in a test file with no obvious relationship to the PR's actual changes.
|
||||
- Requires either a hotfix PR (touching a file the failing PRs have no business touching) or a manual rebase of every affected branch.
|
||||
- Blocks work that has already been reviewed and approved.
|
||||
|
||||
In this repo the effect was measurable: at least five PRs stalled across a two-day window, one hotfix PR was shipped just to unblock the queue, and multiple authors spent time debugging a failure completely unrelated to their changes.
|
||||
|
||||
The broader principle is that tests which gate on *bookkeeping consistency between files* impose their maintenance cost on every contributor, every time, even when those contributors did nothing wrong. That cost compounds with team size and release cadence.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Apply this guidance whenever you find yourself:
|
||||
|
||||
- Writing a test that reads two files and asserts that a string in one matches a value derived from the other.
|
||||
- Adding a CI step labeled "consistency check," "sync check," or "pin check" where the check compares a hardcoded value against a computed one from a separate file.
|
||||
- Working in a repo where a versioned manifest (e.g., `SKILL.md`, `package.json`, `pyproject.toml`) and a deployment artifact (e.g., a shell script, a Dockerfile, a Helm values file) are both maintained by hand.
|
||||
- Reviewing a PR that touches only one of two "paired" files and fails a consistency test for the other.
|
||||
|
||||
It does *not* apply to tests that read a single source of truth and validate its internal structure (e.g., asserting that `SKILL.md`'s frontmatter version is double-quoted, or that `package.json`'s `version` field is a valid semver string). Those tests have one file and one assertion; they cannot cascade across branches.
|
||||
|
||||
## Examples
|
||||
|
||||
### Before — the pattern that caused the cascade
|
||||
|
||||
Original `tests/test_version_consistency.py` (deleted in commit `9fb19ea`):
|
||||
|
||||
```python
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_ROOT = ROOT / "skills" / "last30days"
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
|
||||
if not match:
|
||||
raise AssertionError("SKILL.md version frontmatter not found")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
class TestVersionConsistency(unittest.TestCase):
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # source 1: SKILL.md frontmatter
|
||||
self.assertIn( # assertion: sync.sh must contain
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # source 2: hardcoded string in sync.sh
|
||||
)
|
||||
```
|
||||
|
||||
`sync.sh` contained a line like:
|
||||
|
||||
```bash
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/3.2.0"
|
||||
```
|
||||
|
||||
When SKILL.md bumped to `3.2.1` in a release PR, `sync.sh` was updated in the same PR and CI stayed green. But every PR branched before that release still had `sync.sh` at `3.2.0`. Their CI failed immediately, with an assertion error pointing at the test, not at the release PR.
|
||||
|
||||
### After — what we did: delete both
|
||||
|
||||
PR #405 deleted `sync.sh` (the install workflow replaced it) and dropped `test_sync_cache_path_uses_skill_version` in the same change. No consistency gate, no pin to maintain, no cascade possible.
|
||||
|
||||
### After — what we could have done instead: derive at runtime
|
||||
|
||||
If `sync.sh` had still been needed, the right fix would have been to remove the hardcoded version from the script and derive it from `SKILL.md`:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# sync.sh — no hardcoded version; reads SKILL.md as single source of truth
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "${SCRIPT_DIR}/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
|
||||
if [ -z "$SKILL_VERSION" ]; then
|
||||
echo "error: could not parse version from SKILL.md" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/${SKILL_VERSION}"
|
||||
# ... rest of sync logic
|
||||
```
|
||||
|
||||
With this in place, `test_sync_cache_path_uses_skill_version` has no reason to exist — there is nothing to assert. Delete it. If the version parsing breaks, `sync.sh` itself exits non-zero with a clear message.
|
||||
|
||||
## Related
|
||||
|
||||
- **PR #397** (merged) — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`. The hotfix that unblocked the cascade temporarily by bumping the pin.
|
||||
- **PR #400** (merged) — caught the same cascade during rebase; had to bump the pin to clear CI.
|
||||
- **PR #390** (closed) and **PR #392** (rebased + merged) — OpenClaw `SCRAPECREATORS_API_KEY` fix; both blocked by the cascade until rebased onto post-#405 main.
|
||||
- **PR #405** (merged) — the permanent fix: deleted `sync.sh` + `test_sync_cache_path_uses_skill_version` together.
|
||||
- **PR #412** (merged) — adjacent work that consolidated SKILL.md version parsing into `lib/skill_meta.py`, reducing future drift risk by giving the version field one canonical reader.
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"version": "3.2.4",
|
||||
"version": "3.0.5",
|
||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||
"settings": [
|
||||
{
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"triggerOnUpdates": true,
|
||||
"statusCheck": true
|
||||
}
|
||||
+2
-1
@@ -6,7 +6,8 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash \"${CLAUDE_PLUGIN_ROOT:-${extensionPath:-.}}/hooks/scripts/check-config.sh\""
|
||||
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-config.sh",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -33,13 +33,8 @@ load_env_vars() {
|
||||
[[ -z "$key" ]] && continue
|
||||
key=$(echo "$key" | xargs)
|
||||
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||
# Strip inline comments (# preceded by whitespace) to prevent
|
||||
# command substitution in backtick-containing comments
|
||||
value="${value%%[[:space:]]#*}"
|
||||
if [[ -n "$key" && -n "$value" ]]; then
|
||||
# printf -v writes via assignment semantics (global from inside a
|
||||
# function), works on macOS's /bin/bash 3.2 — `declare -g` is 4.2+.
|
||||
printf -v "ENV_${key}" '%s' "$value"
|
||||
eval "ENV_${key}=\"${value}\""
|
||||
fi
|
||||
done < "$file"
|
||||
fi
|
||||
@@ -63,53 +58,14 @@ fi
|
||||
# Check SETUP_COMPLETE (from file or env)
|
||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||
|
||||
# Compute last-run summary line (if last-run.json exists)
|
||||
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
||||
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
||||
LAST_RUN_FILE="$LAST30DAYS_CONFIG_DIR/last-run.json"
|
||||
else
|
||||
LAST_RUN_FILE=""
|
||||
fi
|
||||
else
|
||||
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
|
||||
fi
|
||||
LAST_RUN_LINE=""
|
||||
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
|
||||
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 - <<'PY' 2>/dev/null || true
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
|
||||
path = os.environ["LAST_RUN_FILE"]
|
||||
try:
|
||||
with open(path) as fh:
|
||||
d = json.load(fh)
|
||||
topic = (d.get("topic") or "?")[:60]
|
||||
ts = d.get("timestamp", "")
|
||||
dt = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
delta = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
|
||||
if delta < 60: ago = f"{int(delta)}s ago"
|
||||
elif delta < 3600: ago = f"{int(delta//60)}m ago"
|
||||
elif delta < 86400: ago = f"{int(delta//3600)}h ago"
|
||||
else: ago = f"{int(delta//86400)}d ago"
|
||||
total = d.get("total", 0)
|
||||
print(f" Last run: \"{topic}\" · {ago} · {total} results")
|
||||
except Exception:
|
||||
pass
|
||||
PY
|
||||
)
|
||||
fi
|
||||
|
||||
# If setup has never been run, show welcome message for new users
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||
cat <<'EOF'
|
||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||
Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).
|
||||
|
||||
Reddit, Hacker News, and Polymarket work out of the box.
|
||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||
EOF
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -141,33 +97,16 @@ if [[ -n "$HAS_BSKY" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
|
||||
# Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer
|
||||
# whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing.
|
||||
SC_ADD=3
|
||||
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
|
||||
EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \
|
||||
| sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//')
|
||||
if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||
fi
|
||||
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Fully configured — compact ready message
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
else
|
||||
# Setup done but missing ScrapeCreators — recommend it
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
||||
echo " 10,000 free API calls, no credit card — scrapecreators.com"
|
||||
echo " last30days has no affiliation with any API provider."
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# /last30days v3.1 Launch Video
|
||||
|
||||
30-second Remotion-rendered MP4 announcing **v3.1: Competitors mode** for posting on X.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
cd marketing/v3.1-launch
|
||||
npm install # one-time, ~200MB of node_modules
|
||||
npm run preview # opens Remotion Studio at localhost:3000 to scrub frames
|
||||
npm run render # writes out/last30days-v3.1-launch.mp4 (high quality, CRF 18)
|
||||
npm run render:fast # writes a CRF 23 preview for fast iteration
|
||||
```
|
||||
|
||||
## Specs
|
||||
|
||||
- 1920×1080, 30fps, 30 seconds (900 frames)
|
||||
- H.264 / MP4
|
||||
- Silent (autoplay-muted-friendly; captions baked in)
|
||||
- Marketing label `v3.1` (engine code version stays 3.0.14)
|
||||
|
||||
## Scene timing (single source of truth: `src/lib/timing.ts`)
|
||||
|
||||
| Scene | Frames | Time | What |
|
||||
|-------|--------|------|------|
|
||||
| 1. Hook | 0-89 | 0.0-3.0s | Badge animates in + "What if one search ran 3 at once?" |
|
||||
| 2. Old way | 90-239 | 3.0-8.0s | Single terminal: `/last30days OpenAI` |
|
||||
| 3. Fan-out | 240-419 | 8.0-14.0s | `--competitors` types in → splits into 3 panes |
|
||||
| 4. Comparison | 420-629 | 14.0-21.0s | 3 panes collapse into Head-to-Head table |
|
||||
| 5. How | 630-779 | 21.0-26.0s | 3-line text card |
|
||||
| 6. CTA | 780-899 | 26.0-30.0s | Install command + repo URL |
|
||||
|
||||
Edit `src/lib/timing.ts` to retime scenes; the `LaunchVideo` composition reads from there.
|
||||
|
||||
## Output
|
||||
|
||||
Rendered MP4 lives at `out/last30days-v3.1-launch.mp4` (gitignored). Upload directly to X.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "last30days-v3-1-launch-video",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "30s Remotion launch video for /last30days v3.1 (competitors mode).",
|
||||
"scripts": {
|
||||
"preview": "remotion studio src/index.ts",
|
||||
"render": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18",
|
||||
"render:fast": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch-preview.mp4 --codec=h264 --crf=23"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "19.0.0",
|
||||
"react-dom": "19.0.0",
|
||||
"remotion": "4.0.250",
|
||||
"@remotion/cli": "4.0.250",
|
||||
"@remotion/google-fonts": "4.0.250"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "19.0.0",
|
||||
"@types/node": "22.10.0",
|
||||
"typescript": "5.6.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { Config } from "@remotion/cli/config";
|
||||
|
||||
Config.setVideoImageFormat("jpeg");
|
||||
Config.setOverwriteOutput(true);
|
||||
Config.setConcurrency(null);
|
||||
Config.setCodec("h264");
|
||||
@@ -0,0 +1,35 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, Sequence } from "remotion";
|
||||
import { SCENES } from "./lib/timing";
|
||||
import { Scene1Hook } from "./scenes/Scene1Hook";
|
||||
import { Scene2OldWay } from "./scenes/Scene2OldWay";
|
||||
import { Scene3FanOut } from "./scenes/Scene3FanOut";
|
||||
import { Scene4Comparison } from "./scenes/Scene4Comparison";
|
||||
import { Scene5HowItWorks } from "./scenes/Scene5HowItWorks";
|
||||
import { Scene6CTA } from "./scenes/Scene6CTA";
|
||||
import { COLORS } from "./lib/colors";
|
||||
|
||||
export const LaunchVideo: React.FC = () => {
|
||||
return (
|
||||
<AbsoluteFill style={{ background: COLORS.bgDeep }}>
|
||||
<Sequence from={SCENES.hook.from} durationInFrames={SCENES.hook.durationInFrames}>
|
||||
<Scene1Hook />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.oldWay.from} durationInFrames={SCENES.oldWay.durationInFrames}>
|
||||
<Scene2OldWay />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.fanOut.from} durationInFrames={SCENES.fanOut.durationInFrames}>
|
||||
<Scene3FanOut />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.comparison.from} durationInFrames={SCENES.comparison.durationInFrames}>
|
||||
<Scene4Comparison />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.howItWorks.from} durationInFrames={SCENES.howItWorks.durationInFrames}>
|
||||
<Scene5HowItWorks />
|
||||
</Sequence>
|
||||
<Sequence from={SCENES.cta.from} durationInFrames={SCENES.cta.durationInFrames}>
|
||||
<Scene6CTA />
|
||||
</Sequence>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
import React from "react";
|
||||
import { Composition } from "remotion";
|
||||
import { LaunchVideo } from "./LaunchVideo";
|
||||
import { FPS, TOTAL_FRAMES } from "./lib/timing";
|
||||
|
||||
export const RemotionRoot: React.FC = () => {
|
||||
return (
|
||||
<>
|
||||
<Composition
|
||||
id="LaunchVideo"
|
||||
component={LaunchVideo}
|
||||
durationInFrames={TOTAL_FRAMES}
|
||||
fps={FPS}
|
||||
width={1920}
|
||||
height={1080}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,68 @@
|
||||
import React from "react";
|
||||
import { spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
startFrame?: number;
|
||||
size?: "small" | "large";
|
||||
};
|
||||
|
||||
export const BadgeBar: React.FC<Props> = ({ startFrame = 0, size = "large" }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
const elapsed = Math.max(0, frame - startFrame);
|
||||
|
||||
const scale = spring({
|
||||
frame: elapsed,
|
||||
fps,
|
||||
config: { damping: 12, stiffness: 90 },
|
||||
from: 0.85,
|
||||
to: 1,
|
||||
});
|
||||
const opacity = spring({
|
||||
frame: elapsed,
|
||||
fps,
|
||||
config: { damping: 20 },
|
||||
from: 0,
|
||||
to: 1,
|
||||
});
|
||||
|
||||
const fontSize = size === "large" ? 56 : 28;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
transform: `scale(${scale})`,
|
||||
opacity,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize,
|
||||
color: COLORS.fgPrimary,
|
||||
letterSpacing: 0.5,
|
||||
}}
|
||||
>
|
||||
<span style={{ fontSize: fontSize * 1.1, marginRight: 16 }}>🌐</span>
|
||||
<span>last30days</span>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 14,
|
||||
color: COLORS.accentCyan,
|
||||
fontWeight: 600,
|
||||
}}
|
||||
>
|
||||
v3.1
|
||||
</span>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 16,
|
||||
color: COLORS.fgDim,
|
||||
fontSize: fontSize * 0.55,
|
||||
}}
|
||||
>
|
||||
· synced 2026-04-22
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,135 @@
|
||||
import React from "react";
|
||||
import { interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Row = {
|
||||
dimension: string;
|
||||
cells: [string, string, string];
|
||||
};
|
||||
|
||||
type Props = {
|
||||
startFrame: number;
|
||||
entities: [string, string, string];
|
||||
rows: Row[];
|
||||
rowStaggerFrames?: number;
|
||||
};
|
||||
|
||||
export const ComparisonTable: React.FC<Props> = ({
|
||||
startFrame,
|
||||
entities,
|
||||
rows,
|
||||
rowStaggerFrames = 18,
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const headerOpacity = interpolate(
|
||||
frame - startFrame,
|
||||
[0, 12],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
|
||||
const colTemplate = "1.4fr 1fr 1fr 1fr";
|
||||
const cellPad = "16px 22px";
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
background: COLORS.bgPanel,
|
||||
borderRadius: 16,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
overflow: "hidden",
|
||||
fontFamily: FONT_MONO,
|
||||
boxShadow: "0 24px 60px rgba(0,0,0,0.6)",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: colTemplate,
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderBottom: `1px solid ${COLORS.border}`,
|
||||
opacity: headerOpacity,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 22,
|
||||
fontWeight: 500,
|
||||
}}
|
||||
>
|
||||
Dimension
|
||||
</div>
|
||||
{entities.map((entity, idx) => (
|
||||
<div
|
||||
key={entity}
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: idx === 0 ? COLORS.accentCyan : COLORS.fgPrimary,
|
||||
fontSize: 26,
|
||||
fontWeight: 600,
|
||||
borderLeft: `1px solid ${COLORS.border}`,
|
||||
}}
|
||||
>
|
||||
{entity}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{rows.map((row, idx) => {
|
||||
const rowStart = startFrame + 12 + idx * rowStaggerFrames;
|
||||
const rowOpacity = interpolate(
|
||||
frame - rowStart,
|
||||
[0, 14],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
const rowSlide = interpolate(
|
||||
frame - rowStart,
|
||||
[0, 14],
|
||||
[12, 0],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
return (
|
||||
<div
|
||||
key={row.dimension}
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: colTemplate,
|
||||
borderBottom:
|
||||
idx === rows.length - 1 ? "none" : `1px solid ${COLORS.border}`,
|
||||
opacity: rowOpacity,
|
||||
transform: `translateY(${rowSlide}px)`,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 22,
|
||||
}}
|
||||
>
|
||||
{row.dimension}
|
||||
</div>
|
||||
{row.cells.map((cell, cellIdx) => (
|
||||
<div
|
||||
key={cellIdx}
|
||||
style={{
|
||||
padding: cellPad,
|
||||
color: COLORS.fgPrimary,
|
||||
fontSize: 22,
|
||||
borderLeft: `1px solid ${COLORS.border}`,
|
||||
lineHeight: 1.35,
|
||||
}}
|
||||
>
|
||||
{cell}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,95 @@
|
||||
import React from "react";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
title?: string;
|
||||
width?: number | string;
|
||||
height?: number | string;
|
||||
children?: React.ReactNode;
|
||||
glow?: boolean;
|
||||
};
|
||||
|
||||
export const TerminalWindow: React.FC<Props> = ({
|
||||
title = "/last30days",
|
||||
width = "100%",
|
||||
height = "100%",
|
||||
children,
|
||||
glow = false,
|
||||
}) => {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width,
|
||||
height,
|
||||
background: COLORS.bgPanel,
|
||||
borderRadius: 16,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
boxShadow: glow
|
||||
? `0 0 60px ${COLORS.accentCyan}33, 0 24px 60px rgba(0,0,0,0.6)`
|
||||
: "0 24px 60px rgba(0,0,0,0.6)",
|
||||
overflow: "hidden",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
fontFamily: FONT_MONO,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
height: 36,
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderBottom: `1px solid ${COLORS.border}`,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
padding: "0 16px",
|
||||
gap: 8,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficRed,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficYellow,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: 12,
|
||||
background: COLORS.trafficGreen,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 16,
|
||||
color: COLORS.fgMuted,
|
||||
fontSize: 14,
|
||||
fontFamily: FONT_MONO,
|
||||
letterSpacing: 0.5,
|
||||
}}
|
||||
>
|
||||
{title}
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
padding: "20px 28px",
|
||||
color: COLORS.fgPrimary,
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
import React from "react";
|
||||
import { interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||
|
||||
type Props = {
|
||||
text: string;
|
||||
startFrame: number;
|
||||
charsPerSec?: number;
|
||||
fontSize?: number;
|
||||
color?: string;
|
||||
prefix?: string;
|
||||
prefixColor?: string;
|
||||
showCursor?: boolean;
|
||||
fps?: number;
|
||||
};
|
||||
|
||||
export const TypedLine: React.FC<Props> = ({
|
||||
text,
|
||||
startFrame,
|
||||
charsPerSec = 28,
|
||||
fontSize = 32,
|
||||
color = COLORS.fgPrimary,
|
||||
prefix,
|
||||
prefixColor = COLORS.accentGreen,
|
||||
showCursor = true,
|
||||
fps = 30,
|
||||
}) => {
|
||||
const frame = useCurrentFrame();
|
||||
const elapsed = Math.max(0, frame - startFrame);
|
||||
const totalChars = text.length;
|
||||
const lengthFrames = Math.ceil((totalChars / charsPerSec) * fps);
|
||||
const visibleChars = Math.round(
|
||||
interpolate(elapsed, [0, lengthFrames], [0, totalChars], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
}),
|
||||
);
|
||||
const visible = text.slice(0, visibleChars);
|
||||
const done = visibleChars >= totalChars;
|
||||
const cursorOn = showCursor && Math.floor(frame / 15) % 2 === 0;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize,
|
||||
color,
|
||||
whiteSpace: "pre",
|
||||
lineHeight: 1.4,
|
||||
}}
|
||||
>
|
||||
{prefix ? (
|
||||
<span style={{ color: prefixColor, marginRight: 12 }}>{prefix}</span>
|
||||
) : null}
|
||||
<span>{visible}</span>
|
||||
{(!done || cursorOn) && (
|
||||
<span
|
||||
style={{
|
||||
display: "inline-block",
|
||||
width: fontSize * 0.55,
|
||||
height: fontSize * 0.95,
|
||||
background: color,
|
||||
verticalAlign: "text-bottom",
|
||||
marginLeft: 2,
|
||||
opacity: cursorOn ? 0.85 : 0,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
import { registerRoot } from "remotion";
|
||||
import { RemotionRoot } from "./Root";
|
||||
|
||||
registerRoot(RemotionRoot);
|
||||
@@ -0,0 +1,20 @@
|
||||
export const COLORS = {
|
||||
bgDeep: "#0a0e14",
|
||||
bgPanel: "#11161d",
|
||||
bgPanelSoft: "#161c25",
|
||||
fgPrimary: "#e6e6e6",
|
||||
fgMuted: "#8a93a3",
|
||||
fgDim: "#5b6573",
|
||||
border: "#2a3340",
|
||||
accentCyan: "#36d6f7",
|
||||
accentMagenta: "#ff55a3",
|
||||
accentGreen: "#5fff9f",
|
||||
accentAmber: "#ffc857",
|
||||
trafficRed: "#ff5f57",
|
||||
trafficYellow: "#febc2e",
|
||||
trafficGreen: "#28c840",
|
||||
} as const;
|
||||
|
||||
export const FONT_MONO = '"JetBrains Mono", "SF Mono", "Menlo", monospace';
|
||||
export const FONT_SANS =
|
||||
'"Inter", "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif';
|
||||
@@ -0,0 +1,13 @@
|
||||
// Single source of truth for scene frame ranges.
|
||||
// 30fps × 30s = 900 frames total.
|
||||
export const FPS = 30;
|
||||
export const TOTAL_FRAMES = 900;
|
||||
|
||||
export const SCENES = {
|
||||
hook: { from: 0, durationInFrames: 90 },
|
||||
oldWay: { from: 90, durationInFrames: 150 },
|
||||
fanOut: { from: 240, durationInFrames: 180 },
|
||||
comparison: { from: 420, durationInFrames: 210 },
|
||||
howItWorks: { from: 630, durationInFrames: 150 },
|
||||
cta: { from: 780, durationInFrames: 120 },
|
||||
} as const;
|
||||
@@ -0,0 +1,61 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { BadgeBar } from "../components/BadgeBar";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene1Hook: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const captionOpacity = interpolate(frame, [20, 35, 75, 90], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const captionLift = spring({
|
||||
frame: frame - 20,
|
||||
fps,
|
||||
config: { damping: 15, stiffness: 70 },
|
||||
from: 16,
|
||||
to: 0,
|
||||
});
|
||||
|
||||
const badgeFadeOut = interpolate(frame, [70, 90], [1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 40%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 60%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 64,
|
||||
}}
|
||||
>
|
||||
<div style={{ opacity: badgeFadeOut }}>
|
||||
<BadgeBar />
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionOpacity,
|
||||
transform: `translateY(${captionLift}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 96,
|
||||
fontWeight: 600,
|
||||
color: COLORS.fgPrimary,
|
||||
textAlign: "center",
|
||||
letterSpacing: -1.5,
|
||||
lineHeight: 1.1,
|
||||
maxWidth: 1400,
|
||||
}}
|
||||
>
|
||||
What if one search
|
||||
<br />
|
||||
ran <span style={{ color: COLORS.accentCyan }}>3 at once?</span>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,100 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { TerminalWindow } from "../components/TerminalWindow";
|
||||
import { TypedLine } from "../components/TypedLine";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene2OldWay: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const enter = interpolate(frame, [0, 20], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const slide = interpolate(frame, [0, 20], [40, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Result card fades in after type completes (~70 frames)
|
||||
const resultFade = interpolate(frame, [70, 95], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Caption appears late
|
||||
const captionFade = interpolate(frame, [110, 130, 150], [0, 1, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: 80,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 1400,
|
||||
height: 520,
|
||||
opacity: enter,
|
||||
transform: `translateY(${slide}px)`,
|
||||
}}
|
||||
>
|
||||
<TerminalWindow title="bash">
|
||||
<TypedLine
|
||||
text="/last30days OpenAI"
|
||||
startFrame={20}
|
||||
prefix="$"
|
||||
fontSize={42}
|
||||
/>
|
||||
<div
|
||||
style={{
|
||||
opacity: resultFade,
|
||||
marginTop: 36,
|
||||
padding: "20px 24px",
|
||||
background: COLORS.bgPanelSoft,
|
||||
borderRadius: 12,
|
||||
border: `1px solid ${COLORS.border}`,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 26,
|
||||
color: COLORS.fgPrimary,
|
||||
lineHeight: 1.6,
|
||||
}}
|
||||
>
|
||||
<div style={{ color: COLORS.accentGreen }}>
|
||||
✅ All agents reported back!
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted, marginTop: 6 }}>
|
||||
├─ 🟠 Reddit: 14 threads
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted }}>
|
||||
├─ 🔵 X: 22 posts
|
||||
</div>
|
||||
<div style={{ color: COLORS.fgMuted }}>
|
||||
└─ 🟡 HN: 1 story
|
||||
</div>
|
||||
</div>
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
marginTop: 60,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 38,
|
||||
color: COLORS.fgMuted,
|
||||
}}
|
||||
>
|
||||
The old way: <span style={{ color: COLORS.fgPrimary }}>one topic.</span>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,221 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { TerminalWindow } from "../components/TerminalWindow";
|
||||
import { TypedLine } from "../components/TypedLine";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const PROGRESS_LINES = [
|
||||
{ source: "Reddit", color: "#ff6a3d" },
|
||||
{ source: "X", color: "#36d6f7" },
|
||||
{ source: "YouTube", color: "#ff5757" },
|
||||
{ source: "TikTok", color: "#5fff9f" },
|
||||
{ source: "Instagram", color: "#ff55a3" },
|
||||
];
|
||||
|
||||
const ENTITIES: { label: string; tag: string; accent: string }[] = [
|
||||
{ label: "OpenAI", tag: "$ /last30days OpenAI", accent: COLORS.accentCyan },
|
||||
{ label: "Anthropic", tag: "$ /last30days Anthropic", accent: COLORS.accentMagenta },
|
||||
{ label: "xAI", tag: "$ /last30days xAI", accent: COLORS.accentAmber },
|
||||
];
|
||||
|
||||
const FanPane: React.FC<{
|
||||
label: string;
|
||||
tag: string;
|
||||
accent: string;
|
||||
panelStart: number;
|
||||
}> = ({ label, tag, accent, panelStart }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
const localFrame = Math.max(0, frame - panelStart);
|
||||
|
||||
const enter = spring({
|
||||
frame: localFrame,
|
||||
fps,
|
||||
config: { damping: 18, stiffness: 80 },
|
||||
from: 0,
|
||||
to: 1,
|
||||
});
|
||||
const slide = interpolate(localFrame, [0, 20], [40, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
opacity: enter,
|
||||
transform: `translateY(${slide}px)`,
|
||||
height: 480,
|
||||
}}
|
||||
>
|
||||
<TerminalWindow title={label} glow>
|
||||
<div
|
||||
style={{
|
||||
color: accent,
|
||||
fontSize: 18,
|
||||
fontFamily: FONT_MONO,
|
||||
marginBottom: 14,
|
||||
}}
|
||||
>
|
||||
{tag}
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
fontSize: 22,
|
||||
color: COLORS.accentGreen,
|
||||
fontFamily: FONT_MONO,
|
||||
marginBottom: 12,
|
||||
}}
|
||||
>
|
||||
[Competitors] running...
|
||||
</div>
|
||||
{PROGRESS_LINES.map((line, idx) => {
|
||||
const lineStart = panelStart + 16 + idx * 6;
|
||||
const lineFade = interpolate(
|
||||
frame - lineStart,
|
||||
[0, 8],
|
||||
[0, 1],
|
||||
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||
);
|
||||
// pulse the in-progress dot
|
||||
const dotOn = Math.floor((frame - lineStart) / 6) % 2 === 0;
|
||||
return (
|
||||
<div
|
||||
key={line.source}
|
||||
style={{
|
||||
opacity: lineFade,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 20,
|
||||
color: COLORS.fgMuted,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
marginBottom: 6,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
display: "inline-block",
|
||||
width: 10,
|
||||
height: 10,
|
||||
borderRadius: 10,
|
||||
background: dotOn ? line.color : COLORS.bgPanelSoft,
|
||||
marginRight: 12,
|
||||
boxShadow: dotOn ? `0 0 10px ${line.color}` : "none",
|
||||
}}
|
||||
/>
|
||||
<span style={{ color: line.color, marginRight: 8 }}>
|
||||
►
|
||||
</span>
|
||||
<span>{line.source}</span>
|
||||
<span style={{ marginLeft: "auto", color: COLORS.fgDim }}>
|
||||
{dotOn ? "..." : "·"}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const Scene3FanOut: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
// Phase 1 (0-30 frames): single terminal types --competitors flag
|
||||
// Phase 2 (30+): split into 3 panes
|
||||
const splitProgress = interpolate(frame, [30, 50], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const singleOpacity = interpolate(frame, [0, 8, 30, 45], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
// Caption fades in shortly after panes settle so it has time to read.
|
||||
const captionFade = interpolate(frame, [60, 80], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: 60,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
textAlign: "center",
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 42,
|
||||
color: COLORS.fgPrimary,
|
||||
marginBottom: 32,
|
||||
}}
|
||||
>
|
||||
Now it discovers competitors
|
||||
<br />
|
||||
<span style={{ color: COLORS.accentCyan }}>and runs all 3.</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
position: "relative",
|
||||
}}
|
||||
>
|
||||
{/* Single terminal during phase 1, fades out as panes appear */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: 0,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
opacity: singleOpacity,
|
||||
}}
|
||||
>
|
||||
<div style={{ width: 1200, height: 380 }}>
|
||||
<TerminalWindow title="bash">
|
||||
<TypedLine
|
||||
text="/last30days OpenAI --competitors"
|
||||
startFrame={0}
|
||||
prefix="$"
|
||||
fontSize={42}
|
||||
/>
|
||||
</TerminalWindow>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Three panes fade in starting frame ~30 */}
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: 0,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 24,
|
||||
opacity: splitProgress,
|
||||
}}
|
||||
>
|
||||
{ENTITIES.map((entity, idx) => (
|
||||
<FanPane
|
||||
key={entity.label}
|
||||
label={entity.label}
|
||||
tag={entity.tag}
|
||||
accent={entity.accent}
|
||||
panelStart={45 + idx * 8}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,105 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { ComparisonTable } from "../components/ComparisonTable";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const ROWS = [
|
||||
{
|
||||
dimension: "What it is",
|
||||
cells: [
|
||||
"GPT-5 leader, Plus + API",
|
||||
"Claude 4, safety-first",
|
||||
"Grok, X-native, fast",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "30-day momentum",
|
||||
cells: [
|
||||
"GPT-5 launch wave",
|
||||
"Claude 4.7 1M context",
|
||||
"Grok 5 reveal",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "Community vibe",
|
||||
cells: [
|
||||
"Defensive but deep",
|
||||
"Quiet, devs-only",
|
||||
"Loud, meme-rich",
|
||||
] as [string, string, string],
|
||||
},
|
||||
{
|
||||
dimension: "Best for",
|
||||
cells: [
|
||||
"Mainstream + tools",
|
||||
"Long-context coding",
|
||||
"Live X intel",
|
||||
] as [string, string, string],
|
||||
},
|
||||
];
|
||||
|
||||
export const Scene4Comparison: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const captionFade = interpolate(frame, [0, 12, 180, 210], [0, 1, 1, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const captionLift = interpolate(frame, [0, 14], [16, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const tableFade = interpolate(frame, [16, 32], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: COLORS.bgDeep,
|
||||
padding: "48px 80px",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
opacity: captionFade,
|
||||
transform: `translateY(${captionLift}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 38,
|
||||
color: COLORS.fgMuted,
|
||||
textAlign: "center",
|
||||
marginBottom: 36,
|
||||
}}
|
||||
>
|
||||
<span style={{ color: COLORS.accentCyan, fontWeight: 600 }}>
|
||||
3 full passes.
|
||||
</span>
|
||||
<span style={{ marginLeft: 18, color: COLORS.accentMagenta, fontWeight: 600 }}>
|
||||
3 save files.
|
||||
</span>
|
||||
<span style={{ marginLeft: 18, color: COLORS.fgPrimary, fontWeight: 600 }}>
|
||||
1 comparison.
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
maxWidth: 1640,
|
||||
opacity: tableFade,
|
||||
}}
|
||||
>
|
||||
<ComparisonTable
|
||||
startFrame={20}
|
||||
entities={["OpenAI", "Anthropic", "xAI"]}
|
||||
rows={ROWS}
|
||||
rowStaggerFrames={22}
|
||||
/>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||
|
||||
const LINES = [
|
||||
{ text: "You pick the topic.", color: COLORS.fgPrimary },
|
||||
{ text: "The agent picks the peers.", color: COLORS.accentCyan },
|
||||
{ text: "The engine fans out.", color: COLORS.accentMagenta },
|
||||
];
|
||||
|
||||
export const Scene5HowItWorks: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 60%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 48,
|
||||
}}
|
||||
>
|
||||
{LINES.map((line, idx) => {
|
||||
const start = 10 + idx * 28;
|
||||
const fade = interpolate(frame, [start, start + 14], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
const slide = interpolate(frame, [start, start + 18], [24, 0], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
return (
|
||||
<div
|
||||
key={line.text}
|
||||
style={{
|
||||
opacity: fade,
|
||||
transform: `translateY(${slide}px)`,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 78,
|
||||
fontWeight: 600,
|
||||
color: line.color,
|
||||
letterSpacing: -1,
|
||||
}}
|
||||
>
|
||||
{line.text}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,73 @@
|
||||
import React from "react";
|
||||
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||
import { BadgeBar } from "../components/BadgeBar";
|
||||
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||
|
||||
export const Scene6CTA: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const installFade = interpolate(frame, [20, 40], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
// Stronger 1Hz pulse on the install line for end-of-video emphasis
|
||||
const pulse = 0.8 + 0.2 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||
const glowPulse = 0.4 + 0.4 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||
|
||||
const repoFade = interpolate(frame, [50, 70], [0, 1], {
|
||||
extrapolateLeft: "clamp",
|
||||
extrapolateRight: "clamp",
|
||||
});
|
||||
|
||||
const enterScale = spring({
|
||||
frame,
|
||||
fps,
|
||||
config: { damping: 18, stiffness: 90 },
|
||||
from: 0.95,
|
||||
to: 1,
|
||||
});
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: `radial-gradient(circle at 50% 50%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
gap: 56,
|
||||
transform: `scale(${enterScale})`,
|
||||
}}
|
||||
>
|
||||
<BadgeBar />
|
||||
<div
|
||||
style={{
|
||||
opacity: installFade * pulse,
|
||||
padding: "18px 36px",
|
||||
background: COLORS.bgPanel,
|
||||
border: `1px solid ${COLORS.accentCyan}`,
|
||||
borderRadius: 14,
|
||||
boxShadow: `0 0 ${40 + glowPulse * 60}px ${COLORS.accentCyan}${Math.round(40 + glowPulse * 80).toString(16)}`,
|
||||
fontFamily: FONT_MONO,
|
||||
fontSize: 44,
|
||||
color: COLORS.fgPrimary,
|
||||
}}
|
||||
>
|
||||
<span style={{ color: COLORS.accentGreen, marginRight: 18 }}>$</span>
|
||||
/last30days <span style={{ color: COLORS.fgMuted }}>{"{topic}"}</span>{" "}
|
||||
<span style={{ color: COLORS.accentCyan }}>--competitors</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
opacity: repoFade,
|
||||
fontFamily: FONT_SANS,
|
||||
fontSize: 28,
|
||||
color: COLORS.fgMuted,
|
||||
}}
|
||||
>
|
||||
github.com/mvanhorn/last30days-skill
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.4 MiB |
+9
-6
@@ -1,14 +1,16 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.3.0"
|
||||
version = "3.0.0"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
dependencies = [
|
||||
"requests>=2.32,<3",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.0.3,<10",
|
||||
"pytest>=9,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
]
|
||||
|
||||
@@ -22,9 +24,9 @@ addopts = [
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["skills/last30days/scripts", "tests"]
|
||||
source = ["scripts", "tests"]
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
@@ -32,6 +34,7 @@ omit = [
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
|
||||
+43
-47
@@ -1,64 +1,52 @@
|
||||
## v3.3.0 — install everywhere, ship the reliability sweep
|
||||
|
||||
The AI world reinvents itself every month. This skill keeps you current.
|
||||
|
||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, Digg, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||
|
||||
## What's new in v3.3.0
|
||||
## v3 is the intelligent search release
|
||||
|
||||
### Install everywhere with one command
|
||||
v3 is a ground-up engine rewrite by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
|
||||
`npx skills add mvanhorn/last30days-skill -g -y` is now the canonical install path for **every harness** — Claude Code, OpenAI Codex CLI, Cursor, Gemini CLI, GitHub Copilot, Windsurf, and 50+ other Agent Skills hosts. The skill auto-detects each harness's skills directory and symlinks in place, so edits propagate live. No more per-harness manual paths in the README.
|
||||
Type "OpenClaw" and v3 resolves @steipete, r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags before a single API call fires. Type "Peter Steinberger" and it resolves his X handle and GitHub profile, switches to person mode, and shows what he shipped this month at 85% merge rate across 22 PRs. None of that was on Google.
|
||||
|
||||
### New emit mode: `--emit=html`
|
||||
## Headline features
|
||||
|
||||
Shareable, print-friendly HTML briefs. Drop the file in Slack, mail it to a stakeholder, or print it for the meeting. Same data as compact mode, structured for human reading.
|
||||
### Intelligent pre-research
|
||||
|
||||
### New source: Digg
|
||||
The killer feature. A new Python pre-research brain resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags, all resolved before a single API call.
|
||||
|
||||
Digg surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes directly into the brief. Auto-enabled when `digg-pp-cli` is on PATH. Footer line: `⛏️ Digg: N clusters │ K posts │ M authors`. No X auth required for the inline quotes.
|
||||
### Best Takes
|
||||
|
||||
### YouTube residential-IP routing (`LAST30DAYS_YOUTUBE_SSH_HOST`)
|
||||
A second LLM judge scores every result for humor, wit, and virality alongside relevance. Every brief now ends with a Best Takes section surfacing the cleverest one-liners and most viral quotes. The Reddit and X people are funny, and the old engine buried their best stuff.
|
||||
|
||||
Running on a datacenter VPS (Hetzner, DigitalOcean, AWS, etc.)? YouTube's bot-wall fingerprints datacenter IP ranges before any cookie check. Set `LAST30DAYS_YOUTUBE_SSH_HOST=<ssh-alias>` and yt-dlp runs over SSH against a residential-IP host instead. One env var, no proxy service required.
|
||||
### Cross-source cluster merging
|
||||
|
||||
### macOS Keychain credential source
|
||||
When the same story hits Reddit, X, and YouTube, v3 merges them into one cluster instead of three duplicates. Entity-based overlap detection catches matches even when the titles use different words.
|
||||
|
||||
When env vars and config files aren't set, the engine now reads credentials from the macOS Keychain. Stores secrets where macOS expects them; nothing on disk in plaintext.
|
||||
### Single-pass comparisons
|
||||
|
||||
### `EXCLUDE_SOURCES` env var
|
||||
"X vs Y" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides at once. Same depth, 3 minutes.
|
||||
|
||||
The inverse of `INCLUDE_SOURCES`. Useful for "everything except TikTok" or "everything except the slow ones."
|
||||
### GitHub person-mode and project-mode
|
||||
|
||||
## Reliability sweep
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. PR velocity, top repos by stars, release notes for what shipped this month, woven into the narrative alongside X posts and Reddit threads.
|
||||
|
||||
This release closes a long tail of platform-specific issues that have been accumulating:
|
||||
When the topic is a project, it pulls live star counts, READMEs, releases, and top issues from the GitHub API. No stale blog posts.
|
||||
|
||||
- **Reddit**: subreddits starting with `r` no longer get mangled by `lstrip("r/")`. Browser-like headers + gzip handling fix urllib 403s on the public JSON endpoint. HTTP 402 now triggers the OpenAI/public-JSON fallback chain when ScrapeCreators credits are exhausted.
|
||||
- **xAI**: empty or malformed responses now surface in `errors_by_source` instead of silently returning zero results.
|
||||
- **Windows**: process cleanup no longer crashes on `os.killpg`. POSIX-style secret-permission warnings skipped. Save-path footer uses forward slashes.
|
||||
- **Auth**: comma-separated `SCRAPECREATORS_API_KEY=key1,key2` rotation restored (accidentally dropped in v3.0.6).
|
||||
- **YouTube + HN**: SC YouTube + multi-token HN searches unblocked. Transcript-fetch ratio surfaced.
|
||||
- **HTTP**: retry budget expanded with exponential backoff on DNS failure. Parallel AI search aligned with current API schema.
|
||||
- **OpenClaw**: now works without a ScrapeCreators key. Poll-timing initialized once.
|
||||
### ELI5 mode
|
||||
|
||||
## Multi-harness reframe
|
||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations, just clearer. Say "eli5 off" to go back.
|
||||
|
||||
`AGENTS.md` is now the canonical project doc; `CLAUDE.md` points at it. The skill is positioned as a multi-harness Agent Skills package, not a Claude-Code-specific tool. SKILL.md's path resolution rewrote `SKILL_ROOT` → `SKILL_DIR`, removing ~80 lines of bash and fixing a real spec-vs-engine divergence bug.
|
||||
### 13+ sources
|
||||
|
||||
## Breaking change
|
||||
v3 adds Threads, Pinterest, Perplexity, Bluesky, and Parallel AI grounding to the existing Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and Web lineup. Perplexity Deep Research (`--deep-research`) gives you 50+ citation reports for serious investigation.
|
||||
|
||||
**`.codex-plugin/plugin.json` removed.** Codex native-plugin users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`. The `npx skills add` path now reaches every harness uniformly.
|
||||
### Per-author cap and entity disambiguation
|
||||
|
||||
Max 3 items per author prevents single-voice dominance. Synthesis trusts resolved handles over fuzzy keyword matches.
|
||||
|
||||
## Install
|
||||
|
||||
Any harness (recommended):
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -y
|
||||
```
|
||||
|
||||
Claude Code marketplace:
|
||||
Claude Code:
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
@@ -70,21 +58,29 @@ OpenClaw:
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
OpenAI Codex CLI: run `codex` from a checkout of this repo and v3's skill at `.agents/skills/last30days/SKILL.md` will be discovered automatically. Or copy `SKILL.md` to `~/.agents/skills/last30days/SKILL.md` for a global install.
|
||||
|
||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
## Contributors
|
||||
## v3 Community
|
||||
|
||||
First-time contributors whose fixes shipped in v3.3.0 (most via PR triage salvage — the fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
||||
v3 was shaped by community contributors whose PRs and issues inspired core features. Their code wasn't merged directly (v3 was a ground-up rewrite), but their ideas drove what shipped.
|
||||
|
||||
- Dave Morin — portable test-harness paths
|
||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
||||
- Eric Oberhofer — multi-key rotation restored
|
||||
- gujishh — Windows process cleanup
|
||||
- Franco Carballar — Reddit browser-like headers
|
||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
||||
- Kaustav Mishra — xAI error surfacing
|
||||
- [@thinkun](https://github.com/thinkun) — OpenClaw ScrapeCreators-key-optional fix
|
||||
Thanks to @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, and @hnshah. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list.
|
||||
|
||||
Plus every contributor who shipped one of the ~75 PRs merged this cycle. See [CHANGELOG.md](CHANGELOG.md) under `[3.3.0]` for the full PR list and `git log v3.2.0..v3.3.0` for the complete commit graph.
|
||||
Contributors who shaped the release itself:
|
||||
|
||||
- @Jah-yee (#153) surfaced the need for a real Codex CLI integration, which shipped in #219
|
||||
- @Cody-Coyote (#204) reported the marketplace validation bug that needed fixing before v3 could ship cleanly
|
||||
- @dannyshmueli pushed for v3 and Codex family support publicly on X
|
||||
|
||||
Full Added / Changed / Fixed detail lives in [CHANGELOG.md](CHANGELOG.md) under `[3.0.0]`.
|
||||
|
||||
## Earlier contributors
|
||||
|
||||
From the v1 and v2 lineage:
|
||||
|
||||
- [@galligan](https://github.com/galligan) for marketplace plugin inspiration
|
||||
- [@hutchins](https://x.com/hutchins) for pushing the YouTube feature
|
||||
|
||||
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
||||
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root)
|
||||
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||
#
|
||||
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
||||
# directory containing SKILL.md and the scripts/ runtime from skills/last30days.
|
||||
# See
|
||||
# directory containing SKILL.md and the scripts/ runtime. See
|
||||
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
@@ -18,7 +17,14 @@ fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/last30days.skill"
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD
|
||||
|
||||
# claude.ai's .skill bundle only needs the root SKILL.md + scripts/ runtime.
|
||||
# Claude Code needs skills/ and .claude-plugin/ in the git archive
|
||||
# (that's why they're NOT in .gitattributes export-ignore), but the .skill
|
||||
# bundle must strip them to keep a single canonical SKILL.md and stay under
|
||||
# the 200-file cap.
|
||||
zip -d "$OUT" "last30days/skills/*" "last30days/.claude-plugin/*" > /dev/null 2>&1 || true
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# A/B test runner: public release vs private beta
|
||||
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West"
|
||||
# Usage: bash scripts/compare.sh "Kanye West"
|
||||
#
|
||||
# Runs /last30days (public release) and /last30days-beta (private beta)
|
||||
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
||||
@@ -9,8 +9,8 @@
|
||||
set -e
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>"
|
||||
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose"
|
||||
echo "Usage: bash scripts/compare.sh <topic>"
|
||||
echo " Example: bash scripts/compare.sh Kevin Rose"
|
||||
exit 1
|
||||
fi
|
||||
TOPIC="$*"
|
||||
+3
-8
@@ -20,11 +20,9 @@ sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from lib import env as envlib
|
||||
from lib import schema
|
||||
from lib.providers import GEMINI_FLASH_LITE
|
||||
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
||||
|
||||
|
||||
@@ -44,7 +42,7 @@ def _load_default_topics() -> list[tuple[str, str]]:
|
||||
|
||||
DEFAULT_TOPICS = _load_default_topics()
|
||||
DEFAULT_SEARCH = ""
|
||||
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
|
||||
DEFAULT_JUDGE_MODEL = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||
|
||||
|
||||
@@ -309,10 +307,7 @@ def create_eval_env() -> dict[str, str]:
|
||||
|
||||
|
||||
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
||||
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py"
|
||||
if not engine.exists():
|
||||
engine = repo_dir / "scripts" / "last30days.py"
|
||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
||||
cmd = [sys.executable, "scripts/last30days.py", topic, "--emit=json"]
|
||||
if search:
|
||||
cmd.extend(["--search", search])
|
||||
if quick:
|
||||
@@ -1,12 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# ruff: noqa: E402
|
||||
"""last30days CLI."""
|
||||
"""last30days v3.0.0 CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -42,7 +41,7 @@ if os.name == "nt":
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from lib import env, html_render, pipeline, render, schema, ui
|
||||
from lib import env, pipeline, render, schema, ui
|
||||
|
||||
_child_pids: set[int] = set()
|
||||
_child_pids_lock = threading.Lock()
|
||||
@@ -63,10 +62,7 @@ def _cleanup_children() -> None:
|
||||
pids = list(_child_pids)
|
||||
for pid in pids:
|
||||
try:
|
||||
if hasattr(os, "killpg"):
|
||||
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||
else:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
continue
|
||||
|
||||
@@ -95,50 +91,30 @@ def slugify(value: str) -> str:
|
||||
return slug or "last30days"
|
||||
|
||||
|
||||
def save_output(
|
||||
report: schema.Report,
|
||||
emit: str,
|
||||
save_dir: str,
|
||||
suffix: str = "",
|
||||
synthesis_md: str | None = None,
|
||||
topic_override: str | None = None,
|
||||
rendered_content: str | None = None,
|
||||
) -> Path:
|
||||
def save_output(report: schema.Report, emit: str, save_dir: str, suffix: str = "") -> Path:
|
||||
from datetime import datetime
|
||||
path = Path(save_dir).expanduser().resolve()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
slug = slugify(topic_override or report.topic)
|
||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
||||
raw_label = "raw-html" if emit == "html" else "raw"
|
||||
slug = slugify(report.topic)
|
||||
extension = "json" if emit == "json" else "md"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
out_path = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
||||
out_path = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
if out_path.exists():
|
||||
out_path = path / f"{slug}-{raw_label}{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Markdown saves keep the complete debug artifact. JSON and HTML preserve
|
||||
# their requested wire format so file extensions match their content.
|
||||
if rendered_content is not None:
|
||||
content = rendered_content
|
||||
elif emit in {"json", "html"}:
|
||||
content = emit_output(report, emit, synthesis_md=synthesis_md)
|
||||
out_path = path / f"{slug}-raw{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Always save the FULL dump to disk (all items, all sources, transcripts).
|
||||
# Claude sees compact clusters via --emit=compact on stdout.
|
||||
# The saved file is the complete debug artifact.
|
||||
if emit == "json":
|
||||
content = emit_output(report, emit)
|
||||
else:
|
||||
content = render.render_full(report)
|
||||
out_path.write_text(content, encoding="utf-8")
|
||||
return out_path
|
||||
|
||||
|
||||
def emit_output(
|
||||
report: schema.Report,
|
||||
emit: str,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
def emit_output(report: schema.Report, emit: str, fun_level: str = "medium", save_path: str | None = None) -> str:
|
||||
if emit == "json":
|
||||
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
|
||||
if emit == "html":
|
||||
return html_render.render_html(
|
||||
report, fun_level=fun_level, save_path=save_path, synthesis_md=synthesis_md,
|
||||
)
|
||||
if emit in {"compact", "md"}:
|
||||
return render.render_compact(report, fun_level=fun_level, save_path=save_path)
|
||||
if emit == "context":
|
||||
@@ -151,7 +127,6 @@ def emit_comparison_output(
|
||||
emit: str,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
synthesis_md: str | None = None,
|
||||
) -> str:
|
||||
if emit == "json":
|
||||
payload = {
|
||||
@@ -163,13 +138,6 @@ def emit_comparison_output(
|
||||
],
|
||||
}
|
||||
return json.dumps(payload, indent=2, sort_keys=True)
|
||||
if emit == "html":
|
||||
return html_render.render_html_comparison(
|
||||
entity_reports,
|
||||
fun_level=fun_level,
|
||||
save_path=save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
if emit in {"compact", "md"}:
|
||||
return render.render_comparison_multi(
|
||||
entity_reports, fun_level=fun_level, save_path=save_path,
|
||||
@@ -179,10 +147,6 @@ def emit_comparison_output(
|
||||
raise SystemExit(f"Unsupported emit mode: {emit}")
|
||||
|
||||
|
||||
def comparison_topic(entity_reports: list[tuple[str, schema.Report]]) -> str:
|
||||
return " vs ".join(label for label, _ in entity_reports)
|
||||
|
||||
|
||||
def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str) -> str:
|
||||
"""Compute the user-friendly save path string that will be shown in the footer.
|
||||
|
||||
@@ -192,24 +156,15 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
||||
from pathlib import Path as _Path
|
||||
path = _Path(save_dir).expanduser().resolve()
|
||||
slug = slugify(topic)
|
||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
||||
raw_label = "raw-html" if emit == "html" else "raw"
|
||||
extension = "json" if emit == "json" else "md"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
raw = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
||||
raw = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
try:
|
||||
home = _Path.home().resolve()
|
||||
relative = raw.relative_to(home)
|
||||
return f"~/{relative.as_posix()}"
|
||||
return f"~/{relative}"
|
||||
except ValueError:
|
||||
return raw.as_posix()
|
||||
|
||||
|
||||
def read_synthesis_file(path: str) -> str:
|
||||
try:
|
||||
return Path(path).expanduser().read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
sys.stderr.write(f"[last30days] Cannot read --synthesis-file: {exc}\n")
|
||||
raise SystemExit(2)
|
||||
return str(raw)
|
||||
|
||||
|
||||
def persist_report(report: schema.Report) -> dict[str, int]:
|
||||
@@ -238,7 +193,7 @@ def persist_report(report: schema.Report) -> dict[str, int]:
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
|
||||
parser.add_argument("topic", nargs="*", help="Research topic")
|
||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md", "html"])
|
||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md"])
|
||||
parser.add_argument("--search", help="Comma-separated source list")
|
||||
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
|
||||
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
|
||||
@@ -246,7 +201,6 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
||||
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
||||
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
||||
parser.add_argument("--synthesis-file", help="Markdown synthesis to embed in --emit=html output")
|
||||
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
|
||||
parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
|
||||
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
|
||||
@@ -391,7 +345,7 @@ def subrun_kwargs_for(
|
||||
|
||||
subreddits = _choose("subreddits", "subreddits")
|
||||
if isinstance(subreddits, list):
|
||||
subreddits = [s.strip().removeprefix("r/") for s in subreddits if s.strip()] or None
|
||||
subreddits = [s.strip().lstrip("r/") for s in subreddits if s.strip()] or None
|
||||
|
||||
x_related = plan_entry.get("x_related")
|
||||
if isinstance(x_related, list):
|
||||
@@ -535,24 +489,6 @@ def _show_runtime_ui(
|
||||
progress.show_promo(promo, diag=diag)
|
||||
|
||||
|
||||
def _write_last_run(topic: str, report: "schema.Report") -> None:
|
||||
try:
|
||||
if env.CONFIG_DIR is None:
|
||||
return
|
||||
target = env.CONFIG_DIR
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
counts = {source: len(items) for source, items in report.items_by_source.items()}
|
||||
payload = {
|
||||
"topic": topic,
|
||||
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"sources": counts,
|
||||
"total": sum(counts.values()),
|
||||
}
|
||||
(target / "last-run.json").write_text(json.dumps(payload, indent=2))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
||||
@@ -563,13 +499,6 @@ def main() -> int:
|
||||
|
||||
config = env.get_config()
|
||||
|
||||
# Surface SSH-routing config as an env var so library modules (e.g.
|
||||
# youtube_yt) can read it without taking a config dependency. This
|
||||
# routes yt-dlp through `ssh <host>` to bypass YouTube's bot-wall on
|
||||
# datacenter IPs (see lib/youtube_yt.py for details).
|
||||
if config.get("LAST30DAYS_YOUTUBE_SSH_HOST") and "LAST30DAYS_YOUTUBE_SSH_HOST" not in os.environ:
|
||||
os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"]
|
||||
|
||||
# Handle setup subcommand
|
||||
topic = " ".join(args.topic).strip()
|
||||
if topic.lower() == "setup":
|
||||
@@ -608,13 +537,6 @@ def main() -> int:
|
||||
parser.print_usage(sys.stderr)
|
||||
return 2
|
||||
|
||||
synthesis_md = None
|
||||
if args.synthesis_file:
|
||||
if args.emit == "html":
|
||||
synthesis_md = read_synthesis_file(args.synthesis_file)
|
||||
else:
|
||||
sys.stderr.write("[last30days] Warning: --synthesis-file is only used with --emit=html; ignoring.\n")
|
||||
|
||||
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
|
||||
from lib import preflight
|
||||
refuse_msg = preflight.check_class_1_trap(topic)
|
||||
@@ -628,7 +550,7 @@ def main() -> int:
|
||||
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
||||
try:
|
||||
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
||||
subreddits = [s.strip().removeprefix("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||
subreddits = [s.strip().lstrip("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
||||
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
||||
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
||||
@@ -647,7 +569,6 @@ def main() -> int:
|
||||
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
||||
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
||||
# without WebSearch (OpenClaw, Codex, raw CLI).
|
||||
repos_from_auto_resolve = False
|
||||
if args.auto_resolve and not external_plan:
|
||||
from lib import resolve
|
||||
resolution = resolve.auto_resolve(topic, config)
|
||||
@@ -662,9 +583,6 @@ def main() -> int:
|
||||
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
||||
if resolution.get("github_repos") and not args.github_repo:
|
||||
args.github_repo = ",".join(resolution["github_repos"])
|
||||
# auto_resolve already canonicalized via canonicalize_github_repos(cap=5);
|
||||
# mark so we don't re-canonicalize below and clobber its relevance order.
|
||||
repos_from_auto_resolve = True
|
||||
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
||||
if resolution.get("context"):
|
||||
# Inject context into external_plan metadata for the planner to use
|
||||
@@ -677,20 +595,6 @@ def main() -> int:
|
||||
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
||||
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
||||
|
||||
# Only canonicalize when repos came from a user-supplied --github-repo flag.
|
||||
# When repos_from_auto_resolve is True, auto_resolve already ran
|
||||
# canonicalize_github_repos(cap=5) and ranked by relevance; re-running here
|
||||
# with cap=None can re-sort by topic-slug match and lose that ordering.
|
||||
if github_repos and not repos_from_auto_resolve:
|
||||
from lib import resolve as resolve_lib
|
||||
original_github_repos = github_repos[:]
|
||||
github_repos = resolve_lib.canonicalize_github_repos(topic, github_repos, cap=None)
|
||||
if github_repos != original_github_repos:
|
||||
sys.stderr.write(
|
||||
"[GitHub] Canonicalized repos: "
|
||||
f"{','.join(original_github_repos)} -> {','.join(github_repos)}\n"
|
||||
)
|
||||
|
||||
# --deep-research: auto-enable perplexity source and set deep flag
|
||||
if args.deep_research:
|
||||
if not config.get("OPENROUTER_API_KEY"):
|
||||
@@ -910,18 +814,7 @@ def main() -> int:
|
||||
report, progress, diag,
|
||||
suppress_web_promo=bool(external_plan or comp_plan),
|
||||
)
|
||||
_write_last_run(topic, report)
|
||||
# LAST30DAYS_STORE env var = persistence default-on. Read both os.environ
|
||||
# (for shell-exported users) and config (for users who set it in
|
||||
# ~/.config/last30days/.env, which env.py loads but does not propagate
|
||||
# to os.environ). Mirrors the LAST30DAYS_DEBUG / LAST30DAYS_SKIP_PREFLIGHT
|
||||
# convention; env-var or config wins, with `--store` flag still working.
|
||||
_store_env = (
|
||||
os.environ.get("LAST30DAYS_STORE")
|
||||
or config.get("LAST30DAYS_STORE")
|
||||
or ""
|
||||
).lower()
|
||||
if args.store or _store_env in ("1", "true", "yes"):
|
||||
if args.store:
|
||||
counts = persist_report(report)
|
||||
sys.stderr.write(
|
||||
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
||||
@@ -931,32 +824,7 @@ def main() -> int:
|
||||
# Show quality nudge if applicable
|
||||
try:
|
||||
from lib import quality_nudge
|
||||
# Populate transcript-fetch ratio so quality_nudge can detect the
|
||||
# degraded-YouTube failure mode (videos returned but transcripts
|
||||
# silently failed - typically a stale yt-dlp binary).
|
||||
youtube_items = report.items_by_source.get("youtube") or []
|
||||
instagram_items = report.items_by_source.get("instagram") or []
|
||||
research_results = {
|
||||
"youtube_videos_count": len(youtube_items),
|
||||
"youtube_transcripts_count": sum(
|
||||
1 for it in youtube_items
|
||||
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
||||
),
|
||||
"youtube_error": report.errors_by_source.get("youtube"),
|
||||
"x_error": report.errors_by_source.get("x"),
|
||||
# Captions-disabled videos can never produce a transcript regardless
|
||||
# of yt-dlp version; subtract them from the degraded-ratio
|
||||
# denominator so a single uploader-disabled video does not trip the
|
||||
# "stale yt-dlp" nudge.
|
||||
"youtube_captions_disabled_count": sum(
|
||||
1 for it in youtube_items if it.metadata.get("captions_disabled")
|
||||
),
|
||||
# Track Instagram returned-zero-items so quality_nudge can detect
|
||||
# the silent-failure case (SC configured but the v2 reels endpoint
|
||||
# 500'd through both the original query and the hashtag retry).
|
||||
"instagram_items_count": len(instagram_items),
|
||||
}
|
||||
quality = quality_nudge.compute_quality_score(config, research_results)
|
||||
quality = quality_nudge.compute_quality_score(config, {})
|
||||
if quality.get("nudge_text"):
|
||||
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
||||
sys.stderr.flush()
|
||||
@@ -964,15 +832,10 @@ def main() -> int:
|
||||
pass
|
||||
|
||||
fun_level = config.get("FUN_LEVEL", "medium").lower()
|
||||
# Comparison HTML is the one case where the saved file's title and content
|
||||
# have to be overridden away from the leading entity's report. Compute the
|
||||
# gate once so the footer-display and save-output paths can't disagree.
|
||||
is_comparison_html = bool(entity_reports) and args.emit == "html"
|
||||
footer_save_path = None
|
||||
if args.save_dir:
|
||||
save_topic_for_display = comparison_topic(entity_reports) if is_comparison_html else report.topic
|
||||
footer_save_path = compute_save_path_display(
|
||||
args.save_dir, save_topic_for_display, args.save_suffix or "", args.emit
|
||||
args.save_dir, report.topic, args.save_suffix or "", args.emit
|
||||
)
|
||||
|
||||
# Signal to render_compact whether pre-research flags were supplied.
|
||||
@@ -991,31 +854,15 @@ def main() -> int:
|
||||
|
||||
if entity_reports:
|
||||
rendered = emit_comparison_output(
|
||||
entity_reports,
|
||||
args.emit,
|
||||
fun_level=fun_level,
|
||||
save_path=footer_save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
entity_reports, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||
)
|
||||
else:
|
||||
rendered = emit_output(
|
||||
report,
|
||||
args.emit,
|
||||
fun_level=fun_level,
|
||||
save_path=footer_save_path,
|
||||
synthesis_md=synthesis_md,
|
||||
report, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||
)
|
||||
if args.save_dir:
|
||||
# Save the main topic's raw file (single-entity or comparison main).
|
||||
save_path = save_output(
|
||||
report,
|
||||
args.emit,
|
||||
args.save_dir,
|
||||
suffix=args.save_suffix or "",
|
||||
synthesis_md=synthesis_md,
|
||||
topic_override=comparison_topic(entity_reports) if is_comparison_html else None,
|
||||
rendered_content=rendered if is_comparison_html else None,
|
||||
)
|
||||
save_path = save_output(report, args.emit, args.save_dir, suffix=args.save_suffix or "")
|
||||
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
||||
# Competitor / vs-mode: also save a per-entity raw file for each peer.
|
||||
# Matches historical vs-mode behavior (N passes → N save files).
|
||||
@@ -1024,7 +871,6 @@ def main() -> int:
|
||||
peer_path = save_output(
|
||||
entity_report, args.emit, args.save_dir,
|
||||
suffix=args.save_suffix or "",
|
||||
synthesis_md=synthesis_md,
|
||||
)
|
||||
sys.stderr.write(f"[last30days] Saved output to {peer_path}\n")
|
||||
sys.stderr.flush()
|
||||
@@ -7,22 +7,18 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log, subproc
|
||||
from . import http, log
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
# How many times to retry the bird-search subprocess when stdout is non-JSON
|
||||
# (typically an HTML anti-bot interstitial from Twitter's edge).
|
||||
MAX_JSON_DECODE_RETRIES = 2
|
||||
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
|
||||
|
||||
|
||||
def _first_of(*values):
|
||||
"""Return first value that is not None."""
|
||||
@@ -154,14 +150,16 @@ def get_bird_status() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"""Invoke the vendored bird-search.mjs subprocess once.
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
Returns (result, error_dict). If error_dict is non-None, treat it as the
|
||||
final result and do not retry — those errors are terminal (timeout,
|
||||
spawn failure). If error_dict is None, the subprocess ran to completion
|
||||
and `result` is the SubprocResult; the caller decides whether to retry
|
||||
based on the result.stdout content.
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
@@ -170,109 +168,62 @@ def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"--json",
|
||||
]
|
||||
|
||||
pid_holder: list[int] = []
|
||||
# Use process groups for clean cleanup on timeout/kill
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
|
||||
def _register(pid: int) -> None:
|
||||
pid_holder.append(pid)
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
# Register for cleanup tracking (if available)
|
||||
try:
|
||||
from last30days import register_child_pid
|
||||
register_child_pid(pid)
|
||||
from last30days import register_child_pid, unregister_child_pid
|
||||
register_child_pid(proc.pid)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(
|
||||
cmd,
|
||||
timeout=timeout,
|
||||
env=_subprocess_env(),
|
||||
on_pid=_register,
|
||||
)
|
||||
except subproc.SubprocTimeout:
|
||||
return None, {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
except Exception as e:
|
||||
return None, {"error": str(e), "items": []}
|
||||
finally:
|
||||
if pid_holder:
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
# Kill the entire process group
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
finally:
|
||||
try:
|
||||
from last30days import unregister_child_pid
|
||||
unregister_child_pid(pid_holder[0])
|
||||
unregister_child_pid(proc.pid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result, None
|
||||
|
||||
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
Retries the subprocess on JSON-decode failure (typically a Twitter
|
||||
anti-bot HTML interstitial in stdout) up to MAX_JSON_DECODE_RETRIES
|
||||
times with JSON_DECODE_RETRY_DELAY seconds between attempts. Terminal
|
||||
errors (subprocess timeout, non-zero return code) are returned
|
||||
immediately without retry.
|
||||
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds (per attempt)
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
last_decode_error: Optional[str] = None
|
||||
|
||||
for attempt in range(MAX_JSON_DECODE_RETRIES):
|
||||
result, terminal_error = _invoke_bird_subprocess(query, count, timeout)
|
||||
if terminal_error is not None:
|
||||
return terminal_error
|
||||
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
if proc.returncode != 0:
|
||||
error = stderr.strip() if stderr else "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
output = result.stdout.strip()
|
||||
output = stdout.strip() if stdout else ""
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError as e:
|
||||
# Twitter's edge sometimes serves an HTML anti-bot interstitial
|
||||
# in place of JSON. Tag the failure shape so it's distinguishable
|
||||
# from "no results" in logs, then retry the subprocess.
|
||||
looks_html = output.lstrip().lower().startswith(("<!doctype", "<html", "<"))
|
||||
attempt_num = attempt + 1
|
||||
log_msg = (
|
||||
f"Bird search returned non-JSON stdout "
|
||||
f"(looks_html={looks_html}, attempt {attempt_num}/{MAX_JSON_DECODE_RETRIES}, "
|
||||
f"first 80 chars: {output[:80]!r})"
|
||||
)
|
||||
last_decode_error = str(e)
|
||||
if attempt_num < MAX_JSON_DECODE_RETRIES:
|
||||
log.source_log(
|
||||
"X/bird",
|
||||
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
|
||||
)
|
||||
time.sleep(JSON_DECODE_RETRY_DELAY)
|
||||
continue
|
||||
log.source_log("X/bird", log_msg)
|
||||
return {
|
||||
"error": (
|
||||
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
|
||||
f"(likely Twitter anti-bot interstitial): {e}"
|
||||
),
|
||||
"items": [],
|
||||
}
|
||||
|
||||
parsed = json.loads(output)
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
# Defensive fallthrough — loop should always return above.
|
||||
return {
|
||||
"error": f"Bird search exhausted retries: {last_decode_error}",
|
||||
"items": [],
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "items": []}
|
||||
|
||||
|
||||
def search_x(
|
||||
@@ -379,29 +330,47 @@ def search_handles(
|
||||
"--json",
|
||||
]
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
||||
except subproc.SubprocTimeout:
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
except OSError as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
if result.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return []
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
|
||||
if proc.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||
return []
|
||||
|
||||
output = (stdout or "").strip()
|
||||
if not output:
|
||||
return []
|
||||
|
||||
response = json.loads(output)
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
return []
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
"""Bluesky search via AT Protocol (requires app password).
|
||||
|
||||
Uses bsky.social for auth and api.bsky.app for post search (the canonical
|
||||
authenticated AppView). The previous default `public.api.bsky.app` is the
|
||||
unauthenticated public mirror, which BunnyCDN now blocks for searchPosts
|
||||
regardless of auth header (verified 2026-05-04). Override the search host
|
||||
via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again.
|
||||
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are
|
||||
19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords.
|
||||
The createSession endpoint accepts main-account passwords too, but they're
|
||||
bad hygiene (no scope, can't revoke individually).
|
||||
Uses bsky.social for auth and public.api.bsky.app for post search.
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
@@ -23,64 +14,7 @@ from typing import Any, Dict, List, Optional
|
||||
from . import http, log
|
||||
|
||||
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
||||
_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
|
||||
|
||||
|
||||
def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""Resolve the Bluesky search URL with BSKY_SEARCH_HOST override.
|
||||
|
||||
Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or
|
||||
.env file. The project's env.py loads .env into config but not into
|
||||
os.environ, so check both — same hybrid pattern as last30days.py for
|
||||
LAST30DAYS_STORE.
|
||||
|
||||
Hardens user-supplied host values against three common mis-configurations:
|
||||
whitespace (e.g. " api.bsky.app "), embedded path components (e.g.
|
||||
"api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and
|
||||
embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these
|
||||
we log a warning and fall back to the default rather than building an
|
||||
invalid URL with an opaque downstream error.
|
||||
"""
|
||||
config = config or {}
|
||||
raw = (
|
||||
os.environ.get("BSKY_SEARCH_HOST")
|
||||
or config.get("BSKY_SEARCH_HOST")
|
||||
or _DEFAULT_BSKY_SEARCH_HOST
|
||||
)
|
||||
host = raw.strip().rstrip("/")
|
||||
# Strip embedded scheme so users who paste full URLs do not break the f-string.
|
||||
for prefix in ("https://", "http://"):
|
||||
if host.lower().startswith(prefix):
|
||||
host = host[len(prefix):]
|
||||
break
|
||||
if not host or "/" in host or " " in host:
|
||||
# Embedded path or whitespace remains — don't trust it. Default + log.
|
||||
if raw != _DEFAULT_BSKY_SEARCH_HOST:
|
||||
_log(
|
||||
f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; "
|
||||
f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}"
|
||||
)
|
||||
host = _DEFAULT_BSKY_SEARCH_HOST
|
||||
return f"https://{host}/xrpc/app.bsky.feed.searchPosts"
|
||||
|
||||
|
||||
# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric
|
||||
# with three hyphens at fixed positions).
|
||||
_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$")
|
||||
|
||||
|
||||
def _validate_app_password_format(value) -> bool:
|
||||
"""Return True if value matches Bluesky's 19-char app-password format.
|
||||
|
||||
False for non-strings (None, int, list) so callers passing config dict
|
||||
values directly don't crash. Detect-but-not-gate: the createSession
|
||||
endpoint also accepts main-account passwords, so failing this check is
|
||||
a hygiene smell, not a hard error.
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return bool(_APP_PASSWORD_RE.fullmatch(value))
|
||||
|
||||
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
|
||||
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 15,
|
||||
@@ -210,20 +144,6 @@ def search_bluesky(
|
||||
if not handle or not app_password:
|
||||
return {"posts": [], "error": "Bluesky credentials not configured"}
|
||||
|
||||
# One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password
|
||||
# form. createSession accepts main-account passwords too — but main
|
||||
# passwords have no scope (full account access), can't be revoked
|
||||
# individually, and rotating them breaks every service that holds them.
|
||||
# We warn but do not gate, matching the project's detect-don't-block
|
||||
# philosophy elsewhere.
|
||||
if not _validate_app_password_format(app_password):
|
||||
_log(
|
||||
"BSKY_APP_PASSWORD does not look like an app password "
|
||||
"(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main "
|
||||
"account password — those work but are bad hygiene. Generate "
|
||||
"an app password at https://bsky.app/settings/app-passwords"
|
||||
)
|
||||
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
@@ -235,7 +155,7 @@ def search_bluesky(
|
||||
"limit": str(min(count, 100)),
|
||||
"sort": "top",
|
||||
}
|
||||
url = f"{_resolve_search_url(config)}?{urlencode(params)}"
|
||||
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
|
||||
|
||||
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
token = _create_session(handle, app_password)
|
||||
@@ -1,12 +1,9 @@
|
||||
"""Chrome and Brave cookie extraction for macOS.
|
||||
"""Chrome cookie extraction for macOS.
|
||||
|
||||
Extracts cookies from Chromium-based browser SQLite databases using only
|
||||
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
|
||||
dependencies.
|
||||
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
|
||||
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
|
||||
|
||||
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
Chrome and Brave share the same algorithm; only the DB path and Keychain
|
||||
service name differ.
|
||||
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||
"""
|
||||
|
||||
@@ -21,11 +18,10 @@ from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cookie DB locations on macOS
|
||||
# Chrome cookie DB location on macOS
|
||||
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||
BRAVE_BASE_DIR = Path.home() / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser"
|
||||
|
||||
# Chromium v10 encryption constants (shared by Chrome and Brave)
|
||||
# Chrome v10 encryption constants
|
||||
CHROME_SALT = b"saltysalt"
|
||||
CHROME_PBKDF2_ITERATIONS = 1003
|
||||
CHROME_KEY_LENGTH = 16
|
||||
@@ -33,8 +29,8 @@ CHROME_KEY_LENGTH = 16
|
||||
CHROME_IV_HEX = "20" * 16
|
||||
|
||||
|
||||
def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||
"""Retrieve the encryption passphrase for a Chromium-based browser from macOS Keychain.
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
|
||||
|
||||
Calls `security find-generic-password` which may trigger a system dialog
|
||||
on first access.
|
||||
@@ -43,34 +39,30 @@ def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["security", "find-generic-password", "-w", "-s", service_name],
|
||||
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.info("%s Keychain access denied or browser not installed: %s", service_name, result.stderr.strip())
|
||||
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
|
||||
return None
|
||||
passphrase = result.stdout.strip()
|
||||
if not passphrase:
|
||||
logger.info("%s Keychain returned empty passphrase", service_name)
|
||||
logger.info("Chrome Keychain returned empty passphrase")
|
||||
return None
|
||||
return passphrase.encode("utf-8")
|
||||
except FileNotFoundError:
|
||||
logger.info("'security' command not found — not on macOS?")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.info("%s Keychain access timed out", service_name)
|
||||
logger.info("Chrome Keychain access timed out")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Failed to get %s encryption key: %s", service_name, e)
|
||||
logger.info("Failed to get Chrome encryption key: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
return _get_chromium_encryption_key("Chrome Safe Storage")
|
||||
|
||||
|
||||
def _derive_aes_key(passphrase: bytes) -> bytes:
|
||||
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
||||
return hashlib.pbkdf2_hmac(
|
||||
@@ -173,42 +165,36 @@ def _get_db_version(cursor: sqlite3.Cursor) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_chromium_cookies_macos(
|
||||
db_path: Path,
|
||||
keychain_service: str,
|
||||
domain: str,
|
||||
cookie_names: list[str],
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from any Chromium-based browser on macOS.
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS.
|
||||
|
||||
Copies the locked Cookies database to a temp file, reads specified cookies,
|
||||
and decrypts v10-encrypted values using the Keychain-stored key.
|
||||
|
||||
Args:
|
||||
db_path: Path to the browser's Cookies SQLite file.
|
||||
keychain_service: macOS Keychain service name (e.g. "Chrome Safe Storage").
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
|
||||
cookie_names: List of cookie names to extract
|
||||
|
||||
Returns:
|
||||
Dict mapping cookie name to decrypted value, or None on failure.
|
||||
Only includes cookies that were successfully found and decrypted.
|
||||
"""
|
||||
if not db_path.exists():
|
||||
logger.info("%s cookies database not found at %s", keychain_service, db_path)
|
||||
if not CHROME_COOKIES_DB.exists():
|
||||
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
|
||||
return None
|
||||
|
||||
passphrase = _get_chromium_encryption_key(keychain_service)
|
||||
# Get encryption key from Keychain
|
||||
passphrase = _get_chrome_encryption_key()
|
||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||
|
||||
# Copy DB to temp file (browser locks the original while running)
|
||||
# Copy DB to temp file (Chrome locks the original)
|
||||
tmp_fd = None
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(db_path), tmp_path)
|
||||
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
|
||||
except Exception as e:
|
||||
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
|
||||
logger.info("Failed to copy Chrome cookies database: %s", e)
|
||||
if tmp_path:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
@@ -225,22 +211,26 @@ def _extract_chromium_cookies_macos(
|
||||
cursor = conn.cursor()
|
||||
|
||||
db_version = _get_db_version(cursor)
|
||||
logger.debug("%s cookie DB version: %d", keychain_service, db_version)
|
||||
logger.debug("Chrome cookie DB version: %d", db_version)
|
||||
|
||||
# Build query with placeholders for cookie names
|
||||
placeholders = ",".join("?" for _ in cookie_names)
|
||||
query = (
|
||||
f"SELECT name, value, encrypted_value FROM cookies "
|
||||
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
||||
)
|
||||
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
|
||||
params = [f"%{domain}"] + list(cookie_names)
|
||||
cursor.execute(query, params)
|
||||
|
||||
results: dict[str, str] = {}
|
||||
for name, value, encrypted_value in cursor.fetchall():
|
||||
# Prefer unencrypted value if present
|
||||
if value:
|
||||
results[name] = value
|
||||
continue
|
||||
|
||||
# Handle encrypted value
|
||||
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||
if aes_key is None:
|
||||
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||
@@ -251,72 +241,25 @@ def _extract_chromium_cookies_macos(
|
||||
else:
|
||||
logger.debug("Failed to decrypt cookie %s", name)
|
||||
elif encrypted_value:
|
||||
# Unknown encryption version
|
||||
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
||||
|
||||
conn.close()
|
||||
|
||||
if not results:
|
||||
logger.info("No matching cookies found in %s for domain %s", keychain_service, domain)
|
||||
logger.info("No matching cookies found in Chrome for domain %s", domain)
|
||||
return None
|
||||
|
||||
return results
|
||||
|
||||
except sqlite3.Error as e:
|
||||
logger.info("Failed to read %s cookies database: %s", keychain_service, e)
|
||||
logger.info("Failed to read Chrome cookies database: %s", e)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Unexpected error reading %s cookies: %s", keychain_service, e)
|
||||
logger.info("Unexpected error reading Chrome cookies: %s", e)
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS."""
|
||||
return _extract_chromium_cookies_macos(
|
||||
CHROME_COOKIES_DB, "Chrome Safe Storage", domain, cookie_names
|
||||
)
|
||||
|
||||
|
||||
def _find_brave_cookies_db() -> Optional[Path]:
|
||||
"""Find Brave's Cookies database on macOS.
|
||||
|
||||
Tries the Default profile first, then scans numbered Profile directories
|
||||
by most-recently-modified. Brave creates extra profiles as "Profile 1",
|
||||
"Profile 2", etc. alongside Default; the most recently used one is the
|
||||
likeliest to hold current cookies. Lexicographic sort would visit
|
||||
"Profile 10" before "Profile 2", which can return the wrong profile.
|
||||
"""
|
||||
default = BRAVE_BASE_DIR / "Default" / "Cookies"
|
||||
if default.exists():
|
||||
return default
|
||||
|
||||
try:
|
||||
candidates = [
|
||||
child for child in BRAVE_BASE_DIR.iterdir()
|
||||
if child.is_dir() and child.name.startswith("Profile ")
|
||||
]
|
||||
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
||||
candidate = child / "Cookies"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Brave on macOS.
|
||||
|
||||
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
|
||||
path and Keychain service name differ.
|
||||
"""
|
||||
db_path = _find_brave_cookies_db()
|
||||
if db_path is None:
|
||||
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
|
||||
return None
|
||||
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Browser cookie extraction for last30days.
|
||||
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Brave, Safari)
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
|
||||
to enable zero-config authentication for services like X/Twitter.
|
||||
|
||||
Only uses Python stdlib — no external dependencies.
|
||||
@@ -255,29 +255,6 @@ def extract_chrome_cookies(
|
||||
return None
|
||||
|
||||
|
||||
def extract_brave_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Brave for the given domain and cookie names.
|
||||
|
||||
macOS only — Brave uses the same v10 AES-128-CBC encryption as Chrome,
|
||||
with a different DB path and Keychain service name ("Brave Safe Storage").
|
||||
Tries the Default profile first, then scans numbered Profile directories.
|
||||
|
||||
Returns:
|
||||
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||
"""
|
||||
if platform.system() != "Darwin":
|
||||
logger.debug("Brave cookie extraction only supported on macOS")
|
||||
return None
|
||||
try:
|
||||
from .chrome_cookies import extract_brave_cookies_macos
|
||||
return extract_brave_cookies_macos(domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("Brave cookie extraction failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def extract_safari_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
@@ -305,9 +282,9 @@ def extract_cookies(
|
||||
"""Extract cookies from the specified browser.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
'auto' tries browsers in platform-appropriate order:
|
||||
- macOS: Chrome -> Brave -> Firefox -> Safari
|
||||
- macOS: Chrome -> Firefox -> Safari
|
||||
- Linux: Firefox only
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
@@ -356,7 +333,7 @@ def extract_cookies_with_source(
|
||||
so callers can track the source.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
|
||||
@@ -367,7 +344,6 @@ def extract_cookies_with_source(
|
||||
extractors = {
|
||||
"firefox": extract_firefox_cookies,
|
||||
"chrome": extract_chrome_cookies,
|
||||
"brave": extract_brave_cookies,
|
||||
"safari": extract_safari_cookies,
|
||||
}
|
||||
|
||||
@@ -384,7 +360,7 @@ def extract_cookies_with_source(
|
||||
# Auto mode: try browsers in platform-appropriate order
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
order = ["chrome", "brave", "firefox", "safari"]
|
||||
order = ["chrome", "firefox", "safari"]
|
||||
elif system == "Linux":
|
||||
order = ["firefox"]
|
||||
else:
|
||||
@@ -39,14 +39,11 @@ def normalize_text(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
|
||||
if len(norm) < n:
|
||||
return {norm} if norm else set()
|
||||
return {norm[index:index + n] for index in range(len(norm) - n + 1)}
|
||||
|
||||
|
||||
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
||||
return _ngrams_of_normalized(normalize_text(text), n)
|
||||
text = normalize_text(text)
|
||||
if len(text) < n:
|
||||
return {text} if text else set()
|
||||
return {text[index:index + n] for index in range(len(text) - n + 1)}
|
||||
|
||||
|
||||
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
||||
@@ -93,7 +90,7 @@ class _PreparedText:
|
||||
|
||||
def __init__(self, raw: str) -> None:
|
||||
norm = normalize_text(raw)
|
||||
self.ngrams = _ngrams_of_normalized(norm)
|
||||
self.ngrams = get_ngrams(norm) if norm else set()
|
||||
self.tokens = _tokenize(norm)
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ def _extract_subreddits(reddit_items: List[Dict[str, Any]]) -> List[str]:
|
||||
|
||||
for item in reddit_items:
|
||||
# Primary subreddit
|
||||
sub = item.get("subreddit", "").strip().removeprefix("r/")
|
||||
sub = item.get("subreddit", "").strip().lstrip("r/")
|
||||
if sub:
|
||||
sub_counts[sub] += 1
|
||||
|
||||
@@ -29,23 +29,6 @@ else:
|
||||
|
||||
CODEX_AUTH_FILE = Path(os.environ.get("CODEX_AUTH_FILE", str(Path.home() / ".codex" / "auth.json")))
|
||||
|
||||
# macOS Keychain integration: items stored with this service prefix are picked
|
||||
# up automatically on Darwin as the lowest-priority credential source.
|
||||
# Example: `security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."`.
|
||||
KEYCHAIN_SERVICE_PREFIX = "last30days-"
|
||||
|
||||
# Single source of truth for which credentials the Keychain loader looks up.
|
||||
# The setup-keychain.sh helper mirrors this list and is held in sync via
|
||||
# tests/test_env_keychain.py::test_keychain_keys_match_setup_script.
|
||||
KEYCHAIN_KEYS = (
|
||||
"OPENAI_API_KEY", "XAI_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY", "SCRAPECREATORS_API_KEY", "APIFY_API_TOKEN",
|
||||
"AUTH_TOKEN", "CT0", "BSKY_HANDLE", "BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN", "BRAVE_API_KEY", "EXA_API_KEY", "SERPER_API_KEY",
|
||||
"OPENROUTER_API_KEY", "PARALLEL_API_KEY", "XQUIK_API_KEY",
|
||||
"XIAOHONGSHU_API_BASE",
|
||||
)
|
||||
|
||||
AuthSource = Literal["api_key", "codex", "none"]
|
||||
AuthStatus = Literal["ok", "missing", "expired", "missing_account_id"]
|
||||
|
||||
@@ -70,10 +53,6 @@ class OpenAIAuth:
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file has overly permissive permissions."""
|
||||
if os.name == "nt":
|
||||
# Windows reports synthesized POSIX mode bits that do not reflect NTFS ACLs.
|
||||
return
|
||||
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
# Check if group or other can read (bits 0o044)
|
||||
@@ -112,46 +91,6 @@ def load_env_file(path: Path) -> dict[str, str]:
|
||||
return env
|
||||
|
||||
|
||||
def _load_keychain(keys: list[str]) -> dict[str, str]:
|
||||
"""Load credentials from macOS Keychain (no-op on other platforms).
|
||||
|
||||
Each key is looked up as a generic password with service name
|
||||
``f"{KEYCHAIN_SERVICE_PREFIX}{key}"`` for the current user. Missing items
|
||||
and lookup failures are silent — Keychain is the lowest-priority source
|
||||
and is meant to be additive over `.env` files and process environment.
|
||||
"""
|
||||
import platform
|
||||
if platform.system() != "Darwin":
|
||||
return {}
|
||||
|
||||
import shutil
|
||||
security = shutil.which("security")
|
||||
if not security:
|
||||
return {}
|
||||
|
||||
import subprocess
|
||||
import pwd
|
||||
# USER can be unset under sudo, in Docker without --env USER, or in some CI
|
||||
# runners; fall back to the OS user record so lookups still match items
|
||||
# stored by setup-keychain.sh (which uses $USER).
|
||||
user = os.environ.get("USER") or pwd.getpwuid(os.getuid()).pw_name
|
||||
env: dict[str, str] = {}
|
||||
for key in keys:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[security, "find-generic-password",
|
||||
"-a", user,
|
||||
"-s", f"{KEYCHAIN_SERVICE_PREFIX}{key}",
|
||||
"-w"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
continue
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
env[key] = result.stdout.strip()
|
||||
return env
|
||||
|
||||
|
||||
def _decode_jwt_payload(token: str) -> dict[str, Any] | None:
|
||||
"""Decode JWT payload without verification."""
|
||||
try:
|
||||
@@ -275,7 +214,6 @@ def get_config() -> dict[str, Any]:
|
||||
1. Environment variables (os.environ)
|
||||
2. .claude/last30days.env (per-project config)
|
||||
3. ~/.config/last30days/.env (global config)
|
||||
4. macOS Keychain items prefixed ``last30days-`` (Darwin only)
|
||||
"""
|
||||
# Load from global config file
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||
@@ -284,14 +222,9 @@ def get_config() -> dict[str, Any]:
|
||||
project_env_path = _find_project_env()
|
||||
project_env = load_env_file(project_env_path) if project_env_path else {}
|
||||
|
||||
# Merge file sources: project > global
|
||||
# Merge: project overrides global
|
||||
merged_env = {**file_env, **project_env}
|
||||
|
||||
# Keychain is the lowest-priority source (Darwin only; no-op elsewhere).
|
||||
# Loaded before openai_auth so OPENAI_API_KEY can come from Keychain too.
|
||||
keychain_env = _load_keychain(list(KEYCHAIN_KEYS))
|
||||
merged_env = {**keychain_env, **merged_env}
|
||||
|
||||
openai_auth = get_openai_auth(merged_env)
|
||||
|
||||
# Build config: Codex/OpenAI auth + process.env > project .env > global .env
|
||||
@@ -314,7 +247,6 @@ def get_config() -> dict[str, Any]:
|
||||
('LAST30DAYS_RERANK_MODEL', None),
|
||||
('LAST30DAYS_X_MODEL', None),
|
||||
('LAST30DAYS_X_BACKEND', None),
|
||||
('LAST30DAYS_STORE', None),
|
||||
('OPENAI_MODEL_PIN', None),
|
||||
('XAI_MODEL_PIN', None),
|
||||
('SCRAPECREATORS_API_KEY', None),
|
||||
@@ -323,7 +255,6 @@ def get_config() -> dict[str, Any]:
|
||||
('CT0', None),
|
||||
('BSKY_HANDLE', None),
|
||||
('BSKY_APP_PASSWORD', None),
|
||||
('BSKY_SEARCH_HOST', None),
|
||||
('TRUTHSOCIAL_TOKEN', None),
|
||||
('BRAVE_API_KEY', None),
|
||||
('EXA_API_KEY', None),
|
||||
@@ -334,41 +265,16 @@ def get_config() -> dict[str, Any]:
|
||||
('FROM_BROWSER', None),
|
||||
('SETUP_COMPLETE', None),
|
||||
('INCLUDE_SOURCES', ''),
|
||||
('EXCLUDE_SOURCES', ''),
|
||||
('LAST30DAYS_YOUTUBE_SSH_HOST', None),
|
||||
('LAST30DAYS_TRANSCRIPT_TIMEOUT', None),
|
||||
]
|
||||
|
||||
for key, default in keys:
|
||||
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
||||
|
||||
# Backward-compat: ScrapeCreators' own examples and tutorials use the
|
||||
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and
|
||||
# CREATORS). Accept that form too so users who follow the vendor's docs
|
||||
# don't silently end up with has_scrapecreators=False. Canonical name
|
||||
# wins when both are set.
|
||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||
legacy = os.environ.get('SCRAPE_CREATORS_API_KEY') or merged_env.get('SCRAPE_CREATORS_API_KEY')
|
||||
if legacy:
|
||||
config['SCRAPECREATORS_API_KEY'] = legacy
|
||||
|
||||
# Multi-key rotation: comma-separated SCRAPECREATORS_API_KEY round-robins
|
||||
# via random.choice per run. Originally added in #268, accidentally dropped
|
||||
# in v3.0.6, restored here.
|
||||
sc_key_raw = config.get('SCRAPECREATORS_API_KEY') or ''
|
||||
if ',' in sc_key_raw:
|
||||
import random
|
||||
sc_keys = [k.strip() for k in sc_key_raw.split(',') if k.strip()]
|
||||
config['SCRAPECREATORS_API_KEY'] = random.choice(sc_keys) if sc_keys else ''
|
||||
|
||||
# Track which config source was used (highest-priority file source wins
|
||||
# the label; keychain is only reported when nothing else is configured).
|
||||
# Track which config source was used
|
||||
if project_env_path:
|
||||
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
||||
elif CONFIG_FILE and CONFIG_FILE.exists():
|
||||
config['_CONFIG_SOURCE'] = f'global:{CONFIG_FILE}'
|
||||
elif keychain_env:
|
||||
config['_CONFIG_SOURCE'] = 'keychain'
|
||||
else:
|
||||
config['_CONFIG_SOURCE'] = 'env_only'
|
||||
|
||||
@@ -466,6 +372,14 @@ def config_exists() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def is_reddit_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Reddit search is available.
|
||||
|
||||
v3 uses ScrapeCreators only.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
|
||||
|
||||
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
||||
"""Determine which Reddit backend to use.
|
||||
|
||||
@@ -611,12 +525,12 @@ def _parse_include_sources(config: dict[str, Any]) -> set[str]:
|
||||
def is_threads_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Threads source is available.
|
||||
|
||||
Returns True when SCRAPECREATORS_API_KEY is set. Threads runs alongside
|
||||
TikTok and Instagram as part of the SC family — same key, same per-call
|
||||
cost shape, so the same default-on rule applies. Suppress via
|
||||
EXCLUDE_SOURCES=threads.
|
||||
Requires SCRAPECREATORS_API_KEY AND 'threads' in INCLUDE_SOURCES.
|
||||
Threads is an opt-in source - it is not activated by default.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||
return False
|
||||
return 'threads' in _parse_include_sources(config)
|
||||
|
||||
|
||||
def is_instagram_available(config: dict[str, Any]) -> bool:
|
||||
@@ -116,8 +116,6 @@ def weighted_rrf(
|
||||
"""Fuse ranked lists into a single candidate pool."""
|
||||
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
||||
candidates: dict[str, schema.Candidate] = {}
|
||||
# Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
|
||||
seen_source_items: dict[str, set[tuple[str, str]]] = {}
|
||||
|
||||
for (label, source), items in streams.items():
|
||||
subquery = subqueries[label]
|
||||
@@ -156,7 +154,6 @@ def weighted_rrf(
|
||||
]
|
||||
},
|
||||
)
|
||||
seen_source_items[key] = {(item.source, item.item_id)}
|
||||
continue
|
||||
|
||||
candidate = candidates[key]
|
||||
@@ -182,9 +179,7 @@ def weighted_rrf(
|
||||
candidate.subquery_labels.append(label)
|
||||
if item.source not in candidate.sources:
|
||||
candidate.sources.append(item.source)
|
||||
source_item_key = (item.source, item.item_id)
|
||||
if source_item_key not in seen_source_items[key]:
|
||||
seen_source_items[key].add(source_item_key)
|
||||
if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
|
||||
candidate.source_items.append(item)
|
||||
candidate.metadata.setdefault("provenance", []).append(
|
||||
{
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
@@ -140,10 +139,7 @@ def parallel_search(
|
||||
data = http.request(
|
||||
"POST", "https://api.parallel.ai/v1/search",
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
json_data={
|
||||
"search_queries": [query],
|
||||
"advanced_settings": {"max_results": count},
|
||||
},
|
||||
json_data={"query": query, "max_results": count},
|
||||
timeout=15,
|
||||
)
|
||||
items = []
|
||||
@@ -153,7 +149,7 @@ def parallel_search(
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
raw_date = r.get("publish_date") or ""
|
||||
raw_date = r.get("published_date") or ""
|
||||
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
|
||||
if not _in_date_range(pub_date, date_range):
|
||||
continue
|
||||
@@ -162,7 +158,7 @@ def parallel_search(
|
||||
"title": r.get("title", ""),
|
||||
"url": url,
|
||||
"source_domain": _domain(url),
|
||||
"snippet": ((r.get("excerpts") or [""])[0] or "")[:500],
|
||||
"snippet": r.get("snippet", ""),
|
||||
"date": pub_date,
|
||||
"relevance": 0.8,
|
||||
"why_relevant": "Parallel AI web search",
|
||||
@@ -209,90 +205,29 @@ def web_search(
|
||||
backend = "parallel"
|
||||
else:
|
||||
return [], {}
|
||||
items: list[dict] = []
|
||||
artifact: dict = {}
|
||||
if backend == "brave":
|
||||
key = config.get("BRAVE_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
|
||||
items, artifact = brave_search(query, date_range, key)
|
||||
elif backend == "exa":
|
||||
return brave_search(query, date_range, key)
|
||||
if backend == "exa":
|
||||
key = config.get("EXA_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
|
||||
items, artifact = exa_search(query, date_range, key)
|
||||
elif backend == "serper":
|
||||
return exa_search(query, date_range, key)
|
||||
if backend == "serper":
|
||||
key = config.get("SERPER_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
|
||||
items, artifact = serper_search(query, date_range, key)
|
||||
elif backend == "parallel":
|
||||
return serper_search(query, date_range, key)
|
||||
if backend == "parallel":
|
||||
key = config.get("PARALLEL_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
|
||||
items, artifact = parallel_search(query, date_range, key)
|
||||
elif backend != "none":
|
||||
return parallel_search(query, date_range, key)
|
||||
if backend != "none":
|
||||
raise ValueError(f"Unsupported web backend: {backend!r}")
|
||||
else:
|
||||
return [], {}
|
||||
if items and not _reddit_excluded(config):
|
||||
items = _enrich_reddit_items(items)
|
||||
return items, artifact
|
||||
|
||||
|
||||
def _reddit_excluded(config: dict) -> bool:
|
||||
"""Return True when EXCLUDE_SOURCES contains 'reddit'.
|
||||
|
||||
Respects the same suppression knob the pipeline uses for source gating,
|
||||
so a user who set EXCLUDE_SOURCES=reddit doesn't get Reddit content
|
||||
smuggled back in via web-search URLs.
|
||||
"""
|
||||
raw = (config.get("EXCLUDE_SOURCES") or "").split(",")
|
||||
return any(s.strip().lower() == "reddit" for s in raw)
|
||||
|
||||
|
||||
def _enrich_reddit_items(items: list[dict]) -> list[dict]:
|
||||
"""Enrich web search results that are Reddit URLs with thread body and comments.
|
||||
|
||||
Claude Code's WebFetch blocks reddit.com, so the model can't retrieve
|
||||
Reddit content from web search results. This fetches it via the public
|
||||
JSON API (reddit.com/.../.json) which bypasses that restriction.
|
||||
|
||||
Callers should gate this with EXCLUDE_SOURCES=reddit handling (see
|
||||
`_reddit_excluded`) so a user who explicitly excluded Reddit doesn't
|
||||
get Reddit content via web-search URLs.
|
||||
"""
|
||||
from . import reddit_enrich
|
||||
from .reddit_enrich import RedditRateLimitError
|
||||
|
||||
for item in items:
|
||||
url = item.get("url", "")
|
||||
if "reddit.com" not in url or "/comments/" not in url:
|
||||
continue
|
||||
try:
|
||||
thread_data = reddit_enrich.fetch_thread_data(url, timeout=8)
|
||||
if not thread_data:
|
||||
continue
|
||||
parsed = reddit_enrich.parse_thread_data(thread_data)
|
||||
# selftext lives under parsed["submission"], not at the top level
|
||||
selftext = (parsed.get("submission") or {}).get("selftext", "")
|
||||
if selftext:
|
||||
item["snippet"] = selftext[:2000]
|
||||
comments = parsed.get("comments", [])
|
||||
top = reddit_enrich.get_top_comments(comments)
|
||||
if top:
|
||||
item["top_comments"] = [
|
||||
{"score": c.get("score", 0), "excerpt": (c.get("body") or "")[:200]}
|
||||
for c in top[:5]
|
||||
]
|
||||
item["enriched_via"] = "reddit_json_api"
|
||||
except RedditRateLimitError as exc:
|
||||
# Stop iterating to avoid flooding more 429s
|
||||
sys.stderr.write(f"[Web] Reddit rate-limited, halting enrichment: {exc}\n")
|
||||
break
|
||||
except Exception as exc:
|
||||
sys.stderr.write(f"[Web] Reddit enrichment failed for {url}: {exc}\n")
|
||||
return items
|
||||
return [], {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -88,26 +88,17 @@ def search_hackernews(
|
||||
|
||||
# Use extracted core subject instead of raw topic for cleaner Algolia matching
|
||||
core = extract_core_subject(topic)
|
||||
# Hyphens and commas tokenize awkwardly in Algolia; flatten them so themed
|
||||
# queries like "ts-bun-node" or "claude, personal agents" become plain words.
|
||||
core_flat = _flatten_query_for_algolia(core)
|
||||
_log(f"Searching for '{core_flat}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
|
||||
# Use relevance-sorted search with minimum engagement filter.
|
||||
# NOTE: restrictSearchableAttributes=title omitted intentionally — it would
|
||||
# miss Ask HN/Show HN threads where the topic appears in the body.
|
||||
params = {
|
||||
"query": core_flat,
|
||||
"query": core,
|
||||
"tags": "story",
|
||||
"numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
|
||||
"hitsPerPage": str(count),
|
||||
}
|
||||
# Algolia defaults to AND across query tokens, so a 4-5 word theme query
|
||||
# matches no stories. Mark all-but-the-first token as optional so Algolia
|
||||
# ranks by how many tokens match instead of requiring every one.
|
||||
tokens = core_flat.split()
|
||||
if len(tokens) > 1:
|
||||
params["optionalWords"] = " ".join(tokens[1:])
|
||||
|
||||
from urllib.parse import urlencode
|
||||
url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
|
||||
@@ -126,56 +117,28 @@ def search_hackernews(
|
||||
return response
|
||||
|
||||
|
||||
_WORD_BOUNDARY_RE_CACHE: Dict[str, "re.Pattern[str]"] = {}
|
||||
|
||||
|
||||
def _flatten_query_for_algolia(text: str) -> str:
|
||||
"""Normalise query for Algolia + post-filter comparison.
|
||||
|
||||
Multi-keyword theme queries frequently contain commas (delimiters) or
|
||||
hyphens (compound terms like ``ts-bun-node``); both tokenize awkwardly.
|
||||
Flatten them to spaces and collapse runs of whitespace so the search
|
||||
parameter and the post-filter operate on the same shape.
|
||||
"""
|
||||
return " ".join(text.replace(",", " ").replace("-", " ").split())
|
||||
|
||||
|
||||
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
|
||||
"""Check if any query token appears as a whole word in the title.
|
||||
"""Check if the query term appears in the title content, not just an HN prefix or author.
|
||||
|
||||
Returns True when the query is empty (no filter), or when at least one
|
||||
query token matches as a whole word in the title after stripping
|
||||
"Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes.
|
||||
|
||||
We previously required *every* token to appear (all-words), which killed
|
||||
every Algolia hit on multi-keyword themes like "claude, personal agents,
|
||||
agentic infra" because real HN titles never contain all five tokens
|
||||
verbatim. Relaxing to any-word matches Algolia's `optionalWords` behaviour
|
||||
in `search_hackernews`. Token-overlap relevance scoring at parse time
|
||||
demotes hits where only one weak token matched, so the loosened gate
|
||||
won't surface noise to the top of the ranking.
|
||||
|
||||
Word-boundary matching (rather than naive substring) prevents short
|
||||
tokens like ``ai`` or ``ts`` from matching unrelated words like
|
||||
``email`` or ``artists``.
|
||||
Returns True if the query (or any multi-word token) appears in the title
|
||||
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
|
||||
and ignoring the author name. Returns True when query is empty (no filter).
|
||||
"""
|
||||
if not query:
|
||||
return True
|
||||
stripped = _HN_PREFIXES.sub("", title).strip()
|
||||
# Also check that the match isn't solely in the author's username
|
||||
check_text = stripped.lower()
|
||||
# Normalise the query the same way search_hackernews does so post-filter
|
||||
# tokens line up with what Algolia actually saw.
|
||||
query_words = [w for w in _flatten_query_for_algolia(query.lower()).split() if w]
|
||||
if not query_words:
|
||||
return True
|
||||
query_lower = query.lower()
|
||||
# Check each word of the query independently; all must appear somewhere
|
||||
# in the stripped title (not just the prefix).
|
||||
query_words = query_lower.split()
|
||||
for word in query_words:
|
||||
pattern = _WORD_BOUNDARY_RE_CACHE.get(word)
|
||||
if pattern is None:
|
||||
pattern = re.compile(rf"\b{re.escape(word)}\b")
|
||||
_WORD_BOUNDARY_RE_CACHE[word] = pattern
|
||||
if pattern.search(check_text):
|
||||
return True
|
||||
return False
|
||||
if word in check_text:
|
||||
continue
|
||||
# Word not found in stripped title — reject
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -23,19 +22,9 @@ def log(msg: str):
|
||||
MAX_RETRIES = 5
|
||||
MAX_429_RETRIES = 2
|
||||
RETRY_DELAY = 2.0
|
||||
# DNS resolution failures (gaierror) are transient — typically resolved by a
|
||||
# brief backoff and retry. Use a dedicated minimum attempt count + exponential
|
||||
# delays (1s, 2s, 4s) so callers that pass a small `retries` value still get a
|
||||
# meaningful chance to recover from a transient resolution failure.
|
||||
MIN_DNS_RETRIES = 3
|
||||
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
||||
|
||||
|
||||
def _is_dns_failure(err: urllib.error.URLError) -> bool:
|
||||
"""Return True if a URLError was caused by DNS resolution (gaierror)."""
|
||||
return isinstance(getattr(err, "reason", None), socket.gaierror)
|
||||
|
||||
|
||||
class HTTPError(Exception):
|
||||
"""HTTP request error with status code."""
|
||||
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
||||
@@ -96,13 +85,7 @@ def request(
|
||||
|
||||
last_error = None
|
||||
rate_limit_count = 0
|
||||
# DNS failures get a dedicated minimum attempt count + exponential backoff.
|
||||
# `effective_retries` is the actual loop bound; we expand it on the first
|
||||
# gaierror if the caller passed a smaller `retries` value than MIN_DNS_RETRIES.
|
||||
effective_retries = retries
|
||||
dns_attempts = 0
|
||||
attempt = 0
|
||||
while attempt < effective_retries:
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
body = response.read().decode('utf-8')
|
||||
@@ -132,8 +115,6 @@ def request(
|
||||
if rate_limit_count >= max_429_retries:
|
||||
raise last_error
|
||||
|
||||
# HTTP errors respect the caller's original `retries`; only DNS
|
||||
# failures get the widened `effective_retries` budget.
|
||||
if attempt < retries - 1:
|
||||
if e.code == 429:
|
||||
# Respect Retry-After header, fall back to exponential backoff
|
||||
@@ -149,43 +130,11 @@ def request(
|
||||
else:
|
||||
delay = RETRY_DELAY * (2 ** attempt)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure may have widened `effective_retries`, but that
|
||||
# widening is DNS-only — don't grant extra HTTP attempts.
|
||||
break
|
||||
except urllib.error.URLError as e:
|
||||
log(f"URL Error: {e.reason}")
|
||||
last_error = HTTPError(f"URL Error: {e.reason}")
|
||||
if _is_dns_failure(e):
|
||||
# DNS resolution failures are transient; expand the retry budget
|
||||
# to MIN_DNS_RETRIES if the caller passed fewer, and use
|
||||
# exponential backoff (1s, 2s, 4s, ...) instead of the linear
|
||||
# default. Counts DNS attempts separately so other URLError
|
||||
# causes don't bypass the regular retry budget.
|
||||
dns_attempts += 1
|
||||
if effective_retries < MIN_DNS_RETRIES:
|
||||
log(
|
||||
f"DNS resolution failed; expanding retry budget from "
|
||||
f"{effective_retries} to {MIN_DNS_RETRIES}"
|
||||
)
|
||||
effective_retries = MIN_DNS_RETRIES
|
||||
if attempt < effective_retries - 1:
|
||||
delay = 2 ** (dns_attempts - 1) # 1s, 2s, 4s, 8s, ...
|
||||
log(
|
||||
f"DNS resolution failure (attempt {dns_attempts}); "
|
||||
f"retrying in {delay:.1f}s"
|
||||
)
|
||||
time.sleep(delay)
|
||||
elif attempt < retries - 1:
|
||||
# Non-DNS URLError (e.g. ConnectionRefused) respects the
|
||||
# caller's original retry budget, not the DNS-widened bound.
|
||||
if attempt < retries - 1:
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure widening `effective_retries` does not carry over
|
||||
# to non-DNS error paths.
|
||||
break
|
||||
except json.JSONDecodeError as e:
|
||||
log(f"JSON decode error: {e}")
|
||||
last_error = HTTPError(f"Invalid JSON response: {e}")
|
||||
@@ -195,13 +144,7 @@ def request(
|
||||
log(f"Connection error: {type(e).__name__}: {e}")
|
||||
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
||||
if attempt < retries - 1:
|
||||
# Socket errors respect the caller's original retry budget.
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Original budget exhausted; DNS widening doesn't apply here.
|
||||
break
|
||||
|
||||
attempt += 1
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
@@ -7,14 +7,17 @@ Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
|
||||
API docs: https://scrapecreators.com/docs
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import dates, http, log
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||
|
||||
@@ -28,42 +31,7 @@ DEPTH_CONFIG = {
|
||||
# Max words to keep from each caption
|
||||
CAPTION_MAX_WORDS = 500
|
||||
|
||||
# Default transcript fetch timeout (seconds). SC's
|
||||
# /v2/instagram/media/transcript regularly takes >15s on real workloads,
|
||||
# so the default is generous; override via LAST30DAYS_TRANSCRIPT_TIMEOUT.
|
||||
DEFAULT_TRANSCRIPT_TIMEOUT = 30
|
||||
|
||||
|
||||
def _resolve_transcript_timeout(
|
||||
timeout: Optional[float] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> float:
|
||||
"""Resolve the IG transcript-fetch timeout.
|
||||
|
||||
Priority (highest wins):
|
||||
1. Explicit ``timeout`` kwarg
|
||||
2. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in os.environ
|
||||
3. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in caller-supplied config dict
|
||||
4. ``DEFAULT_TRANSCRIPT_TIMEOUT`` (30s)
|
||||
|
||||
Mirrors the ``os.environ.get(X) or config.get(X)`` pattern used for
|
||||
LAST30DAYS_STORE in last30days.py so the env var works whether it's
|
||||
shell-exported or set in ~/.config/last30days/.env.
|
||||
"""
|
||||
if timeout is not None:
|
||||
try:
|
||||
return float(timeout)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
raw = os.environ.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||
if not raw and config:
|
||||
raw = config.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||
if raw:
|
||||
try:
|
||||
return float(raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return float(DEFAULT_TRANSCRIPT_TIMEOUT)
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
@@ -81,17 +49,6 @@ def _extract_core_subject(topic: str) -> str:
|
||||
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
||||
|
||||
|
||||
def _to_hashtag_form(query: str) -> str:
|
||||
"""Collapse a multi-word query to hashtag form (no spaces, lowercase).
|
||||
|
||||
SC's /v2/instagram/reels/search wraps Google Search and is documented
|
||||
to be flaky on multi-token queries. Single-token queries map to a
|
||||
hashtag page lookup which is the stable path. Used as a 500-retry
|
||||
fallback before the request bubbles up as a silent failure.
|
||||
"""
|
||||
return ''.join(query.split()).lower()
|
||||
|
||||
|
||||
def _infer_query_intent(topic: str) -> str:
|
||||
"""Tiny local intent classifier for Instagram query expansion."""
|
||||
text = topic.lower().strip()
|
||||
@@ -279,17 +236,30 @@ def _user_reels(
|
||||
"""
|
||||
_log(f"User reels: @{handle}")
|
||||
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
|
||||
try:
|
||||
data = http.get(
|
||||
reels_url,
|
||||
params={"handle": handle},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"User reels error for @{handle}: {e}")
|
||||
return []
|
||||
if not _requests:
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"handle": handle})
|
||||
url = f"{reels_url}?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"User reels error (urllib) for @{handle}: {e}")
|
||||
return []
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
reels_url,
|
||||
params={"handle": handle},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"User reels error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
|
||||
_log(f" -> {len(raw_items)} reels from @{handle}")
|
||||
@@ -323,37 +293,31 @@ def search_instagram(
|
||||
|
||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except http.HTTPError as e:
|
||||
# SC's v2 reels search wraps Google Search and 500s frequently on
|
||||
# multi-token queries. Single tokens hit the stable hashtag-page
|
||||
# path. Retry once with hashtag form before bubbling up.
|
||||
if getattr(e, "status_code", None) == 500 and ' ' in core_topic:
|
||||
_log(f"IG search 500 on '{core_topic}', retrying with hashtag form")
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": _to_hashtag_form(core_topic)},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as retry_e:
|
||||
_log(f"IG search retry failed: {retry_e}")
|
||||
return {"items": [], "error": f"{type(retry_e).__name__}: {retry_e}"}
|
||||
else:
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"query": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
||||
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
||||
@@ -385,8 +349,6 @@ def fetch_captions(
|
||||
video_items: List[Dict[str, Any]],
|
||||
token: str,
|
||||
depth: str = "default",
|
||||
timeout: Optional[float] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
||||
|
||||
@@ -398,21 +360,14 @@ def fetch_captions(
|
||||
video_items: Items from search_instagram()
|
||||
token: ScrapeCreators API key
|
||||
depth: Depth level for caption limit
|
||||
timeout: Optional per-request transcript timeout in seconds. When
|
||||
None, resolves from LAST30DAYS_TRANSCRIPT_TIMEOUT (env or
|
||||
config), defaulting to DEFAULT_TRANSCRIPT_TIMEOUT (30s).
|
||||
config: Optional config dict (from env.get_config()) used as a
|
||||
fallback source for LAST30DAYS_TRANSCRIPT_TIMEOUT when the
|
||||
value is not exported in os.environ.
|
||||
|
||||
Returns:
|
||||
Dict mapping video_id -> caption text (truncated to 500 words)
|
||||
"""
|
||||
depth_cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = depth_cfg["max_captions"]
|
||||
transcript_timeout = _resolve_transcript_timeout(timeout, config)
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = config["max_captions"]
|
||||
|
||||
if not video_items or not token:
|
||||
if not video_items or not token or not _requests:
|
||||
return {}
|
||||
|
||||
top_items = video_items[:max_captions]
|
||||
@@ -437,24 +392,26 @@ def fetch_captions(
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
data = http.get(
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||
params={"url": url},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=transcript_timeout,
|
||||
retries=1,
|
||||
timeout=15,
|
||||
)
|
||||
transcripts = data.get("transcripts") or []
|
||||
if transcripts and isinstance(transcripts, list):
|
||||
transcript_text = " ".join(
|
||||
t.get("text", "") for t in transcripts
|
||||
if isinstance(t, dict) and t.get("text")
|
||||
)
|
||||
if transcript_text:
|
||||
words = transcript_text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = transcript_text
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
transcripts = data.get("transcripts") or []
|
||||
if transcripts and isinstance(transcripts, list):
|
||||
# Combine all transcript segments
|
||||
transcript_text = " ".join(
|
||||
t.get("text", "") for t in transcripts
|
||||
if isinstance(t, dict) and t.get("text")
|
||||
)
|
||||
if transcript_text:
|
||||
words = transcript_text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = transcript_text
|
||||
except Exception as e:
|
||||
_log(f"Transcript fetch failed for {vid}: {e}")
|
||||
|
||||
@@ -49,7 +49,6 @@ def normalize_source_items(
|
||||
"xquik": _normalize_x,
|
||||
"pinterest": _normalize_pinterest,
|
||||
"polymarket": _normalize_polymarket,
|
||||
"digg": _normalize_digg,
|
||||
"grounding": _normalize_grounding,
|
||||
"xiaohongshu": _normalize_grounding,
|
||||
"github": _normalize_github,
|
||||
@@ -111,19 +110,6 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
|
||||
return default
|
||||
|
||||
|
||||
def _join_comment_excerpts(
|
||||
top_comments: list[Any],
|
||||
key: str,
|
||||
limit: int = 3,
|
||||
) -> str:
|
||||
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
|
||||
return " ".join(
|
||||
str(comment.get(key) or "").strip()
|
||||
for comment in top_comments[:limit]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
|
||||
|
||||
def _domain_from_url(url: str) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
@@ -183,7 +169,11 @@ def _normalize_reddit(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
body = "\n".join(
|
||||
part
|
||||
for part in [
|
||||
@@ -251,11 +241,6 @@ def _normalize_youtube(
|
||||
metadata: dict[str, Any] = {}
|
||||
if highlights:
|
||||
metadata["transcript_highlights"] = highlights
|
||||
if item.get("captions_disabled"):
|
||||
# Surfaced for quality_nudge: uploader disabled captions, so this
|
||||
# video should be subtracted from the degraded-transcript-ratio
|
||||
# denominator (it was never going to produce a transcript).
|
||||
metadata["captions_disabled"] = True
|
||||
metadata["top_comments"] = _remap_comments(
|
||||
item.get("top_comments") or [],
|
||||
score_keys=("score", "likes"),
|
||||
@@ -353,7 +338,11 @@ def _normalize_hackernews(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "text")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("text") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
title = str(item.get("title") or "").strip()
|
||||
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
||||
return _source_item(
|
||||
@@ -405,53 +394,6 @@ def _normalize_microblog(
|
||||
)
|
||||
|
||||
|
||||
def _normalize_digg(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
index: int,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
"""Normalizer for Digg AI 1000 clusters.
|
||||
|
||||
Each cluster is one item. The TLDR carries the most useful body for
|
||||
rerank and synthesis. Top-ranked X posts attached at search time are
|
||||
passed through under metadata['posts'] so render can emit them as
|
||||
inline 'via Digg' quotes.
|
||||
"""
|
||||
title = str(item.get("title") or "").strip()
|
||||
tldr = str(item.get("tldr") or "").strip()
|
||||
body = "\n\n".join(part for part in [title, tldr] if part)
|
||||
posts = item.get("posts") or []
|
||||
if not isinstance(posts, list):
|
||||
posts = []
|
||||
cluster_url_id = str(item.get("id") or f"DG{index + 1}")
|
||||
return _source_item(
|
||||
item_id=cluster_url_id,
|
||||
source=source,
|
||||
title=title or f"Digg cluster {index + 1}",
|
||||
body=body,
|
||||
url=str(item.get("url") or f"https://di.gg/ai/{cluster_url_id}"),
|
||||
author="",
|
||||
container="Digg",
|
||||
published_at=item.get("date"),
|
||||
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
|
||||
engagement=item.get("engagement") or {},
|
||||
relevance_hint=item.get("relevance", 0.5),
|
||||
why_relevant=str(item.get("why_relevant") or ""),
|
||||
snippet=tldr[:400],
|
||||
metadata={
|
||||
"clusterUrlId": cluster_url_id,
|
||||
"tldr": tldr,
|
||||
"rank": (item.get("engagement") or {}).get("rank"),
|
||||
"uniqueAuthors": (item.get("engagement") or {}).get("uniqueAuthors"),
|
||||
"postCount": (item.get("engagement") or {}).get("postCount"),
|
||||
"firstPostAge": item.get("first_post_age"),
|
||||
"posts": posts,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalize_polymarket(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
@@ -499,7 +441,11 @@ def _normalize_github(
|
||||
title = str(item.get("title") or "").strip()
|
||||
snippet_text = str(item.get("snippet") or "").strip()
|
||||
top_comments = item.get("metadata", {}).get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
||||
metadata = item.get("metadata") or {}
|
||||
return _source_item(
|
||||
@@ -11,6 +11,11 @@ import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import dates, http, log
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
|
||||
@@ -135,17 +140,31 @@ def search_pinterest(
|
||||
|
||||
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"keyword": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/search?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Extract items from response - try common SC response shapes
|
||||
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
|
||||
@@ -15,7 +15,6 @@ from . import (
|
||||
bluesky,
|
||||
dates,
|
||||
dedupe,
|
||||
digg,
|
||||
entity_extract,
|
||||
env,
|
||||
github,
|
||||
@@ -31,7 +30,6 @@ from . import (
|
||||
query,
|
||||
reddit,
|
||||
reddit_public,
|
||||
relevance,
|
||||
rerank,
|
||||
schema,
|
||||
signals,
|
||||
@@ -79,10 +77,7 @@ MOCK_AVAILABLE_SOURCES = [
|
||||
"xiaohongshu",
|
||||
"github",
|
||||
"perplexity",
|
||||
"threads",
|
||||
"pinterest",
|
||||
"xquik",
|
||||
"digg",
|
||||
]
|
||||
|
||||
|
||||
@@ -110,8 +105,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.extend(["hackernews", "polymarket"])
|
||||
if config.get("GITHUB_TOKEN") or which("gh"):
|
||||
available.append("github")
|
||||
if which("digg-pp-cli"):
|
||||
available.append("digg")
|
||||
if env.is_bluesky_available(config):
|
||||
available.append("bluesky")
|
||||
if env.is_truthsocial_available(config):
|
||||
@@ -120,9 +113,7 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("grounding")
|
||||
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
|
||||
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
|
||||
if config.get("OPENROUTER_API_KEY") and (
|
||||
"perplexity" in include_sources or (requested_sources and "perplexity" in requested_sources)
|
||||
):
|
||||
if config.get("OPENROUTER_API_KEY") and "perplexity" in include_sources:
|
||||
available.append("perplexity")
|
||||
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
|
||||
available.append("xiaohongshu")
|
||||
@@ -132,9 +123,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("pinterest")
|
||||
if env.is_xquik_available(config):
|
||||
available.append("xquik")
|
||||
exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()}
|
||||
if exclude:
|
||||
available = [s for s in available if s not in exclude]
|
||||
return available
|
||||
|
||||
|
||||
@@ -207,7 +195,7 @@ def run(
|
||||
available = [source for source in available if source in requested_sources]
|
||||
if web_backend == "none":
|
||||
available = [s for s in available if s != "grounding"]
|
||||
elif web_backend in ("brave", "exa", "serper", "parallel") and "grounding" not in available:
|
||||
elif web_backend in ("brave", "exa", "serper") and "grounding" not in available:
|
||||
available.append("grounding")
|
||||
if not available:
|
||||
raise RuntimeError("No sources are available for this run.")
|
||||
@@ -512,12 +500,11 @@ def _normalize_score_dedupe(
|
||||
source, raw_items, from_date, to_date,
|
||||
freshness_mode=freshness_mode,
|
||||
)
|
||||
prepared_query = relevance.PreparedQuery(ranking_query)
|
||||
normalized = signals.annotate_stream(normalized, prepared_query, freshness_mode)
|
||||
normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
|
||||
normalized = signals.prune_low_relevance(normalized)
|
||||
normalized = dedupe.dedupe_items(normalized)
|
||||
for item in normalized:
|
||||
item.snippet = snippet.extract_best_snippet(item, prepared_query)
|
||||
item.snippet = snippet.extract_best_snippet(item, ranking_query)
|
||||
return normalized
|
||||
|
||||
|
||||
@@ -542,12 +529,6 @@ def _finalize_items_by_source(
|
||||
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
|
||||
if keywords:
|
||||
items = polymarket.filter_items_against_keywords(items, keywords)
|
||||
if source == "digg" and items:
|
||||
# Pull top-ranked X posts only for the survivors that will appear
|
||||
# in the brief. Spending the enrichment budget here (rather than
|
||||
# at retrieval time) keeps the inline 'via Digg' quotes
|
||||
# paired with the clusters dedupe actually kept.
|
||||
digg.enrich_source_items(items, top_k=3)
|
||||
finalized[source] = items
|
||||
return finalized
|
||||
|
||||
@@ -983,13 +964,6 @@ def _retrieve_stream(
|
||||
if source == "hackernews":
|
||||
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
||||
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
||||
if source == "digg":
|
||||
result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
|
||||
items = digg.parse_digg_response(result, query=subquery.search_query)
|
||||
# Enrichment with attached X posts is deferred to
|
||||
# _finalize_items_by_source so it runs on the items that actually
|
||||
# survive dedupe rather than on top-K of the raw fanout.
|
||||
return items, {}
|
||||
if source == "bluesky":
|
||||
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
||||
return bluesky.parse_bluesky_response(result), {}
|
||||
@@ -1082,45 +1056,6 @@ def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[d
|
||||
"why_relevant": "Brave web search",
|
||||
}
|
||||
],
|
||||
"digg": [
|
||||
{
|
||||
"id": "mock1abc",
|
||||
"title": f"Digg cluster about {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock1abc",
|
||||
"tldr": f"Curated cluster summarizing recent {subquery.search_query} discussion across the AI 1000.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(3)[0],
|
||||
"engagement": {"postCount": 8, "uniqueAuthors": 5, "rank": 2, "rank_score": 49.0},
|
||||
"first_post_age": "3d",
|
||||
"posts": [
|
||||
{
|
||||
"username": "exampledev",
|
||||
"display_name": "Example Dev",
|
||||
"category": "Engineer",
|
||||
"rank": 142,
|
||||
"body": f"Quote from the AI 1000 about {subquery.search_query}.",
|
||||
"post_type": "tweet",
|
||||
"x_url": "https://x.com/exampledev/status/1",
|
||||
"posted_at": dates.get_date_range(3)[0],
|
||||
},
|
||||
],
|
||||
"relevance": 0.84,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
{
|
||||
"id": "mock2def",
|
||||
"title": f"Second Digg cluster on {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock2def",
|
||||
"tldr": f"Another angle on {subquery.search_query}.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(8)[0],
|
||||
"engagement": {"postCount": 3, "uniqueAuthors": 2, "rank": 18, "rank_score": 33.0},
|
||||
"first_post_age": "8d",
|
||||
"posts": [],
|
||||
"relevance": 0.71,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
],
|
||||
}
|
||||
if source == "grounding":
|
||||
return payloads.get(source, []), {
|
||||
@@ -19,14 +19,14 @@ ALLOWED_INTENTS = {
|
||||
}
|
||||
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
|
||||
QUICK_SOURCE_PRIORITY = {
|
||||
"factual": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||
"product": ["youtube", "reddit", "x", "xquik", "tiktok"],
|
||||
"concept": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||
"opinion": ["reddit", "x", "xquik", "youtube", "hackernews"],
|
||||
"how_to": ["youtube", "reddit", "x", "xquik", "hackernews"],
|
||||
"comparison": ["reddit", "x", "xquik", "hackernews", "youtube"],
|
||||
"breaking_news": ["x", "xquik", "reddit", "hackernews", "youtube", "polymarket"],
|
||||
"prediction": ["polymarket", "x", "xquik", "hackernews", "reddit", "youtube"],
|
||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||
"product": ["youtube", "reddit", "x", "tiktok"],
|
||||
"concept": ["hackernews", "reddit", "x", "youtube"],
|
||||
"opinion": ["reddit", "x", "youtube", "hackernews"],
|
||||
"how_to": ["youtube", "reddit", "x", "hackernews"],
|
||||
"comparison": ["reddit", "x", "hackernews", "youtube"],
|
||||
"breaking_news": ["x", "reddit", "hackernews", "youtube", "polymarket"],
|
||||
"prediction": ["polymarket", "x", "hackernews", "reddit", "youtube"],
|
||||
}
|
||||
SOURCE_PRIORITY = {
|
||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||
@@ -60,7 +60,6 @@ INTENT_SOURCE_EXCLUSIONS: dict[str, set[str]] = {
|
||||
SOURCE_CAPABILITIES = {
|
||||
"reddit": {"discussion", "social"},
|
||||
"x": {"discussion", "social"},
|
||||
"xquik": {"discussion", "social"},
|
||||
"youtube": {"video", "video_longform", "discussion"},
|
||||
"tiktok": {"video", "video_shortform", "social"},
|
||||
"instagram": {"video", "video_shortform", "social"},
|
||||
@@ -68,7 +67,6 @@ SOURCE_CAPABILITIES = {
|
||||
"bluesky": {"discussion", "social"},
|
||||
"truthsocial": {"discussion", "social"},
|
||||
"polymarket": {"market"},
|
||||
"digg": {"discussion", "social", "link"},
|
||||
"xiaohongshu": {"video", "video_shortform", "social"},
|
||||
"github": {"discussion", "link"},
|
||||
"grounding": {"web", "reference", "link"},
|
||||
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
from . import env, http, schema
|
||||
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite"
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_PRO = "gemini-3.1-pro-preview"
|
||||
OPENAI_DEFAULT = "gpt-5.4-nano"
|
||||
XAI_DEFAULT = "grok-4-1-fast"
|
||||
@@ -19,11 +19,7 @@ OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
||||
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
|
||||
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
|
||||
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||
# OpenRouter routes the Gemini Flash Lite tier as the -preview slug; that is the
|
||||
# stable form on that routing layer even though native Gemini's GEMINI_FLASH_LITE
|
||||
# constant is suffix-free. If GEMINI_FLASH_LITE moves to a non-preview stable ID,
|
||||
# double-check that OpenRouter's slug still maps to the same upstream model.
|
||||
OPENROUTER_DEFAULT = "google/gemini-3.1-flash-lite-preview"
|
||||
OPENROUTER_DEFAULT = "google/gemini-flash-2.0"
|
||||
|
||||
|
||||
class ReasoningClient:
|
||||
@@ -97,6 +93,13 @@ class GeminiClient(ReasoningClient):
|
||||
)
|
||||
return extract_gemini_text(payload)
|
||||
|
||||
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
|
||||
|
||||
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
|
||||
|
||||
|
||||
class OpenAIClient(ReasoningClient):
|
||||
name = "openai"
|
||||
|
||||
@@ -236,8 +239,8 @@ def _resolve_model_pins(config: dict[str, Any], depth: str, provider_name: str)
|
||||
rerank_model = config.get("LAST30DAYS_RERANK_MODEL") or default_rerank
|
||||
|
||||
if provider_name == "gemini":
|
||||
_require_gemini_31(planner_model, role="planner")
|
||||
_require_gemini_31(rerank_model, role="rerank")
|
||||
_require_gemini_31_preview(planner_model, role="planner")
|
||||
_require_gemini_31_preview(rerank_model, role="rerank")
|
||||
|
||||
return planner_model, rerank_model
|
||||
|
||||
@@ -348,11 +351,11 @@ def _resolve_x_backend(config: dict[str, Any]) -> str | None:
|
||||
return env.get_x_source(config)
|
||||
|
||||
|
||||
def _require_gemini_31(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-"):
|
||||
def _require_gemini_31_preview(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-") and model.endswith("-preview"):
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"{role} must use a Gemini 3.1 model. Got: {model}"
|
||||
f"{role} must use a Gemini 3.1 preview model. Got: {model}"
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Post-research quality score and upgrade nudge.
|
||||
|
||||
Computes a quality score based on 5 core sources and builds
|
||||
a nudge message describing what the user missed and how to fix it.
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
|
||||
|
||||
# The 5 core sources
|
||||
CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit"]
|
||||
|
||||
# Labels for display
|
||||
SOURCE_LABELS = {
|
||||
"hn": "Hacker News",
|
||||
"polymarket": "Polymarket",
|
||||
"x": "X/Twitter",
|
||||
"youtube": "YouTube",
|
||||
"reddit": "Reddit",
|
||||
}
|
||||
|
||||
|
||||
def _is_x_active(config: dict, research_results: dict) -> bool:
|
||||
"""Check if X source is active (has credentials AND didn't error)."""
|
||||
has_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||
if not has_creds:
|
||||
return False
|
||||
# If X errored this run, it's configured but broken
|
||||
if research_results.get("x_error"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_youtube_active(config: dict, research_results: dict) -> bool:
|
||||
"""Check if YouTube source is active (yt-dlp installed)."""
|
||||
try:
|
||||
from . import youtube_yt
|
||||
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||
except Exception:
|
||||
has_ytdlp = False
|
||||
if not has_ytdlp:
|
||||
return False
|
||||
if research_results.get("youtube_error"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||
"""Compute research quality score based on 5 core sources.
|
||||
|
||||
Args:
|
||||
config: Configuration dict from env.get_config()
|
||||
research_results: Dict with keys like x_error, youtube_error,
|
||||
reddit_error reflecting what happened this run.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"score_pct": 40-100,
|
||||
"core_active": ["hn", "polymarket", ...],
|
||||
"core_missing": ["x", "youtube"],
|
||||
"core_errored": [], # configured but errored
|
||||
"nudge_text": "..." or None if 100%
|
||||
}
|
||||
"""
|
||||
core_active: List[str] = []
|
||||
core_missing: List[str] = []
|
||||
core_errored: List[str] = []
|
||||
|
||||
# HN, Polymarket, and Reddit are always active
|
||||
core_active.append("hn")
|
||||
core_active.append("polymarket")
|
||||
core_active.append("reddit")
|
||||
|
||||
# X
|
||||
has_x_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||
if _is_x_active(config, research_results):
|
||||
core_active.append("x")
|
||||
else:
|
||||
core_missing.append("x")
|
||||
if has_x_creds and research_results.get("x_error"):
|
||||
core_errored.append("x")
|
||||
|
||||
# YouTube
|
||||
yt_active = _is_youtube_active(config, research_results)
|
||||
if yt_active:
|
||||
core_active.append("youtube")
|
||||
else:
|
||||
core_missing.append("youtube")
|
||||
# Check if configured but errored (yt-dlp installed but failed this run)
|
||||
try:
|
||||
from . import youtube_yt
|
||||
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||
except Exception:
|
||||
has_ytdlp = False
|
||||
if has_ytdlp and research_results.get("youtube_error"):
|
||||
core_errored.append("youtube")
|
||||
|
||||
score_pct = int(len(core_active) / 5 * 100)
|
||||
|
||||
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||
active_sources = research_results.get("active_sources") or []
|
||||
nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None
|
||||
|
||||
return {
|
||||
"score_pct": score_pct,
|
||||
"core_active": core_active,
|
||||
"core_missing": core_missing,
|
||||
"core_errored": core_errored,
|
||||
"nudge_text": nudge_text,
|
||||
}
|
||||
|
||||
|
||||
def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str:
|
||||
"""Build human-readable nudge text describing what was missed.
|
||||
|
||||
Prioritizes free suggestions. Optionally mentions bonus sources
|
||||
(TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured.
|
||||
"""
|
||||
lines: List[str] = []
|
||||
|
||||
# Describe what was missed
|
||||
missed_parts: List[str] = []
|
||||
for src in core_missing:
|
||||
label = SOURCE_LABELS[src]
|
||||
if src in core_errored:
|
||||
missed_parts.append(f"{label} (errored this run)")
|
||||
else:
|
||||
missed_parts.append(label)
|
||||
|
||||
active_count = 5 - len(core_missing)
|
||||
lines.append(f"Research quality: {active_count}/5 core sources.")
|
||||
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
||||
lines.append("")
|
||||
|
||||
# Free suggestions
|
||||
free_suggestions: List[str] = []
|
||||
|
||||
if "x" in core_missing:
|
||||
if "x" in core_errored:
|
||||
free_suggestions.append(
|
||||
"X/Twitter errored - log into x.com in your browser, then re-run."
|
||||
)
|
||||
else:
|
||||
free_suggestions.append(
|
||||
"X/Twitter: real-time posts with likes and reposts - the fastest "
|
||||
"signal for breaking topics. Two options: log into x.com in your "
|
||||
"browser and re-run (cookies detected automatically), or add "
|
||||
"XAI_API_KEY to your .env (no browser access, get key at api.x.ai)."
|
||||
)
|
||||
|
||||
if "youtube" in core_missing:
|
||||
if "youtube" in core_errored:
|
||||
free_suggestions.append(
|
||||
"YouTube errored - update yt-dlp: brew upgrade yt-dlp"
|
||||
)
|
||||
else:
|
||||
free_suggestions.append(
|
||||
"YouTube: video transcripts with key moments - often the deepest "
|
||||
"explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
|
||||
)
|
||||
|
||||
# Mention bonus opt-in sources when SC key is present
|
||||
if has_sc:
|
||||
bonus_hints = []
|
||||
if "threads" not in (active_sources or []):
|
||||
bonus_hints.append("Threads")
|
||||
if "pinterest" not in (active_sources or []):
|
||||
bonus_hints.append("Pinterest")
|
||||
if bonus_hints:
|
||||
free_suggestions.append(
|
||||
f"Your SC key also powers {', '.join(bonus_hints)} and YouTube comments. "
|
||||
"Add them to INCLUDE_SOURCES in your .env to enable."
|
||||
)
|
||||
|
||||
if free_suggestions:
|
||||
lines.append("Free fixes:")
|
||||
for s in free_suggestions:
|
||||
lines.append(f" - {s}")
|
||||
lines.append("")
|
||||
|
||||
# Bonus sources mention (non-blocking)
|
||||
if not has_sc:
|
||||
lines.append(
|
||||
"Bonus: TikTok and Instagram are available with a free "
|
||||
"ScrapeCreators key at scrapecreators.com (no affiliation)."
|
||||
)
|
||||
else:
|
||||
lines.append("last30days has no affiliation with any API provider.")
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -334,7 +334,7 @@ def _global_search(
|
||||
)
|
||||
return data.get("posts", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
if e.status_code in (401, 403):
|
||||
raise
|
||||
_log(f"Global search error: {e}")
|
||||
return []
|
||||
@@ -376,11 +376,6 @@ def _subreddit_search(
|
||||
retries=2,
|
||||
)
|
||||
return data.get("posts", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
raise
|
||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||
return []
|
||||
@@ -408,11 +403,6 @@ def fetch_post_comments(
|
||||
retries=2,
|
||||
)
|
||||
return data.get("comments", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
raise
|
||||
_log(f"Comment fetch error: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
_log(f"Comment fetch error: {e}")
|
||||
return []
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user