Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b31ec05c74 |
@@ -1,20 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "last30days-skill",
|
|
||||||
"interface": {
|
|
||||||
"displayName": "Last 30 Days"
|
|
||||||
},
|
|
||||||
"plugins": [
|
|
||||||
{
|
|
||||||
"name": "last30days",
|
|
||||||
"source": {
|
|
||||||
"source": "local",
|
|
||||||
"path": "./"
|
|
||||||
},
|
|
||||||
"policy": {
|
|
||||||
"installation": "AVAILABLE",
|
|
||||||
"authentication": "ON_INSTALL"
|
|
||||||
},
|
|
||||||
"category": "Research"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,17 +1,16 @@
|
|||||||
{
|
{
|
||||||
|
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||||
"name": "last30days-skill",
|
"name": "last30days-skill",
|
||||||
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||||
"owner": {
|
"owner": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
"url": "https://github.com/mvanhorn"
|
"url": "https://github.com/mvanhorn"
|
||||||
},
|
},
|
||||||
"metadata": {
|
|
||||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
|
||||||
},
|
|
||||||
"plugins": [
|
"plugins": [
|
||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||||
"version": "3.3.0",
|
"version": "3.0.9",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
"url": "https://github.com/mvanhorn"
|
"url": "https://github.com/mvanhorn"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"version": "3.3.0",
|
"version": "3.0.14",
|
||||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
@@ -10,5 +10,6 @@
|
|||||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"],
|
||||||
|
"hooks": {}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"name": "last30days"
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
# Exclude non-runtime files from `git archive` output.
|
# Exclude non-runtime files from `git archive` output.
|
||||||
# Used by skills/last30days/scripts/build-skill.sh to produce a
|
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||||
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
|
|
||||||
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||||
|
|
||||||
# Anthropic canonical skill-packaging excludes
|
# Anthropic canonical skill-packaging excludes
|
||||||
@@ -33,8 +32,9 @@ release-notes.md export-ignore
|
|||||||
CHANGELOG.md export-ignore
|
CHANGELOG.md export-ignore
|
||||||
uv.lock export-ignore
|
uv.lock export-ignore
|
||||||
|
|
||||||
# Platform adapters are kept in git archives because Claude Code and Codex
|
# Platform adapters - skill-upload path is platform-agnostic
|
||||||
# plugin installs use the same repository archive as their source payload.
|
.agents/ export-ignore
|
||||||
|
.codex-plugin/ export-ignore
|
||||||
.hermes-plugin/ export-ignore
|
.hermes-plugin/ export-ignore
|
||||||
|
|
||||||
# CI workflows - repo-only, not needed at skill runtime
|
# CI workflows - repo-only, not needed at skill runtime
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
name: Bug Report
|
|
||||||
description: Report a bug or unexpected behavior
|
|
||||||
labels: [bug]
|
|
||||||
body:
|
|
||||||
- type: textarea
|
|
||||||
id: summary
|
|
||||||
attributes:
|
|
||||||
label: Summary
|
|
||||||
description: What happened?
|
|
||||||
placeholder: Describe the bug in 1-2 sentences.
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: repro
|
|
||||||
attributes:
|
|
||||||
label: Steps to Reproduce
|
|
||||||
description: How can we reproduce this?
|
|
||||||
placeholder: |
|
|
||||||
1. Run `python3 skills/last30days/scripts/last30days.py "topic" --emit=compact`
|
|
||||||
2. ...
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: expected
|
|
||||||
attributes:
|
|
||||||
label: Expected Behavior
|
|
||||||
description: What should have happened?
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: traceback
|
|
||||||
attributes:
|
|
||||||
label: Error / Traceback
|
|
||||||
description: Paste the full traceback or error output.
|
|
||||||
render: text
|
|
||||||
- type: dropdown
|
|
||||||
id: install
|
|
||||||
attributes:
|
|
||||||
label: Install Method
|
|
||||||
options:
|
|
||||||
- Claude Code plugin
|
|
||||||
- Gemini CLI extension
|
|
||||||
- Codex plugin
|
|
||||||
- Hermes skill
|
|
||||||
- Manual (git clone)
|
|
||||||
- Other
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: input
|
|
||||||
id: os
|
|
||||||
attributes:
|
|
||||||
label: OS
|
|
||||||
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
name: Feature Request
|
|
||||||
description: Suggest a new feature or improvement
|
|
||||||
labels: [enhancement]
|
|
||||||
body:
|
|
||||||
- type: textarea
|
|
||||||
id: problem
|
|
||||||
attributes:
|
|
||||||
label: Problem
|
|
||||||
description: What problem does this solve?
|
|
||||||
placeholder: When I try to ..., I can't ...
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: solution
|
|
||||||
attributes:
|
|
||||||
label: Proposed Solution
|
|
||||||
description: How should this work?
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
|
||||||
id: alternatives
|
|
||||||
attributes:
|
|
||||||
label: Alternatives Considered
|
|
||||||
description: Other approaches you thought of (optional).
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
## Summary
|
|
||||||
|
|
||||||
<!-- What does this PR do? 1-3 sentences. -->
|
|
||||||
|
|
||||||
## Changes
|
|
||||||
|
|
||||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
|
||||||
|
|
||||||
-
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
<!-- How did you verify this works? -->
|
|
||||||
|
|
||||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
|
||||||
|
|
||||||
## Related Issues
|
|
||||||
|
|
||||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
|
||||||
@@ -19,7 +19,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Build .skill artifact
|
- name: Build .skill artifact
|
||||||
run: |
|
run: |
|
||||||
bash skills/last30days/scripts/build-skill.sh
|
bash scripts/build-skill.sh
|
||||||
test -f dist/last30days.skill
|
test -f dist/last30days.skill
|
||||||
|
|
||||||
- name: Create GitHub release
|
- name: Create GitHub release
|
||||||
|
|||||||
@@ -1,67 +0,0 @@
|
|||||||
name: Security
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
dependency-audit:
|
|
||||||
name: Dependency audit
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v5
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
run: uv python install 3.12
|
|
||||||
|
|
||||||
- name: Export locked dependency set
|
|
||||||
run: |
|
|
||||||
uv export \
|
|
||||||
--locked \
|
|
||||||
--all-groups \
|
|
||||||
--no-hashes \
|
|
||||||
--format requirements.txt \
|
|
||||||
--output-file /tmp/last30days-requirements.txt
|
|
||||||
|
|
||||||
# Advisory-first: visibility before enforcement. This repo handles API keys,
|
|
||||||
# cookies, browser tokens, and local env files, so dependency CVEs should be
|
|
||||||
# visible in CI logs even before the project has a clean blocking baseline.
|
|
||||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
|
||||||
- name: Run pip-audit against locked dependencies
|
|
||||||
continue-on-error: true
|
|
||||||
run: uvx --python 3.12 pip-audit -r /tmp/last30days-requirements.txt --progress-spinner=off
|
|
||||||
|
|
||||||
secret-scan:
|
|
||||||
name: Secret scan
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout full history for diff-aware scanning
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
# Advisory-first: this reports verified secrets in pull requests and pushes to
|
|
||||||
# main, but does not block merges until maintainers confirm a clean baseline.
|
|
||||||
# The TruffleHog action automatically scans the PR range for pull_request
|
|
||||||
# events and the pushed commit range for push events.
|
|
||||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
|
||||||
# Contributor policy: never commit real secrets in fixtures, tests, docs, or
|
|
||||||
# examples; use obvious dummy values and env-based auth patterns instead.
|
|
||||||
- name: Run TruffleHog OSS secret scan
|
|
||||||
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
|
||||||
uses: trufflesecurity/trufflehog@v3.95.2
|
|
||||||
continue-on-error: true
|
|
||||||
with:
|
|
||||||
path: ./
|
|
||||||
version: v3.95.2
|
|
||||||
extra_args: --only-verified
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
name: Validate
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v5
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
run: uv python install 3.12
|
|
||||||
|
|
||||||
- name: Run test suite
|
|
||||||
run: uv run pytest
|
|
||||||
@@ -28,7 +28,9 @@ htmlcov/
|
|||||||
|
|
||||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||||
docs/plans/
|
docs/plans/
|
||||||
.context/
|
|
||||||
|
|
||||||
/work
|
# Marketing video build artifacts
|
||||||
/print
|
marketing/v3.1-launch/node_modules/
|
||||||
|
marketing/v3.1-launch/out/
|
||||||
|
marketing/*/node_modules/
|
||||||
|
marketing/*/out/
|
||||||
|
|||||||
@@ -1,54 +0,0 @@
|
|||||||
# last30days Skill
|
|
||||||
|
|
||||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
|
||||||
|
|
||||||
## Structure
|
|
||||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
|
||||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
|
||||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
|
||||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
|
||||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
|
||||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
|
||||||
|
|
||||||
## Orientation
|
|
||||||
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
|
||||||
- Feature design starts from the slash-command UX. A new engine flag with no SKILL.md integration is incomplete — the model invoking the skill won't know the flag exists.
|
|
||||||
- README and PR examples show `/last30days <topic>` first. Direct CLI invocation (`python3 scripts/last30days.py ...`) is a fallback for scripting, cron, and dev-time engine testing; label it as such, never as the primary path.
|
|
||||||
- Slash commands don't pass shell mechanics through. `/last30days OpenClaw --emit=html | pbcopy` is invalid in any harness — either use the slash form (no flags or pipes; let the model translate user intent into engine flags) or use the direct CLI form (full `python3 ...` with explicit flags and a real shell).
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
```bash
|
|
||||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
|
||||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
|
||||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
|
||||||
|
|
||||||
## Rules
|
|
||||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
|
||||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
|
||||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
|
||||||
|
|
||||||
## Security hygiene
|
|
||||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
|
||||||
- Use the env-based auth patterns in `skills/last30days/scripts/lib/env.py`; tests and fixtures must use obvious dummy values only.
|
|
||||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
|
||||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
|
||||||
|
|
||||||
## Maintaining CONFIGURATION.md
|
|
||||||
|
|
||||||
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
|
|
||||||
|
|
||||||
Update `CONFIGURATION.md` when:
|
|
||||||
|
|
||||||
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
|
|
||||||
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
|
|
||||||
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Hermes…)
|
|
||||||
- adding a new optional source that requires its own credential
|
|
||||||
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
|
|
||||||
|
|
||||||
Keep the existing structure organized by how often each layer is touched: per-run flags → env vars / `.env` → optional trend-monitoring stack → per-client patterns. Add new content into the right section rather than appending at the end.
|
|
||||||
|
|
||||||
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
|
|
||||||
|
|
||||||
## Beta channel
|
|
||||||
|
|
||||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
|
||||||
@@ -5,186 +5,6 @@ All notable changes to this project will be documented in this file.
|
|||||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
## [Unreleased]
|
|
||||||
|
|
||||||
## [3.3.0] - 2026-05-17
|
|
||||||
|
|
||||||
A week-long shipping cycle: ~75 PRs merged plus 7 community fixes salvaged through PR triage. Big themes: install story modernized for the multi-harness world (Claude Code, Codex, Cursor, Gemini CLI, Copilot, Windsurf, and 50+ Agent Skills hosts), new emit and source modes, and a substantial reliability sweep across Reddit, X, Windows, YouTube, and the planner.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
**Emit modes and sources**
|
|
||||||
|
|
||||||
- `--emit=html` for shareable, print-friendly HTML research briefs ([#332](https://github.com/mvanhorn/last30days-skill/pull/332)).
|
|
||||||
- **Digg AI 1000 source**, auto-enabled when `digg-pp-cli` is on PATH ([#370](https://github.com/mvanhorn/last30days-skill/pull/370)). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief.
|
|
||||||
|
|
||||||
**Configuration knobs**
|
|
||||||
|
|
||||||
- `EXCLUDE_SOURCES` env var — the inverse of `INCLUDE_SOURCES`, honored in source count and pipeline filter ([#399](https://github.com/mvanhorn/last30days-skill/pull/399)).
|
|
||||||
- `LAST30DAYS_YOUTUBE_SSH_HOST` — opt-in SSH routing for `yt-dlp` through a residential-IP host, for users on datacenter VPS hit by YouTube's bot-wall ([#376](https://github.com/mvanhorn/last30days-skill/pull/376)). Host validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection. Transcript path unchanged (uses HTTP fallback).
|
|
||||||
- macOS Keychain as a credential source — reads from the system keychain when env vars and config files aren't set ([#407](https://github.com/mvanhorn/last30days-skill/pull/407)).
|
|
||||||
- Configuration enablement: env-var defaults and source-resilience patterns across the config layer ([#344](https://github.com/mvanhorn/last30days-skill/pull/344)).
|
|
||||||
|
|
||||||
**Pipeline and storage**
|
|
||||||
|
|
||||||
- Reddit URL auto-enrichment from web search via the public JSON API ([#366](https://github.com/mvanhorn/last30days-skill/pull/366)).
|
|
||||||
- Per-run finding sightings recorded in the SQLite store ([#373](https://github.com/mvanhorn/last30days-skill/pull/373)).
|
|
||||||
- Brave browser support for X/Twitter cookie extraction ([#320](https://github.com/mvanhorn/last30days-skill/pull/320)).
|
|
||||||
|
|
||||||
**Tests and CI**
|
|
||||||
|
|
||||||
- Full pytest suite restored to CI; 13 rotted tests repaired ([#416](https://github.com/mvanhorn/last30days-skill/pull/416)).
|
|
||||||
- `greptile.json` added with `triggerOnUpdates` + `statusCheck` ([#418](https://github.com/mvanhorn/last30days-skill/pull/418)).
|
|
||||||
- Advisory security workflow ([#368](https://github.com/mvanhorn/last30days-skill/pull/368)).
|
|
||||||
- Parallel grounding backend test coverage ([#355](https://github.com/mvanhorn/last30days-skill/pull/355)).
|
|
||||||
|
|
||||||
**Docs**
|
|
||||||
|
|
||||||
- New `CONFIGURATION.md` with README pointers ([#339](https://github.com/mvanhorn/last30days-skill/pull/339)).
|
|
||||||
- `docs/solutions/` learning capture for release-time consistency-test cascades ([#413](https://github.com/mvanhorn/last30days-skill/pull/413)) and the eval-not-in-CI design decision ([#417](https://github.com/mvanhorn/last30days-skill/pull/417)).
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
**Install story modernized**
|
|
||||||
|
|
||||||
- `npx skills add` is now the canonical install path for every harness ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). README and SKILL.md flipped to recommend `npx skills add . -g -y` over per-harness manual instructions. Surfaces Gemini CLI, Copilot, Windsurf, and 50+ other Agent Skills hosts that the install pattern reaches.
|
|
||||||
- README dropped the Gemini CLI native-extension install path (now covered by `npx skills add`).
|
|
||||||
- `hooks.json` made polyglot for Gemini CLI + Claude Code compatibility ([#318](https://github.com/mvanhorn/last30days-skill/pull/318)).
|
|
||||||
|
|
||||||
**Skill semantics and multi-harness reframe**
|
|
||||||
|
|
||||||
- `AGENTS.md` is now canonical; `CLAUDE.md` points at it ([#410](https://github.com/mvanhorn/last30days-skill/pull/410)). Reframes the project as a multi-harness Agent Skills package rather than a Claude-Code-specific tool.
|
|
||||||
- SKILL.md path resolution rewritten: STEP 0 narrows to a Claude-Code-marketplaces-only stale-clone guard; Step 1 walks a single `SKILL_DIR` substitution pattern ([#400](https://github.com/mvanhorn/last30days-skill/pull/400), [#409](https://github.com/mvanhorn/last30days-skill/pull/409)). Removes ~80 lines of bash and fixes a real spec-vs-engine divergence where the previous resolver could pick a different install than the SKILL.md the model loaded from.
|
|
||||||
- SKILL.md version regex consolidated into `lib/skill_meta.py` ([#412](https://github.com/mvanhorn/last30days-skill/pull/412)).
|
|
||||||
- `--plan` / `--competitors-plan` invocation templates switched from inline single-quoted JSON to heredoc-written tmpfiles ([#404](https://github.com/mvanhorn/last30days-skill/pull/404), fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403)). Apostrophes in resolved context strings ("McDonald's", "people's choice") no longer break shell parsing.
|
|
||||||
- `POSTS_PER_CLUSTER` raised 3→5 and render-side display limit 2→3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous caps routinely truncated cluster context.
|
|
||||||
- Digg AI 1000 renamed to "Digg" in user-facing output ([#372](https://github.com/mvanhorn/last30days-skill/pull/372)) — footer line, source label, inline-quote suffix, why_relevant, container attribution. Internal references retain the upstream product name.
|
|
||||||
- GitHub repo resolution canonicalized for ambiguous product comparisons ([#302](https://github.com/mvanhorn/last30days-skill/pull/302)).
|
|
||||||
|
|
||||||
**Dependencies and tooling**
|
|
||||||
|
|
||||||
- Dropped `requests` runtime dependency. All providers route through stdlib `urllib` via the `lib/http` wrapper ([#393](https://github.com/mvanhorn/last30days-skill/pull/393)).
|
|
||||||
- Migrated to `gemini-3.1-flash-lite` GA model ([#378](https://github.com/mvanhorn/last30days-skill/pull/378)).
|
|
||||||
- Aligned Codex/Claude plugin manifests + added Codex `AGENTS.md` ([#321](https://github.com/mvanhorn/last30days-skill/pull/321)).
|
|
||||||
- pytest dev dep bumped 9.0.2 → 9.0.3 ([#414](https://github.com/mvanhorn/last30days-skill/pull/414)).
|
|
||||||
|
|
||||||
### Removed
|
|
||||||
|
|
||||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1 ([#400](https://github.com/mvanhorn/last30days-skill/pull/400)). Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
|
||||||
- **`skills/last30days/scripts/sync.sh`** — maintainer dev-deploy script ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). Replaced by `npx skills add . -g -y` (live-symlink into every detected harness's skill dir — better than sync.sh's copy model since edits propagate live). Hermes uses `hermes skills install mvanhorn/last30days-skill --force`; OpenClaw uses `clawhub install last30days-official`.
|
|
||||||
- Orphaned `SPEC.md` and `TASKS.md` ([#419](https://github.com/mvanhorn/last30days-skill/pull/419)).
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
**Reddit**
|
|
||||||
|
|
||||||
- `lstrip("r/")` mangled subreddits starting with `r` (`r/robotics` → `obotics`, `r/ruby` → `uby`); replaced with `removeprefix("r/")` at 4 sites (Alex Key, salvaged from #288).
|
|
||||||
- Browser-like User-Agent + `Accept-Language`/`Accept-Encoding`/`Connection` headers + gzip decompression to fix `urllib` 403s on Reddit's public JSON endpoint (Franco Carballar, salvaged from #199).
|
|
||||||
- HTTP 402 re-raised across all three ScrapeCreators paths (`_global_search`, `_subreddit_search`, `fetch_post_comments`) so the OpenAI/public-JSON fallback chain triggers when credits are exhausted (Jonathan Oppenheim, salvaged from #170).
|
|
||||||
|
|
||||||
**Authentication and credentials**
|
|
||||||
|
|
||||||
- Restored multi-key rotation for `SCRAPECREATORS_API_KEY` accidentally dropped in v3.0.6 (Eric Oberhofer, salvaged from #287). Comma-separated keys round-robin via `random.choice` per run.
|
|
||||||
|
|
||||||
**Windows compatibility**
|
|
||||||
|
|
||||||
- `os.killpg` in `_cleanup_children()` guarded with `hasattr(os, "killpg")`, falls back to `os.kill(SIGTERM)` (gujishh, salvaged from #226).
|
|
||||||
- POSIX-style secret-permission warning skipped on Windows ([#357](https://github.com/mvanhorn/last30days-skill/pull/357)).
|
|
||||||
- Render uses forward slashes in save-path footer for Windows ([#338](https://github.com/mvanhorn/last30days-skill/pull/338)).
|
|
||||||
|
|
||||||
**xAI / X / xurl**
|
|
||||||
|
|
||||||
- `parse_x_response` now raises `http.HTTPError` on empty output, missing JSON, or decode failure — surfaces in `errors_by_source` instead of silently returning an empty result list (Kaustav Mishra, salvaged from #155).
|
|
||||||
- `xurl` treats `PermissionError` from PATH lookup as unavailable ([#322](https://github.com/mvanhorn/last30days-skill/pull/322)).
|
|
||||||
|
|
||||||
**YouTube**
|
|
||||||
|
|
||||||
- SC YouTube + multi-token HN searches unblocked ([#388](https://github.com/mvanhorn/last30days-skill/pull/388)).
|
|
||||||
- Transcript-fetch ratio surfaced + degraded-run nudge for stale `yt-dlp` ([#340](https://github.com/mvanhorn/last30days-skill/pull/340)).
|
|
||||||
|
|
||||||
**bird_x / HTTP**
|
|
||||||
|
|
||||||
- Subprocess retry on non-JSON stdout to handle X anti-bot HTML interstitials ([#383](https://github.com/mvanhorn/last30days-skill/pull/383)).
|
|
||||||
- HTTP retry budget expanded + exponential backoff on DNS resolution failure ([#382](https://github.com/mvanhorn/last30days-skill/pull/382)).
|
|
||||||
- Parallel AI search aligned with current API schema ([#341](https://github.com/mvanhorn/last30days-skill/pull/341)).
|
|
||||||
- Parallel web backend routed through grounding ([#354](https://github.com/mvanhorn/last30days-skill/pull/354)).
|
|
||||||
|
|
||||||
**Planner and sources**
|
|
||||||
|
|
||||||
- `xquik` registered in `SOURCE_CAPABILITIES` ([#336](https://github.com/mvanhorn/last30days-skill/pull/336), fixes [#319](https://github.com/mvanhorn/last30days-skill/issues/319)).
|
|
||||||
- Honor explicit optional source requests ([#356](https://github.com/mvanhorn/last30days-skill/pull/356)).
|
|
||||||
- ScrapeCreators source-gating aligned between code and docs ([#415](https://github.com/mvanhorn/last30days-skill/pull/415)).
|
|
||||||
- OpenClaw works without ScrapeCreators key ([#392](https://github.com/mvanhorn/last30days-skill/pull/392), by @thinkun).
|
|
||||||
|
|
||||||
**Render, version display, hosting paths**
|
|
||||||
|
|
||||||
- Hardcoded `v3.0.0` in render replaced with dynamic `_skill_version()` ([#365](https://github.com/mvanhorn/last30days-skill/pull/365)).
|
|
||||||
- Comparison HTML artifacts saved correctly ([#389](https://github.com/mvanhorn/last30days-skill/pull/389)).
|
|
||||||
- `OPENROUTER_DEFAULT` model ID corrected ([#323](https://github.com/mvanhorn/last30days-skill/pull/323)).
|
|
||||||
- OpenClaw poll-timing initialized once ([#358](https://github.com/mvanhorn/last30days-skill/pull/358)).
|
|
||||||
- Prefer sandboxed Safari cookie path ([#343](https://github.com/mvanhorn/last30days-skill/pull/343)).
|
|
||||||
- Preserve clean mode for last-run state ([#334](https://github.com/mvanhorn/last30days-skill/pull/334)).
|
|
||||||
- Replaced hardcoded `/Users/mvanhorn/...` paths in `test-v1-vs-v2.sh` with portable env-var overrides (Dave Morin, salvaged from #297).
|
|
||||||
|
|
||||||
**Hooks**
|
|
||||||
|
|
||||||
- `check-config.sh` path-quoting fix for paths with spaces ([#337](https://github.com/mvanhorn/last30days-skill/pull/337)).
|
|
||||||
- Replaced unsafe `eval` with `declare` in `check-config.sh` ([#364](https://github.com/mvanhorn/last30days-skill/pull/364)).
|
|
||||||
|
|
||||||
**Sync and version metadata**
|
|
||||||
|
|
||||||
- `sync.sh` pointed at this repo's plugin cache, not the private repo's ([#402](https://github.com/mvanhorn/last30days-skill/pull/402)).
|
|
||||||
- Sync cache target bumped to 3.2.1 to match SKILL.md ([#397](https://github.com/mvanhorn/last30days-skill/pull/397)).
|
|
||||||
- ScrapeCreators free-tier credit count corrected to 100 in docs ([#369](https://github.com/mvanhorn/last30days-skill/pull/369), fixes [#367](https://github.com/mvanhorn/last30days-skill/issues/367)).
|
|
||||||
- Gemini extension version synced ([#349](https://github.com/mvanhorn/last30days-skill/pull/349)).
|
|
||||||
- Various stale path/link fixes ([#345](https://github.com/mvanhorn/last30days-skill/pull/345), [#346](https://github.com/mvanhorn/last30days-skill/pull/346), [#347](https://github.com/mvanhorn/last30days-skill/pull/347), [#348](https://github.com/mvanhorn/last30days-skill/pull/348), [#351](https://github.com/mvanhorn/last30days-skill/pull/351)).
|
|
||||||
|
|
||||||
### Contributors
|
|
||||||
|
|
||||||
First-time contributors whose fixes shipped in this release (most via PR triage salvage — fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
|
||||||
|
|
||||||
- Dave Morin — portable test-harness paths
|
|
||||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
|
||||||
- Eric Oberhofer — multi-key rotation restored
|
|
||||||
- gujishh — Windows process cleanup
|
|
||||||
- Franco Carballar — Reddit browser-like headers
|
|
||||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
|
||||||
- Kaustav Mishra — xAI error surfacing
|
|
||||||
- [@thinkun](https://github.com/thinkun) ([#363](https://github.com/mvanhorn/last30days-skill/pull/363)) — OpenClaw ScrapeCreators-key-optional fix
|
|
||||||
|
|
||||||
Full PR list at [github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0](https://github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0).
|
|
||||||
|
|
||||||
## [3.2.0] - 2026-05-09
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
|
|
||||||
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
|
|
||||||
|
|
||||||
## [3.1.1] - 2026-04-24
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
|
|
||||||
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
|
|
||||||
|
|
||||||
## [3.1.0] - 2026-04-22
|
|
||||||
|
|
||||||
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
|
||||||
- **Broken README link.** The README's "source of truth" link pointed at root `SKILL.md`, which is no longer maintained after the plugin-layout restructure. Fixed to point at `skills/last30days/SKILL.md`.
|
|
||||||
|
|
||||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
|
||||||
|
|
||||||
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
|
|
||||||
|
|
||||||
## [3.0.14] - 2026-04-22
|
## [3.0.14] - 2026-04-22
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
@@ -241,7 +61,6 @@ Individual changelog entries for 3.0.10 through 3.0.14 below document the increm
|
|||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
||||||
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
|
|
||||||
|
|
||||||
### Behavior fallback
|
### Behavior fallback
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1,25 @@
|
|||||||
@AGENTS.md
|
# last30days Skill
|
||||||
|
|
||||||
|
Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||||
|
Python scripts with multi-source search aggregation.
|
||||||
|
|
||||||
|
## Structure
|
||||||
|
- `scripts/last30days.py` — main research engine
|
||||||
|
- `scripts/lib/` — search, enrichment, rendering modules
|
||||||
|
- `scripts/lib/vendor/bird-search/` — vendored X search client
|
||||||
|
- `SKILL.md` — skill definition (deployed to ~/.claude/skills/last30days/)
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
```bash
|
||||||
|
python3 scripts/last30days.py "test query" --emit=compact # Run research
|
||||||
|
bash scripts/sync.sh # Deploy to ~/.claude, ~/.agents, ~/.codex
|
||||||
|
```
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||||
|
- After edits: run `bash scripts/sync.sh` to deploy
|
||||||
|
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||||
|
|
||||||
|
## Beta channel
|
||||||
|
|
||||||
|
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
# Concepts
|
|
||||||
|
|
||||||
Shared vocabulary for `last30days-skill`. Terms here have a precise project-specific meaning — distinct enough from their general technical sense that a new contributor would need them defined to follow conversations, PR descriptions, or the SKILL.md contract.
|
|
||||||
|
|
||||||
## The package
|
|
||||||
|
|
||||||
### Skill
|
|
||||||
|
|
||||||
A self-contained agent-instructions package consisting of a `SKILL.md` prose contract plus a sibling `scripts/` directory containing the executable code the SKILL.md invokes. The package conforms to the [Agent Skills](https://agentskills.io) open format and installs across every major harness (Claude Code, Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ others) via `npx skills add`, harness-native plugin installers, or per-harness skill directories. A Skill is the unit of distribution; the Skill is the product.
|
|
||||||
|
|
||||||
### Engine
|
|
||||||
|
|
||||||
The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do the actual research work. The Engine and SKILL.md have a contract: SKILL.md tells the model which flags to pass (`--plan`, `--competitors-plan`, `--x-handle`, `--subreddits`, `--emit=compact`, etc.), and the Engine produces a specific output shape (badge line, ranked evidence clusters, emoji-tree footer) that the model is contractually required to pass through. The Engine is implementation; the SKILL.md prose is the agent-facing surface.
|
|
||||||
|
|
||||||
### Harness
|
|
||||||
|
|
||||||
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
|
||||||
|
|
||||||
## Distribution
|
|
||||||
|
|
||||||
### Beta channel
|
|
||||||
|
|
||||||
A parallel install of the Skill, sourced from the private `mvanhorn/last30days-skill-private` repo and installed as `/last30days-beta` rather than `/last30days`. The Beta channel exists so experimental changes can be tested by real users before they ship to the public `/last30days`. Promotion from Beta to public happens via a review PR against this (public) repo — Beta-only changes never ship to public without that PR. The Beta channel workflow guide lives in `BETA.md` in the private repo.
|
|
||||||
@@ -1,268 +0,0 @@
|
|||||||
# Configuration
|
|
||||||
|
|
||||||
Everything you can tune in `/last30days` without editing the engine source.
|
|
||||||
Three layers, in order of how often you'll touch them:
|
|
||||||
|
|
||||||
1. **Per-run flags** - what you pass on the command line.
|
|
||||||
2. **Environment variables and `.env`** - what's enabled across all runs.
|
|
||||||
3. **Optional trend-monitoring stack** - SQLite store, watchlist, briefings.
|
|
||||||
|
|
||||||
Per-client patterns and the experimental beta channel are at the bottom.
|
|
||||||
|
|
||||||
> Skip ahead: [Where output is saved](#where-output-is-saved) - [API keys](#api-keys-env) - [Reasoning provider](#reasoning-provider-priority) - [Web search backend](#web-search-backend-priority) - [Trend monitoring](#trend-monitoring-store--watchlist--briefings) - [Per-client patterns](#per-client-patterns) - [Beta channel](#beta-channel)
|
|
||||||
|
|
||||||
## Why this document exists
|
|
||||||
|
|
||||||
This is a focused **configuration reference** maintained alongside the engine. The runtime contract (the voice rules, the planner protocol, the LAWs the synthesizing model follows) lives in [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md) - that file is authoritative when the two ever differ. This file's job is narrower: surface every knob a user or operator can turn, in one place, kept current with the code so client-facing setups stay reliable. New configuration knobs added to the engine should be reflected here in the same PR.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Where output is saved
|
|
||||||
|
|
||||||
| Platform | Default path | Override |
|
|
||||||
|---|---|---|
|
|
||||||
| Linux / macOS | `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` | set `LAST30DAYS_MEMORY_DIR=/path` |
|
|
||||||
| Windows | `LAST30DAYS_MEMORY_DIR` defaults to `C:\Users\<you>\Documents\Last30Days\` | set `LAST30DAYS_MEMORY_DIR=C:\path` |
|
|
||||||
|
|
||||||
Each run produces one file per topic, slug-named:
|
|
||||||
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
|
|
||||||
|
|
||||||
**Per-run overrides:**
|
|
||||||
- `--save-dir <path>` - one-off output location.
|
|
||||||
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
|
||||||
|
|
||||||
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## API keys (`.env`)
|
|
||||||
|
|
||||||
The skill reads keys from a `.env` file. Two locations are supported, in priority order:
|
|
||||||
|
|
||||||
1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present.
|
|
||||||
2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback.
|
|
||||||
|
|
||||||
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
|
||||||
|
|
||||||
The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case.
|
|
||||||
|
|
||||||
**Source-by-source** - what each key unlocks:
|
|
||||||
|
|
||||||
| Source | Key(s) | Required for | Free tier |
|
|
||||||
|---|---|---|---|
|
|
||||||
| Reddit (public) | none | always on | yes |
|
|
||||||
| Hacker News | none | always on | yes |
|
|
||||||
| Polymarket | none | always on | yes |
|
|
||||||
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
|
|
||||||
| YouTube | `yt-dlp` CLI installed | always on if `yt-dlp` present | yes |
|
|
||||||
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; xAI / ScrapeCreators = paid |
|
|
||||||
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
|
|
||||||
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
|
|
||||||
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
|
|
||||||
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
|
|
||||||
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
|
|
||||||
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
|
|
||||||
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
|
|
||||||
| Perplexity Deep Research | `OPENROUTER_API_KEY` | `--deep-research` flag (~$0.90/query) | no |
|
|
||||||
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
|
|
||||||
|
|
||||||
**Example `.env` skeleton** (placeholders only - replace with your own values):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Reasoning + planning (one provider; see priority below)
|
|
||||||
GOOGLE_API_KEY=<your-gemini-key>
|
|
||||||
|
|
||||||
# Web search backend (one is enough; Brave is the cheapest)
|
|
||||||
BRAVE_API_KEY=<your-brave-key>
|
|
||||||
|
|
||||||
# Optional sources
|
|
||||||
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
|
|
||||||
INCLUDE_SOURCES=tiktok,instagram
|
|
||||||
|
|
||||||
# X authentication (one option only)
|
|
||||||
XAI_API_KEY=<your-xai-key>
|
|
||||||
# OR cookie-jar (no key needed; logs in via your browser session)
|
|
||||||
# FROM_BROWSER=firefox
|
|
||||||
|
|
||||||
# Bluesky
|
|
||||||
BSKY_HANDLE=<your-handle>.bsky.social
|
|
||||||
BSKY_APP_PASSWORD=<your-app-password>
|
|
||||||
```
|
|
||||||
|
|
||||||
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
|
||||||
|
|
||||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search.
|
|
||||||
|
|
||||||
### Bluesky app-password format and search host
|
|
||||||
|
|
||||||
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
|
|
||||||
|
|
||||||
The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Reasoning provider priority
|
|
||||||
|
|
||||||
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
|
||||||
|
|
||||||
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
|
||||||
2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`)
|
|
||||||
3. **xAI** - `XAI_API_KEY`
|
|
||||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (also unlocks `--deep-research`)
|
|
||||||
5. **Local / deterministic** - always available, lowest quality
|
|
||||||
|
|
||||||
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Web search backend priority
|
|
||||||
|
|
||||||
Used by `--auto-resolve` (when WebSearch isn't available from the host) and Step 2 supplements. Auto-detect priority (override per-run with `--web-backend=<name>`):
|
|
||||||
|
|
||||||
1. **Brave** - `BRAVE_API_KEY`
|
|
||||||
2. **Exa** - `EXA_API_KEY`
|
|
||||||
3. **Serper** - `SERPER_API_KEY`
|
|
||||||
4. **Parallel** - `PARALLEL_API_KEY`
|
|
||||||
5. **Host's native WebSearch** - Claude Code, Codex, Gemini all have one built in
|
|
||||||
|
|
||||||
Visible quality difference between hosts with vs without a configured backend. If your client setup produces thinner results than yours, this is usually why.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Trend monitoring (`--store` + watchlist + briefings)
|
|
||||||
|
|
||||||
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
|
|
||||||
|
|
||||||
### `--store` flag
|
|
||||||
|
|
||||||
Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate.
|
|
||||||
|
|
||||||
**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`.
|
|
||||||
|
|
||||||
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
|
|
||||||
|
|
||||||
### `watchlist.py` - recurring topics
|
|
||||||
|
|
||||||
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
|
|
||||||
|
|
||||||
Two-step flow (the watchlist holds the topic; an external scheduler invokes the run):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Add the topic to the watchlist
|
|
||||||
# Default schedule daily 8am; --weekly switches to Mondays 8am
|
|
||||||
python3 scripts/watchlist.py add "british airways middle east" --weekly
|
|
||||||
|
|
||||||
# 2. Configure delivery and budget (optional)
|
|
||||||
python3 scripts/watchlist.py config delivery "https://hooks.slack.com/services/..."
|
|
||||||
python3 scripts/watchlist.py config budget 5.00
|
|
||||||
|
|
||||||
# 3. Trigger via cron / Task Scheduler / GitHub Actions
|
|
||||||
python3 scripts/watchlist.py run-one "british airways middle east"
|
|
||||||
# or run every enabled topic, gated by daily_budget
|
|
||||||
python3 scripts/watchlist.py run-all
|
|
||||||
```
|
|
||||||
|
|
||||||
The schedule field stored on each topic is metadata - the actual cron / Task Scheduler invocation is your responsibility. Watchlist runs hardcode `--quick` and `--lookback-days 90` when spawning the underlying engine.
|
|
||||||
|
|
||||||
### `briefing.py` - daily / weekly digests
|
|
||||||
|
|
||||||
[`scripts/briefing.py`](skills/last30days/scripts/briefing.py) reads the SQLite store and emits structured data the agent then synthesizes into prose. Modes: `generate` (daily), `generate --weekly`, `show [--date DATE]` (display a saved briefing). Briefs save to `~/.local/share/last30days/briefs/`.
|
|
||||||
|
|
||||||
### Recommended cadence pattern
|
|
||||||
|
|
||||||
| Step | Cadence | Command |
|
|
||||||
|---|---|---|
|
|
||||||
| Baseline | one-time per topic | `/last30days "<topic>" --days=30 --store` |
|
|
||||||
| Add to watchlist | one-time per topic | `python3 scripts/watchlist.py add "<topic>" --weekly` |
|
|
||||||
| Recurring run | daily or weekly (external scheduler) | `python3 scripts/watchlist.py run-all` |
|
|
||||||
| Digest | weekly | `python3 scripts/briefing.py generate --weekly` |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Per-client patterns
|
|
||||||
|
|
||||||
The skill is built to flex around different client environments. Four patterns that compose well:
|
|
||||||
|
|
||||||
### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders)
|
|
||||||
|
|
||||||
The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
|
||||||
SCRAPECREATORS_API_KEY=<acme-scoped-key-or-shared>
|
|
||||||
INCLUDE_SOURCES=tiktok,instagram
|
|
||||||
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
|
||||||
```
|
|
||||||
|
|
||||||
`cd` into the client folder, run `/last30days <topic>` as normal, no flags or wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
|
||||||
|
|
||||||
### 2. Per-client save dir + suffix wrapper
|
|
||||||
|
|
||||||
For workflows where you don't `cd` into a client folder (running from anywhere, scripted batches), a tiny shell function isolates each client's research without engine changes.
|
|
||||||
|
|
||||||
PowerShell example:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
function Run-L30D-Client {
|
|
||||||
param([string]$ClientSlug, [Parameter(ValueFromRemainingArguments=$true)]$Args)
|
|
||||||
$env:LAST30DAYS_MEMORY_DIR = "C:\Users\$env:USERNAME\Clients\$ClientSlug\Research\Last30Days"
|
|
||||||
/last30days @Args --save-suffix=$ClientSlug
|
|
||||||
}
|
|
||||||
# Usage: Run-L30D-Client acme "british airways middle east"
|
|
||||||
```
|
|
||||||
|
|
||||||
Bash example:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
l30d-client() {
|
|
||||||
local client=$1; shift
|
|
||||||
LAST30DAYS_MEMORY_DIR="$HOME/Clients/$client/Research/Last30Days" \
|
|
||||||
/last30days "$@" --save-suffix="$client"
|
|
||||||
}
|
|
||||||
# Usage: l30d-client acme "british airways middle east"
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Custom category-peer subreddits
|
|
||||||
|
|
||||||
[`scripts/lib/categories.py`](skills/last30days/scripts/lib/categories.py) holds a table of `(category_id, trigger_keywords, peer_subreddits)`. If a client lives in a vertical that isn't covered (legal-tech, real-estate-tech, B2B HR SaaS), add a row. Pure data, no logic.
|
|
||||||
|
|
||||||
Section 2a of `SKILL.md` documents the merging rule the skill applies when your topic matches a category.
|
|
||||||
|
|
||||||
### 4. Pre-built `--competitors-plan` JSON
|
|
||||||
|
|
||||||
For competitor-vs-comparisons that recur, a pre-written JSON skeleton per client industry saves real time:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"Competitor B": {
|
|
||||||
"x_handle": "competitor_b_handle",
|
|
||||||
"subreddits": ["sub1", "sub2"],
|
|
||||||
"github_user": "competitor-b-org",
|
|
||||||
"context": "Founded 2019, focused on ..."
|
|
||||||
},
|
|
||||||
"Competitor C": { ... }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Pass as `--competitors-plan @client/competitors-plan.json` (or as a string). See `SKILL.md` section "If QUERY_TYPE = COMPARISON" for the full schema.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Beta channel
|
|
||||||
|
|
||||||
Experimental customizations live on a private companion repo (`mvanhorn/last30days-skill-private`) installed as `/last30days-beta`. Never ship beta-only changes to the public marketplace without a review PR against the public repo. Workflow guide: `BETA.md` in the private repo.
|
|
||||||
|
|
||||||
This is the right home for client-specific changes you don't intend to upstream - custom category rows, internal subreddit lists, per-vertical plan templates.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Cross-references
|
|
||||||
|
|
||||||
- The CLI flag surface: `python3 scripts/last30days.py --help`
|
|
||||||
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
|
|
||||||
- Engine spec (some sections stale; SKILL.md wins on conflicts): [`SPEC.md`](SPEC.md)
|
|
||||||
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
|
|
||||||
@@ -23,7 +23,7 @@ v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos
|
|||||||
### @thinkun
|
### @thinkun
|
||||||
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
||||||
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
||||||
> Thinker, technologist, AI expert, music-tinkerer. Founder of [Thinkun](https://thinkun.com). [@thinkun on GitHub](https://github.com/thinkun) · [@unthink on X](https://x.com/unthink)
|
> _Add your bio, website, or anything you'd like here._
|
||||||
|
|
||||||
### @thomasmktong
|
### @thomasmktong
|
||||||
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
||||||
|
|||||||
@@ -10,20 +10,28 @@ This guide covers installing last30days on Hermes AI Agent.
|
|||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
|
### Option 1: Via sync.sh (Recommended)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
hermes skills install mvanhorn/last30days-skill --force
|
# Clone the repo
|
||||||
|
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||||
|
cd last30days-skill
|
||||||
|
|
||||||
|
# Run the sync script
|
||||||
|
bash scripts/sync.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days/`
|
||||||
|
|
||||||
### Developer / live-edit alternative
|
### Option 2: Manual Copy
|
||||||
|
|
||||||
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
# Create directory
|
||||||
mkdir -p ~/.hermes/skills/research
|
mkdir -p ~/.hermes/skills/research/last30days
|
||||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
|
|
||||||
|
# Copy files
|
||||||
|
cp -r scripts ~/.hermes/skills/research/last30days/
|
||||||
|
cp .hermes-plugin/SKILL.md ~/.hermes/skills/research/last30days/
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
@@ -51,7 +59,7 @@ On first run, the skill will guide you through setup:
|
|||||||
|
|
||||||
2. **Optional: ScrapeCreators**
|
2. **Optional: ScrapeCreators**
|
||||||
- Adds TikTok, Instagram, Reddit backup
|
- Adds TikTok, Instagram, Reddit backup
|
||||||
- 100 free credits (no expiration)
|
- 10,000 free API calls
|
||||||
- Sign up at scrapecreators.com
|
- Sign up at scrapecreators.com
|
||||||
|
|
||||||
3. **Optional: API Keys**
|
3. **Optional: API Keys**
|
||||||
@@ -98,11 +106,13 @@ python3.12 scripts/last30days.py --diagnose
|
|||||||
|
|
||||||
## Updating
|
## Updating
|
||||||
|
|
||||||
```bash
|
To update to the latest version:
|
||||||
hermes skills install mvanhorn/last30days-skill --force
|
|
||||||
```
|
|
||||||
|
|
||||||
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step.
|
```bash
|
||||||
|
cd last30days-skill
|
||||||
|
git pull
|
||||||
|
bash scripts/sync.sh
|
||||||
|
```
|
||||||
|
|
||||||
## Support
|
## Support
|
||||||
|
|
||||||
|
|||||||
@@ -14,19 +14,21 @@
|
|||||||
|
|
||||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||||
|
|
||||||
**Claude Code (recommended — auto-updates via marketplace):**
|
Claude Code:
|
||||||
```
|
```
|
||||||
/plugin marketplace add mvanhorn/last30days-skill
|
/plugin marketplace add mvanhorn/last30days-skill
|
||||||
/plugin install last30days
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
OpenClaw:
|
||||||
```
|
```
|
||||||
npx skills add mvanhorn/last30days-skill -g
|
clawhub install last30days-official
|
||||||
```
|
```
|
||||||
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
|
|
||||||
|
|
||||||
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
|
Hermes:
|
||||||
|
```
|
||||||
|
# The skill auto-deploys when you run sync.sh
|
||||||
|
# Or manually copy to ~/.hermes/skills/research/last30days/
|
||||||
|
```
|
||||||
|
|
||||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||||
|
|
||||||
@@ -66,7 +68,6 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
|||||||
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
||||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||||
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
|
||||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||||
@@ -95,28 +96,6 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
|||||||
|
|
||||||
## What v3 Changed
|
## What v3 Changed
|
||||||
|
|
||||||
### Shareable HTML briefs
|
|
||||||
|
|
||||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
|
||||||
|
|
||||||
```
|
|
||||||
/last30days OpenClaw --emit=html
|
|
||||||
```
|
|
||||||
|
|
||||||
or just ask in plain language:
|
|
||||||
|
|
||||||
```
|
|
||||||
/last30days OpenClaw, give me a shareable HTML brief
|
|
||||||
/last30days Cursor IDE for slack
|
|
||||||
/last30days Anthropic earnings export as html
|
|
||||||
```
|
|
||||||
|
|
||||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
|
||||||
|
|
||||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
|
||||||
|
|
||||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
|
||||||
|
|
||||||
### Intelligent search: the killer feature
|
### Intelligent search: the killer feature
|
||||||
|
|
||||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||||
@@ -153,10 +132,8 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
|||||||
|
|
||||||
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
||||||
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
||||||
- **TikTok, Instagram, Threads.** All three activate automatically once `SCRAPECREATORS_API_KEY` is set — same key, same per-call cost. Suppress any of them with `EXCLUDE_SOURCES=tiktok,instagram,threads` (any comma-separated subset).
|
- **Threads, Pinterest, YouTube + TikTok comments.** Opt-in sources via ScrapeCreators. Set `INCLUDE_SOURCES=tiktok,instagram` and add threads, pinterest, youtube_comments, tiktok_comments for more. `youtube_comments` and `tiktok_comments` surface top comments with vote counts the same way Reddit does.
|
||||||
- **Pinterest.** Per-query opt-in (visual pins, narrow utility): the model passes `--search=pinterest` for the runs that need it. Requires `SCRAPECREATORS_API_KEY`.
|
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` to unlock.
|
||||||
- **YouTube + TikTok comments.** Persistent opt-in via `INCLUDE_SOURCES=youtube_comments,tiktok_comments` because each video pulls N extra ScrapeCreators calls on top of the base search. Surface top comments with vote counts the same way Reddit does.
|
|
||||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` and `INCLUDE_SOURCES=perplexity` (it's a separate paid API — opt-in keeps you from being surprise-billed).
|
|
||||||
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
||||||
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
||||||
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
||||||
@@ -168,61 +145,12 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
|||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
| Surface | Install | Updates |
|
| Surface | Install |
|
||||||
|---------|---------|---------|
|
|---------|---------|
|
||||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + |
|
||||||
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
| **Claude Code** | `/plugin marketplace add mvanhorn/last30days-skill` |
|
||||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
| **OpenClaw** | `clawhub install last30days-official` |
|
||||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
| **Gemini CLI** | Clone then `gemini extensions install ./last30days-skill` (see below) |
|
||||||
|
|
||||||
### Claude Code (recommended)
|
|
||||||
|
|
||||||
```
|
|
||||||
/plugin marketplace add mvanhorn/last30days-skill
|
|
||||||
```
|
|
||||||
|
|
||||||
Recommended because the Claude Code marketplace handles updates for you — the plugin cache is versioned and auto-refreshes when a new release publishes. Run `claude plugin update last30days@last30days-skill` to force a check.
|
|
||||||
|
|
||||||
If you'd rather use the agent-skills install path on Claude Code, that's also supported:
|
|
||||||
|
|
||||||
```
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
|
||||||
```
|
|
||||||
|
|
||||||
The native plugin and the `npx skills` install can coexist; Claude Code dedupes the slash command.
|
|
||||||
|
|
||||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
|
||||||
|
|
||||||
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills add mvanhorn/last30days-skill -g
|
|
||||||
```
|
|
||||||
|
|
||||||
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
|
|
||||||
|
|
||||||
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
|
||||||
```
|
|
||||||
|
|
||||||
Update later with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills update last30days -g
|
|
||||||
```
|
|
||||||
|
|
||||||
Or update everything you've installed globally via `npx skills`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npx skills update -g
|
|
||||||
```
|
|
||||||
|
|
||||||
List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
|
||||||
|
|
||||||
### claude.ai (web)
|
### claude.ai (web)
|
||||||
|
|
||||||
@@ -230,7 +158,15 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
|||||||
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
||||||
3. Click the `+` button in the Skills panel and drop the file in
|
3. Click the `+` button in the Skills panel and drop the file in
|
||||||
|
|
||||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
Enable "Code execution and file creation" under Capabilities first - skills won't run without it.
|
||||||
|
|
||||||
|
### Claude Code
|
||||||
|
|
||||||
|
```
|
||||||
|
/plugin marketplace add mvanhorn/last30days-skill
|
||||||
|
```
|
||||||
|
|
||||||
|
Update later with `claude plugin update last30days@last30days-skill`.
|
||||||
|
|
||||||
### OpenClaw
|
### OpenClaw
|
||||||
|
|
||||||
@@ -238,14 +174,22 @@ Enable "Code execution and file creation" under Capabilities first — skills wo
|
|||||||
clawhub install last30days-official
|
clawhub install last30days-official
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Gemini CLI
|
||||||
|
|
||||||
|
Gemini CLI v0.9.0 has an upstream installer bug that can fail with `Configuration file not found at /tmp/gemini-extensionXXXXXX/gemini-extension.json` ([upstream issue](https://github.com/google-gemini/gemini-cli/issues/11452)). Workaround:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/mvanhorn/last30days-skill
|
||||||
|
gemini extensions install ./last30days-skill
|
||||||
|
```
|
||||||
|
|
||||||
### Manual (developer)
|
### Manual (developer)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
Or build the claude.ai `.skill` file from source: `bash scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||||
|
|
||||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||||
|
|
||||||
@@ -259,40 +203,10 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
|||||||
| X / Twitter | Log into x.com in any browser | Free |
|
| X / Twitter | Log into x.com in any browser | Free |
|
||||||
| YouTube | `brew install yt-dlp` | Free |
|
| YouTube | `brew install yt-dlp` | Free |
|
||||||
| Bluesky | App password from bsky.app | Free |
|
| Bluesky | App password from bsky.app | Free |
|
||||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG |
|
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls |
|
||||||
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
||||||
| Web search | Brave Search key | 2,000 free queries/month |
|
| Web search | Brave Search key | 2,000 free queries/month |
|
||||||
|
|
||||||
### macOS Keychain (optional)
|
|
||||||
|
|
||||||
On macOS you can store keys in the system Keychain instead of a `.env` file. The skill picks them up automatically as the lowest-priority source — `.env` files and process environment still win on collision.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Interactive setup — prompts for each known key, skip with empty input
|
|
||||||
skills/last30days/scripts/setup-keychain.sh
|
|
||||||
|
|
||||||
# Or store a single key by hand
|
|
||||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
|
||||||
|
|
||||||
# Inspect / clean up
|
|
||||||
skills/last30days/scripts/setup-keychain.sh --list
|
|
||||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
|
||||||
```
|
|
||||||
|
|
||||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
|
||||||
|
|
||||||
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Two things you'll likely want to know on day one:
|
|
||||||
|
|
||||||
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each run produces `<slug>-raw[-suffix].md`.
|
|
||||||
|
|
||||||
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
|
||||||
|
|
||||||
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
|
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
name: last30days
|
name: last30days
|
||||||
version: "3.3.0"
|
version: "3.0.1"
|
||||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||||
@@ -13,9 +13,9 @@ metadata:
|
|||||||
openclaw:
|
openclaw:
|
||||||
emoji: "📰"
|
emoji: "📰"
|
||||||
requires:
|
requires:
|
||||||
env: []
|
env:
|
||||||
optionalEnv:
|
|
||||||
- SCRAPECREATORS_API_KEY
|
- SCRAPECREATORS_API_KEY
|
||||||
|
optionalEnv:
|
||||||
- OPENAI_API_KEY
|
- OPENAI_API_KEY
|
||||||
- XAI_API_KEY
|
- XAI_API_KEY
|
||||||
- OPENROUTER_API_KEY
|
- OPENROUTER_API_KEY
|
||||||
@@ -45,7 +45,6 @@ metadata:
|
|||||||
- instagram
|
- instagram
|
||||||
- hackernews
|
- hackernews
|
||||||
- polymarket
|
- polymarket
|
||||||
- digg
|
|
||||||
- bluesky
|
- bluesky
|
||||||
- truthsocial
|
- truthsocial
|
||||||
- trends
|
- trends
|
||||||
@@ -60,32 +59,23 @@ metadata:
|
|||||||
- clawhub
|
- clawhub
|
||||||
---
|
---
|
||||||
|
|
||||||
# STEP 0: STALE-CLONE SELF-CHECK — RUN BEFORE READING BELOW
|
# STEP 0: CANONICAL PATH SELF-CHECK — RUN BEFORE READING BELOW
|
||||||
|
|
||||||
Before reading anything else in this file, check whether you loaded SKILL.md from the one known stale-clone location: Claude Code's marketplaces directory.
|
Before reading anything else in this file, verify that the SKILL.md you just loaded is the versioned cache copy, not a stale marketplace clone:
|
||||||
|
|
||||||
**Run this check:**
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
CLAUDE_CACHE_LATEST=$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)
|
CANONICAL_ROOT="$HOME/.claude/plugins/cache/last30days-skill/last30days"
|
||||||
# Two cache layouts ship in the wild — nested ({version}/skills/last30days/SKILL.md)
|
CANONICAL_LATEST=$(ls -d "$CANONICAL_ROOT"/*/ 2>/dev/null | sort -V | tail -1)
|
||||||
# and flat ({version}/SKILL.md). Resolve to whichever shape actually exists.
|
CANONICAL_LATEST="${CANONICAL_LATEST%/}"
|
||||||
CLAUDE_CACHE_SKILL_MD=""
|
|
||||||
if [ -n "$CLAUDE_CACHE_LATEST" ]; then
|
|
||||||
if [ -f "$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md" ]; then
|
|
||||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md"
|
|
||||||
elif [ -f "$CLAUDE_CACHE_LATEST/SKILL.md" ]; then
|
|
||||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/SKILL.md"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
echo "CLAUDE_CACHE_SKILL_MD=$CLAUDE_CACHE_SKILL_MD"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
If the SKILL.md path you just Read contains `/.claude/plugins/marketplaces/` AND `$CLAUDE_CACHE_SKILL_MD` is non-empty, STOP and re-read `$CLAUDE_CACHE_SKILL_MD` before proceeding. Otherwise the SKILL.md you have is fine — continue.
|
If the SKILL.md file you just read is NOT under `$CANONICAL_LATEST/`, STOP and re-read `$CANONICAL_LATEST/SKILL.md` before proceeding. Do not run `--help`, do not plan, do not invoke the engine against any other path.
|
||||||
|
|
||||||
**Why this specific check:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test. STEP 0 defends against that one Claude Code-specific bug.
|
**Why:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test.
|
||||||
|
|
||||||
**Other install paths are fine:** `~/.codex/skills/`, `~/.agents/skills/`, an `npx skills add` install dir, or a repo checkout are all valid load points - the resolver in Step 1 picks them up. Do NOT abort or hop on those paths.
|
**How to self-check:** the file path you used in your last Read tool call should match `$CANONICAL_LATEST/SKILL.md`. If it contains `marketplaces/` or any other prefix, that is the stale-path failure mode. Re-read from `$CANONICAL_LATEST/SKILL.md` and restart this contract from the top.
|
||||||
|
|
||||||
|
The same pinned resolver appears later in Step 1 for the engine Bash invocation. That guard is necessary but insufficient — by the time you reach Step 1, you may have already internalized an out-of-date flag list from the stale SKILL.md above it. This STEP 0 runs first so the CONTRACT itself is read from the right file.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -97,7 +87,7 @@ You are inside the `/last30days` SKILL. This is a specific research tool with a
|
|||||||
|
|
||||||
**How v3.0.7 fixes it:** three structural anchors.
|
**How v3.0.7 fixes it:** three structural anchors.
|
||||||
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
||||||
2. **The SKILL_DIR substitution** in the engine Bash calls uses the directory of the SKILL.md the model just Read — no resolver list, no precedence walk. Whichever install the harness loaded SKILL.md from is the install whose engine runs. Aligns spec-with-code and works for any harness without enumerating its install path.
|
2. **The pinned SKILL_ROOT resolution** in the engine Bash calls always points to the public plugin cache, never `~/.openclaw/` or other stale copies.
|
||||||
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
||||||
|
|
||||||
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
||||||
@@ -114,7 +104,7 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
|
|||||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||||
```
|
```
|
||||||
|
|
||||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_DIR/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_DIR/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||||
|
|
||||||
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
||||||
|
|
||||||
@@ -186,13 +176,13 @@ The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses
|
|||||||
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
|
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
|
||||||
```
|
```
|
||||||
|
|
||||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan "$QUERY_PLAN_FILE"` (the path to a tmpfile you wrote via heredoc — see Step 1 for the pattern; never inline `--plan '$JSON'`, apostrophes in search/ranking strings break shell parsing).
|
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan '$JSON'`.
|
||||||
|
|
||||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan "$QUERY_PLAN_FILE"` (or any path the engine can read). A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan '$JSON'`. A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||||
|
|
||||||
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
|
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
|
||||||
|
|
||||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan "$QUERY_PLAN_FILE"` (or another path the engine can read)? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan, then write it to a tmpfile per the Step 1 pattern. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan '$JSON'`? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||||
|
|
||||||
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
|
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
|
||||||
|
|
||||||
@@ -243,7 +233,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# last30days v3.3.0: Research Any Topic from the Last 30 Days
|
# last30days v3.0.1: Research Any Topic from the Last 30 Days
|
||||||
|
|
||||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||||
|
|
||||||
@@ -327,14 +317,14 @@ Common patterns:
|
|||||||
|
|
||||||
- Always active: Reddit, Hacker News, Polymarket
|
- Always active: Reddit, Hacker News, Polymarket
|
||||||
- If gh CLI is installed (check `which gh`): add GitHub
|
- If gh CLI is installed (check `which gh`): add GitHub
|
||||||
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg
|
|
||||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
||||||
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
||||||
- If SCRAPECREATORS_API_KEY is set: add TikTok, Instagram, Threads (suppress any of these via EXCLUDE_SOURCES)
|
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
||||||
- If SCRAPECREATORS_API_KEY is set and the user explicitly requested pinterest for this query (e.g. via `--search=pinterest`): add Pinterest
|
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram
|
||||||
|
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains threads: add Threads
|
||||||
|
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest
|
||||||
- If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky
|
- If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky
|
||||||
- If OPENROUTER_API_KEY is set and INCLUDE_SOURCES contains perplexity: add Perplexity
|
- If OPENROUTER_API_KEY is set: add Perplexity
|
||||||
- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying
|
|
||||||
|
|
||||||
Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma):
|
Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma):
|
||||||
|
|
||||||
@@ -591,50 +581,18 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe
|
|||||||
|
|
||||||
**Invocation:**
|
**Invocation:**
|
||||||
```bash
|
```bash
|
||||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||||
# Substitute the actual path below — your harness told you where this file lives via
|
|
||||||
# the Read tool result. Examples:
|
|
||||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
|
||||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
|
||||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days/SKILL.md
|
|
||||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days
|
|
||||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
|
||||||
# packages SKILL.md and scripts/ as siblings).
|
|
||||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
|
||||||
|
|
||||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
|
||||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
|
||||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Write the per-entity plan to a tmpfile and pass the path to the engine.
|
|
||||||
# The engine's parse_competitors_plan() reads file paths transparently. This
|
|
||||||
# avoids the inline-single-quoted-JSON apostrophe trap (resolved context
|
|
||||||
# strings like "people's choice" or "McDonald's" otherwise close the outer
|
|
||||||
# single-quote and break shell parsing before the engine is even invoked).
|
|
||||||
# Trailing XXXXXX (no .json suffix) so BSD/macOS mktemp works the same as
|
|
||||||
# GNU; BSD only substitutes X's at the end of the template.
|
|
||||||
COMPETITORS_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-competitors.XXXXXX")
|
|
||||||
trap 'rm -f "$COMPETITORS_PLAN_FILE"' EXIT
|
|
||||||
cat > "$COMPETITORS_PLAN_FILE" <<'PLAN_EOF'
|
|
||||||
{
|
|
||||||
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
|
||||||
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
|
||||||
}
|
|
||||||
PLAN_EOF
|
|
||||||
|
|
||||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
|
||||||
--emit=compact \
|
--emit=compact \
|
||||||
--save-dir="${LAST30DAYS_MEMORY_DIR}" \
|
--save-dir="${LAST30DAYS_MEMORY_DIR}" \
|
||||||
--save-suffix=v3 \
|
--save-suffix=v3 \
|
||||||
--x-handle={TOPIC_A_HANDLE} \
|
--x-handle={TOPIC_A_HANDLE} \
|
||||||
--subreddits={TOPIC_A_SUBS} \
|
--subreddits={TOPIC_A_SUBS} \
|
||||||
--competitors-plan "$COMPETITORS_PLAN_FILE"
|
--competitors-plan '{
|
||||||
|
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
||||||
|
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
||||||
|
}'
|
||||||
```
|
```
|
||||||
|
|
||||||
**The quoted heredoc marker `'PLAN_EOF'` is load-bearing** — quoting suppresses shell interpolation so apostrophes, `$`, backticks, etc. pass through verbatim. If you ever switch to an unquoted `<<PLAN_EOF`, every variable reference and apostrophe inside the JSON becomes a parse hazard.
|
|
||||||
|
|
||||||
Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive).
|
Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive).
|
||||||
|
|
||||||
**Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan.
|
**Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan.
|
||||||
@@ -866,7 +824,7 @@ Only show lines for platforms where something was resolved. Skip empty lines. On
|
|||||||
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
||||||
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
||||||
|
|
||||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key), digg (Digg clusters - only if `digg-pp-cli` is on PATH)
|
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key)
|
||||||
|
|
||||||
**Intent → freshness_mode mapping:**
|
**Intent → freshness_mode mapping:**
|
||||||
- breaking_news, prediction → `strict_recent`
|
- breaking_news, prediction → `strict_recent`
|
||||||
@@ -909,45 +867,34 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
|||||||
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
# PIN SKILL_ROOT to the public plugin cache (highest-version dir wins on upgrade).
|
||||||
# Substitute the actual path below — your harness told you where this file lives via
|
# DO NOT write your own path-discovery loop. The 2026-04-18 Peter Steinberger run 1
|
||||||
# the Read tool result. Examples:
|
# regression was caused by a custom discovery loop landing on ~/.openclaw/skills/last30days/
|
||||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
# (a stale copy from a private-repo sync pattern). That path contains a pre-plan-007
|
||||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
# engine and produces non-canonical output. This pinned resolution ignores every stale
|
||||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days/SKILL.md
|
# copy (~/.openclaw/, ~/.agents/, ~/.codex/) and picks the plugin cache exclusively.
|
||||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days
|
SKILL_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
|
||||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
SKILL_ROOT="${SKILL_ROOT%/}"
|
||||||
# packages SKILL.md and scripts/ as siblings).
|
|
||||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
|
||||||
|
|
||||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
# Fallback for repo checkout / Gemini / Codex hosts where the plugin cache does not exist.
|
||||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
# Only runs if the public plugin cache is missing entirely.
|
||||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||||
|
for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
|
||||||
|
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||||
|
echo "ERROR: Could not find scripts/last30days.py in public plugin cache or repo checkout" >&2
|
||||||
|
echo "Expected: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/scripts/last30days.py" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||||
```
|
```
|
||||||
|
|
||||||
**If you ran Steps 0.55 and 0.75 (agent planning), pass the plan via a tmpfile and add the targeting flags:**
|
**If you ran Steps 0.55 and 0.75 (agent planning), add these flags:**
|
||||||
|
- `--plan 'QUERY_PLAN_JSON'` (replace with actual JSON from Step 0.75)
|
||||||
```bash
|
|
||||||
# Write QUERY_PLAN_JSON to a tmpfile before the engine invocation above.
|
|
||||||
# parse_plan() reads file paths transparently; this avoids inline-JSON
|
|
||||||
# shell-quoting hazards (apostrophes in search_query / ranking_query
|
|
||||||
# strings break single-quoted command-line JSON). Trailing XXXXXX (no
|
|
||||||
# .json suffix) for BSD/macOS portability — BSD mktemp only substitutes
|
|
||||||
# X's at the end of the template.
|
|
||||||
QUERY_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-plan.XXXXXX")
|
|
||||||
trap 'rm -f "$QUERY_PLAN_FILE"' EXIT
|
|
||||||
cat > "$QUERY_PLAN_FILE" <<'PLAN_EOF'
|
|
||||||
{QUERY_PLAN_JSON_FROM_STEP_0.75}
|
|
||||||
PLAN_EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
Then add to the engine command:
|
|
||||||
|
|
||||||
- `--plan "$QUERY_PLAN_FILE"` (path to the file you just wrote)
|
|
||||||
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
|
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
|
||||||
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
|
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
|
||||||
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
|
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
|
||||||
@@ -1550,33 +1497,6 @@ Close with `I have all the links to the {N} {source list} I pulled from. Just as
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## SHAREABLE HTML BRIEF (when the user asked for one)
|
|
||||||
|
|
||||||
**This section fires if EITHER trigger is true:**
|
|
||||||
|
|
||||||
- `$ARGUMENTS` contains `--emit=html`, `--emit:html`, or `--html` as a flag
|
|
||||||
- The user's natural-language request asks for an HTML brief, shareable doc, or file for sharing (Slack, email, Notion, "export as HTML", etc). Use your judgment for phrasing variants.
|
|
||||||
|
|
||||||
**If neither trigger fires, skip this entire section and proceed to WAIT FOR USER'S RESPONSE.** No HTML save flow, no reference read needed.
|
|
||||||
|
|
||||||
**When triggered, you MUST:**
|
|
||||||
|
|
||||||
- Read `references/save-html-brief.md` BEFORE proceeding to WAIT FOR USER'S RESPONSE
|
|
||||||
- Follow that file's instructions exactly - it is the canonical source for the save flow
|
|
||||||
- Append the confirmation line (`📎 Shareable brief saved to <path>`) to your already-emitted chat response
|
|
||||||
|
|
||||||
**You MUST NOT:**
|
|
||||||
|
|
||||||
- Improvise the HTML save flow from memory or from instructions you've seen before
|
|
||||||
- Skip the reference read because the steps "look familiar"
|
|
||||||
- Save to a different path than the reference specifies
|
|
||||||
- Add data quality warnings, debug headers, or safety notes to the saved HTML
|
|
||||||
- Re-research the topic for the HTML render - the engine cache covers the second invocation
|
|
||||||
|
|
||||||
**Why the directive is forceful:** the reference file is the only source of truth for the save flow. Skipping it produces broken artifacts - wrong path conventions, missing synthesis content, leaked engine debug output, or warnings that don't belong in shareable docs.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WAIT FOR USER'S RESPONSE
|
## WAIT FOR USER'S RESPONSE
|
||||||
|
|
||||||
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.
|
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.
|
||||||
@@ -1688,7 +1608,7 @@ Want another prompt? Just tell me what you're creating next.
|
|||||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 100 free credits)
|
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 10,000 free API calls)
|
||||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||||
- Stores research findings in local SQLite database (watchlist mode only)
|
- Stores research findings in local SQLite database (watchlist mode only)
|
||||||
@@ -1701,7 +1621,7 @@ Want another prompt? Just tell me what you're creating next.
|
|||||||
- Does not log, cache, or write API keys to output files
|
- Does not log, cache, or write API keys to output files
|
||||||
- Does not send data to any endpoint not listed above
|
- Does not send data to any endpoint not listed above
|
||||||
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
||||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (100 free credits one-time, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (10,000 free API calls, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||||
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
||||||
|
|
||||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# last30days Skill Specification
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. OpenAI auth can come from `OPENAI_API_KEY` or Codex login credentials.
|
||||||
|
|
||||||
|
The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`:
|
||||||
|
|
||||||
|
- **env.py**: Load API keys from `~/.config/last30days/.env` and Codex auth from `~/.codex/auth.json`
|
||||||
|
- **dates.py**: Date range calculation and confidence scoring
|
||||||
|
- **cache.py**: 24-hour TTL caching keyed by topic + date range
|
||||||
|
- **http.py**: stdlib-only HTTP client with retry logic
|
||||||
|
- **models.py**: Auto-selection of OpenAI/xAI models with 7-day caching
|
||||||
|
- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit
|
||||||
|
- **xai_x.py**: xAI Responses API + x_search for X
|
||||||
|
- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics
|
||||||
|
- **hackernews.py**: Hacker News search via Algolia API (free, no auth)
|
||||||
|
- **polymarket.py**: Polymarket prediction market search via Gamma API (free, no auth)
|
||||||
|
- **normalize.py**: Convert raw API responses to canonical schema
|
||||||
|
- **score.py**: Compute popularity-aware scores (relevance + recency + engagement)
|
||||||
|
- **dedupe.py**: Near-duplicate detection via text similarity
|
||||||
|
- **render.py**: Generate markdown and JSON outputs
|
||||||
|
- **schema.py**: Type definitions and validation
|
||||||
|
|
||||||
|
## Embedding in Other Skills
|
||||||
|
|
||||||
|
Other skills can import the research context in several ways:
|
||||||
|
|
||||||
|
### Inline Context Injection
|
||||||
|
```markdown
|
||||||
|
## Recent Research Context
|
||||||
|
!python3 ~/.claude/skills/last30days/scripts/last30days.py "your topic" --emit=context
|
||||||
|
```
|
||||||
|
|
||||||
|
### Read from File
|
||||||
|
```markdown
|
||||||
|
## Research Context
|
||||||
|
!cat ~/.local/share/last30days/out/last30days.context.md
|
||||||
|
```
|
||||||
|
|
||||||
|
### Get Path for Dynamic Loading
|
||||||
|
```bash
|
||||||
|
CONTEXT_PATH=$(python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=path)
|
||||||
|
cat "$CONTEXT_PATH"
|
||||||
|
```
|
||||||
|
|
||||||
|
### JSON for Programmatic Use
|
||||||
|
```bash
|
||||||
|
python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=json > research.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## CLI Reference
|
||||||
|
|
||||||
|
```
|
||||||
|
python3 ~/.claude/skills/last30days/scripts/last30days.py <topic> [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--refresh Bypass cache and fetch fresh data
|
||||||
|
--mock Use fixtures instead of real API calls
|
||||||
|
--emit=MODE Output mode: compact|json|md|context|path (default: compact)
|
||||||
|
--sources=MODE Source selection: auto|reddit|x|both (default: auto)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Output Files
|
||||||
|
|
||||||
|
All outputs are written to `~/.local/share/last30days/out/`:
|
||||||
|
|
||||||
|
- `report.md` - Human-readable full report
|
||||||
|
- `report.json` - Normalized data with scores
|
||||||
|
- `last30days.context.md` - Compact reusable snippet for other skills
|
||||||
|
- `raw_openai.json` - Raw OpenAI API response
|
||||||
|
- `raw_xai.json` - Raw xAI API response
|
||||||
|
- `raw_reddit_threads_enriched.json` - Enriched Reddit thread data
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
# last30days Implementation Tasks
|
||||||
|
|
||||||
|
## Setup & Configuration
|
||||||
|
- [x] Create directory structure
|
||||||
|
- [x] Write SPEC.md
|
||||||
|
- [x] Write TASKS.md
|
||||||
|
- [x] Write SKILL.md with proper frontmatter
|
||||||
|
|
||||||
|
## Core Library Modules
|
||||||
|
- [x] scripts/lib/env.py - Environment and API key loading
|
||||||
|
- [x] scripts/lib/dates.py - Date range and confidence utilities
|
||||||
|
- [x] scripts/lib/cache.py - TTL-based caching
|
||||||
|
- [x] scripts/lib/http.py - HTTP client with retry
|
||||||
|
- [x] scripts/lib/models.py - Auto model selection
|
||||||
|
- [x] scripts/lib/schema.py - Data structures
|
||||||
|
- [x] scripts/lib/openai_reddit.py - OpenAI Responses API
|
||||||
|
- [x] scripts/lib/xai_x.py - xAI Responses API
|
||||||
|
- [x] scripts/lib/reddit_enrich.py - Reddit thread JSON fetcher
|
||||||
|
- [x] scripts/lib/normalize.py - Schema normalization
|
||||||
|
- [x] scripts/lib/score.py - Popularity scoring
|
||||||
|
- [x] scripts/lib/dedupe.py - Near-duplicate detection
|
||||||
|
- [x] scripts/lib/render.py - Output rendering
|
||||||
|
|
||||||
|
## Main Script
|
||||||
|
- [x] scripts/last30days.py - CLI orchestrator
|
||||||
|
|
||||||
|
## Fixtures
|
||||||
|
- [x] fixtures/openai_sample.json
|
||||||
|
- [x] fixtures/xai_sample.json
|
||||||
|
- [x] fixtures/reddit_thread_sample.json
|
||||||
|
- [x] fixtures/models_openai_sample.json
|
||||||
|
- [x] fixtures/models_xai_sample.json
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
- [x] tests/test_dates.py
|
||||||
|
- [x] tests/test_cache.py
|
||||||
|
- [x] tests/test_models.py
|
||||||
|
- [x] tests/test_score.py
|
||||||
|
- [x] tests/test_dedupe.py
|
||||||
|
- [x] tests/test_normalize.py
|
||||||
|
- [x] tests/test_render.py
|
||||||
|
|
||||||
|
## Validation
|
||||||
|
- [x] Run tests in mock mode
|
||||||
|
- [x] Demo --emit=compact
|
||||||
|
- [x] Demo --emit=context
|
||||||
|
- [x] Verify file tree
|
||||||
|
Before Width: | Height: | Size: 2.7 MiB After Width: | Height: | Size: 2.7 MiB |
|
Before Width: | Height: | Size: 2.3 MiB After Width: | Height: | Size: 2.3 MiB |
|
Before Width: | Height: | Size: 3.8 MiB After Width: | Height: | Size: 3.8 MiB |
|
Before Width: | Height: | Size: 2.6 MiB After Width: | Height: | Size: 2.6 MiB |
@@ -142,7 +142,7 @@ The repo vendors a search-only subset of Bird's Twitter GraphQL client and shell
|
|||||||
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
||||||
| Replies/quotes | Real | Real |
|
| Replies/quotes | Real | Real |
|
||||||
| Author handle | Real | Real |
|
| Author handle | Real | Real |
|
||||||
| Relevance score | Default 0.7 (re-ranked by relevance.py) | AI-assessed 0.0-1.0 |
|
| Relevance score | Default 0.7 (re-ranked by score.py) | AI-assessed 0.0-1.0 |
|
||||||
|
|
||||||
### Depth settings
|
### Depth settings
|
||||||
|
|
||||||
@@ -183,14 +183,13 @@ After both searches complete:
|
|||||||
|
|
||||||
| File | Purpose |
|
| File | Purpose |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `skills/last30days/scripts/last30days.py` | Main CLI entry point |
|
| `scripts/last30days.py` | Main orchestrator, concurrent execution |
|
||||||
| `skills/last30days/scripts/lib/pipeline.py` | Multi-source retrieval orchestration |
|
| `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API |
|
||||||
| `skills/last30days/scripts/lib/reddit_public.py` | Reddit public JSON search |
|
| `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||||
| `skills/last30days/scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
| `scripts/lib/xai_x.py` | X search via xAI API |
|
||||||
| `skills/last30days/scripts/lib/xai_x.py` | X search via xAI API |
|
| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||||
| `skills/last30days/scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
| `scripts/lib/models.py` | Auto-select best available model |
|
||||||
| `skills/last30days/scripts/lib/providers.py` | Reasoning provider and model selection |
|
| `scripts/lib/env.py` | API key loading, source detection |
|
||||||
| `skills/last30days/scripts/lib/env.py` | API key loading, source detection |
|
| `scripts/lib/http.py` | HTTP transport with retries |
|
||||||
| `skills/last30days/scripts/lib/http.py` | HTTP transport with retries |
|
| `scripts/lib/score.py` | Relevance scoring |
|
||||||
| `skills/last30days/scripts/lib/relevance.py` | Query matching and relevance scoring |
|
| `scripts/lib/dedupe.py` | URL-based deduplication |
|
||||||
| `skills/last30days/scripts/lib/dedupe.py` | URL-based deduplication |
|
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
---
|
---
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
||||||
type: feat
|
type: feat
|
||||||
status: active
|
status: active
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
---
|
---
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
||||||
type: fix
|
type: fix
|
||||||
status: active
|
status: active
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
---
|
---
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
||||||
type: feat
|
type: feat
|
||||||
status: active
|
status: active
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
---
|
---
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
||||||
type: fix
|
type: fix
|
||||||
status: active
|
status: active
|
||||||
|
|||||||
@@ -0,0 +1,396 @@
|
|||||||
|
---
|
||||||
|
title: "marketing: v3.1 launch video — 30s Remotion piece for X"
|
||||||
|
type: feat
|
||||||
|
status: active
|
||||||
|
date: 2026-04-22
|
||||||
|
---
|
||||||
|
|
||||||
|
# marketing: v3.1 launch video — 30s Remotion piece for X
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Build a 30-second Remotion-rendered MP4 marketing video announcing **v3.1: Competitors mode** for `/last30days`. Posts to X. Frames the new vs-mode-with-auto-discovery as the headline — three full passes, three save files, one comparison — without burying it in CLI minutiae.
|
||||||
|
|
||||||
|
Marketing release tag is **v3.1** (rebrands the 3.0.11-3.0.14 bundle into one shippable narrative for the launch tweet). Engine version stays 3.0.14 — `3.1` is the marketing version, not a code version bump.
|
||||||
|
|
||||||
|
## Script (60 frames per second × 30 seconds = 900 frames; this version assumes 30fps × 30s = 900 frames at 30fps)
|
||||||
|
|
||||||
|
Total runtime: 30.0 seconds @ 30fps = 900 frames. Six scenes, on-screen text only (silent autoplay-friendly).
|
||||||
|
|
||||||
|
### Scene 1 — Hook (0:00 – 0:03 | frames 0-90)
|
||||||
|
|
||||||
|
**Visual:** Black background. The `/last30days` badge animates in (the literal `🌐 last30days v3.1 · synced 2026-04-22` line) with the spring scale-in used in slick devtool intros.
|
||||||
|
|
||||||
|
**Caption (overlay, large):**
|
||||||
|
```
|
||||||
|
What if one search
|
||||||
|
ran 3 at once?
|
||||||
|
```
|
||||||
|
|
||||||
|
### Scene 2 — Set the world (0:03 – 0:08 | frames 90-240)
|
||||||
|
|
||||||
|
**Visual:** Single mac terminal window center-stage. Type-on animation:
|
||||||
|
```
|
||||||
|
$ /last30days OpenAI
|
||||||
|
```
|
||||||
|
Below it, a simple result card stub appears (Reddit upvote count + X likes), then static.
|
||||||
|
|
||||||
|
**Caption (small bottom-left):**
|
||||||
|
```
|
||||||
|
The old way: one topic.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Scene 3 — The reveal (0:08 – 0:14 | frames 240-420)
|
||||||
|
|
||||||
|
**Visual:** The terminal types one more flag:
|
||||||
|
```
|
||||||
|
$ /last30days OpenAI --competitors
|
||||||
|
```
|
||||||
|
Hard cut → the single terminal **splits into 3 panes** side by side. Each pane shows a different topic header animating in, in this order:
|
||||||
|
- Left: `OpenAI`
|
||||||
|
- Middle: `vs Anthropic`
|
||||||
|
- Right: `vs xAI`
|
||||||
|
|
||||||
|
Pane content scrolls fake "search progress" lines (Reddit, X, YouTube indicators) in parallel, like a live fan-out.
|
||||||
|
|
||||||
|
**Caption (top center):**
|
||||||
|
```
|
||||||
|
Now it discovers competitors
|
||||||
|
and runs all 3.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Scene 4 — Result reveal (0:14 – 0:21 | frames 420-630)
|
||||||
|
|
||||||
|
**Visual:** The 3 panes collapse into a single comparison surface — the `## Head-to-Head` table from the actual engine output, with rows fading in one by one (What it is, Streams, Best for, Trajectory). Each entity column lights up as its row populates.
|
||||||
|
|
||||||
|
**Caption (bottom):**
|
||||||
|
```
|
||||||
|
3 full passes. 3 save files.
|
||||||
|
1 comparison.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Scene 5 — How it's special (0:21 – 0:26 | frames 630-780)
|
||||||
|
|
||||||
|
**Visual:** Cut to a clean text card, large mono font:
|
||||||
|
```
|
||||||
|
You pick the topic.
|
||||||
|
The agent picks the peers.
|
||||||
|
The engine fans out.
|
||||||
|
```
|
||||||
|
Each line fades in 1.5s apart.
|
||||||
|
|
||||||
|
### Scene 6 — CTA (0:26 – 0:30 | frames 780-900)
|
||||||
|
|
||||||
|
**Visual:** Black background. Centered:
|
||||||
|
- Top: `🌐 last30days v3.1`
|
||||||
|
- Middle: `/last30days {topic} --competitors`
|
||||||
|
- Bottom: `github.com/mvanhorn/last30days-skill`
|
||||||
|
|
||||||
|
Subtle pulse on the install line.
|
||||||
|
|
||||||
|
**End frame holds for ~0.5s.**
|
||||||
|
|
||||||
|
## Problem Frame
|
||||||
|
|
||||||
|
The 3.0.11-3.0.14 release bundle ships a transformative feature (per-entity vs-mode fanout + `--competitors` shortcut + per-entity save files), but the value lands flat in a tweet thread or screenshot. A 30s video does what static text cannot: shows the fan-out happening in real time and the 3 → 1 collapse into a comparison. Higher tweet engagement, easier to RT/QT.
|
||||||
|
|
||||||
|
## Requirements Trace
|
||||||
|
|
||||||
|
- R1. Final artifact: a single MP4 file, 1920×1080, 30fps, ~30 seconds (±0.5s), under 30MB, suitable for direct X upload.
|
||||||
|
- R2. Six-scene script as defined above, with text/visuals/timing matching to within 5 frames.
|
||||||
|
- R3. Branded look: matches the `🌐 last30days v3.1` badge style (terminal aesthetic, mono font, dark background).
|
||||||
|
- R4. Silent — no voiceover, no music in v1. Captions baked in. Designed for autoplay-muted feeds.
|
||||||
|
- R5. Reproducible: another contributor (or a future-me) can re-render with one command. Project lives in-repo so the source is versioned.
|
||||||
|
|
||||||
|
## Scope Boundaries
|
||||||
|
|
||||||
|
- No voiceover. Text-on-screen only. (Voice can be a v1.1 if engagement is high.)
|
||||||
|
- No background music in the rendered MP4. (Music can be added in post via QuickTime/iMovie if desired before posting.)
|
||||||
|
- No localization. English captions only.
|
||||||
|
- No A/B test variants. One video.
|
||||||
|
- No 9:16 vertical version. 16:9 only. (Vertical can be a separate render after launch validates the format.)
|
||||||
|
|
||||||
|
### Deferred to Separate Tasks
|
||||||
|
|
||||||
|
- Voiceover variant: defer to follow-up if v1 lands well.
|
||||||
|
- 9:16 mobile cut: defer; same source compositions can re-render at 1080×1920 in a follow-up.
|
||||||
|
- Animated GIF for embedding in README.md: defer; can be ffmpeg-extracted from the MP4.
|
||||||
|
|
||||||
|
## Context & Research
|
||||||
|
|
||||||
|
### Relevant Code and Patterns
|
||||||
|
|
||||||
|
- `SKILL.md` — the comparison render scaffold (`## Head-to-Head` table) is the visual reference for Scene 4's table look.
|
||||||
|
- `scripts/lib/render.py` `_render_comparison_scaffold` — emits the 9-axis table whose visual style we're recreating in a more polished form.
|
||||||
|
- `CHANGELOG.md` 3.0.11-3.0.14 entries — the prose source for the script's beats.
|
||||||
|
- `.claude-plugin/plugin.json` version 3.0.14 — current code version (marketing version is 3.1 for the launch).
|
||||||
|
|
||||||
|
### External References
|
||||||
|
|
||||||
|
- Remotion 4.x docs (https://www.remotion.dev/docs/) — current API for compositions, sequences, springs, and render CLI.
|
||||||
|
- X video specs 2026: max 2 min 20 s, ≤512MB, MP4 with H.264 + AAC, recommended 1920×1080 for landscape autoplay.
|
||||||
|
|
||||||
|
### Institutional Learnings
|
||||||
|
|
||||||
|
- No prior `marketing/` dir or video plans in `docs/plans/`. This is a greenfield asset directory.
|
||||||
|
|
||||||
|
## Key Technical Decisions
|
||||||
|
|
||||||
|
- **Remotion, not ffmpeg-only.** Remotion's React-based compositions handle the typed-on terminal effect, spring-animated badges, and scene transitions far more cleanly than raw ffmpeg filtergraphs. Render output is still MP4 via Remotion's bundled ffmpeg.
|
||||||
|
- **In-repo asset directory at `marketing/v3.1-launch/`.** Lives with the product so future versions can fork the project. Adds `marketing/` to `.gitignore` exceptions only for source files; rendered MP4 stays out of git (uploaded separately).
|
||||||
|
- **16:9 1920×1080 @ 30fps.** Best fit for X landscape autoplay on desktop and mobile feed. 30fps is plenty for typed-text + UI animation; 60fps doubles render time without obvious quality gain.
|
||||||
|
- **Silent + captions.** X autoplay defaults to muted. Sound-off is the realistic viewing condition. Captions baked into the visual.
|
||||||
|
- **Six scenes, one composition.** Single Remotion composition with sequenced child compositions per scene. Easier to re-time than scene-files. Frame-numbered timing in the script enables precise edits.
|
||||||
|
- **Mono font (JetBrains Mono or Geist Mono).** Matches the terminal aesthetic of the actual `/last30days` output. Available via Google Fonts or @remotion/google-fonts.
|
||||||
|
- **Marketing version 3.1 ≠ engine version 3.0.14.** `3.1` is the launch label. Engine stays 3.0.14. Avoids confusion in CHANGELOG.
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
### Resolved During Planning
|
||||||
|
|
||||||
|
- **Aspect ratio?** 16:9 1080p. Best X autoplay format; vertical can be a follow-up.
|
||||||
|
- **Voiceover or silent?** Silent + on-screen captions. Autoplay-muted is the realistic condition.
|
||||||
|
- **In-repo or separate repo?** In-repo at `marketing/v3.1-launch/`. Rendered MP4 not committed; source compositions are.
|
||||||
|
- **Length?** Exactly 30s (900 frames @ 30fps). No flex.
|
||||||
|
- **Marketing version label?** `v3.1`. Engine code version stays 3.0.14.
|
||||||
|
|
||||||
|
### Deferred to Implementation
|
||||||
|
|
||||||
|
- Exact animation easing curves per scene — pick during build via Remotion preview iteration.
|
||||||
|
- Whether the comparison-table mock in Scene 4 uses canned text or pulls from the actual saved `*-raw.md` files. Probably canned for visual control.
|
||||||
|
- Whether Scene 3's "search progress" lines are typed individually or use a marquee scroll. Pick during preview.
|
||||||
|
- Exact accent color palette beyond "terminal dark." Iterate against preview.
|
||||||
|
|
||||||
|
## Output Structure
|
||||||
|
|
||||||
|
marketing/
|
||||||
|
v3.1-launch/
|
||||||
|
package.json # Remotion dependency manifest
|
||||||
|
tsconfig.json # TypeScript config
|
||||||
|
remotion.config.ts # Remotion render config (codec, fps, resolution)
|
||||||
|
src/
|
||||||
|
index.ts # Remotion entry — registers compositions
|
||||||
|
Root.tsx # Root composition definition
|
||||||
|
LaunchVideo.tsx # Main 30s composition that sequences scenes
|
||||||
|
scenes/
|
||||||
|
Scene1Hook.tsx
|
||||||
|
Scene2OldWay.tsx
|
||||||
|
Scene3FanOut.tsx
|
||||||
|
Scene4Comparison.tsx
|
||||||
|
Scene5HowItWorks.tsx
|
||||||
|
Scene6CTA.tsx
|
||||||
|
components/
|
||||||
|
TerminalWindow.tsx # Reusable mac-style terminal frame
|
||||||
|
TypedLine.tsx # Type-on animation primitive
|
||||||
|
ComparisonTable.tsx # The Head-to-Head table mock
|
||||||
|
BadgeBar.tsx # The 🌐 last30days v3.1 badge
|
||||||
|
lib/
|
||||||
|
timing.ts # Frame ranges per scene (single source of truth)
|
||||||
|
colors.ts # Brand palette
|
||||||
|
public/ # Static assets (logo, fonts if local)
|
||||||
|
out/ # Rendered MP4 lives here (gitignored)
|
||||||
|
README.md # How to preview/render
|
||||||
|
|
||||||
|
## High-Level Technical Design
|
||||||
|
|
||||||
|
> *Directional guidance for review — not implementation specification.*
|
||||||
|
|
||||||
|
```
|
||||||
|
LaunchVideo (durationInFrames = 900)
|
||||||
|
├── <Sequence from={0} durationInFrames={90}> <Scene1Hook />
|
||||||
|
├── <Sequence from={90} durationInFrames={150}> <Scene2OldWay />
|
||||||
|
├── <Sequence from={240} durationInFrames={180}> <Scene3FanOut />
|
||||||
|
├── <Sequence from={420} durationInFrames={210}> <Scene4Comparison />
|
||||||
|
├── <Sequence from={630} durationInFrames={150}> <Scene5HowItWorks />
|
||||||
|
└── <Sequence from={780} durationInFrames={120}> <Scene6CTA />
|
||||||
|
```
|
||||||
|
|
||||||
|
Per-scene components use `useCurrentFrame()` + `interpolate()` + `spring()` for timing. `TerminalWindow` is the dominant motif across scenes 2-4.
|
||||||
|
|
||||||
|
## Implementation Units
|
||||||
|
|
||||||
|
- [ ] **Unit 1: Remotion project scaffold**
|
||||||
|
|
||||||
|
**Goal:** Spin up a working Remotion project at `marketing/v3.1-launch/` that previews a blank composition and renders to MP4.
|
||||||
|
|
||||||
|
**Requirements:** R1, R5
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `marketing/v3.1-launch/package.json`
|
||||||
|
- Create: `marketing/v3.1-launch/tsconfig.json`
|
||||||
|
- Create: `marketing/v3.1-launch/remotion.config.ts`
|
||||||
|
- Create: `marketing/v3.1-launch/src/index.ts`
|
||||||
|
- Create: `marketing/v3.1-launch/src/Root.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/README.md`
|
||||||
|
- Modify: `.gitignore` (add `marketing/v3.1-launch/out/`, `marketing/v3.1-launch/node_modules/`)
|
||||||
|
|
||||||
|
**Approach:**
|
||||||
|
- Use `npx create-video@latest --blank` (Remotion 4.x scaffolding) targeting `marketing/v3.1-launch/`. Strip the demo composition.
|
||||||
|
- Configure: 1920×1080, 30fps, H.264, AAC (audio codec needed even for silent — empty track).
|
||||||
|
- README documents `npm install`, `npm run preview` (Remotion Studio), `npm run render` (one-command MP4).
|
||||||
|
|
||||||
|
**Test scenarios:**
|
||||||
|
- Test expectation: none for scaffold, but verification = `npm run preview` opens Remotion Studio with a blank 30s composition; `npm run render` produces a black MP4 at the right resolution.
|
||||||
|
|
||||||
|
**Verification:**
|
||||||
|
- Studio loads at localhost:3000 with the empty `LaunchVideo` composition listed.
|
||||||
|
- A render produces `out/launch-video.mp4` at 1920×1080, 30s, valid MP4.
|
||||||
|
|
||||||
|
- [ ] **Unit 2: Reusable components (Terminal, TypedLine, BadgeBar)**
|
||||||
|
|
||||||
|
**Goal:** Build the three primitive components scenes 2-6 will compose. Each is independently previewable.
|
||||||
|
|
||||||
|
**Requirements:** R3, R5
|
||||||
|
|
||||||
|
**Dependencies:** Unit 1
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `marketing/v3.1-launch/src/components/TerminalWindow.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/components/TypedLine.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/components/BadgeBar.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/lib/colors.ts`
|
||||||
|
- Create: `marketing/v3.1-launch/src/lib/timing.ts`
|
||||||
|
|
||||||
|
**Approach:**
|
||||||
|
- `TerminalWindow`: mac-style traffic-light header, dark gradient background, mono content area. Accepts children.
|
||||||
|
- `TypedLine`: takes a string and a `startFrame`, renders character-by-character at ~30 chars/sec. Reuses Remotion's `interpolate(useCurrentFrame() - startFrame, [0, lengthFrames], [0, text.length])` clamped.
|
||||||
|
- `BadgeBar`: renders the literal `🌐 last30days v3.1 · synced 2026-04-22` line in mono with the same gradient color treatment as the engine's compact emit.
|
||||||
|
- `colors.ts`: 5-7 brand colors (terminal-bg, terminal-fg, accent-cyan, accent-magenta, muted, success-green, warning-amber).
|
||||||
|
- `timing.ts`: exports the scene frame ranges as named constants. Single source of truth for any retiming.
|
||||||
|
|
||||||
|
**Test scenarios:**
|
||||||
|
- Test expectation: none — visual components verified in Remotion Studio.
|
||||||
|
|
||||||
|
**Verification:**
|
||||||
|
- Each component renders standalone in Studio when wrapped in a temporary preview composition.
|
||||||
|
- TypedLine animates character-by-character without flicker.
|
||||||
|
|
||||||
|
- [ ] **Unit 3: Scenes 1-3 (Hook, Old way, Fan-out reveal)**
|
||||||
|
|
||||||
|
**Goal:** Build the first half of the video (frames 0-420). The narrative arc up through the visual fan-out.
|
||||||
|
|
||||||
|
**Requirements:** R2
|
||||||
|
|
||||||
|
**Dependencies:** Unit 2
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `marketing/v3.1-launch/src/scenes/Scene1Hook.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/scenes/Scene2OldWay.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/scenes/Scene3FanOut.tsx`
|
||||||
|
- Modify: `marketing/v3.1-launch/src/Root.tsx` (register sequences)
|
||||||
|
|
||||||
|
**Approach:**
|
||||||
|
- Scene 1: spring-in BadgeBar, large overlay caption, 3-second hold.
|
||||||
|
- Scene 2: TerminalWindow with TypedLine (`$ /last30days OpenAI`), then a single result-card mock fading in.
|
||||||
|
- Scene 3: typing animation appends `--competitors`, hard cut, three TerminalWindow components arranged in a row with staggered fade-in. Each pane shows a different entity header + scrolling progress lines.
|
||||||
|
|
||||||
|
**Test scenarios:**
|
||||||
|
- Test expectation: none — verified visually in Studio.
|
||||||
|
|
||||||
|
**Verification:**
|
||||||
|
- Scrub the 0-14s range in Studio; visuals match the script timing within 5 frames.
|
||||||
|
- The fan-out moment (frame 240) lands cleanly; no jank in the transition from 1 → 3 panes.
|
||||||
|
|
||||||
|
- [ ] **Unit 4: Scenes 4-6 (Comparison reveal, How it works, CTA)**
|
||||||
|
|
||||||
|
**Goal:** Build the back half of the video (frames 420-900). Resolution + payoff + call to action.
|
||||||
|
|
||||||
|
**Requirements:** R2
|
||||||
|
|
||||||
|
**Dependencies:** Unit 2
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `marketing/v3.1-launch/src/scenes/Scene4Comparison.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/scenes/Scene5HowItWorks.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/scenes/Scene6CTA.tsx`
|
||||||
|
- Create: `marketing/v3.1-launch/src/components/ComparisonTable.tsx`
|
||||||
|
|
||||||
|
**Approach:**
|
||||||
|
- Scene 4: ComparisonTable component renders 3-column markdown-style table; rows fade in one by one (stagger 15-20 frames). Uses canned data — OpenAI / Anthropic / xAI with believable cell content drawn from real 3.0.13 outputs.
|
||||||
|
- Scene 5: three-line text card; lines fade in 45 frames apart.
|
||||||
|
- Scene 6: three centered text blocks; install line gets a 1Hz subtle opacity pulse for emphasis.
|
||||||
|
|
||||||
|
**Test scenarios:**
|
||||||
|
- Test expectation: none — verified visually.
|
||||||
|
|
||||||
|
**Verification:**
|
||||||
|
- Scrub 14-30s; table reveal feels paced (not too slow, not strobed); CTA holds long enough to read (~3-4s).
|
||||||
|
- ComparisonTable cells are legible at 1920×1080 (mono font ≥ 28px).
|
||||||
|
|
||||||
|
- [ ] **Unit 5: Final composition wiring + render**
|
||||||
|
|
||||||
|
**Goal:** Wire the six scenes into the master `LaunchVideo` composition, render to MP4, verify against X upload constraints.
|
||||||
|
|
||||||
|
**Requirements:** R1, R2, R5
|
||||||
|
|
||||||
|
**Dependencies:** Units 3, 4
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `marketing/v3.1-launch/src/LaunchVideo.tsx` (sequence all 6 scenes)
|
||||||
|
- Modify: `marketing/v3.1-launch/README.md` (add render command + verification checklist)
|
||||||
|
|
||||||
|
**Approach:**
|
||||||
|
- `LaunchVideo` is a single Composition that imports `Scene1Hook` … `Scene6CTA` and wraps each in `<Sequence from=… durationInFrames=…>` matching `lib/timing.ts`.
|
||||||
|
- Run `npx remotion render LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18`.
|
||||||
|
- Verify output: 30.0s ±0.1s, 1920×1080, file size <30MB, opens in QuickTime, plays without dropped frames.
|
||||||
|
|
||||||
|
**Test scenarios:**
|
||||||
|
- Test expectation: none — verification is the rendered MP4 itself.
|
||||||
|
|
||||||
|
**Verification:**
|
||||||
|
- `ffprobe out/last30days-v3.1-launch.mp4` reports 1920×1080, 30fps, ~30.0s, h264, faststart-friendly.
|
||||||
|
- Manual play-through end-to-end in QuickTime feels coherent and on-pace.
|
||||||
|
- File <30MB so X upload is instant.
|
||||||
|
|
||||||
|
- [ ] **Unit 6: Polish pass + ship**
|
||||||
|
|
||||||
|
**Goal:** Watch the full render, fix obvious jank, do a second render, and stage for X posting.
|
||||||
|
|
||||||
|
**Requirements:** R1, R2, R3
|
||||||
|
|
||||||
|
**Dependencies:** Unit 5
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Possibly modify: any scene file based on watch-through findings.
|
||||||
|
|
||||||
|
**Approach:**
|
||||||
|
- Watch the rendered MP4 at full size. Note: timing felt off, transitions too fast, captions overflow, color clash, anything visibly broken.
|
||||||
|
- Iterate: edit scene component → re-preview in Studio → re-render full MP4.
|
||||||
|
- Cap at 2 polish passes; ship the better of the two renders.
|
||||||
|
- Final MP4 sits at `marketing/v3.1-launch/out/last30days-v3.1-launch.mp4` ready for X upload.
|
||||||
|
|
||||||
|
**Test scenarios:**
|
||||||
|
- Test expectation: none — pure subjective polish.
|
||||||
|
|
||||||
|
**Verification:**
|
||||||
|
- User watches the final render and approves.
|
||||||
|
- No glaring visual bugs (overflowing text, frozen frames, color clashes).
|
||||||
|
|
||||||
|
## System-Wide Impact
|
||||||
|
|
||||||
|
- **Interaction graph:** None — this is a standalone marketing artifact. Doesn't touch the Python engine, doesn't change any user-facing behavior.
|
||||||
|
- **State lifecycle risks:** None.
|
||||||
|
- **API surface parity:** N/A.
|
||||||
|
- **Unchanged invariants:** The shipped 3.0.14 engine is untouched.
|
||||||
|
|
||||||
|
## Risks & Dependencies
|
||||||
|
|
||||||
|
| Risk | Mitigation |
|
||||||
|
|------|------------|
|
||||||
|
| Remotion install pulls 200MB+ of node_modules. | `marketing/v3.1-launch/node_modules/` in `.gitignore`; checked-in source stays small. |
|
||||||
|
| Render time blows past patience (>5 min for 30s @ 1080p30). | Bun-based render or `--concurrency` flag. Default Remotion is fast enough on M-series Macs. If slow, lower preview to 720p, render final at 1080p. |
|
||||||
|
| Captions overflow the 1920px width on certain fonts. | Use a known mono font with a measured per-character width; cap caption lines at 36 chars. |
|
||||||
|
| The "fan-out" visual in Scene 3 looks confusing instead of magical. | Polish pass (Unit 6) is the safety net; if still bad, fall back to a simpler "1 → 3 panes wipe" instead of typed split. |
|
||||||
|
| File size >30MB hits X upload friction. | Use `--crf=18` (high quality, reasonable size); fall back to `--crf=23` if over. 30s @ 1080p30 H.264 is normally 5-15MB. |
|
||||||
|
|
||||||
|
## Documentation / Operational Notes
|
||||||
|
|
||||||
|
- README at `marketing/v3.1-launch/README.md` documents preview / render commands.
|
||||||
|
- After render, the MP4 is uploaded directly to X. Tweet copy is the user's call (this plan stops at the rendered file).
|
||||||
|
|
||||||
|
## Sources & References
|
||||||
|
|
||||||
|
- Related code: `scripts/lib/render.py` (`_render_comparison_scaffold` is the visual model for Scene 4); `SKILL.md` Competitor mode section (the narrative source).
|
||||||
|
- Related PRs: #308, #311, #312 (the 3.0.11 → 3.0.14 release bundle this video markets as "v3.1").
|
||||||
|
- External docs: https://www.remotion.dev/docs/ (Remotion 4.x API).
|
||||||
|
- X video specs: https://help.x.com/en/using-x/twitter-videos.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Search Quality Eval
|
# Search Quality Eval
|
||||||
|
|
||||||
`skills/last30days/scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||||
|
|
||||||
What it does:
|
What it does:
|
||||||
|
|
||||||
@@ -18,13 +18,13 @@ What it does:
|
|||||||
Recommended usage:
|
Recommended usage:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv run python skills/last30days/scripts/evaluate_search_quality.py
|
uv run python scripts/evaluate_search_quality.py
|
||||||
```
|
```
|
||||||
|
|
||||||
Useful flags:
|
Useful flags:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv run python skills/last30days/scripts/evaluate_search_quality.py \
|
uv run python scripts/evaluate_search_quality.py \
|
||||||
--baseline-rev origin/main \
|
--baseline-rev origin/main \
|
||||||
--candidate-rev HEAD \
|
--candidate-rev HEAD \
|
||||||
--no-default-topics \
|
--no-default-topics \
|
||||||
|
|||||||
@@ -1,82 +0,0 @@
|
|||||||
---
|
|
||||||
title: Search-quality eval is manual by default, not a CI gate on every PR
|
|
||||||
date: 2026-05-10
|
|
||||||
category: docs/solutions/architecture
|
|
||||||
module: skills/last30days/scripts/evaluate_search_quality.py
|
|
||||||
problem_type: design_decision
|
|
||||||
component: ci_policy
|
|
||||||
severity: low
|
|
||||||
applies_when:
|
|
||||||
- a contributor proposes wiring search-quality eval into PR CI
|
|
||||||
- a change affects retrieval, ranking, grounding, or synthesis quality and a reviewer asks "why aren't we testing this in CI?"
|
|
||||||
- someone is deciding whether a new evaluator-style script belongs in the default CI workflow
|
|
||||||
related_components:
|
|
||||||
- search_quality_evaluation
|
|
||||||
- ci_workflow
|
|
||||||
- llm_judging
|
|
||||||
tags:
|
|
||||||
- ci-policy
|
|
||||||
- eval
|
|
||||||
- design-decision
|
|
||||||
- cost-vs-signal
|
|
||||||
- non-determinism
|
|
||||||
- manual-gates
|
|
||||||
---
|
|
||||||
|
|
||||||
# Search-quality eval is manual by default, not a CI gate on every PR
|
|
||||||
|
|
||||||
## Context
|
|
||||||
|
|
||||||
`skills/last30days/scripts/evaluate_search_quality.py` compares a baseline revision against a candidate revision across a fixed pool of reviewer topics. It produces two flavors of metrics: deterministic overlap (Jaccard, retention) and LLM-judged quality scores. The natural impulse on seeing an evaluator script is to wire it into CI on every PR — "regression catcher, run it automatically." We deliberately don't.
|
|
||||||
|
|
||||||
Three properties of this particular evaluator make CI-on-every-PR the wrong default:
|
|
||||||
|
|
||||||
1. **Live API access.** The candidate revision typically needs the engine to actually run, which means real ScrapeCreators calls, real reddit fetches, real YouTube searches. CI runs would either need production credentials or a record/replay fixture set that drifts almost immediately as external APIs change shape.
|
|
||||||
|
|
||||||
2. **Cost and latency.** A full eval pass runs the pipeline N times across reviewer topics. Multiplied by every PR (including doc-only PRs), the spend is meaningful and the wall-clock pushes CI from ~30s to many minutes.
|
|
||||||
|
|
||||||
3. **Non-determinism in the judging path.** The LLM-judged metrics are valuable for review but depend on judge-model behavior on a given day. A flaky eval that fails 1 PR in 20 because the judge re-scored an item differently is a worse CI signal than no eval at all — it teaches contributors to retry rather than read the result.
|
|
||||||
|
|
||||||
The deterministic overlap metrics are useful regression signals but they are not the same as user-facing correctness. A change that improves overlap can degrade synthesis quality; a change that drops overlap can be a deliberate improvement. So even the deterministic side isn't safe to auto-fail on.
|
|
||||||
|
|
||||||
## Guidance
|
|
||||||
|
|
||||||
### 1. Keep search-quality eval available, just not automatic
|
|
||||||
|
|
||||||
The script stays runnable by maintainers and contributors. The pattern is:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
LAST30DAYS_PYTHON=python3.13 \
|
|
||||||
python3 skills/last30days/scripts/evaluate_search_quality.py \
|
|
||||||
--baseline main --candidate HEAD
|
|
||||||
```
|
|
||||||
|
|
||||||
Reviewers can request a manual eval run when a PR is in the retrieval/ranking/synthesis path and the risk warrants it. Contributors can run it locally before submitting if they want signal upfront.
|
|
||||||
|
|
||||||
### 2. Standard PR CI gates remain deterministic and contract-shaped
|
|
||||||
|
|
||||||
`pytest` (offline-safe), plugin-contract checks, version-consistency contracts, ruff/lint. Anything that returns the same answer twice for the same input. Quality-of-output assessment lives outside that loop.
|
|
||||||
|
|
||||||
### 3. The middle ground is `workflow_dispatch`, not auto-PR-gating
|
|
||||||
|
|
||||||
If maintainers want a GitHub-triggered eval that doesn't make every PR pay the live-API cost, the right shape is a manually-dispatched workflow (or a label-triggered one) — not a `pull_request:` workflow that runs unconditionally. That keeps the cost knob in human hands.
|
|
||||||
|
|
||||||
### 4. Revisit if the eval can ever be made offline-deterministic
|
|
||||||
|
|
||||||
The blocker is the live-API + non-determinism combination. If a future iteration of the script can compute meaningful Jaccard/retention metrics against static fixtures (no live API calls, no LLM judging), the decision flips and it becomes a candidate for default CI. The decision below tracks that condition; revisit when it's met.
|
|
||||||
|
|
||||||
## What this means in practice
|
|
||||||
|
|
||||||
- Don't merge PRs that wire `evaluate_search_quality.py` into the default `validate.yml` workflow.
|
|
||||||
- Do merge PRs that add `workflow_dispatch` triggers or label-gated runs.
|
|
||||||
- When reviewing a retrieval/ranking change, request a manual eval if the diff suggests it could regress quality — don't expect CI to catch it.
|
|
||||||
|
|
||||||
## Links
|
|
||||||
|
|
||||||
- `skills/last30days/scripts/evaluate_search_quality.py` — the evaluator script
|
|
||||||
- `docs/search-quality-eval.md` — user-facing usage documentation
|
|
||||||
- `.github/workflows/validate.yml` — the default CI workflow (deterministic gates only)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
*Adapted from a draft ADR proposed by @hnshah in [#374](https://github.com/mvanhorn/last30days-skill/pull/374), restructured into the `docs/solutions/` convention. The original ADR text correctly identified the constraint; this version adds the "why workflow_dispatch is the middle ground" framing and the revisit-condition.*
|
|
||||||
@@ -1,219 +0,0 @@
|
|||||||
---
|
|
||||||
title: Release-time consistency tests cause cascade CI failures across all open PRs
|
|
||||||
date: 2026-05-16
|
|
||||||
category: docs/solutions/workflow-issues
|
|
||||||
module: ci-release-engineering
|
|
||||||
problem_type: workflow_issue
|
|
||||||
component: testing_framework
|
|
||||||
severity: high
|
|
||||||
applies_when:
|
|
||||||
- a test asserts consistency between two release-time artifacts (e.g., SKILL.md version and a hardcoded pin in a shell script)
|
|
||||||
- one artifact is updated as part of a version bump and the other requires a manual lockstep update
|
|
||||||
- multiple long-lived PRs are open simultaneously against the same base branch
|
|
||||||
symptoms:
|
|
||||||
- every open PR's CI fails after a version bump even though the PRs are unrelated to versioning
|
|
||||||
- the failing test references a stale hardcoded value that was not updated alongside the bumped version
|
|
||||||
- PR authors must rebase and manually fix an artifact they did not touch
|
|
||||||
root_cause: missing_workflow_step
|
|
||||||
resolution_type: code_fix
|
|
||||||
related_components:
|
|
||||||
- development_workflow
|
|
||||||
- documentation
|
|
||||||
tags:
|
|
||||||
- ci
|
|
||||||
- release-engineering
|
|
||||||
- consistency-test
|
|
||||||
- version-pin
|
|
||||||
- cascade-failure
|
|
||||||
- test-design
|
|
||||||
- workflow
|
|
||||||
---
|
|
||||||
|
|
||||||
# Release-time consistency tests cause cascade CI failures across all open PRs
|
|
||||||
|
|
||||||
## Context
|
|
||||||
|
|
||||||
A `tests/test_version_consistency.py::test_sync_cache_path_uses_skill_version` test was added to enforce that the version string embedded in `skills/last30days/scripts/sync.sh` (a hardcoded plugin-cache path segment) matched the version frontmatter in `skills/last30days/SKILL.md`. The intention was sound: the cache path had to stay in lockstep with the skill version or the sync would silently pull stale files.
|
|
||||||
|
|
||||||
The test worked as designed until a release shipped. At that point it turned into a cascade-failure machine:
|
|
||||||
|
|
||||||
1. A release PR bumps `SKILL.md` version (e.g., 3.2.0 → 3.2.1) **and** bumps the `sync.sh` pin. That PR's CI is green.
|
|
||||||
2. The release PR merges to `main`.
|
|
||||||
3. Every PR that was open at merge time was branched from pre-release `main`. Those PRs have `SKILL.md` 3.2.1 (inherited via merge-base with `main`) but their branch never touched `sync.sh`.
|
|
||||||
4. CI for those PRs runs the consistency test against the new `main` — `SKILL.md` says 3.2.1, `sync.sh` still says 3.2.0 — and fails.
|
|
||||||
5. All open PRs are now red simultaneously, with a failure that has nothing to do with their changes.
|
|
||||||
|
|
||||||
This affected at least five PRs during the 2026-05-13 to 2026-05-15 window: PR #400 (caught during rebase, required a manual pin bump), PRs #390 and #392 (OpenClaw `SCRAPECREATORS_API_KEY` fix, both stalled for the same stale-pin reason), and at least two others. A follow-up hotfix PR (#397 — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`) was required just to unblock the queue.
|
|
||||||
|
|
||||||
The permanent fix was PR #405: delete `sync.sh` entirely (the install workflow made it redundant) and drop `test_sync_cache_path_uses_skill_version`. Once both were gone, no version-consistency cascade was possible.
|
|
||||||
|
|
||||||
## Guidance
|
|
||||||
|
|
||||||
### 1. Don't write consistency tests that read two files and assert one matches a substring derived from the other
|
|
||||||
|
|
||||||
This pattern looks safe but is not:
|
|
||||||
|
|
||||||
```python
|
|
||||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
|
||||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
|
||||||
version = _skill_version() # reads SKILL.md
|
|
||||||
self.assertIn(
|
|
||||||
f'last30days-skill/last30days/{version}"',
|
|
||||||
sync_text, # asserts sync.sh contains that string
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
It encodes the assumption that both files are always updated together, in the same commit, on the same branch. That assumption breaks the moment two files have independent lifecycle owners — a versioned manifest and a deployment script are archetypal examples.
|
|
||||||
|
|
||||||
### 2. If the values genuinely need to stay in sync, derive one from the other at runtime
|
|
||||||
|
|
||||||
Remove the hardcoded pin from `sync.sh` and compute it:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# sync.sh — derive version from SKILL.md at runtime, no pin to maintain
|
|
||||||
SKILL_VERSION=$(grep -m1 '^version:' "$(dirname "$0")/../SKILL.md" \
|
|
||||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
|
||||||
CACHE_PATH="last30days-skill/last30days/${SKILL_VERSION}"
|
|
||||||
```
|
|
||||||
|
|
||||||
Now there is only one source of truth (`SKILL.md`). The test that asserted they matched becomes vacuous and should be deleted. If `SKILL.md` is wrong, the sync itself will fail loudly — which is better feedback than a CI gate on a different PR.
|
|
||||||
|
|
||||||
### 3. If two values must stay independent for legitimate reasons, update them together and make the test self-skip if either source is missing
|
|
||||||
|
|
||||||
If separate versioning is genuinely required (e.g., SKILL.md versions for harness consumers, sync.sh versions a private artifact store with its own cadence), update both in the same PR — never staggered — and write the test to self-skip rather than error when either file is absent:
|
|
||||||
|
|
||||||
```python
|
|
||||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
|
||||||
sync_sh = SKILL_ROOT / "scripts" / "sync.sh"
|
|
||||||
if not sync_sh.exists():
|
|
||||||
self.skipTest("sync.sh not present; skipping pin consistency check")
|
|
||||||
sync_text = sync_sh.read_text(encoding="utf-8")
|
|
||||||
version = _skill_version()
|
|
||||||
self.assertIn(
|
|
||||||
f'last30days-skill/last30days/{version}"',
|
|
||||||
sync_text,
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
Self-skipping means deleting the file is a non-event in CI — no cascading red, no hotfix PR to the queue.
|
|
||||||
|
|
||||||
### 4. Run consistency tests against the merge-base diff, not main
|
|
||||||
|
|
||||||
If you keep a two-file consistency test, scope it so it only fails when the PR itself modifies one of the two files but not the other. A GitHub Actions step can do this:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- name: Check sync.sh version pin consistency
|
|
||||||
run: |
|
|
||||||
BASE=$(git merge-base HEAD origin/main)
|
|
||||||
SKILL_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'SKILL\.md' || true)
|
|
||||||
SYNC_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'sync\.sh' || true)
|
|
||||||
if [ "$SKILL_CHANGED" -gt 0 ] && [ "$SYNC_CHANGED" -eq 0 ]; then
|
|
||||||
echo "SKILL.md version bumped but sync.sh pin was not updated"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
This only fires when your PR touched `SKILL.md` and left `sync.sh` alone — never because a release merged to `main` after you branched.
|
|
||||||
|
|
||||||
### 5. Ask whether you actually need this test
|
|
||||||
|
|
||||||
If the values are wrong, downstream tooling will fail loudly: the sync will fetch the wrong artifact, installs will break, or the harness will reject the version. A test that exists only to catch a human-bookkeeping error at release time adds cascade-fail risk without offering a meaningfully earlier signal. Weigh that cost before adding any two-file consistency gate.
|
|
||||||
|
|
||||||
## Why This Matters
|
|
||||||
|
|
||||||
The damage from a stale-pin consistency test is asymmetric. It:
|
|
||||||
|
|
||||||
- Fails on every open PR simultaneously the moment a release lands on `main` — not just the PR that forgot to update the pin.
|
|
||||||
- Produces a failure message that points at a line in a test file with no obvious relationship to the PR's actual changes.
|
|
||||||
- Requires either a hotfix PR (touching a file the failing PRs have no business touching) or a manual rebase of every affected branch.
|
|
||||||
- Blocks work that has already been reviewed and approved.
|
|
||||||
|
|
||||||
In this repo the effect was measurable: at least five PRs stalled across a two-day window, one hotfix PR was shipped just to unblock the queue, and multiple authors spent time debugging a failure completely unrelated to their changes.
|
|
||||||
|
|
||||||
The broader principle is that tests which gate on *bookkeeping consistency between files* impose their maintenance cost on every contributor, every time, even when those contributors did nothing wrong. That cost compounds with team size and release cadence.
|
|
||||||
|
|
||||||
## When to Apply
|
|
||||||
|
|
||||||
Apply this guidance whenever you find yourself:
|
|
||||||
|
|
||||||
- Writing a test that reads two files and asserts that a string in one matches a value derived from the other.
|
|
||||||
- Adding a CI step labeled "consistency check," "sync check," or "pin check" where the check compares a hardcoded value against a computed one from a separate file.
|
|
||||||
- Working in a repo where a versioned manifest (e.g., `SKILL.md`, `package.json`, `pyproject.toml`) and a deployment artifact (e.g., a shell script, a Dockerfile, a Helm values file) are both maintained by hand.
|
|
||||||
- Reviewing a PR that touches only one of two "paired" files and fails a consistency test for the other.
|
|
||||||
|
|
||||||
It does *not* apply to tests that read a single source of truth and validate its internal structure (e.g., asserting that `SKILL.md`'s frontmatter version is double-quoted, or that `package.json`'s `version` field is a valid semver string). Those tests have one file and one assertion; they cannot cascade across branches.
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
### Before — the pattern that caused the cascade
|
|
||||||
|
|
||||||
Original `tests/test_version_consistency.py` (deleted in commit `9fb19ea`):
|
|
||||||
|
|
||||||
```python
|
|
||||||
import re
|
|
||||||
import unittest
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
SKILL_ROOT = ROOT / "skills" / "last30days"
|
|
||||||
|
|
||||||
|
|
||||||
def _skill_version() -> str:
|
|
||||||
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
|
||||||
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
|
|
||||||
if not match:
|
|
||||||
raise AssertionError("SKILL.md version frontmatter not found")
|
|
||||||
return match.group(1)
|
|
||||||
|
|
||||||
|
|
||||||
class TestVersionConsistency(unittest.TestCase):
|
|
||||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
|
||||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
|
||||||
version = _skill_version() # source 1: SKILL.md frontmatter
|
|
||||||
self.assertIn( # assertion: sync.sh must contain
|
|
||||||
f'last30days-skill/last30days/{version}"',
|
|
||||||
sync_text, # source 2: hardcoded string in sync.sh
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
`sync.sh` contained a line like:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/3.2.0"
|
|
||||||
```
|
|
||||||
|
|
||||||
When SKILL.md bumped to `3.2.1` in a release PR, `sync.sh` was updated in the same PR and CI stayed green. But every PR branched before that release still had `sync.sh` at `3.2.0`. Their CI failed immediately, with an assertion error pointing at the test, not at the release PR.
|
|
||||||
|
|
||||||
### After — what we did: delete both
|
|
||||||
|
|
||||||
PR #405 deleted `sync.sh` (the install workflow replaced it) and dropped `test_sync_cache_path_uses_skill_version` in the same change. No consistency gate, no pin to maintain, no cascade possible.
|
|
||||||
|
|
||||||
### After — what we could have done instead: derive at runtime
|
|
||||||
|
|
||||||
If `sync.sh` had still been needed, the right fix would have been to remove the hardcoded version from the script and derive it from `SKILL.md`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
#!/usr/bin/env bash
|
|
||||||
# sync.sh — no hardcoded version; reads SKILL.md as single source of truth
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
||||||
SKILL_VERSION=$(grep -m1 '^version:' "${SCRIPT_DIR}/../SKILL.md" \
|
|
||||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
|
||||||
|
|
||||||
if [ -z "$SKILL_VERSION" ]; then
|
|
||||||
echo "error: could not parse version from SKILL.md" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/${SKILL_VERSION}"
|
|
||||||
# ... rest of sync logic
|
|
||||||
```
|
|
||||||
|
|
||||||
With this in place, `test_sync_cache_path_uses_skill_version` has no reason to exist — there is nothing to assert. Delete it. If the version parsing breaks, `sync.sh` itself exits non-zero with a clear message.
|
|
||||||
|
|
||||||
## Related
|
|
||||||
|
|
||||||
- **PR #397** (merged) — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`. The hotfix that unblocked the cascade temporarily by bumping the pin.
|
|
||||||
- **PR #400** (merged) — caught the same cascade during rebase; had to bump the pin to clear CI.
|
|
||||||
- **PR #390** (closed) and **PR #392** (rebased + merged) — OpenClaw `SCRAPECREATORS_API_KEY` fix; both blocked by the cascade until rebased onto post-#405 main.
|
|
||||||
- **PR #405** (merged) — the permanent fix: deleted `sync.sh` + `test_sync_cache_path_uses_skill_version` together.
|
|
||||||
- **PR #412** (merged) — adjacent work that consolidated SKILL.md version parsing into `lib/skill_meta.py`, reducing future drift risk by giving the version field one canonical reader.
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days-skill",
|
"name": "last30days-skill",
|
||||||
"version": "3.2.4",
|
"version": "3.0.5",
|
||||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||||
"settings": [
|
"settings": [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,4 +0,0 @@
|
|||||||
{
|
|
||||||
"triggerOnUpdates": true,
|
|
||||||
"statusCheck": true
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,8 @@
|
|||||||
"hooks": [
|
"hooks": [
|
||||||
{
|
{
|
||||||
"type": "command",
|
"type": "command",
|
||||||
"command": "bash \"${CLAUDE_PLUGIN_ROOT:-${extensionPath:-.}}/hooks/scripts/check-config.sh\""
|
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-config.sh",
|
||||||
|
"timeout": 5
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,13 +33,8 @@ load_env_vars() {
|
|||||||
[[ -z "$key" ]] && continue
|
[[ -z "$key" ]] && continue
|
||||||
key=$(echo "$key" | xargs)
|
key=$(echo "$key" | xargs)
|
||||||
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||||
# Strip inline comments (# preceded by whitespace) to prevent
|
|
||||||
# command substitution in backtick-containing comments
|
|
||||||
value="${value%%[[:space:]]#*}"
|
|
||||||
if [[ -n "$key" && -n "$value" ]]; then
|
if [[ -n "$key" && -n "$value" ]]; then
|
||||||
# printf -v writes via assignment semantics (global from inside a
|
eval "ENV_${key}=\"${value}\""
|
||||||
# function), works on macOS's /bin/bash 3.2 — `declare -g` is 4.2+.
|
|
||||||
printf -v "ENV_${key}" '%s' "$value"
|
|
||||||
fi
|
fi
|
||||||
done < "$file"
|
done < "$file"
|
||||||
fi
|
fi
|
||||||
@@ -63,53 +58,14 @@ fi
|
|||||||
# Check SETUP_COMPLETE (from file or env)
|
# Check SETUP_COMPLETE (from file or env)
|
||||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||||
|
|
||||||
# Compute last-run summary line (if last-run.json exists)
|
|
||||||
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
|
||||||
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
|
||||||
LAST_RUN_FILE="$LAST30DAYS_CONFIG_DIR/last-run.json"
|
|
||||||
else
|
|
||||||
LAST_RUN_FILE=""
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
|
|
||||||
fi
|
|
||||||
LAST_RUN_LINE=""
|
|
||||||
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
|
|
||||||
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 - <<'PY' 2>/dev/null || true
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
|
|
||||||
path = os.environ["LAST_RUN_FILE"]
|
|
||||||
try:
|
|
||||||
with open(path) as fh:
|
|
||||||
d = json.load(fh)
|
|
||||||
topic = (d.get("topic") or "?")[:60]
|
|
||||||
ts = d.get("timestamp", "")
|
|
||||||
dt = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
||||||
delta = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
|
|
||||||
if delta < 60: ago = f"{int(delta)}s ago"
|
|
||||||
elif delta < 3600: ago = f"{int(delta//60)}m ago"
|
|
||||||
elif delta < 86400: ago = f"{int(delta//3600)}h ago"
|
|
||||||
else: ago = f"{int(delta//86400)}d ago"
|
|
||||||
total = d.get("total", 0)
|
|
||||||
print(f" Last run: \"{topic}\" · {ago} · {total} results")
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
PY
|
|
||||||
)
|
|
||||||
fi
|
|
||||||
|
|
||||||
# If setup has never been run, show welcome message for new users
|
# If setup has never been run, show welcome message for new users
|
||||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||||
Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).
|
|
||||||
|
|
||||||
Reddit, Hacker News, and Polymarket work out of the box.
|
Reddit, Hacker News, and Polymarket work out of the box.
|
||||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||||
EOF
|
EOF
|
||||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -141,33 +97,16 @@ if [[ -n "$HAS_BSKY" ]]; then
|
|||||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||||
fi
|
fi
|
||||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||||
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
|
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||||
# Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer
|
|
||||||
# whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing.
|
|
||||||
SC_ADD=3
|
|
||||||
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
|
|
||||||
EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \
|
|
||||||
| sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//')
|
|
||||||
if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then
|
|
||||||
SC_ADD=$((SC_ADD - 1))
|
|
||||||
fi
|
|
||||||
if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then
|
|
||||||
SC_ADD=$((SC_ADD - 1))
|
|
||||||
fi
|
|
||||||
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||||
# Fully configured — compact ready message
|
# Fully configured — compact ready message
|
||||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
|
||||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
|
||||||
else
|
else
|
||||||
# Setup done but missing ScrapeCreators — recommend it
|
# Setup done but missing ScrapeCreators — recommend it
|
||||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
|
||||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
|
||||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
echo " 10,000 free API calls, no credit card — scrapecreators.com"
|
||||||
echo " last30days has no affiliation with any API provider."
|
echo " last30days has no affiliation with any API provider."
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# /last30days v3.1 Launch Video
|
||||||
|
|
||||||
|
30-second Remotion-rendered MP4 announcing **v3.1: Competitors mode** for posting on X.
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd marketing/v3.1-launch
|
||||||
|
npm install # one-time, ~200MB of node_modules
|
||||||
|
npm run preview # opens Remotion Studio at localhost:3000 to scrub frames
|
||||||
|
npm run render # writes out/last30days-v3.1-launch.mp4 (high quality, CRF 18)
|
||||||
|
npm run render:fast # writes a CRF 23 preview for fast iteration
|
||||||
|
```
|
||||||
|
|
||||||
|
## Specs
|
||||||
|
|
||||||
|
- 1920×1080, 30fps, 30 seconds (900 frames)
|
||||||
|
- H.264 / MP4
|
||||||
|
- Silent (autoplay-muted-friendly; captions baked in)
|
||||||
|
- Marketing label `v3.1` (engine code version stays 3.0.14)
|
||||||
|
|
||||||
|
## Scene timing (single source of truth: `src/lib/timing.ts`)
|
||||||
|
|
||||||
|
| Scene | Frames | Time | What |
|
||||||
|
|-------|--------|------|------|
|
||||||
|
| 1. Hook | 0-89 | 0.0-3.0s | Badge animates in + "What if one search ran 3 at once?" |
|
||||||
|
| 2. Old way | 90-239 | 3.0-8.0s | Single terminal: `/last30days OpenAI` |
|
||||||
|
| 3. Fan-out | 240-419 | 8.0-14.0s | `--competitors` types in → splits into 3 panes |
|
||||||
|
| 4. Comparison | 420-629 | 14.0-21.0s | 3 panes collapse into Head-to-Head table |
|
||||||
|
| 5. How | 630-779 | 21.0-26.0s | 3-line text card |
|
||||||
|
| 6. CTA | 780-899 | 26.0-30.0s | Install command + repo URL |
|
||||||
|
|
||||||
|
Edit `src/lib/timing.ts` to retime scenes; the `LaunchVideo` composition reads from there.
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
Rendered MP4 lives at `out/last30days-v3.1-launch.mp4` (gitignored). Upload directly to X.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"name": "last30days-v3-1-launch-video",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"description": "30s Remotion launch video for /last30days v3.1 (competitors mode).",
|
||||||
|
"scripts": {
|
||||||
|
"preview": "remotion studio src/index.ts",
|
||||||
|
"render": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch.mp4 --codec=h264 --crf=18",
|
||||||
|
"render:fast": "remotion render src/index.ts LaunchVideo out/last30days-v3.1-launch-preview.mp4 --codec=h264 --crf=23"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"react": "19.0.0",
|
||||||
|
"react-dom": "19.0.0",
|
||||||
|
"remotion": "4.0.250",
|
||||||
|
"@remotion/cli": "4.0.250",
|
||||||
|
"@remotion/google-fonts": "4.0.250"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/react": "19.0.0",
|
||||||
|
"@types/node": "22.10.0",
|
||||||
|
"typescript": "5.6.3"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
import { Config } from "@remotion/cli/config";
|
||||||
|
|
||||||
|
Config.setVideoImageFormat("jpeg");
|
||||||
|
Config.setOverwriteOutput(true);
|
||||||
|
Config.setConcurrency(null);
|
||||||
|
Config.setCodec("h264");
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { AbsoluteFill, Sequence } from "remotion";
|
||||||
|
import { SCENES } from "./lib/timing";
|
||||||
|
import { Scene1Hook } from "./scenes/Scene1Hook";
|
||||||
|
import { Scene2OldWay } from "./scenes/Scene2OldWay";
|
||||||
|
import { Scene3FanOut } from "./scenes/Scene3FanOut";
|
||||||
|
import { Scene4Comparison } from "./scenes/Scene4Comparison";
|
||||||
|
import { Scene5HowItWorks } from "./scenes/Scene5HowItWorks";
|
||||||
|
import { Scene6CTA } from "./scenes/Scene6CTA";
|
||||||
|
import { COLORS } from "./lib/colors";
|
||||||
|
|
||||||
|
export const LaunchVideo: React.FC = () => {
|
||||||
|
return (
|
||||||
|
<AbsoluteFill style={{ background: COLORS.bgDeep }}>
|
||||||
|
<Sequence from={SCENES.hook.from} durationInFrames={SCENES.hook.durationInFrames}>
|
||||||
|
<Scene1Hook />
|
||||||
|
</Sequence>
|
||||||
|
<Sequence from={SCENES.oldWay.from} durationInFrames={SCENES.oldWay.durationInFrames}>
|
||||||
|
<Scene2OldWay />
|
||||||
|
</Sequence>
|
||||||
|
<Sequence from={SCENES.fanOut.from} durationInFrames={SCENES.fanOut.durationInFrames}>
|
||||||
|
<Scene3FanOut />
|
||||||
|
</Sequence>
|
||||||
|
<Sequence from={SCENES.comparison.from} durationInFrames={SCENES.comparison.durationInFrames}>
|
||||||
|
<Scene4Comparison />
|
||||||
|
</Sequence>
|
||||||
|
<Sequence from={SCENES.howItWorks.from} durationInFrames={SCENES.howItWorks.durationInFrames}>
|
||||||
|
<Scene5HowItWorks />
|
||||||
|
</Sequence>
|
||||||
|
<Sequence from={SCENES.cta.from} durationInFrames={SCENES.cta.durationInFrames}>
|
||||||
|
<Scene6CTA />
|
||||||
|
</Sequence>
|
||||||
|
</AbsoluteFill>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { Composition } from "remotion";
|
||||||
|
import { LaunchVideo } from "./LaunchVideo";
|
||||||
|
import { FPS, TOTAL_FRAMES } from "./lib/timing";
|
||||||
|
|
||||||
|
export const RemotionRoot: React.FC = () => {
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Composition
|
||||||
|
id="LaunchVideo"
|
||||||
|
component={LaunchVideo}
|
||||||
|
durationInFrames={TOTAL_FRAMES}
|
||||||
|
fps={FPS}
|
||||||
|
width={1920}
|
||||||
|
height={1080}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||||
|
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
startFrame?: number;
|
||||||
|
size?: "small" | "large";
|
||||||
|
};
|
||||||
|
|
||||||
|
export const BadgeBar: React.FC<Props> = ({ startFrame = 0, size = "large" }) => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
const { fps } = useVideoConfig();
|
||||||
|
const elapsed = Math.max(0, frame - startFrame);
|
||||||
|
|
||||||
|
const scale = spring({
|
||||||
|
frame: elapsed,
|
||||||
|
fps,
|
||||||
|
config: { damping: 12, stiffness: 90 },
|
||||||
|
from: 0.85,
|
||||||
|
to: 1,
|
||||||
|
});
|
||||||
|
const opacity = spring({
|
||||||
|
frame: elapsed,
|
||||||
|
fps,
|
||||||
|
config: { damping: 20 },
|
||||||
|
from: 0,
|
||||||
|
to: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const fontSize = size === "large" ? 56 : 28;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
transform: `scale(${scale})`,
|
||||||
|
opacity,
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
fontSize,
|
||||||
|
color: COLORS.fgPrimary,
|
||||||
|
letterSpacing: 0.5,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span style={{ fontSize: fontSize * 1.1, marginRight: 16 }}>🌐</span>
|
||||||
|
<span>last30days</span>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
marginLeft: 14,
|
||||||
|
color: COLORS.accentCyan,
|
||||||
|
fontWeight: 600,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
v3.1
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
marginLeft: 16,
|
||||||
|
color: COLORS.fgDim,
|
||||||
|
fontSize: fontSize * 0.55,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
· synced 2026-04-22
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { interpolate, useCurrentFrame } from "remotion";
|
||||||
|
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||||
|
|
||||||
|
type Row = {
|
||||||
|
dimension: string;
|
||||||
|
cells: [string, string, string];
|
||||||
|
};
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
startFrame: number;
|
||||||
|
entities: [string, string, string];
|
||||||
|
rows: Row[];
|
||||||
|
rowStaggerFrames?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const ComparisonTable: React.FC<Props> = ({
|
||||||
|
startFrame,
|
||||||
|
entities,
|
||||||
|
rows,
|
||||||
|
rowStaggerFrames = 18,
|
||||||
|
}) => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
|
||||||
|
const headerOpacity = interpolate(
|
||||||
|
frame - startFrame,
|
||||||
|
[0, 12],
|
||||||
|
[0, 1],
|
||||||
|
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||||
|
);
|
||||||
|
|
||||||
|
const colTemplate = "1.4fr 1fr 1fr 1fr";
|
||||||
|
const cellPad = "16px 22px";
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
width: "100%",
|
||||||
|
background: COLORS.bgPanel,
|
||||||
|
borderRadius: 16,
|
||||||
|
border: `1px solid ${COLORS.border}`,
|
||||||
|
overflow: "hidden",
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
boxShadow: "0 24px 60px rgba(0,0,0,0.6)",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
display: "grid",
|
||||||
|
gridTemplateColumns: colTemplate,
|
||||||
|
background: COLORS.bgPanelSoft,
|
||||||
|
borderBottom: `1px solid ${COLORS.border}`,
|
||||||
|
opacity: headerOpacity,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
padding: cellPad,
|
||||||
|
color: COLORS.fgMuted,
|
||||||
|
fontSize: 22,
|
||||||
|
fontWeight: 500,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Dimension
|
||||||
|
</div>
|
||||||
|
{entities.map((entity, idx) => (
|
||||||
|
<div
|
||||||
|
key={entity}
|
||||||
|
style={{
|
||||||
|
padding: cellPad,
|
||||||
|
color: idx === 0 ? COLORS.accentCyan : COLORS.fgPrimary,
|
||||||
|
fontSize: 26,
|
||||||
|
fontWeight: 600,
|
||||||
|
borderLeft: `1px solid ${COLORS.border}`,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{entity}
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
{rows.map((row, idx) => {
|
||||||
|
const rowStart = startFrame + 12 + idx * rowStaggerFrames;
|
||||||
|
const rowOpacity = interpolate(
|
||||||
|
frame - rowStart,
|
||||||
|
[0, 14],
|
||||||
|
[0, 1],
|
||||||
|
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||||
|
);
|
||||||
|
const rowSlide = interpolate(
|
||||||
|
frame - rowStart,
|
||||||
|
[0, 14],
|
||||||
|
[12, 0],
|
||||||
|
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||||
|
);
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={row.dimension}
|
||||||
|
style={{
|
||||||
|
display: "grid",
|
||||||
|
gridTemplateColumns: colTemplate,
|
||||||
|
borderBottom:
|
||||||
|
idx === rows.length - 1 ? "none" : `1px solid ${COLORS.border}`,
|
||||||
|
opacity: rowOpacity,
|
||||||
|
transform: `translateY(${rowSlide}px)`,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
padding: cellPad,
|
||||||
|
color: COLORS.fgMuted,
|
||||||
|
fontSize: 22,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{row.dimension}
|
||||||
|
</div>
|
||||||
|
{row.cells.map((cell, cellIdx) => (
|
||||||
|
<div
|
||||||
|
key={cellIdx}
|
||||||
|
style={{
|
||||||
|
padding: cellPad,
|
||||||
|
color: COLORS.fgPrimary,
|
||||||
|
fontSize: 22,
|
||||||
|
borderLeft: `1px solid ${COLORS.border}`,
|
||||||
|
lineHeight: 1.35,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{cell}
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
title?: string;
|
||||||
|
width?: number | string;
|
||||||
|
height?: number | string;
|
||||||
|
children?: React.ReactNode;
|
||||||
|
glow?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const TerminalWindow: React.FC<Props> = ({
|
||||||
|
title = "/last30days",
|
||||||
|
width = "100%",
|
||||||
|
height = "100%",
|
||||||
|
children,
|
||||||
|
glow = false,
|
||||||
|
}) => {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
background: COLORS.bgPanel,
|
||||||
|
borderRadius: 16,
|
||||||
|
border: `1px solid ${COLORS.border}`,
|
||||||
|
boxShadow: glow
|
||||||
|
? `0 0 60px ${COLORS.accentCyan}33, 0 24px 60px rgba(0,0,0,0.6)`
|
||||||
|
: "0 24px 60px rgba(0,0,0,0.6)",
|
||||||
|
overflow: "hidden",
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
height: 36,
|
||||||
|
background: COLORS.bgPanelSoft,
|
||||||
|
borderBottom: `1px solid ${COLORS.border}`,
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
padding: "0 16px",
|
||||||
|
gap: 8,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
width: 12,
|
||||||
|
height: 12,
|
||||||
|
borderRadius: 12,
|
||||||
|
background: COLORS.trafficRed,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
width: 12,
|
||||||
|
height: 12,
|
||||||
|
borderRadius: 12,
|
||||||
|
background: COLORS.trafficYellow,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
width: 12,
|
||||||
|
height: 12,
|
||||||
|
borderRadius: 12,
|
||||||
|
background: COLORS.trafficGreen,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
marginLeft: 16,
|
||||||
|
color: COLORS.fgMuted,
|
||||||
|
fontSize: 14,
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
letterSpacing: 0.5,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{title}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
flex: 1,
|
||||||
|
padding: "20px 28px",
|
||||||
|
color: COLORS.fgPrimary,
|
||||||
|
overflow: "hidden",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{children}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { interpolate, useCurrentFrame } from "remotion";
|
||||||
|
import { COLORS, FONT_MONO } from "../lib/colors";
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
text: string;
|
||||||
|
startFrame: number;
|
||||||
|
charsPerSec?: number;
|
||||||
|
fontSize?: number;
|
||||||
|
color?: string;
|
||||||
|
prefix?: string;
|
||||||
|
prefixColor?: string;
|
||||||
|
showCursor?: boolean;
|
||||||
|
fps?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const TypedLine: React.FC<Props> = ({
|
||||||
|
text,
|
||||||
|
startFrame,
|
||||||
|
charsPerSec = 28,
|
||||||
|
fontSize = 32,
|
||||||
|
color = COLORS.fgPrimary,
|
||||||
|
prefix,
|
||||||
|
prefixColor = COLORS.accentGreen,
|
||||||
|
showCursor = true,
|
||||||
|
fps = 30,
|
||||||
|
}) => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
const elapsed = Math.max(0, frame - startFrame);
|
||||||
|
const totalChars = text.length;
|
||||||
|
const lengthFrames = Math.ceil((totalChars / charsPerSec) * fps);
|
||||||
|
const visibleChars = Math.round(
|
||||||
|
interpolate(elapsed, [0, lengthFrames], [0, totalChars], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const visible = text.slice(0, visibleChars);
|
||||||
|
const done = visibleChars >= totalChars;
|
||||||
|
const cursorOn = showCursor && Math.floor(frame / 15) % 2 === 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
fontSize,
|
||||||
|
color,
|
||||||
|
whiteSpace: "pre",
|
||||||
|
lineHeight: 1.4,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{prefix ? (
|
||||||
|
<span style={{ color: prefixColor, marginRight: 12 }}>{prefix}</span>
|
||||||
|
) : null}
|
||||||
|
<span>{visible}</span>
|
||||||
|
{(!done || cursorOn) && (
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
display: "inline-block",
|
||||||
|
width: fontSize * 0.55,
|
||||||
|
height: fontSize * 0.95,
|
||||||
|
background: color,
|
||||||
|
verticalAlign: "text-bottom",
|
||||||
|
marginLeft: 2,
|
||||||
|
opacity: cursorOn ? 0.85 : 0,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
import { registerRoot } from "remotion";
|
||||||
|
import { RemotionRoot } from "./Root";
|
||||||
|
|
||||||
|
registerRoot(RemotionRoot);
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
export const COLORS = {
|
||||||
|
bgDeep: "#0a0e14",
|
||||||
|
bgPanel: "#11161d",
|
||||||
|
bgPanelSoft: "#161c25",
|
||||||
|
fgPrimary: "#e6e6e6",
|
||||||
|
fgMuted: "#8a93a3",
|
||||||
|
fgDim: "#5b6573",
|
||||||
|
border: "#2a3340",
|
||||||
|
accentCyan: "#36d6f7",
|
||||||
|
accentMagenta: "#ff55a3",
|
||||||
|
accentGreen: "#5fff9f",
|
||||||
|
accentAmber: "#ffc857",
|
||||||
|
trafficRed: "#ff5f57",
|
||||||
|
trafficYellow: "#febc2e",
|
||||||
|
trafficGreen: "#28c840",
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export const FONT_MONO = '"JetBrains Mono", "SF Mono", "Menlo", monospace';
|
||||||
|
export const FONT_SANS =
|
||||||
|
'"Inter", "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif';
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
// Single source of truth for scene frame ranges.
|
||||||
|
// 30fps × 30s = 900 frames total.
|
||||||
|
export const FPS = 30;
|
||||||
|
export const TOTAL_FRAMES = 900;
|
||||||
|
|
||||||
|
export const SCENES = {
|
||||||
|
hook: { from: 0, durationInFrames: 90 },
|
||||||
|
oldWay: { from: 90, durationInFrames: 150 },
|
||||||
|
fanOut: { from: 240, durationInFrames: 180 },
|
||||||
|
comparison: { from: 420, durationInFrames: 210 },
|
||||||
|
howItWorks: { from: 630, durationInFrames: 150 },
|
||||||
|
cta: { from: 780, durationInFrames: 120 },
|
||||||
|
} as const;
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||||
|
import { BadgeBar } from "../components/BadgeBar";
|
||||||
|
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||||
|
|
||||||
|
export const Scene1Hook: React.FC = () => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
const { fps } = useVideoConfig();
|
||||||
|
|
||||||
|
const captionOpacity = interpolate(frame, [20, 35, 75, 90], [0, 1, 1, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
const captionLift = spring({
|
||||||
|
frame: frame - 20,
|
||||||
|
fps,
|
||||||
|
config: { damping: 15, stiffness: 70 },
|
||||||
|
from: 16,
|
||||||
|
to: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const badgeFadeOut = interpolate(frame, [70, 90], [1, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<AbsoluteFill
|
||||||
|
style={{
|
||||||
|
background: `radial-gradient(circle at 50% 40%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 60%)`,
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
gap: 64,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div style={{ opacity: badgeFadeOut }}>
|
||||||
|
<BadgeBar />
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
opacity: captionOpacity,
|
||||||
|
transform: `translateY(${captionLift}px)`,
|
||||||
|
fontFamily: FONT_SANS,
|
||||||
|
fontSize: 96,
|
||||||
|
fontWeight: 600,
|
||||||
|
color: COLORS.fgPrimary,
|
||||||
|
textAlign: "center",
|
||||||
|
letterSpacing: -1.5,
|
||||||
|
lineHeight: 1.1,
|
||||||
|
maxWidth: 1400,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
What if one search
|
||||||
|
<br />
|
||||||
|
ran <span style={{ color: COLORS.accentCyan }}>3 at once?</span>
|
||||||
|
</div>
|
||||||
|
</AbsoluteFill>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||||
|
import { TerminalWindow } from "../components/TerminalWindow";
|
||||||
|
import { TypedLine } from "../components/TypedLine";
|
||||||
|
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||||
|
|
||||||
|
export const Scene2OldWay: React.FC = () => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
|
||||||
|
const enter = interpolate(frame, [0, 20], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
const slide = interpolate(frame, [0, 20], [40, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
// Result card fades in after type completes (~70 frames)
|
||||||
|
const resultFade = interpolate(frame, [70, 95], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
// Caption appears late
|
||||||
|
const captionFade = interpolate(frame, [110, 130, 150], [0, 1, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<AbsoluteFill
|
||||||
|
style={{
|
||||||
|
background: COLORS.bgDeep,
|
||||||
|
padding: 80,
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
width: 1400,
|
||||||
|
height: 520,
|
||||||
|
opacity: enter,
|
||||||
|
transform: `translateY(${slide}px)`,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<TerminalWindow title="bash">
|
||||||
|
<TypedLine
|
||||||
|
text="/last30days OpenAI"
|
||||||
|
startFrame={20}
|
||||||
|
prefix="$"
|
||||||
|
fontSize={42}
|
||||||
|
/>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
opacity: resultFade,
|
||||||
|
marginTop: 36,
|
||||||
|
padding: "20px 24px",
|
||||||
|
background: COLORS.bgPanelSoft,
|
||||||
|
borderRadius: 12,
|
||||||
|
border: `1px solid ${COLORS.border}`,
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
fontSize: 26,
|
||||||
|
color: COLORS.fgPrimary,
|
||||||
|
lineHeight: 1.6,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div style={{ color: COLORS.accentGreen }}>
|
||||||
|
✅ All agents reported back!
|
||||||
|
</div>
|
||||||
|
<div style={{ color: COLORS.fgMuted, marginTop: 6 }}>
|
||||||
|
├─ 🟠 Reddit: 14 threads
|
||||||
|
</div>
|
||||||
|
<div style={{ color: COLORS.fgMuted }}>
|
||||||
|
├─ 🔵 X: 22 posts
|
||||||
|
</div>
|
||||||
|
<div style={{ color: COLORS.fgMuted }}>
|
||||||
|
└─ 🟡 HN: 1 story
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</TerminalWindow>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
opacity: captionFade,
|
||||||
|
marginTop: 60,
|
||||||
|
fontFamily: FONT_SANS,
|
||||||
|
fontSize: 38,
|
||||||
|
color: COLORS.fgMuted,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
The old way: <span style={{ color: COLORS.fgPrimary }}>one topic.</span>
|
||||||
|
</div>
|
||||||
|
</AbsoluteFill>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||||
|
import { TerminalWindow } from "../components/TerminalWindow";
|
||||||
|
import { TypedLine } from "../components/TypedLine";
|
||||||
|
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||||
|
|
||||||
|
const PROGRESS_LINES = [
|
||||||
|
{ source: "Reddit", color: "#ff6a3d" },
|
||||||
|
{ source: "X", color: "#36d6f7" },
|
||||||
|
{ source: "YouTube", color: "#ff5757" },
|
||||||
|
{ source: "TikTok", color: "#5fff9f" },
|
||||||
|
{ source: "Instagram", color: "#ff55a3" },
|
||||||
|
];
|
||||||
|
|
||||||
|
const ENTITIES: { label: string; tag: string; accent: string }[] = [
|
||||||
|
{ label: "OpenAI", tag: "$ /last30days OpenAI", accent: COLORS.accentCyan },
|
||||||
|
{ label: "Anthropic", tag: "$ /last30days Anthropic", accent: COLORS.accentMagenta },
|
||||||
|
{ label: "xAI", tag: "$ /last30days xAI", accent: COLORS.accentAmber },
|
||||||
|
];
|
||||||
|
|
||||||
|
const FanPane: React.FC<{
|
||||||
|
label: string;
|
||||||
|
tag: string;
|
||||||
|
accent: string;
|
||||||
|
panelStart: number;
|
||||||
|
}> = ({ label, tag, accent, panelStart }) => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
const { fps } = useVideoConfig();
|
||||||
|
const localFrame = Math.max(0, frame - panelStart);
|
||||||
|
|
||||||
|
const enter = spring({
|
||||||
|
frame: localFrame,
|
||||||
|
fps,
|
||||||
|
config: { damping: 18, stiffness: 80 },
|
||||||
|
from: 0,
|
||||||
|
to: 1,
|
||||||
|
});
|
||||||
|
const slide = interpolate(localFrame, [0, 20], [40, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
flex: 1,
|
||||||
|
opacity: enter,
|
||||||
|
transform: `translateY(${slide}px)`,
|
||||||
|
height: 480,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<TerminalWindow title={label} glow>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
color: accent,
|
||||||
|
fontSize: 18,
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
marginBottom: 14,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{tag}
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
fontSize: 22,
|
||||||
|
color: COLORS.accentGreen,
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
marginBottom: 12,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
[Competitors] running...
|
||||||
|
</div>
|
||||||
|
{PROGRESS_LINES.map((line, idx) => {
|
||||||
|
const lineStart = panelStart + 16 + idx * 6;
|
||||||
|
const lineFade = interpolate(
|
||||||
|
frame - lineStart,
|
||||||
|
[0, 8],
|
||||||
|
[0, 1],
|
||||||
|
{ extrapolateLeft: "clamp", extrapolateRight: "clamp" },
|
||||||
|
);
|
||||||
|
// pulse the in-progress dot
|
||||||
|
const dotOn = Math.floor((frame - lineStart) / 6) % 2 === 0;
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={line.source}
|
||||||
|
style={{
|
||||||
|
opacity: lineFade,
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
fontSize: 20,
|
||||||
|
color: COLORS.fgMuted,
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
marginBottom: 6,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
display: "inline-block",
|
||||||
|
width: 10,
|
||||||
|
height: 10,
|
||||||
|
borderRadius: 10,
|
||||||
|
background: dotOn ? line.color : COLORS.bgPanelSoft,
|
||||||
|
marginRight: 12,
|
||||||
|
boxShadow: dotOn ? `0 0 10px ${line.color}` : "none",
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<span style={{ color: line.color, marginRight: 8 }}>
|
||||||
|
►
|
||||||
|
</span>
|
||||||
|
<span>{line.source}</span>
|
||||||
|
<span style={{ marginLeft: "auto", color: COLORS.fgDim }}>
|
||||||
|
{dotOn ? "..." : "·"}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</TerminalWindow>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const Scene3FanOut: React.FC = () => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
|
||||||
|
// Phase 1 (0-30 frames): single terminal types --competitors flag
|
||||||
|
// Phase 2 (30+): split into 3 panes
|
||||||
|
const splitProgress = interpolate(frame, [30, 50], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
const singleOpacity = interpolate(frame, [0, 8, 30, 45], [0, 1, 1, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
// Caption fades in shortly after panes settle so it has time to read.
|
||||||
|
const captionFade = interpolate(frame, [60, 80], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<AbsoluteFill
|
||||||
|
style={{
|
||||||
|
background: COLORS.bgDeep,
|
||||||
|
padding: 60,
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
opacity: captionFade,
|
||||||
|
textAlign: "center",
|
||||||
|
fontFamily: FONT_SANS,
|
||||||
|
fontSize: 42,
|
||||||
|
color: COLORS.fgPrimary,
|
||||||
|
marginBottom: 32,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Now it discovers competitors
|
||||||
|
<br />
|
||||||
|
<span style={{ color: COLORS.accentCyan }}>and runs all 3.</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
flex: 1,
|
||||||
|
position: "relative",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{/* Single terminal during phase 1, fades out as panes appear */}
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
position: "absolute",
|
||||||
|
inset: 0,
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
opacity: singleOpacity,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div style={{ width: 1200, height: 380 }}>
|
||||||
|
<TerminalWindow title="bash">
|
||||||
|
<TypedLine
|
||||||
|
text="/last30days OpenAI --competitors"
|
||||||
|
startFrame={0}
|
||||||
|
prefix="$"
|
||||||
|
fontSize={42}
|
||||||
|
/>
|
||||||
|
</TerminalWindow>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Three panes fade in starting frame ~30 */}
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
position: "absolute",
|
||||||
|
inset: 0,
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
gap: 24,
|
||||||
|
opacity: splitProgress,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{ENTITIES.map((entity, idx) => (
|
||||||
|
<FanPane
|
||||||
|
key={entity.label}
|
||||||
|
label={entity.label}
|
||||||
|
tag={entity.tag}
|
||||||
|
accent={entity.accent}
|
||||||
|
panelStart={45 + idx * 8}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</AbsoluteFill>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||||
|
import { ComparisonTable } from "../components/ComparisonTable";
|
||||||
|
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||||
|
|
||||||
|
const ROWS = [
|
||||||
|
{
|
||||||
|
dimension: "What it is",
|
||||||
|
cells: [
|
||||||
|
"GPT-5 leader, Plus + API",
|
||||||
|
"Claude 4, safety-first",
|
||||||
|
"Grok, X-native, fast",
|
||||||
|
] as [string, string, string],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
dimension: "30-day momentum",
|
||||||
|
cells: [
|
||||||
|
"GPT-5 launch wave",
|
||||||
|
"Claude 4.7 1M context",
|
||||||
|
"Grok 5 reveal",
|
||||||
|
] as [string, string, string],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
dimension: "Community vibe",
|
||||||
|
cells: [
|
||||||
|
"Defensive but deep",
|
||||||
|
"Quiet, devs-only",
|
||||||
|
"Loud, meme-rich",
|
||||||
|
] as [string, string, string],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
dimension: "Best for",
|
||||||
|
cells: [
|
||||||
|
"Mainstream + tools",
|
||||||
|
"Long-context coding",
|
||||||
|
"Live X intel",
|
||||||
|
] as [string, string, string],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
export const Scene4Comparison: React.FC = () => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
|
||||||
|
const captionFade = interpolate(frame, [0, 12, 180, 210], [0, 1, 1, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
const captionLift = interpolate(frame, [0, 14], [16, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
const tableFade = interpolate(frame, [16, 32], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<AbsoluteFill
|
||||||
|
style={{
|
||||||
|
background: COLORS.bgDeep,
|
||||||
|
padding: "48px 80px",
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
alignItems: "center",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
opacity: captionFade,
|
||||||
|
transform: `translateY(${captionLift}px)`,
|
||||||
|
fontFamily: FONT_SANS,
|
||||||
|
fontSize: 38,
|
||||||
|
color: COLORS.fgMuted,
|
||||||
|
textAlign: "center",
|
||||||
|
marginBottom: 36,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span style={{ color: COLORS.accentCyan, fontWeight: 600 }}>
|
||||||
|
3 full passes.
|
||||||
|
</span>
|
||||||
|
<span style={{ marginLeft: 18, color: COLORS.accentMagenta, fontWeight: 600 }}>
|
||||||
|
3 save files.
|
||||||
|
</span>
|
||||||
|
<span style={{ marginLeft: 18, color: COLORS.fgPrimary, fontWeight: 600 }}>
|
||||||
|
1 comparison.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
width: "100%",
|
||||||
|
maxWidth: 1640,
|
||||||
|
opacity: tableFade,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<ComparisonTable
|
||||||
|
startFrame={20}
|
||||||
|
entities={["OpenAI", "Anthropic", "xAI"]}
|
||||||
|
rows={ROWS}
|
||||||
|
rowStaggerFrames={22}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</AbsoluteFill>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { AbsoluteFill, interpolate, useCurrentFrame } from "remotion";
|
||||||
|
import { COLORS, FONT_SANS } from "../lib/colors";
|
||||||
|
|
||||||
|
const LINES = [
|
||||||
|
{ text: "You pick the topic.", color: COLORS.fgPrimary },
|
||||||
|
{ text: "The agent picks the peers.", color: COLORS.accentCyan },
|
||||||
|
{ text: "The engine fans out.", color: COLORS.accentMagenta },
|
||||||
|
];
|
||||||
|
|
||||||
|
export const Scene5HowItWorks: React.FC = () => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<AbsoluteFill
|
||||||
|
style={{
|
||||||
|
background: `radial-gradient(circle at 50% 60%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
gap: 48,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{LINES.map((line, idx) => {
|
||||||
|
const start = 10 + idx * 28;
|
||||||
|
const fade = interpolate(frame, [start, start + 14], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
const slide = interpolate(frame, [start, start + 18], [24, 0], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={line.text}
|
||||||
|
style={{
|
||||||
|
opacity: fade,
|
||||||
|
transform: `translateY(${slide}px)`,
|
||||||
|
fontFamily: FONT_SANS,
|
||||||
|
fontSize: 78,
|
||||||
|
fontWeight: 600,
|
||||||
|
color: line.color,
|
||||||
|
letterSpacing: -1,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{line.text}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</AbsoluteFill>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
import React from "react";
|
||||||
|
import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
|
||||||
|
import { BadgeBar } from "../components/BadgeBar";
|
||||||
|
import { COLORS, FONT_MONO, FONT_SANS } from "../lib/colors";
|
||||||
|
|
||||||
|
export const Scene6CTA: React.FC = () => {
|
||||||
|
const frame = useCurrentFrame();
|
||||||
|
const { fps } = useVideoConfig();
|
||||||
|
|
||||||
|
const installFade = interpolate(frame, [20, 40], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
// Stronger 1Hz pulse on the install line for end-of-video emphasis
|
||||||
|
const pulse = 0.8 + 0.2 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||||
|
const glowPulse = 0.4 + 0.4 * Math.sin((frame / fps) * 2 * Math.PI);
|
||||||
|
|
||||||
|
const repoFade = interpolate(frame, [50, 70], [0, 1], {
|
||||||
|
extrapolateLeft: "clamp",
|
||||||
|
extrapolateRight: "clamp",
|
||||||
|
});
|
||||||
|
|
||||||
|
const enterScale = spring({
|
||||||
|
frame,
|
||||||
|
fps,
|
||||||
|
config: { damping: 18, stiffness: 90 },
|
||||||
|
from: 0.95,
|
||||||
|
to: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<AbsoluteFill
|
||||||
|
style={{
|
||||||
|
background: `radial-gradient(circle at 50% 50%, ${COLORS.bgPanelSoft} 0%, ${COLORS.bgDeep} 70%)`,
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
gap: 56,
|
||||||
|
transform: `scale(${enterScale})`,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<BadgeBar />
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
opacity: installFade * pulse,
|
||||||
|
padding: "18px 36px",
|
||||||
|
background: COLORS.bgPanel,
|
||||||
|
border: `1px solid ${COLORS.accentCyan}`,
|
||||||
|
borderRadius: 14,
|
||||||
|
boxShadow: `0 0 ${40 + glowPulse * 60}px ${COLORS.accentCyan}${Math.round(40 + glowPulse * 80).toString(16)}`,
|
||||||
|
fontFamily: FONT_MONO,
|
||||||
|
fontSize: 44,
|
||||||
|
color: COLORS.fgPrimary,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span style={{ color: COLORS.accentGreen, marginRight: 18 }}>$</span>
|
||||||
|
/last30days <span style={{ color: COLORS.fgMuted }}>{"{topic}"}</span>{" "}
|
||||||
|
<span style={{ color: COLORS.accentCyan }}>--competitors</span>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
opacity: repoFade,
|
||||||
|
fontFamily: FONT_SANS,
|
||||||
|
fontSize: 28,
|
||||||
|
color: COLORS.fgMuted,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
github.com/mvanhorn/last30days-skill
|
||||||
|
</div>
|
||||||
|
</AbsoluteFill>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"jsx": "react-jsx",
|
||||||
|
"strict": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"allowSyntheticDefaultImports": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||||
|
"types": ["node"]
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"]
|
||||||
|
}
|
||||||
|
Before Width: | Height: | Size: 2.4 MiB |
@@ -1,14 +1,16 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "last30days-skill"
|
name = "last30days-skill"
|
||||||
version = "3.3.0"
|
version = "3.0.0"
|
||||||
description = "Multi-source last-30-days research skill"
|
description = "Multi-source last-30-days research skill"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
dependencies = []
|
dependencies = [
|
||||||
|
"requests>=2.32,<3",
|
||||||
|
]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
"pytest>=9.0.3,<10",
|
"pytest>=9,<10",
|
||||||
"pytest-cov>=7,<8",
|
"pytest-cov>=7,<8",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -22,9 +24,9 @@ addopts = [
|
|||||||
|
|
||||||
[tool.coverage.run]
|
[tool.coverage.run]
|
||||||
branch = true
|
branch = true
|
||||||
source = ["skills/last30days/scripts", "tests"]
|
source = ["scripts", "tests"]
|
||||||
omit = [
|
omit = [
|
||||||
"skills/last30days/scripts/lib/vendor/*",
|
"scripts/lib/vendor/*",
|
||||||
"dist/*",
|
"dist/*",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -32,6 +34,7 @@ omit = [
|
|||||||
skip_empty = true
|
skip_empty = true
|
||||||
show_missing = true
|
show_missing = true
|
||||||
omit = [
|
omit = [
|
||||||
"skills/last30days/scripts/lib/vendor/*",
|
"scripts/lib/vendor/*",
|
||||||
"dist/*",
|
"dist/*",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -1,64 +1,52 @@
|
|||||||
## v3.3.0 — install everywhere, ship the reliability sweep
|
|
||||||
|
|
||||||
The AI world reinvents itself every month. This skill keeps you current.
|
The AI world reinvents itself every month. This skill keeps you current.
|
||||||
|
|
||||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, Digg, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||||
|
|
||||||
## What's new in v3.3.0
|
## v3 is the intelligent search release
|
||||||
|
|
||||||
### Install everywhere with one command
|
v3 is a ground-up engine rewrite by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||||
|
|
||||||
`npx skills add mvanhorn/last30days-skill -g -y` is now the canonical install path for **every harness** — Claude Code, OpenAI Codex CLI, Cursor, Gemini CLI, GitHub Copilot, Windsurf, and 50+ other Agent Skills hosts. The skill auto-detects each harness's skills directory and symlinks in place, so edits propagate live. No more per-harness manual paths in the README.
|
Type "OpenClaw" and v3 resolves @steipete, r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags before a single API call fires. Type "Peter Steinberger" and it resolves his X handle and GitHub profile, switches to person mode, and shows what he shipped this month at 85% merge rate across 22 PRs. None of that was on Google.
|
||||||
|
|
||||||
### New emit mode: `--emit=html`
|
## Headline features
|
||||||
|
|
||||||
Shareable, print-friendly HTML briefs. Drop the file in Slack, mail it to a stakeholder, or print it for the meeting. Same data as compact mode, structured for human reading.
|
### Intelligent pre-research
|
||||||
|
|
||||||
### New source: Digg
|
The killer feature. A new Python pre-research brain resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags, all resolved before a single API call.
|
||||||
|
|
||||||
Digg surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes directly into the brief. Auto-enabled when `digg-pp-cli` is on PATH. Footer line: `⛏️ Digg: N clusters │ K posts │ M authors`. No X auth required for the inline quotes.
|
### Best Takes
|
||||||
|
|
||||||
### YouTube residential-IP routing (`LAST30DAYS_YOUTUBE_SSH_HOST`)
|
A second LLM judge scores every result for humor, wit, and virality alongside relevance. Every brief now ends with a Best Takes section surfacing the cleverest one-liners and most viral quotes. The Reddit and X people are funny, and the old engine buried their best stuff.
|
||||||
|
|
||||||
Running on a datacenter VPS (Hetzner, DigitalOcean, AWS, etc.)? YouTube's bot-wall fingerprints datacenter IP ranges before any cookie check. Set `LAST30DAYS_YOUTUBE_SSH_HOST=<ssh-alias>` and yt-dlp runs over SSH against a residential-IP host instead. One env var, no proxy service required.
|
### Cross-source cluster merging
|
||||||
|
|
||||||
### macOS Keychain credential source
|
When the same story hits Reddit, X, and YouTube, v3 merges them into one cluster instead of three duplicates. Entity-based overlap detection catches matches even when the titles use different words.
|
||||||
|
|
||||||
When env vars and config files aren't set, the engine now reads credentials from the macOS Keychain. Stores secrets where macOS expects them; nothing on disk in plaintext.
|
### Single-pass comparisons
|
||||||
|
|
||||||
### `EXCLUDE_SOURCES` env var
|
"X vs Y" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides at once. Same depth, 3 minutes.
|
||||||
|
|
||||||
The inverse of `INCLUDE_SOURCES`. Useful for "everything except TikTok" or "everything except the slow ones."
|
### GitHub person-mode and project-mode
|
||||||
|
|
||||||
## Reliability sweep
|
When the topic is a person, the engine switches from keyword search to author-scoped queries. PR velocity, top repos by stars, release notes for what shipped this month, woven into the narrative alongside X posts and Reddit threads.
|
||||||
|
|
||||||
This release closes a long tail of platform-specific issues that have been accumulating:
|
When the topic is a project, it pulls live star counts, READMEs, releases, and top issues from the GitHub API. No stale blog posts.
|
||||||
|
|
||||||
- **Reddit**: subreddits starting with `r` no longer get mangled by `lstrip("r/")`. Browser-like headers + gzip handling fix urllib 403s on the public JSON endpoint. HTTP 402 now triggers the OpenAI/public-JSON fallback chain when ScrapeCreators credits are exhausted.
|
### ELI5 mode
|
||||||
- **xAI**: empty or malformed responses now surface in `errors_by_source` instead of silently returning zero results.
|
|
||||||
- **Windows**: process cleanup no longer crashes on `os.killpg`. POSIX-style secret-permission warnings skipped. Save-path footer uses forward slashes.
|
|
||||||
- **Auth**: comma-separated `SCRAPECREATORS_API_KEY=key1,key2` rotation restored (accidentally dropped in v3.0.6).
|
|
||||||
- **YouTube + HN**: SC YouTube + multi-token HN searches unblocked. Transcript-fetch ratio surfaced.
|
|
||||||
- **HTTP**: retry budget expanded with exponential backoff on DNS failure. Parallel AI search aligned with current API schema.
|
|
||||||
- **OpenClaw**: now works without a ScrapeCreators key. Poll-timing initialized once.
|
|
||||||
|
|
||||||
## Multi-harness reframe
|
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations, just clearer. Say "eli5 off" to go back.
|
||||||
|
|
||||||
`AGENTS.md` is now the canonical project doc; `CLAUDE.md` points at it. The skill is positioned as a multi-harness Agent Skills package, not a Claude-Code-specific tool. SKILL.md's path resolution rewrote `SKILL_ROOT` → `SKILL_DIR`, removing ~80 lines of bash and fixing a real spec-vs-engine divergence bug.
|
### 13+ sources
|
||||||
|
|
||||||
## Breaking change
|
v3 adds Threads, Pinterest, Perplexity, Bluesky, and Parallel AI grounding to the existing Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and Web lineup. Perplexity Deep Research (`--deep-research`) gives you 50+ citation reports for serious investigation.
|
||||||
|
|
||||||
**`.codex-plugin/plugin.json` removed.** Codex native-plugin users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`. The `npx skills add` path now reaches every harness uniformly.
|
### Per-author cap and entity disambiguation
|
||||||
|
|
||||||
|
Max 3 items per author prevents single-voice dominance. Synthesis trusts resolved handles over fuzzy keyword matches.
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
Any harness (recommended):
|
Claude Code:
|
||||||
|
|
||||||
```
|
|
||||||
npx skills add mvanhorn/last30days-skill -g -y
|
|
||||||
```
|
|
||||||
|
|
||||||
Claude Code marketplace:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
/plugin marketplace add mvanhorn/last30days-skill
|
/plugin marketplace add mvanhorn/last30days-skill
|
||||||
@@ -70,21 +58,29 @@ OpenClaw:
|
|||||||
clawhub install last30days-official
|
clawhub install last30days-official
|
||||||
```
|
```
|
||||||
|
|
||||||
|
OpenAI Codex CLI: run `codex` from a checkout of this repo and v3's skill at `.agents/skills/last30days/SKILL.md` will be discovered automatically. Or copy `SKILL.md` to `~/.agents/skills/last30days/SKILL.md` for a global install.
|
||||||
|
|
||||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||||
|
|
||||||
## Contributors
|
## v3 Community
|
||||||
|
|
||||||
First-time contributors whose fixes shipped in v3.3.0 (most via PR triage salvage — the fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
v3 was shaped by community contributors whose PRs and issues inspired core features. Their code wasn't merged directly (v3 was a ground-up rewrite), but their ideas drove what shipped.
|
||||||
|
|
||||||
- Dave Morin — portable test-harness paths
|
Thanks to @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, and @hnshah. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list.
|
||||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
|
||||||
- Eric Oberhofer — multi-key rotation restored
|
|
||||||
- gujishh — Windows process cleanup
|
|
||||||
- Franco Carballar — Reddit browser-like headers
|
|
||||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
|
||||||
- Kaustav Mishra — xAI error surfacing
|
|
||||||
- [@thinkun](https://github.com/thinkun) — OpenClaw ScrapeCreators-key-optional fix
|
|
||||||
|
|
||||||
Plus every contributor who shipped one of the ~75 PRs merged this cycle. See [CHANGELOG.md](CHANGELOG.md) under `[3.3.0]` for the full PR list and `git log v3.2.0..v3.3.0` for the complete commit graph.
|
Contributors who shaped the release itself:
|
||||||
|
|
||||||
|
- @Jah-yee (#153) surfaced the need for a real Codex CLI integration, which shipped in #219
|
||||||
|
- @Cody-Coyote (#204) reported the marketplace validation bug that needed fixing before v3 could ship cleanly
|
||||||
|
- @dannyshmueli pushed for v3 and Codex family support publicly on X
|
||||||
|
|
||||||
|
Full Added / Changed / Fixed detail lives in [CHANGELOG.md](CHANGELOG.md) under `[3.0.0]`.
|
||||||
|
|
||||||
|
## Earlier contributors
|
||||||
|
|
||||||
|
From the v1 and v2 lineage:
|
||||||
|
|
||||||
|
- [@galligan](https://github.com/galligan) for marketplace plugin inspiration
|
||||||
|
- [@hutchins](https://x.com/hutchins) for pushing the YouTube feature
|
||||||
|
|
||||||
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
||||||
|
|||||||
@@ -1,14 +1,13 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
||||||
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root)
|
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||||
#
|
#
|
||||||
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
||||||
# directory containing SKILL.md and the scripts/ runtime from skills/last30days.
|
# directory containing SKILL.md and the scripts/ runtime. See
|
||||||
# See
|
|
||||||
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
cd "$REPO_ROOT"
|
cd "$REPO_ROOT"
|
||||||
|
|
||||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||||
@@ -18,7 +17,14 @@ fi
|
|||||||
|
|
||||||
mkdir -p dist
|
mkdir -p dist
|
||||||
OUT="dist/last30days.skill"
|
OUT="dist/last30days.skill"
|
||||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days
|
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD
|
||||||
|
|
||||||
|
# claude.ai's .skill bundle only needs the root SKILL.md + scripts/ runtime.
|
||||||
|
# Claude Code needs skills/ and .claude-plugin/ in the git archive
|
||||||
|
# (that's why they're NOT in .gitattributes export-ignore), but the .skill
|
||||||
|
# bundle must strip them to keep a single canonical SKILL.md and stay under
|
||||||
|
# the 200-file cap.
|
||||||
|
zip -d "$OUT" "last30days/skills/*" "last30days/.claude-plugin/*" > /dev/null 2>&1 || true
|
||||||
|
|
||||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||||
SIZE=$(du -h "$OUT" | cut -f1)
|
SIZE=$(du -h "$OUT" | cut -f1)
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# A/B test runner: public release vs private beta
|
# A/B test runner: public release vs private beta
|
||||||
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West"
|
# Usage: bash scripts/compare.sh "Kanye West"
|
||||||
#
|
#
|
||||||
# Runs /last30days (public release) and /last30days-beta (private beta)
|
# Runs /last30days (public release) and /last30days-beta (private beta)
|
||||||
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
||||||
@@ -9,8 +9,8 @@
|
|||||||
set -e
|
set -e
|
||||||
|
|
||||||
if [ $# -eq 0 ]; then
|
if [ $# -eq 0 ]; then
|
||||||
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>"
|
echo "Usage: bash scripts/compare.sh <topic>"
|
||||||
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose"
|
echo " Example: bash scripts/compare.sh Kevin Rose"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
TOPIC="$*"
|
TOPIC="$*"
|
||||||
@@ -20,11 +20,9 @@ sys.path.insert(0, str(Path(__file__).parent))
|
|||||||
|
|
||||||
from lib import env as envlib
|
from lib import env as envlib
|
||||||
from lib import schema
|
from lib import schema
|
||||||
from lib.providers import GEMINI_FLASH_LITE
|
|
||||||
|
|
||||||
|
|
||||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
||||||
|
|
||||||
|
|
||||||
@@ -44,7 +42,7 @@ def _load_default_topics() -> list[tuple[str, str]]:
|
|||||||
|
|
||||||
DEFAULT_TOPICS = _load_default_topics()
|
DEFAULT_TOPICS = _load_default_topics()
|
||||||
DEFAULT_SEARCH = ""
|
DEFAULT_SEARCH = ""
|
||||||
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
|
DEFAULT_JUDGE_MODEL = "gemini-3.1-flash-lite-preview"
|
||||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||||
|
|
||||||
|
|
||||||
@@ -309,10 +307,7 @@ def create_eval_env() -> dict[str, str]:
|
|||||||
|
|
||||||
|
|
||||||
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
||||||
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py"
|
cmd = [sys.executable, "scripts/last30days.py", topic, "--emit=json"]
|
||||||
if not engine.exists():
|
|
||||||
engine = repo_dir / "scripts" / "last30days.py"
|
|
||||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
|
||||||
if search:
|
if search:
|
||||||
cmd.extend(["--search", search])
|
cmd.extend(["--search", search])
|
||||||
if quick:
|
if quick:
|
||||||
@@ -1,12 +1,11 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# ruff: noqa: E402
|
# ruff: noqa: E402
|
||||||
"""last30days CLI."""
|
"""last30days v3.0.0 CLI."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import atexit
|
import atexit
|
||||||
import datetime
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
@@ -42,7 +41,7 @@ if os.name == "nt":
|
|||||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||||
sys.path.insert(0, str(SCRIPT_DIR))
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
from lib import env, html_render, pipeline, render, schema, ui
|
from lib import env, pipeline, render, schema, ui
|
||||||
|
|
||||||
_child_pids: set[int] = set()
|
_child_pids: set[int] = set()
|
||||||
_child_pids_lock = threading.Lock()
|
_child_pids_lock = threading.Lock()
|
||||||
@@ -63,10 +62,7 @@ def _cleanup_children() -> None:
|
|||||||
pids = list(_child_pids)
|
pids = list(_child_pids)
|
||||||
for pid in pids:
|
for pid in pids:
|
||||||
try:
|
try:
|
||||||
if hasattr(os, "killpg"):
|
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||||
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
|
||||||
else:
|
|
||||||
os.kill(pid, signal.SIGTERM)
|
|
||||||
except (ProcessLookupError, PermissionError, OSError):
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -95,50 +91,30 @@ def slugify(value: str) -> str:
|
|||||||
return slug or "last30days"
|
return slug or "last30days"
|
||||||
|
|
||||||
|
|
||||||
def save_output(
|
def save_output(report: schema.Report, emit: str, save_dir: str, suffix: str = "") -> Path:
|
||||||
report: schema.Report,
|
|
||||||
emit: str,
|
|
||||||
save_dir: str,
|
|
||||||
suffix: str = "",
|
|
||||||
synthesis_md: str | None = None,
|
|
||||||
topic_override: str | None = None,
|
|
||||||
rendered_content: str | None = None,
|
|
||||||
) -> Path:
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
path = Path(save_dir).expanduser().resolve()
|
path = Path(save_dir).expanduser().resolve()
|
||||||
path.mkdir(parents=True, exist_ok=True)
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
slug = slugify(topic_override or report.topic)
|
slug = slugify(report.topic)
|
||||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
extension = "json" if emit == "json" else "md"
|
||||||
raw_label = "raw-html" if emit == "html" else "raw"
|
|
||||||
suffix_part = f"-{suffix}" if suffix else ""
|
suffix_part = f"-{suffix}" if suffix else ""
|
||||||
out_path = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
out_path = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||||
if out_path.exists():
|
if out_path.exists():
|
||||||
out_path = path / f"{slug}-{raw_label}{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
out_path = path / f"{slug}-raw{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||||
# Markdown saves keep the complete debug artifact. JSON and HTML preserve
|
# Always save the FULL dump to disk (all items, all sources, transcripts).
|
||||||
# their requested wire format so file extensions match their content.
|
# Claude sees compact clusters via --emit=compact on stdout.
|
||||||
if rendered_content is not None:
|
# The saved file is the complete debug artifact.
|
||||||
content = rendered_content
|
if emit == "json":
|
||||||
elif emit in {"json", "html"}:
|
content = emit_output(report, emit)
|
||||||
content = emit_output(report, emit, synthesis_md=synthesis_md)
|
|
||||||
else:
|
else:
|
||||||
content = render.render_full(report)
|
content = render.render_full(report)
|
||||||
out_path.write_text(content, encoding="utf-8")
|
out_path.write_text(content, encoding="utf-8")
|
||||||
return out_path
|
return out_path
|
||||||
|
|
||||||
|
|
||||||
def emit_output(
|
def emit_output(report: schema.Report, emit: str, fun_level: str = "medium", save_path: str | None = None) -> str:
|
||||||
report: schema.Report,
|
|
||||||
emit: str,
|
|
||||||
fun_level: str = "medium",
|
|
||||||
save_path: str | None = None,
|
|
||||||
synthesis_md: str | None = None,
|
|
||||||
) -> str:
|
|
||||||
if emit == "json":
|
if emit == "json":
|
||||||
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
|
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
|
||||||
if emit == "html":
|
|
||||||
return html_render.render_html(
|
|
||||||
report, fun_level=fun_level, save_path=save_path, synthesis_md=synthesis_md,
|
|
||||||
)
|
|
||||||
if emit in {"compact", "md"}:
|
if emit in {"compact", "md"}:
|
||||||
return render.render_compact(report, fun_level=fun_level, save_path=save_path)
|
return render.render_compact(report, fun_level=fun_level, save_path=save_path)
|
||||||
if emit == "context":
|
if emit == "context":
|
||||||
@@ -151,7 +127,6 @@ def emit_comparison_output(
|
|||||||
emit: str,
|
emit: str,
|
||||||
fun_level: str = "medium",
|
fun_level: str = "medium",
|
||||||
save_path: str | None = None,
|
save_path: str | None = None,
|
||||||
synthesis_md: str | None = None,
|
|
||||||
) -> str:
|
) -> str:
|
||||||
if emit == "json":
|
if emit == "json":
|
||||||
payload = {
|
payload = {
|
||||||
@@ -163,13 +138,6 @@ def emit_comparison_output(
|
|||||||
],
|
],
|
||||||
}
|
}
|
||||||
return json.dumps(payload, indent=2, sort_keys=True)
|
return json.dumps(payload, indent=2, sort_keys=True)
|
||||||
if emit == "html":
|
|
||||||
return html_render.render_html_comparison(
|
|
||||||
entity_reports,
|
|
||||||
fun_level=fun_level,
|
|
||||||
save_path=save_path,
|
|
||||||
synthesis_md=synthesis_md,
|
|
||||||
)
|
|
||||||
if emit in {"compact", "md"}:
|
if emit in {"compact", "md"}:
|
||||||
return render.render_comparison_multi(
|
return render.render_comparison_multi(
|
||||||
entity_reports, fun_level=fun_level, save_path=save_path,
|
entity_reports, fun_level=fun_level, save_path=save_path,
|
||||||
@@ -179,10 +147,6 @@ def emit_comparison_output(
|
|||||||
raise SystemExit(f"Unsupported emit mode: {emit}")
|
raise SystemExit(f"Unsupported emit mode: {emit}")
|
||||||
|
|
||||||
|
|
||||||
def comparison_topic(entity_reports: list[tuple[str, schema.Report]]) -> str:
|
|
||||||
return " vs ".join(label for label, _ in entity_reports)
|
|
||||||
|
|
||||||
|
|
||||||
def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str) -> str:
|
def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str) -> str:
|
||||||
"""Compute the user-friendly save path string that will be shown in the footer.
|
"""Compute the user-friendly save path string that will be shown in the footer.
|
||||||
|
|
||||||
@@ -192,24 +156,15 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
|||||||
from pathlib import Path as _Path
|
from pathlib import Path as _Path
|
||||||
path = _Path(save_dir).expanduser().resolve()
|
path = _Path(save_dir).expanduser().resolve()
|
||||||
slug = slugify(topic)
|
slug = slugify(topic)
|
||||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
extension = "json" if emit == "json" else "md"
|
||||||
raw_label = "raw-html" if emit == "html" else "raw"
|
|
||||||
suffix_part = f"-{suffix}" if suffix else ""
|
suffix_part = f"-{suffix}" if suffix else ""
|
||||||
raw = path / f"{slug}-{raw_label}{suffix_part}.{extension}"
|
raw = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||||
try:
|
try:
|
||||||
home = _Path.home().resolve()
|
home = _Path.home().resolve()
|
||||||
relative = raw.relative_to(home)
|
relative = raw.relative_to(home)
|
||||||
return f"~/{relative.as_posix()}"
|
return f"~/{relative}"
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return raw.as_posix()
|
return str(raw)
|
||||||
|
|
||||||
|
|
||||||
def read_synthesis_file(path: str) -> str:
|
|
||||||
try:
|
|
||||||
return Path(path).expanduser().read_text(encoding="utf-8")
|
|
||||||
except OSError as exc:
|
|
||||||
sys.stderr.write(f"[last30days] Cannot read --synthesis-file: {exc}\n")
|
|
||||||
raise SystemExit(2)
|
|
||||||
|
|
||||||
|
|
||||||
def persist_report(report: schema.Report) -> dict[str, int]:
|
def persist_report(report: schema.Report) -> dict[str, int]:
|
||||||
@@ -238,7 +193,7 @@ def persist_report(report: schema.Report) -> dict[str, int]:
|
|||||||
def build_parser() -> argparse.ArgumentParser:
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
|
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
|
||||||
parser.add_argument("topic", nargs="*", help="Research topic")
|
parser.add_argument("topic", nargs="*", help="Research topic")
|
||||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md", "html"])
|
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md"])
|
||||||
parser.add_argument("--search", help="Comma-separated source list")
|
parser.add_argument("--search", help="Comma-separated source list")
|
||||||
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
|
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
|
||||||
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
|
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
|
||||||
@@ -246,7 +201,6 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
||||||
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
||||||
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
||||||
parser.add_argument("--synthesis-file", help="Markdown synthesis to embed in --emit=html output")
|
|
||||||
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
|
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
|
||||||
parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
|
parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
|
||||||
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
|
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
|
||||||
@@ -391,7 +345,7 @@ def subrun_kwargs_for(
|
|||||||
|
|
||||||
subreddits = _choose("subreddits", "subreddits")
|
subreddits = _choose("subreddits", "subreddits")
|
||||||
if isinstance(subreddits, list):
|
if isinstance(subreddits, list):
|
||||||
subreddits = [s.strip().removeprefix("r/") for s in subreddits if s.strip()] or None
|
subreddits = [s.strip().lstrip("r/") for s in subreddits if s.strip()] or None
|
||||||
|
|
||||||
x_related = plan_entry.get("x_related")
|
x_related = plan_entry.get("x_related")
|
||||||
if isinstance(x_related, list):
|
if isinstance(x_related, list):
|
||||||
@@ -535,24 +489,6 @@ def _show_runtime_ui(
|
|||||||
progress.show_promo(promo, diag=diag)
|
progress.show_promo(promo, diag=diag)
|
||||||
|
|
||||||
|
|
||||||
def _write_last_run(topic: str, report: "schema.Report") -> None:
|
|
||||||
try:
|
|
||||||
if env.CONFIG_DIR is None:
|
|
||||||
return
|
|
||||||
target = env.CONFIG_DIR
|
|
||||||
target.mkdir(parents=True, exist_ok=True)
|
|
||||||
counts = {source: len(items) for source, items in report.items_by_source.items()}
|
|
||||||
payload = {
|
|
||||||
"topic": topic,
|
|
||||||
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
||||||
"sources": counts,
|
|
||||||
"total": sum(counts.values()),
|
|
||||||
}
|
|
||||||
(target / "last-run.json").write_text(json.dumps(payload, indent=2))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = build_parser()
|
parser = build_parser()
|
||||||
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
||||||
@@ -563,13 +499,6 @@ def main() -> int:
|
|||||||
|
|
||||||
config = env.get_config()
|
config = env.get_config()
|
||||||
|
|
||||||
# Surface SSH-routing config as an env var so library modules (e.g.
|
|
||||||
# youtube_yt) can read it without taking a config dependency. This
|
|
||||||
# routes yt-dlp through `ssh <host>` to bypass YouTube's bot-wall on
|
|
||||||
# datacenter IPs (see lib/youtube_yt.py for details).
|
|
||||||
if config.get("LAST30DAYS_YOUTUBE_SSH_HOST") and "LAST30DAYS_YOUTUBE_SSH_HOST" not in os.environ:
|
|
||||||
os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"]
|
|
||||||
|
|
||||||
# Handle setup subcommand
|
# Handle setup subcommand
|
||||||
topic = " ".join(args.topic).strip()
|
topic = " ".join(args.topic).strip()
|
||||||
if topic.lower() == "setup":
|
if topic.lower() == "setup":
|
||||||
@@ -608,13 +537,6 @@ def main() -> int:
|
|||||||
parser.print_usage(sys.stderr)
|
parser.print_usage(sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
|
|
||||||
synthesis_md = None
|
|
||||||
if args.synthesis_file:
|
|
||||||
if args.emit == "html":
|
|
||||||
synthesis_md = read_synthesis_file(args.synthesis_file)
|
|
||||||
else:
|
|
||||||
sys.stderr.write("[last30days] Warning: --synthesis-file is only used with --emit=html; ignoring.\n")
|
|
||||||
|
|
||||||
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
|
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
|
||||||
from lib import preflight
|
from lib import preflight
|
||||||
refuse_msg = preflight.check_class_1_trap(topic)
|
refuse_msg = preflight.check_class_1_trap(topic)
|
||||||
@@ -628,7 +550,7 @@ def main() -> int:
|
|||||||
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
||||||
try:
|
try:
|
||||||
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
||||||
subreddits = [s.strip().removeprefix("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
subreddits = [s.strip().lstrip("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||||
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
||||||
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
||||||
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
||||||
@@ -647,7 +569,6 @@ def main() -> int:
|
|||||||
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
||||||
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
||||||
# without WebSearch (OpenClaw, Codex, raw CLI).
|
# without WebSearch (OpenClaw, Codex, raw CLI).
|
||||||
repos_from_auto_resolve = False
|
|
||||||
if args.auto_resolve and not external_plan:
|
if args.auto_resolve and not external_plan:
|
||||||
from lib import resolve
|
from lib import resolve
|
||||||
resolution = resolve.auto_resolve(topic, config)
|
resolution = resolve.auto_resolve(topic, config)
|
||||||
@@ -662,9 +583,6 @@ def main() -> int:
|
|||||||
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
||||||
if resolution.get("github_repos") and not args.github_repo:
|
if resolution.get("github_repos") and not args.github_repo:
|
||||||
args.github_repo = ",".join(resolution["github_repos"])
|
args.github_repo = ",".join(resolution["github_repos"])
|
||||||
# auto_resolve already canonicalized via canonicalize_github_repos(cap=5);
|
|
||||||
# mark so we don't re-canonicalize below and clobber its relevance order.
|
|
||||||
repos_from_auto_resolve = True
|
|
||||||
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
||||||
if resolution.get("context"):
|
if resolution.get("context"):
|
||||||
# Inject context into external_plan metadata for the planner to use
|
# Inject context into external_plan metadata for the planner to use
|
||||||
@@ -677,20 +595,6 @@ def main() -> int:
|
|||||||
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
||||||
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
||||||
|
|
||||||
# Only canonicalize when repos came from a user-supplied --github-repo flag.
|
|
||||||
# When repos_from_auto_resolve is True, auto_resolve already ran
|
|
||||||
# canonicalize_github_repos(cap=5) and ranked by relevance; re-running here
|
|
||||||
# with cap=None can re-sort by topic-slug match and lose that ordering.
|
|
||||||
if github_repos and not repos_from_auto_resolve:
|
|
||||||
from lib import resolve as resolve_lib
|
|
||||||
original_github_repos = github_repos[:]
|
|
||||||
github_repos = resolve_lib.canonicalize_github_repos(topic, github_repos, cap=None)
|
|
||||||
if github_repos != original_github_repos:
|
|
||||||
sys.stderr.write(
|
|
||||||
"[GitHub] Canonicalized repos: "
|
|
||||||
f"{','.join(original_github_repos)} -> {','.join(github_repos)}\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
# --deep-research: auto-enable perplexity source and set deep flag
|
# --deep-research: auto-enable perplexity source and set deep flag
|
||||||
if args.deep_research:
|
if args.deep_research:
|
||||||
if not config.get("OPENROUTER_API_KEY"):
|
if not config.get("OPENROUTER_API_KEY"):
|
||||||
@@ -910,18 +814,7 @@ def main() -> int:
|
|||||||
report, progress, diag,
|
report, progress, diag,
|
||||||
suppress_web_promo=bool(external_plan or comp_plan),
|
suppress_web_promo=bool(external_plan or comp_plan),
|
||||||
)
|
)
|
||||||
_write_last_run(topic, report)
|
if args.store:
|
||||||
# LAST30DAYS_STORE env var = persistence default-on. Read both os.environ
|
|
||||||
# (for shell-exported users) and config (for users who set it in
|
|
||||||
# ~/.config/last30days/.env, which env.py loads but does not propagate
|
|
||||||
# to os.environ). Mirrors the LAST30DAYS_DEBUG / LAST30DAYS_SKIP_PREFLIGHT
|
|
||||||
# convention; env-var or config wins, with `--store` flag still working.
|
|
||||||
_store_env = (
|
|
||||||
os.environ.get("LAST30DAYS_STORE")
|
|
||||||
or config.get("LAST30DAYS_STORE")
|
|
||||||
or ""
|
|
||||||
).lower()
|
|
||||||
if args.store or _store_env in ("1", "true", "yes"):
|
|
||||||
counts = persist_report(report)
|
counts = persist_report(report)
|
||||||
sys.stderr.write(
|
sys.stderr.write(
|
||||||
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
||||||
@@ -931,32 +824,7 @@ def main() -> int:
|
|||||||
# Show quality nudge if applicable
|
# Show quality nudge if applicable
|
||||||
try:
|
try:
|
||||||
from lib import quality_nudge
|
from lib import quality_nudge
|
||||||
# Populate transcript-fetch ratio so quality_nudge can detect the
|
quality = quality_nudge.compute_quality_score(config, {})
|
||||||
# degraded-YouTube failure mode (videos returned but transcripts
|
|
||||||
# silently failed - typically a stale yt-dlp binary).
|
|
||||||
youtube_items = report.items_by_source.get("youtube") or []
|
|
||||||
instagram_items = report.items_by_source.get("instagram") or []
|
|
||||||
research_results = {
|
|
||||||
"youtube_videos_count": len(youtube_items),
|
|
||||||
"youtube_transcripts_count": sum(
|
|
||||||
1 for it in youtube_items
|
|
||||||
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
|
||||||
),
|
|
||||||
"youtube_error": report.errors_by_source.get("youtube"),
|
|
||||||
"x_error": report.errors_by_source.get("x"),
|
|
||||||
# Captions-disabled videos can never produce a transcript regardless
|
|
||||||
# of yt-dlp version; subtract them from the degraded-ratio
|
|
||||||
# denominator so a single uploader-disabled video does not trip the
|
|
||||||
# "stale yt-dlp" nudge.
|
|
||||||
"youtube_captions_disabled_count": sum(
|
|
||||||
1 for it in youtube_items if it.metadata.get("captions_disabled")
|
|
||||||
),
|
|
||||||
# Track Instagram returned-zero-items so quality_nudge can detect
|
|
||||||
# the silent-failure case (SC configured but the v2 reels endpoint
|
|
||||||
# 500'd through both the original query and the hashtag retry).
|
|
||||||
"instagram_items_count": len(instagram_items),
|
|
||||||
}
|
|
||||||
quality = quality_nudge.compute_quality_score(config, research_results)
|
|
||||||
if quality.get("nudge_text"):
|
if quality.get("nudge_text"):
|
||||||
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
@@ -964,15 +832,10 @@ def main() -> int:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
fun_level = config.get("FUN_LEVEL", "medium").lower()
|
fun_level = config.get("FUN_LEVEL", "medium").lower()
|
||||||
# Comparison HTML is the one case where the saved file's title and content
|
|
||||||
# have to be overridden away from the leading entity's report. Compute the
|
|
||||||
# gate once so the footer-display and save-output paths can't disagree.
|
|
||||||
is_comparison_html = bool(entity_reports) and args.emit == "html"
|
|
||||||
footer_save_path = None
|
footer_save_path = None
|
||||||
if args.save_dir:
|
if args.save_dir:
|
||||||
save_topic_for_display = comparison_topic(entity_reports) if is_comparison_html else report.topic
|
|
||||||
footer_save_path = compute_save_path_display(
|
footer_save_path = compute_save_path_display(
|
||||||
args.save_dir, save_topic_for_display, args.save_suffix or "", args.emit
|
args.save_dir, report.topic, args.save_suffix or "", args.emit
|
||||||
)
|
)
|
||||||
|
|
||||||
# Signal to render_compact whether pre-research flags were supplied.
|
# Signal to render_compact whether pre-research flags were supplied.
|
||||||
@@ -991,31 +854,15 @@ def main() -> int:
|
|||||||
|
|
||||||
if entity_reports:
|
if entity_reports:
|
||||||
rendered = emit_comparison_output(
|
rendered = emit_comparison_output(
|
||||||
entity_reports,
|
entity_reports, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||||
args.emit,
|
|
||||||
fun_level=fun_level,
|
|
||||||
save_path=footer_save_path,
|
|
||||||
synthesis_md=synthesis_md,
|
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
rendered = emit_output(
|
rendered = emit_output(
|
||||||
report,
|
report, args.emit, fun_level=fun_level, save_path=footer_save_path,
|
||||||
args.emit,
|
|
||||||
fun_level=fun_level,
|
|
||||||
save_path=footer_save_path,
|
|
||||||
synthesis_md=synthesis_md,
|
|
||||||
)
|
)
|
||||||
if args.save_dir:
|
if args.save_dir:
|
||||||
# Save the main topic's raw file (single-entity or comparison main).
|
# Save the main topic's raw file (single-entity or comparison main).
|
||||||
save_path = save_output(
|
save_path = save_output(report, args.emit, args.save_dir, suffix=args.save_suffix or "")
|
||||||
report,
|
|
||||||
args.emit,
|
|
||||||
args.save_dir,
|
|
||||||
suffix=args.save_suffix or "",
|
|
||||||
synthesis_md=synthesis_md,
|
|
||||||
topic_override=comparison_topic(entity_reports) if is_comparison_html else None,
|
|
||||||
rendered_content=rendered if is_comparison_html else None,
|
|
||||||
)
|
|
||||||
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
||||||
# Competitor / vs-mode: also save a per-entity raw file for each peer.
|
# Competitor / vs-mode: also save a per-entity raw file for each peer.
|
||||||
# Matches historical vs-mode behavior (N passes → N save files).
|
# Matches historical vs-mode behavior (N passes → N save files).
|
||||||
@@ -1024,7 +871,6 @@ def main() -> int:
|
|||||||
peer_path = save_output(
|
peer_path = save_output(
|
||||||
entity_report, args.emit, args.save_dir,
|
entity_report, args.emit, args.save_dir,
|
||||||
suffix=args.save_suffix or "",
|
suffix=args.save_suffix or "",
|
||||||
synthesis_md=synthesis_md,
|
|
||||||
)
|
)
|
||||||
sys.stderr.write(f"[last30days] Saved output to {peer_path}\n")
|
sys.stderr.write(f"[last30days] Saved output to {peer_path}\n")
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
@@ -7,22 +7,18 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import signal
|
||||||
import shutil
|
import shutil
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from . import http, log, subproc
|
from . import http, log
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from .relevance import token_overlap_relevance as _compute_relevance
|
from .relevance import token_overlap_relevance as _compute_relevance
|
||||||
|
|
||||||
# How many times to retry the bird-search subprocess when stdout is non-JSON
|
|
||||||
# (typically an HTML anti-bot interstitial from Twitter's edge).
|
|
||||||
MAX_JSON_DECODE_RETRIES = 2
|
|
||||||
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
|
|
||||||
|
|
||||||
|
|
||||||
def _first_of(*values):
|
def _first_of(*values):
|
||||||
"""Return first value that is not None."""
|
"""Return first value that is not None."""
|
||||||
@@ -154,14 +150,16 @@ def get_bird_status() -> Dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||||
"""Invoke the vendored bird-search.mjs subprocess once.
|
"""Run a search using the vendored bird-search.mjs module.
|
||||||
|
|
||||||
Returns (result, error_dict). If error_dict is non-None, treat it as the
|
Args:
|
||||||
final result and do not retry — those errors are terminal (timeout,
|
query: Full search query string (including since: filter)
|
||||||
spawn failure). If error_dict is None, the subprocess ran to completion
|
count: Number of results to request
|
||||||
and `result` is the SubprocResult; the caller decides whether to retry
|
timeout: Timeout in seconds
|
||||||
based on the result.stdout content.
|
|
||||||
|
Returns:
|
||||||
|
Raw Bird JSON response or error dict.
|
||||||
"""
|
"""
|
||||||
cmd = [
|
cmd = [
|
||||||
"node", str(_BIRD_SEARCH_MJS),
|
"node", str(_BIRD_SEARCH_MJS),
|
||||||
@@ -170,109 +168,62 @@ def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
|||||||
"--json",
|
"--json",
|
||||||
]
|
]
|
||||||
|
|
||||||
pid_holder: list[int] = []
|
# Use process groups for clean cleanup on timeout/kill
|
||||||
|
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||||
|
|
||||||
def _register(pid: int) -> None:
|
try:
|
||||||
pid_holder.append(pid)
|
proc = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
preexec_fn=preexec,
|
||||||
|
env=_subprocess_env(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Register for cleanup tracking (if available)
|
||||||
try:
|
try:
|
||||||
from last30days import register_child_pid
|
from last30days import register_child_pid, unregister_child_pid
|
||||||
register_child_pid(pid)
|
register_child_pid(proc.pid)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = subproc.run_with_timeout(
|
stdout, stderr = proc.communicate(timeout=timeout)
|
||||||
cmd,
|
except subprocess.TimeoutExpired:
|
||||||
timeout=timeout,
|
# Kill the entire process group
|
||||||
env=_subprocess_env(),
|
try:
|
||||||
on_pid=_register,
|
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||||
)
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
except subproc.SubprocTimeout:
|
proc.kill()
|
||||||
return None, {"error": f"Search timed out after {timeout}s", "items": []}
|
proc.wait(timeout=5)
|
||||||
except Exception as e:
|
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||||
return None, {"error": str(e), "items": []}
|
finally:
|
||||||
finally:
|
|
||||||
if pid_holder:
|
|
||||||
try:
|
try:
|
||||||
from last30days import unregister_child_pid
|
from last30days import unregister_child_pid
|
||||||
unregister_child_pid(pid_holder[0])
|
unregister_child_pid(proc.pid)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
return result, None
|
if proc.returncode != 0:
|
||||||
|
error = stderr.strip() if stderr else "Bird search failed"
|
||||||
|
|
||||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
|
||||||
"""Run a search using the vendored bird-search.mjs module.
|
|
||||||
|
|
||||||
Retries the subprocess on JSON-decode failure (typically a Twitter
|
|
||||||
anti-bot HTML interstitial in stdout) up to MAX_JSON_DECODE_RETRIES
|
|
||||||
times with JSON_DECODE_RETRY_DELAY seconds between attempts. Terminal
|
|
||||||
errors (subprocess timeout, non-zero return code) are returned
|
|
||||||
immediately without retry.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
query: Full search query string (including since: filter)
|
|
||||||
count: Number of results to request
|
|
||||||
timeout: Timeout in seconds (per attempt)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Raw Bird JSON response or error dict.
|
|
||||||
"""
|
|
||||||
last_decode_error: Optional[str] = None
|
|
||||||
|
|
||||||
for attempt in range(MAX_JSON_DECODE_RETRIES):
|
|
||||||
result, terminal_error = _invoke_bird_subprocess(query, count, timeout)
|
|
||||||
if terminal_error is not None:
|
|
||||||
return terminal_error
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
error = result.stderr.strip() or "Bird search failed"
|
|
||||||
return {"error": error, "items": []}
|
return {"error": error, "items": []}
|
||||||
|
|
||||||
output = result.stdout.strip()
|
output = stdout.strip() if stdout else ""
|
||||||
if not output:
|
if not output:
|
||||||
return {"items": []}
|
return {"items": []}
|
||||||
|
|
||||||
try:
|
parsed = json.loads(output)
|
||||||
parsed = json.loads(output)
|
|
||||||
except json.JSONDecodeError as e:
|
|
||||||
# Twitter's edge sometimes serves an HTML anti-bot interstitial
|
|
||||||
# in place of JSON. Tag the failure shape so it's distinguishable
|
|
||||||
# from "no results" in logs, then retry the subprocess.
|
|
||||||
looks_html = output.lstrip().lower().startswith(("<!doctype", "<html", "<"))
|
|
||||||
attempt_num = attempt + 1
|
|
||||||
log_msg = (
|
|
||||||
f"Bird search returned non-JSON stdout "
|
|
||||||
f"(looks_html={looks_html}, attempt {attempt_num}/{MAX_JSON_DECODE_RETRIES}, "
|
|
||||||
f"first 80 chars: {output[:80]!r})"
|
|
||||||
)
|
|
||||||
last_decode_error = str(e)
|
|
||||||
if attempt_num < MAX_JSON_DECODE_RETRIES:
|
|
||||||
log.source_log(
|
|
||||||
"X/bird",
|
|
||||||
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
|
|
||||||
)
|
|
||||||
time.sleep(JSON_DECODE_RETRY_DELAY)
|
|
||||||
continue
|
|
||||||
log.source_log("X/bird", log_msg)
|
|
||||||
return {
|
|
||||||
"error": (
|
|
||||||
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
|
|
||||||
f"(likely Twitter anti-bot interstitial): {e}"
|
|
||||||
),
|
|
||||||
"items": [],
|
|
||||||
}
|
|
||||||
|
|
||||||
if isinstance(parsed, list):
|
if isinstance(parsed, list):
|
||||||
return {"items": parsed}
|
return {"items": parsed}
|
||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
# Defensive fallthrough — loop should always return above.
|
except json.JSONDecodeError as e:
|
||||||
return {
|
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||||
"error": f"Bird search exhausted retries: {last_decode_error}",
|
except Exception as e:
|
||||||
"items": [],
|
return {"error": str(e), "items": []}
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def search_x(
|
def search_x(
|
||||||
@@ -379,29 +330,47 @@ def search_handles(
|
|||||||
"--json",
|
"--json",
|
||||||
]
|
]
|
||||||
|
|
||||||
try:
|
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
|
||||||
except subproc.SubprocTimeout:
|
|
||||||
_log(f"Handle search timed out for @{handle}")
|
|
||||||
return []
|
|
||||||
except OSError as e:
|
|
||||||
_log(f"Handle search error for @{handle}: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
output = result.stdout.strip()
|
|
||||||
if not output:
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
preexec_fn=preexec,
|
||||||
|
env=_subprocess_env(),
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
stdout, stderr = proc.communicate(timeout=15)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||||
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
|
proc.kill()
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
_log(f"Handle search timed out for @{handle}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
if proc.returncode != 0:
|
||||||
|
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
output = (stdout or "").strip()
|
||||||
|
if not output:
|
||||||
|
return []
|
||||||
|
|
||||||
response = json.loads(output)
|
response = json.loads(output)
|
||||||
|
return parse_bird_response(response, query=core_topic)
|
||||||
|
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
_log(f"Invalid JSON from handle search for @{handle}")
|
_log(f"Invalid JSON from handle search for @{handle}")
|
||||||
return []
|
except (OSError, subprocess.SubprocessError) as e:
|
||||||
return parse_bird_response(response, query=core_topic)
|
_log(f"Handle search error for @{handle}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
|
||||||
@@ -1,19 +1,10 @@
|
|||||||
"""Bluesky search via AT Protocol (requires app password).
|
"""Bluesky search via AT Protocol (requires app password).
|
||||||
|
|
||||||
Uses bsky.social for auth and api.bsky.app for post search (the canonical
|
Uses bsky.social for auth and public.api.bsky.app for post search.
|
||||||
authenticated AppView). The previous default `public.api.bsky.app` is the
|
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars.
|
||||||
unauthenticated public mirror, which BunnyCDN now blocks for searchPosts
|
|
||||||
regardless of auth header (verified 2026-05-04). Override the search host
|
|
||||||
via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again.
|
|
||||||
|
|
||||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are
|
|
||||||
19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords.
|
|
||||||
The createSession endpoint accepts main-account passwords too, but they're
|
|
||||||
bad hygiene (no scope, can't revoke individually).
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import math
|
import math
|
||||||
import os
|
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
@@ -23,64 +14,7 @@ from typing import Any, Dict, List, Optional
|
|||||||
from . import http, log
|
from . import http, log
|
||||||
|
|
||||||
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
||||||
_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
|
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
|
||||||
|
|
||||||
|
|
||||||
def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str:
|
|
||||||
"""Resolve the Bluesky search URL with BSKY_SEARCH_HOST override.
|
|
||||||
|
|
||||||
Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or
|
|
||||||
.env file. The project's env.py loads .env into config but not into
|
|
||||||
os.environ, so check both — same hybrid pattern as last30days.py for
|
|
||||||
LAST30DAYS_STORE.
|
|
||||||
|
|
||||||
Hardens user-supplied host values against three common mis-configurations:
|
|
||||||
whitespace (e.g. " api.bsky.app "), embedded path components (e.g.
|
|
||||||
"api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and
|
|
||||||
embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these
|
|
||||||
we log a warning and fall back to the default rather than building an
|
|
||||||
invalid URL with an opaque downstream error.
|
|
||||||
"""
|
|
||||||
config = config or {}
|
|
||||||
raw = (
|
|
||||||
os.environ.get("BSKY_SEARCH_HOST")
|
|
||||||
or config.get("BSKY_SEARCH_HOST")
|
|
||||||
or _DEFAULT_BSKY_SEARCH_HOST
|
|
||||||
)
|
|
||||||
host = raw.strip().rstrip("/")
|
|
||||||
# Strip embedded scheme so users who paste full URLs do not break the f-string.
|
|
||||||
for prefix in ("https://", "http://"):
|
|
||||||
if host.lower().startswith(prefix):
|
|
||||||
host = host[len(prefix):]
|
|
||||||
break
|
|
||||||
if not host or "/" in host or " " in host:
|
|
||||||
# Embedded path or whitespace remains — don't trust it. Default + log.
|
|
||||||
if raw != _DEFAULT_BSKY_SEARCH_HOST:
|
|
||||||
_log(
|
|
||||||
f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; "
|
|
||||||
f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}"
|
|
||||||
)
|
|
||||||
host = _DEFAULT_BSKY_SEARCH_HOST
|
|
||||||
return f"https://{host}/xrpc/app.bsky.feed.searchPosts"
|
|
||||||
|
|
||||||
|
|
||||||
# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric
|
|
||||||
# with three hyphens at fixed positions).
|
|
||||||
_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$")
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_app_password_format(value) -> bool:
|
|
||||||
"""Return True if value matches Bluesky's 19-char app-password format.
|
|
||||||
|
|
||||||
False for non-strings (None, int, list) so callers passing config dict
|
|
||||||
values directly don't crash. Detect-but-not-gate: the createSession
|
|
||||||
endpoint also accepts main-account passwords, so failing this check is
|
|
||||||
a hygiene smell, not a hard error.
|
|
||||||
"""
|
|
||||||
if not isinstance(value, str):
|
|
||||||
return False
|
|
||||||
return bool(_APP_PASSWORD_RE.fullmatch(value))
|
|
||||||
|
|
||||||
|
|
||||||
DEPTH_CONFIG = {
|
DEPTH_CONFIG = {
|
||||||
"quick": 15,
|
"quick": 15,
|
||||||
@@ -210,20 +144,6 @@ def search_bluesky(
|
|||||||
if not handle or not app_password:
|
if not handle or not app_password:
|
||||||
return {"posts": [], "error": "Bluesky credentials not configured"}
|
return {"posts": [], "error": "Bluesky credentials not configured"}
|
||||||
|
|
||||||
# One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password
|
|
||||||
# form. createSession accepts main-account passwords too — but main
|
|
||||||
# passwords have no scope (full account access), can't be revoked
|
|
||||||
# individually, and rotating them breaks every service that holds them.
|
|
||||||
# We warn but do not gate, matching the project's detect-don't-block
|
|
||||||
# philosophy elsewhere.
|
|
||||||
if not _validate_app_password_format(app_password):
|
|
||||||
_log(
|
|
||||||
"BSKY_APP_PASSWORD does not look like an app password "
|
|
||||||
"(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main "
|
|
||||||
"account password — those work but are bad hygiene. Generate "
|
|
||||||
"an app password at https://bsky.app/settings/app-passwords"
|
|
||||||
)
|
|
||||||
|
|
||||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||||
core_topic = _extract_core_subject(topic)
|
core_topic = _extract_core_subject(topic)
|
||||||
|
|
||||||
@@ -235,7 +155,7 @@ def search_bluesky(
|
|||||||
"limit": str(min(count, 100)),
|
"limit": str(min(count, 100)),
|
||||||
"sort": "top",
|
"sort": "top",
|
||||||
}
|
}
|
||||||
url = f"{_resolve_search_url(config)}?{urlencode(params)}"
|
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
|
||||||
|
|
||||||
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
token = _create_session(handle, app_password)
|
token = _create_session(handle, app_password)
|
||||||
@@ -1,12 +1,9 @@
|
|||||||
"""Chrome and Brave cookie extraction for macOS.
|
"""Chrome cookie extraction for macOS.
|
||||||
|
|
||||||
Extracts cookies from Chromium-based browser SQLite databases using only
|
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
|
||||||
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
|
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
|
||||||
dependencies.
|
|
||||||
|
|
||||||
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||||
Chrome and Brave share the same algorithm; only the DB path and Keychain
|
|
||||||
service name differ.
|
|
||||||
This is NOT affected by Windows App-Bound Encryption (v20).
|
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -21,11 +18,10 @@ from typing import Optional
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Cookie DB locations on macOS
|
# Chrome cookie DB location on macOS
|
||||||
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||||
BRAVE_BASE_DIR = Path.home() / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser"
|
|
||||||
|
|
||||||
# Chromium v10 encryption constants (shared by Chrome and Brave)
|
# Chrome v10 encryption constants
|
||||||
CHROME_SALT = b"saltysalt"
|
CHROME_SALT = b"saltysalt"
|
||||||
CHROME_PBKDF2_ITERATIONS = 1003
|
CHROME_PBKDF2_ITERATIONS = 1003
|
||||||
CHROME_KEY_LENGTH = 16
|
CHROME_KEY_LENGTH = 16
|
||||||
@@ -33,8 +29,8 @@ CHROME_KEY_LENGTH = 16
|
|||||||
CHROME_IV_HEX = "20" * 16
|
CHROME_IV_HEX = "20" * 16
|
||||||
|
|
||||||
|
|
||||||
def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||||
"""Retrieve the encryption passphrase for a Chromium-based browser from macOS Keychain.
|
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
|
||||||
|
|
||||||
Calls `security find-generic-password` which may trigger a system dialog
|
Calls `security find-generic-password` which may trigger a system dialog
|
||||||
on first access.
|
on first access.
|
||||||
@@ -43,34 +39,30 @@ def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["security", "find-generic-password", "-w", "-s", service_name],
|
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
logger.info("%s Keychain access denied or browser not installed: %s", service_name, result.stderr.strip())
|
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
|
||||||
return None
|
return None
|
||||||
passphrase = result.stdout.strip()
|
passphrase = result.stdout.strip()
|
||||||
if not passphrase:
|
if not passphrase:
|
||||||
logger.info("%s Keychain returned empty passphrase", service_name)
|
logger.info("Chrome Keychain returned empty passphrase")
|
||||||
return None
|
return None
|
||||||
return passphrase.encode("utf-8")
|
return passphrase.encode("utf-8")
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
logger.info("'security' command not found — not on macOS?")
|
logger.info("'security' command not found — not on macOS?")
|
||||||
return None
|
return None
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
logger.info("%s Keychain access timed out", service_name)
|
logger.info("Chrome Keychain access timed out")
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.info("Failed to get %s encryption key: %s", service_name, e)
|
logger.info("Failed to get Chrome encryption key: %s", e)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
|
||||||
return _get_chromium_encryption_key("Chrome Safe Storage")
|
|
||||||
|
|
||||||
|
|
||||||
def _derive_aes_key(passphrase: bytes) -> bytes:
|
def _derive_aes_key(passphrase: bytes) -> bytes:
|
||||||
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
||||||
return hashlib.pbkdf2_hmac(
|
return hashlib.pbkdf2_hmac(
|
||||||
@@ -173,42 +165,36 @@ def _get_db_version(cursor: sqlite3.Cursor) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _extract_chromium_cookies_macos(
|
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||||
db_path: Path,
|
"""Extract cookies from Chrome on macOS.
|
||||||
keychain_service: str,
|
|
||||||
domain: str,
|
|
||||||
cookie_names: list[str],
|
|
||||||
) -> Optional[dict[str, str]]:
|
|
||||||
"""Extract cookies from any Chromium-based browser on macOS.
|
|
||||||
|
|
||||||
Copies the locked Cookies database to a temp file, reads specified cookies,
|
Copies the locked Cookies database to a temp file, reads specified cookies,
|
||||||
and decrypts v10-encrypted values using the Keychain-stored key.
|
and decrypts v10-encrypted values using the Keychain-stored key.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
db_path: Path to the browser's Cookies SQLite file.
|
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
|
||||||
keychain_service: macOS Keychain service name (e.g. "Chrome Safe Storage").
|
cookie_names: List of cookie names to extract
|
||||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com").
|
|
||||||
cookie_names: List of cookie names to extract.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict mapping cookie name to decrypted value, or None on failure.
|
Dict mapping cookie name to decrypted value, or None on failure.
|
||||||
Only includes cookies that were successfully found and decrypted.
|
Only includes cookies that were successfully found and decrypted.
|
||||||
"""
|
"""
|
||||||
if not db_path.exists():
|
if not CHROME_COOKIES_DB.exists():
|
||||||
logger.info("%s cookies database not found at %s", keychain_service, db_path)
|
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
passphrase = _get_chromium_encryption_key(keychain_service)
|
# Get encryption key from Keychain
|
||||||
|
passphrase = _get_chrome_encryption_key()
|
||||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||||
|
|
||||||
# Copy DB to temp file (browser locks the original while running)
|
# Copy DB to temp file (Chrome locks the original)
|
||||||
tmp_fd = None
|
tmp_fd = None
|
||||||
tmp_path = None
|
tmp_path = None
|
||||||
try:
|
try:
|
||||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||||
shutil.copy2(str(db_path), tmp_path)
|
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
|
logger.info("Failed to copy Chrome cookies database: %s", e)
|
||||||
if tmp_path:
|
if tmp_path:
|
||||||
try:
|
try:
|
||||||
Path(tmp_path).unlink(missing_ok=True)
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
@@ -225,22 +211,26 @@ def _extract_chromium_cookies_macos(
|
|||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
|
|
||||||
db_version = _get_db_version(cursor)
|
db_version = _get_db_version(cursor)
|
||||||
logger.debug("%s cookie DB version: %d", keychain_service, db_version)
|
logger.debug("Chrome cookie DB version: %d", db_version)
|
||||||
|
|
||||||
|
# Build query with placeholders for cookie names
|
||||||
placeholders = ",".join("?" for _ in cookie_names)
|
placeholders = ",".join("?" for _ in cookie_names)
|
||||||
query = (
|
query = (
|
||||||
f"SELECT name, value, encrypted_value FROM cookies "
|
f"SELECT name, value, encrypted_value FROM cookies "
|
||||||
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
||||||
)
|
)
|
||||||
|
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
|
||||||
params = [f"%{domain}"] + list(cookie_names)
|
params = [f"%{domain}"] + list(cookie_names)
|
||||||
cursor.execute(query, params)
|
cursor.execute(query, params)
|
||||||
|
|
||||||
results: dict[str, str] = {}
|
results: dict[str, str] = {}
|
||||||
for name, value, encrypted_value in cursor.fetchall():
|
for name, value, encrypted_value in cursor.fetchall():
|
||||||
|
# Prefer unencrypted value if present
|
||||||
if value:
|
if value:
|
||||||
results[name] = value
|
results[name] = value
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Handle encrypted value
|
||||||
if encrypted_value and encrypted_value[:3] == b"v10":
|
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||||
if aes_key is None:
|
if aes_key is None:
|
||||||
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||||
@@ -251,72 +241,25 @@ def _extract_chromium_cookies_macos(
|
|||||||
else:
|
else:
|
||||||
logger.debug("Failed to decrypt cookie %s", name)
|
logger.debug("Failed to decrypt cookie %s", name)
|
||||||
elif encrypted_value:
|
elif encrypted_value:
|
||||||
|
# Unknown encryption version
|
||||||
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
if not results:
|
if not results:
|
||||||
logger.info("No matching cookies found in %s for domain %s", keychain_service, domain)
|
logger.info("No matching cookies found in Chrome for domain %s", domain)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
except sqlite3.Error as e:
|
except sqlite3.Error as e:
|
||||||
logger.info("Failed to read %s cookies database: %s", keychain_service, e)
|
logger.info("Failed to read Chrome cookies database: %s", e)
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.info("Unexpected error reading %s cookies: %s", keychain_service, e)
|
logger.info("Unexpected error reading Chrome cookies: %s", e)
|
||||||
return None
|
return None
|
||||||
finally:
|
finally:
|
||||||
try:
|
try:
|
||||||
Path(tmp_path).unlink(missing_ok=True)
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
|
||||||
"""Extract cookies from Chrome on macOS."""
|
|
||||||
return _extract_chromium_cookies_macos(
|
|
||||||
CHROME_COOKIES_DB, "Chrome Safe Storage", domain, cookie_names
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _find_brave_cookies_db() -> Optional[Path]:
|
|
||||||
"""Find Brave's Cookies database on macOS.
|
|
||||||
|
|
||||||
Tries the Default profile first, then scans numbered Profile directories
|
|
||||||
by most-recently-modified. Brave creates extra profiles as "Profile 1",
|
|
||||||
"Profile 2", etc. alongside Default; the most recently used one is the
|
|
||||||
likeliest to hold current cookies. Lexicographic sort would visit
|
|
||||||
"Profile 10" before "Profile 2", which can return the wrong profile.
|
|
||||||
"""
|
|
||||||
default = BRAVE_BASE_DIR / "Default" / "Cookies"
|
|
||||||
if default.exists():
|
|
||||||
return default
|
|
||||||
|
|
||||||
try:
|
|
||||||
candidates = [
|
|
||||||
child for child in BRAVE_BASE_DIR.iterdir()
|
|
||||||
if child.is_dir() and child.name.startswith("Profile ")
|
|
||||||
]
|
|
||||||
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
|
||||||
candidate = child / "Cookies"
|
|
||||||
if candidate.exists():
|
|
||||||
return candidate
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
|
||||||
"""Extract cookies from Brave on macOS.
|
|
||||||
|
|
||||||
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
|
|
||||||
path and Keychain service name differ.
|
|
||||||
"""
|
|
||||||
db_path = _find_brave_cookies_db()
|
|
||||||
if db_path is None:
|
|
||||||
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
|
|
||||||
return None
|
|
||||||
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
"""Browser cookie extraction for last30days.
|
"""Browser cookie extraction for last30days.
|
||||||
|
|
||||||
Extracts cookies from local browser databases (Firefox, Chrome, Brave, Safari)
|
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
|
||||||
to enable zero-config authentication for services like X/Twitter.
|
to enable zero-config authentication for services like X/Twitter.
|
||||||
|
|
||||||
Only uses Python stdlib — no external dependencies.
|
Only uses Python stdlib — no external dependencies.
|
||||||
@@ -255,29 +255,6 @@ def extract_chrome_cookies(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def extract_brave_cookies(
|
|
||||||
domain: str, cookie_names: List[str]
|
|
||||||
) -> Optional[Dict[str, str]]:
|
|
||||||
"""Extract cookies from Brave for the given domain and cookie names.
|
|
||||||
|
|
||||||
macOS only — Brave uses the same v10 AES-128-CBC encryption as Chrome,
|
|
||||||
with a different DB path and Keychain service name ("Brave Safe Storage").
|
|
||||||
Tries the Default profile first, then scans numbered Profile directories.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
|
||||||
"""
|
|
||||||
if platform.system() != "Darwin":
|
|
||||||
logger.debug("Brave cookie extraction only supported on macOS")
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
from .chrome_cookies import extract_brave_cookies_macos
|
|
||||||
return extract_brave_cookies_macos(domain, cookie_names)
|
|
||||||
except Exception as exc:
|
|
||||||
logger.debug("Brave cookie extraction failed: %s", exc)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def extract_safari_cookies(
|
def extract_safari_cookies(
|
||||||
domain: str, cookie_names: List[str]
|
domain: str, cookie_names: List[str]
|
||||||
) -> Optional[Dict[str, str]]:
|
) -> Optional[Dict[str, str]]:
|
||||||
@@ -305,9 +282,9 @@ def extract_cookies(
|
|||||||
"""Extract cookies from the specified browser.
|
"""Extract cookies from the specified browser.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||||
'auto' tries browsers in platform-appropriate order:
|
'auto' tries browsers in platform-appropriate order:
|
||||||
- macOS: Chrome -> Brave -> Firefox -> Safari
|
- macOS: Chrome -> Firefox -> Safari
|
||||||
- Linux: Firefox only
|
- Linux: Firefox only
|
||||||
domain: The cookie domain to match (e.g. ".x.com").
|
domain: The cookie domain to match (e.g. ".x.com").
|
||||||
cookie_names: List of cookie names to extract.
|
cookie_names: List of cookie names to extract.
|
||||||
@@ -356,7 +333,7 @@ def extract_cookies_with_source(
|
|||||||
so callers can track the source.
|
so callers can track the source.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||||
domain: The cookie domain to match (e.g. ".x.com").
|
domain: The cookie domain to match (e.g. ".x.com").
|
||||||
cookie_names: List of cookie names to extract.
|
cookie_names: List of cookie names to extract.
|
||||||
|
|
||||||
@@ -367,7 +344,6 @@ def extract_cookies_with_source(
|
|||||||
extractors = {
|
extractors = {
|
||||||
"firefox": extract_firefox_cookies,
|
"firefox": extract_firefox_cookies,
|
||||||
"chrome": extract_chrome_cookies,
|
"chrome": extract_chrome_cookies,
|
||||||
"brave": extract_brave_cookies,
|
|
||||||
"safari": extract_safari_cookies,
|
"safari": extract_safari_cookies,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -384,7 +360,7 @@ def extract_cookies_with_source(
|
|||||||
# Auto mode: try browsers in platform-appropriate order
|
# Auto mode: try browsers in platform-appropriate order
|
||||||
system = platform.system()
|
system = platform.system()
|
||||||
if system == "Darwin":
|
if system == "Darwin":
|
||||||
order = ["chrome", "brave", "firefox", "safari"]
|
order = ["chrome", "firefox", "safari"]
|
||||||
elif system == "Linux":
|
elif system == "Linux":
|
||||||
order = ["firefox"]
|
order = ["firefox"]
|
||||||
else:
|
else:
|
||||||
@@ -39,14 +39,11 @@ def normalize_text(text: str) -> str:
|
|||||||
return re.sub(r"\s+", " ", text).strip()
|
return re.sub(r"\s+", " ", text).strip()
|
||||||
|
|
||||||
|
|
||||||
def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
|
|
||||||
if len(norm) < n:
|
|
||||||
return {norm} if norm else set()
|
|
||||||
return {norm[index:index + n] for index in range(len(norm) - n + 1)}
|
|
||||||
|
|
||||||
|
|
||||||
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
||||||
return _ngrams_of_normalized(normalize_text(text), n)
|
text = normalize_text(text)
|
||||||
|
if len(text) < n:
|
||||||
|
return {text} if text else set()
|
||||||
|
return {text[index:index + n] for index in range(len(text) - n + 1)}
|
||||||
|
|
||||||
|
|
||||||
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
||||||
@@ -93,7 +90,7 @@ class _PreparedText:
|
|||||||
|
|
||||||
def __init__(self, raw: str) -> None:
|
def __init__(self, raw: str) -> None:
|
||||||
norm = normalize_text(raw)
|
norm = normalize_text(raw)
|
||||||
self.ngrams = _ngrams_of_normalized(norm)
|
self.ngrams = get_ngrams(norm) if norm else set()
|
||||||
self.tokens = _tokenize(norm)
|
self.tokens = _tokenize(norm)
|
||||||
|
|
||||||
|
|
||||||
@@ -106,7 +106,7 @@ def _extract_subreddits(reddit_items: List[Dict[str, Any]]) -> List[str]:
|
|||||||
|
|
||||||
for item in reddit_items:
|
for item in reddit_items:
|
||||||
# Primary subreddit
|
# Primary subreddit
|
||||||
sub = item.get("subreddit", "").strip().removeprefix("r/")
|
sub = item.get("subreddit", "").strip().lstrip("r/")
|
||||||
if sub:
|
if sub:
|
||||||
sub_counts[sub] += 1
|
sub_counts[sub] += 1
|
||||||
|
|
||||||
@@ -29,23 +29,6 @@ else:
|
|||||||
|
|
||||||
CODEX_AUTH_FILE = Path(os.environ.get("CODEX_AUTH_FILE", str(Path.home() / ".codex" / "auth.json")))
|
CODEX_AUTH_FILE = Path(os.environ.get("CODEX_AUTH_FILE", str(Path.home() / ".codex" / "auth.json")))
|
||||||
|
|
||||||
# macOS Keychain integration: items stored with this service prefix are picked
|
|
||||||
# up automatically on Darwin as the lowest-priority credential source.
|
|
||||||
# Example: `security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."`.
|
|
||||||
KEYCHAIN_SERVICE_PREFIX = "last30days-"
|
|
||||||
|
|
||||||
# Single source of truth for which credentials the Keychain loader looks up.
|
|
||||||
# The setup-keychain.sh helper mirrors this list and is held in sync via
|
|
||||||
# tests/test_env_keychain.py::test_keychain_keys_match_setup_script.
|
|
||||||
KEYCHAIN_KEYS = (
|
|
||||||
"OPENAI_API_KEY", "XAI_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY",
|
|
||||||
"GOOGLE_GENAI_API_KEY", "SCRAPECREATORS_API_KEY", "APIFY_API_TOKEN",
|
|
||||||
"AUTH_TOKEN", "CT0", "BSKY_HANDLE", "BSKY_APP_PASSWORD",
|
|
||||||
"TRUTHSOCIAL_TOKEN", "BRAVE_API_KEY", "EXA_API_KEY", "SERPER_API_KEY",
|
|
||||||
"OPENROUTER_API_KEY", "PARALLEL_API_KEY", "XQUIK_API_KEY",
|
|
||||||
"XIAOHONGSHU_API_BASE",
|
|
||||||
)
|
|
||||||
|
|
||||||
AuthSource = Literal["api_key", "codex", "none"]
|
AuthSource = Literal["api_key", "codex", "none"]
|
||||||
AuthStatus = Literal["ok", "missing", "expired", "missing_account_id"]
|
AuthStatus = Literal["ok", "missing", "expired", "missing_account_id"]
|
||||||
|
|
||||||
@@ -70,10 +53,6 @@ class OpenAIAuth:
|
|||||||
|
|
||||||
def _check_file_permissions(path: Path) -> None:
|
def _check_file_permissions(path: Path) -> None:
|
||||||
"""Warn to stderr if a secrets file has overly permissive permissions."""
|
"""Warn to stderr if a secrets file has overly permissive permissions."""
|
||||||
if os.name == "nt":
|
|
||||||
# Windows reports synthesized POSIX mode bits that do not reflect NTFS ACLs.
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
mode = path.stat().st_mode
|
mode = path.stat().st_mode
|
||||||
# Check if group or other can read (bits 0o044)
|
# Check if group or other can read (bits 0o044)
|
||||||
@@ -112,46 +91,6 @@ def load_env_file(path: Path) -> dict[str, str]:
|
|||||||
return env
|
return env
|
||||||
|
|
||||||
|
|
||||||
def _load_keychain(keys: list[str]) -> dict[str, str]:
|
|
||||||
"""Load credentials from macOS Keychain (no-op on other platforms).
|
|
||||||
|
|
||||||
Each key is looked up as a generic password with service name
|
|
||||||
``f"{KEYCHAIN_SERVICE_PREFIX}{key}"`` for the current user. Missing items
|
|
||||||
and lookup failures are silent — Keychain is the lowest-priority source
|
|
||||||
and is meant to be additive over `.env` files and process environment.
|
|
||||||
"""
|
|
||||||
import platform
|
|
||||||
if platform.system() != "Darwin":
|
|
||||||
return {}
|
|
||||||
|
|
||||||
import shutil
|
|
||||||
security = shutil.which("security")
|
|
||||||
if not security:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
import subprocess
|
|
||||||
import pwd
|
|
||||||
# USER can be unset under sudo, in Docker without --env USER, or in some CI
|
|
||||||
# runners; fall back to the OS user record so lookups still match items
|
|
||||||
# stored by setup-keychain.sh (which uses $USER).
|
|
||||||
user = os.environ.get("USER") or pwd.getpwuid(os.getuid()).pw_name
|
|
||||||
env: dict[str, str] = {}
|
|
||||||
for key in keys:
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
[security, "find-generic-password",
|
|
||||||
"-a", user,
|
|
||||||
"-s", f"{KEYCHAIN_SERVICE_PREFIX}{key}",
|
|
||||||
"-w"],
|
|
||||||
capture_output=True, text=True, timeout=5,
|
|
||||||
)
|
|
||||||
except (subprocess.TimeoutExpired, OSError):
|
|
||||||
continue
|
|
||||||
if result.returncode == 0 and result.stdout.strip():
|
|
||||||
env[key] = result.stdout.strip()
|
|
||||||
return env
|
|
||||||
|
|
||||||
|
|
||||||
def _decode_jwt_payload(token: str) -> dict[str, Any] | None:
|
def _decode_jwt_payload(token: str) -> dict[str, Any] | None:
|
||||||
"""Decode JWT payload without verification."""
|
"""Decode JWT payload without verification."""
|
||||||
try:
|
try:
|
||||||
@@ -275,7 +214,6 @@ def get_config() -> dict[str, Any]:
|
|||||||
1. Environment variables (os.environ)
|
1. Environment variables (os.environ)
|
||||||
2. .claude/last30days.env (per-project config)
|
2. .claude/last30days.env (per-project config)
|
||||||
3. ~/.config/last30days/.env (global config)
|
3. ~/.config/last30days/.env (global config)
|
||||||
4. macOS Keychain items prefixed ``last30days-`` (Darwin only)
|
|
||||||
"""
|
"""
|
||||||
# Load from global config file
|
# Load from global config file
|
||||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||||
@@ -284,14 +222,9 @@ def get_config() -> dict[str, Any]:
|
|||||||
project_env_path = _find_project_env()
|
project_env_path = _find_project_env()
|
||||||
project_env = load_env_file(project_env_path) if project_env_path else {}
|
project_env = load_env_file(project_env_path) if project_env_path else {}
|
||||||
|
|
||||||
# Merge file sources: project > global
|
# Merge: project overrides global
|
||||||
merged_env = {**file_env, **project_env}
|
merged_env = {**file_env, **project_env}
|
||||||
|
|
||||||
# Keychain is the lowest-priority source (Darwin only; no-op elsewhere).
|
|
||||||
# Loaded before openai_auth so OPENAI_API_KEY can come from Keychain too.
|
|
||||||
keychain_env = _load_keychain(list(KEYCHAIN_KEYS))
|
|
||||||
merged_env = {**keychain_env, **merged_env}
|
|
||||||
|
|
||||||
openai_auth = get_openai_auth(merged_env)
|
openai_auth = get_openai_auth(merged_env)
|
||||||
|
|
||||||
# Build config: Codex/OpenAI auth + process.env > project .env > global .env
|
# Build config: Codex/OpenAI auth + process.env > project .env > global .env
|
||||||
@@ -314,7 +247,6 @@ def get_config() -> dict[str, Any]:
|
|||||||
('LAST30DAYS_RERANK_MODEL', None),
|
('LAST30DAYS_RERANK_MODEL', None),
|
||||||
('LAST30DAYS_X_MODEL', None),
|
('LAST30DAYS_X_MODEL', None),
|
||||||
('LAST30DAYS_X_BACKEND', None),
|
('LAST30DAYS_X_BACKEND', None),
|
||||||
('LAST30DAYS_STORE', None),
|
|
||||||
('OPENAI_MODEL_PIN', None),
|
('OPENAI_MODEL_PIN', None),
|
||||||
('XAI_MODEL_PIN', None),
|
('XAI_MODEL_PIN', None),
|
||||||
('SCRAPECREATORS_API_KEY', None),
|
('SCRAPECREATORS_API_KEY', None),
|
||||||
@@ -323,7 +255,6 @@ def get_config() -> dict[str, Any]:
|
|||||||
('CT0', None),
|
('CT0', None),
|
||||||
('BSKY_HANDLE', None),
|
('BSKY_HANDLE', None),
|
||||||
('BSKY_APP_PASSWORD', None),
|
('BSKY_APP_PASSWORD', None),
|
||||||
('BSKY_SEARCH_HOST', None),
|
|
||||||
('TRUTHSOCIAL_TOKEN', None),
|
('TRUTHSOCIAL_TOKEN', None),
|
||||||
('BRAVE_API_KEY', None),
|
('BRAVE_API_KEY', None),
|
||||||
('EXA_API_KEY', None),
|
('EXA_API_KEY', None),
|
||||||
@@ -334,41 +265,16 @@ def get_config() -> dict[str, Any]:
|
|||||||
('FROM_BROWSER', None),
|
('FROM_BROWSER', None),
|
||||||
('SETUP_COMPLETE', None),
|
('SETUP_COMPLETE', None),
|
||||||
('INCLUDE_SOURCES', ''),
|
('INCLUDE_SOURCES', ''),
|
||||||
('EXCLUDE_SOURCES', ''),
|
|
||||||
('LAST30DAYS_YOUTUBE_SSH_HOST', None),
|
|
||||||
('LAST30DAYS_TRANSCRIPT_TIMEOUT', None),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
for key, default in keys:
|
for key, default in keys:
|
||||||
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
||||||
|
|
||||||
# Backward-compat: ScrapeCreators' own examples and tutorials use the
|
# Track which config source was used
|
||||||
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and
|
|
||||||
# CREATORS). Accept that form too so users who follow the vendor's docs
|
|
||||||
# don't silently end up with has_scrapecreators=False. Canonical name
|
|
||||||
# wins when both are set.
|
|
||||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
|
||||||
legacy = os.environ.get('SCRAPE_CREATORS_API_KEY') or merged_env.get('SCRAPE_CREATORS_API_KEY')
|
|
||||||
if legacy:
|
|
||||||
config['SCRAPECREATORS_API_KEY'] = legacy
|
|
||||||
|
|
||||||
# Multi-key rotation: comma-separated SCRAPECREATORS_API_KEY round-robins
|
|
||||||
# via random.choice per run. Originally added in #268, accidentally dropped
|
|
||||||
# in v3.0.6, restored here.
|
|
||||||
sc_key_raw = config.get('SCRAPECREATORS_API_KEY') or ''
|
|
||||||
if ',' in sc_key_raw:
|
|
||||||
import random
|
|
||||||
sc_keys = [k.strip() for k in sc_key_raw.split(',') if k.strip()]
|
|
||||||
config['SCRAPECREATORS_API_KEY'] = random.choice(sc_keys) if sc_keys else ''
|
|
||||||
|
|
||||||
# Track which config source was used (highest-priority file source wins
|
|
||||||
# the label; keychain is only reported when nothing else is configured).
|
|
||||||
if project_env_path:
|
if project_env_path:
|
||||||
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
||||||
elif CONFIG_FILE and CONFIG_FILE.exists():
|
elif CONFIG_FILE and CONFIG_FILE.exists():
|
||||||
config['_CONFIG_SOURCE'] = f'global:{CONFIG_FILE}'
|
config['_CONFIG_SOURCE'] = f'global:{CONFIG_FILE}'
|
||||||
elif keychain_env:
|
|
||||||
config['_CONFIG_SOURCE'] = 'keychain'
|
|
||||||
else:
|
else:
|
||||||
config['_CONFIG_SOURCE'] = 'env_only'
|
config['_CONFIG_SOURCE'] = 'env_only'
|
||||||
|
|
||||||
@@ -466,6 +372,14 @@ def config_exists() -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_reddit_available(config: dict[str, Any]) -> bool:
|
||||||
|
"""Check if Reddit search is available.
|
||||||
|
|
||||||
|
v3 uses ScrapeCreators only.
|
||||||
|
"""
|
||||||
|
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||||
|
|
||||||
|
|
||||||
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
||||||
"""Determine which Reddit backend to use.
|
"""Determine which Reddit backend to use.
|
||||||
|
|
||||||
@@ -611,12 +525,12 @@ def _parse_include_sources(config: dict[str, Any]) -> set[str]:
|
|||||||
def is_threads_available(config: dict[str, Any]) -> bool:
|
def is_threads_available(config: dict[str, Any]) -> bool:
|
||||||
"""Check if Threads source is available.
|
"""Check if Threads source is available.
|
||||||
|
|
||||||
Returns True when SCRAPECREATORS_API_KEY is set. Threads runs alongside
|
Requires SCRAPECREATORS_API_KEY AND 'threads' in INCLUDE_SOURCES.
|
||||||
TikTok and Instagram as part of the SC family — same key, same per-call
|
Threads is an opt-in source - it is not activated by default.
|
||||||
cost shape, so the same default-on rule applies. Suppress via
|
|
||||||
EXCLUDE_SOURCES=threads.
|
|
||||||
"""
|
"""
|
||||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||||
|
return False
|
||||||
|
return 'threads' in _parse_include_sources(config)
|
||||||
|
|
||||||
|
|
||||||
def is_instagram_available(config: dict[str, Any]) -> bool:
|
def is_instagram_available(config: dict[str, Any]) -> bool:
|
||||||
@@ -116,8 +116,6 @@ def weighted_rrf(
|
|||||||
"""Fuse ranked lists into a single candidate pool."""
|
"""Fuse ranked lists into a single candidate pool."""
|
||||||
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
||||||
candidates: dict[str, schema.Candidate] = {}
|
candidates: dict[str, schema.Candidate] = {}
|
||||||
# Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
|
|
||||||
seen_source_items: dict[str, set[tuple[str, str]]] = {}
|
|
||||||
|
|
||||||
for (label, source), items in streams.items():
|
for (label, source), items in streams.items():
|
||||||
subquery = subqueries[label]
|
subquery = subqueries[label]
|
||||||
@@ -156,7 +154,6 @@ def weighted_rrf(
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
seen_source_items[key] = {(item.source, item.item_id)}
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
candidate = candidates[key]
|
candidate = candidates[key]
|
||||||
@@ -182,9 +179,7 @@ def weighted_rrf(
|
|||||||
candidate.subquery_labels.append(label)
|
candidate.subquery_labels.append(label)
|
||||||
if item.source not in candidate.sources:
|
if item.source not in candidate.sources:
|
||||||
candidate.sources.append(item.source)
|
candidate.sources.append(item.source)
|
||||||
source_item_key = (item.source, item.item_id)
|
if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
|
||||||
if source_item_key not in seen_source_items[key]:
|
|
||||||
seen_source_items[key].add(source_item_key)
|
|
||||||
candidate.source_items.append(item)
|
candidate.source_items.append(item)
|
||||||
candidate.metadata.setdefault("provenance", []).append(
|
candidate.metadata.setdefault("provenance", []).append(
|
||||||
{
|
{
|
||||||
@@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import sys
|
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
@@ -140,10 +139,7 @@ def parallel_search(
|
|||||||
data = http.request(
|
data = http.request(
|
||||||
"POST", "https://api.parallel.ai/v1/search",
|
"POST", "https://api.parallel.ai/v1/search",
|
||||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||||
json_data={
|
json_data={"query": query, "max_results": count},
|
||||||
"search_queries": [query],
|
|
||||||
"advanced_settings": {"max_results": count},
|
|
||||||
},
|
|
||||||
timeout=15,
|
timeout=15,
|
||||||
)
|
)
|
||||||
items = []
|
items = []
|
||||||
@@ -153,7 +149,7 @@ def parallel_search(
|
|||||||
url = r.get("url", "")
|
url = r.get("url", "")
|
||||||
if not url:
|
if not url:
|
||||||
continue
|
continue
|
||||||
raw_date = r.get("publish_date") or ""
|
raw_date = r.get("published_date") or ""
|
||||||
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
|
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
|
||||||
if not _in_date_range(pub_date, date_range):
|
if not _in_date_range(pub_date, date_range):
|
||||||
continue
|
continue
|
||||||
@@ -162,7 +158,7 @@ def parallel_search(
|
|||||||
"title": r.get("title", ""),
|
"title": r.get("title", ""),
|
||||||
"url": url,
|
"url": url,
|
||||||
"source_domain": _domain(url),
|
"source_domain": _domain(url),
|
||||||
"snippet": ((r.get("excerpts") or [""])[0] or "")[:500],
|
"snippet": r.get("snippet", ""),
|
||||||
"date": pub_date,
|
"date": pub_date,
|
||||||
"relevance": 0.8,
|
"relevance": 0.8,
|
||||||
"why_relevant": "Parallel AI web search",
|
"why_relevant": "Parallel AI web search",
|
||||||
@@ -209,90 +205,29 @@ def web_search(
|
|||||||
backend = "parallel"
|
backend = "parallel"
|
||||||
else:
|
else:
|
||||||
return [], {}
|
return [], {}
|
||||||
items: list[dict] = []
|
|
||||||
artifact: dict = {}
|
|
||||||
if backend == "brave":
|
if backend == "brave":
|
||||||
key = config.get("BRAVE_API_KEY")
|
key = config.get("BRAVE_API_KEY")
|
||||||
if not key:
|
if not key:
|
||||||
raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
|
raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
|
||||||
items, artifact = brave_search(query, date_range, key)
|
return brave_search(query, date_range, key)
|
||||||
elif backend == "exa":
|
if backend == "exa":
|
||||||
key = config.get("EXA_API_KEY")
|
key = config.get("EXA_API_KEY")
|
||||||
if not key:
|
if not key:
|
||||||
raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
|
raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
|
||||||
items, artifact = exa_search(query, date_range, key)
|
return exa_search(query, date_range, key)
|
||||||
elif backend == "serper":
|
if backend == "serper":
|
||||||
key = config.get("SERPER_API_KEY")
|
key = config.get("SERPER_API_KEY")
|
||||||
if not key:
|
if not key:
|
||||||
raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
|
raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
|
||||||
items, artifact = serper_search(query, date_range, key)
|
return serper_search(query, date_range, key)
|
||||||
elif backend == "parallel":
|
if backend == "parallel":
|
||||||
key = config.get("PARALLEL_API_KEY")
|
key = config.get("PARALLEL_API_KEY")
|
||||||
if not key:
|
if not key:
|
||||||
raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
|
raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
|
||||||
items, artifact = parallel_search(query, date_range, key)
|
return parallel_search(query, date_range, key)
|
||||||
elif backend != "none":
|
if backend != "none":
|
||||||
raise ValueError(f"Unsupported web backend: {backend!r}")
|
raise ValueError(f"Unsupported web backend: {backend!r}")
|
||||||
else:
|
return [], {}
|
||||||
return [], {}
|
|
||||||
if items and not _reddit_excluded(config):
|
|
||||||
items = _enrich_reddit_items(items)
|
|
||||||
return items, artifact
|
|
||||||
|
|
||||||
|
|
||||||
def _reddit_excluded(config: dict) -> bool:
|
|
||||||
"""Return True when EXCLUDE_SOURCES contains 'reddit'.
|
|
||||||
|
|
||||||
Respects the same suppression knob the pipeline uses for source gating,
|
|
||||||
so a user who set EXCLUDE_SOURCES=reddit doesn't get Reddit content
|
|
||||||
smuggled back in via web-search URLs.
|
|
||||||
"""
|
|
||||||
raw = (config.get("EXCLUDE_SOURCES") or "").split(",")
|
|
||||||
return any(s.strip().lower() == "reddit" for s in raw)
|
|
||||||
|
|
||||||
|
|
||||||
def _enrich_reddit_items(items: list[dict]) -> list[dict]:
|
|
||||||
"""Enrich web search results that are Reddit URLs with thread body and comments.
|
|
||||||
|
|
||||||
Claude Code's WebFetch blocks reddit.com, so the model can't retrieve
|
|
||||||
Reddit content from web search results. This fetches it via the public
|
|
||||||
JSON API (reddit.com/.../.json) which bypasses that restriction.
|
|
||||||
|
|
||||||
Callers should gate this with EXCLUDE_SOURCES=reddit handling (see
|
|
||||||
`_reddit_excluded`) so a user who explicitly excluded Reddit doesn't
|
|
||||||
get Reddit content via web-search URLs.
|
|
||||||
"""
|
|
||||||
from . import reddit_enrich
|
|
||||||
from .reddit_enrich import RedditRateLimitError
|
|
||||||
|
|
||||||
for item in items:
|
|
||||||
url = item.get("url", "")
|
|
||||||
if "reddit.com" not in url or "/comments/" not in url:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
thread_data = reddit_enrich.fetch_thread_data(url, timeout=8)
|
|
||||||
if not thread_data:
|
|
||||||
continue
|
|
||||||
parsed = reddit_enrich.parse_thread_data(thread_data)
|
|
||||||
# selftext lives under parsed["submission"], not at the top level
|
|
||||||
selftext = (parsed.get("submission") or {}).get("selftext", "")
|
|
||||||
if selftext:
|
|
||||||
item["snippet"] = selftext[:2000]
|
|
||||||
comments = parsed.get("comments", [])
|
|
||||||
top = reddit_enrich.get_top_comments(comments)
|
|
||||||
if top:
|
|
||||||
item["top_comments"] = [
|
|
||||||
{"score": c.get("score", 0), "excerpt": (c.get("body") or "")[:200]}
|
|
||||||
for c in top[:5]
|
|
||||||
]
|
|
||||||
item["enriched_via"] = "reddit_json_api"
|
|
||||||
except RedditRateLimitError as exc:
|
|
||||||
# Stop iterating to avoid flooding more 429s
|
|
||||||
sys.stderr.write(f"[Web] Reddit rate-limited, halting enrichment: {exc}\n")
|
|
||||||
break
|
|
||||||
except Exception as exc:
|
|
||||||
sys.stderr.write(f"[Web] Reddit enrichment failed for {url}: {exc}\n")
|
|
||||||
return items
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -88,26 +88,17 @@ def search_hackernews(
|
|||||||
|
|
||||||
# Use extracted core subject instead of raw topic for cleaner Algolia matching
|
# Use extracted core subject instead of raw topic for cleaner Algolia matching
|
||||||
core = extract_core_subject(topic)
|
core = extract_core_subject(topic)
|
||||||
# Hyphens and commas tokenize awkwardly in Algolia; flatten them so themed
|
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
||||||
# queries like "ts-bun-node" or "claude, personal agents" become plain words.
|
|
||||||
core_flat = _flatten_query_for_algolia(core)
|
|
||||||
_log(f"Searching for '{core_flat}' (raw: '{topic}', since {from_date}, count={count})")
|
|
||||||
|
|
||||||
# Use relevance-sorted search with minimum engagement filter.
|
# Use relevance-sorted search with minimum engagement filter.
|
||||||
# NOTE: restrictSearchableAttributes=title omitted intentionally — it would
|
# NOTE: restrictSearchableAttributes=title omitted intentionally — it would
|
||||||
# miss Ask HN/Show HN threads where the topic appears in the body.
|
# miss Ask HN/Show HN threads where the topic appears in the body.
|
||||||
params = {
|
params = {
|
||||||
"query": core_flat,
|
"query": core,
|
||||||
"tags": "story",
|
"tags": "story",
|
||||||
"numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
|
"numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
|
||||||
"hitsPerPage": str(count),
|
"hitsPerPage": str(count),
|
||||||
}
|
}
|
||||||
# Algolia defaults to AND across query tokens, so a 4-5 word theme query
|
|
||||||
# matches no stories. Mark all-but-the-first token as optional so Algolia
|
|
||||||
# ranks by how many tokens match instead of requiring every one.
|
|
||||||
tokens = core_flat.split()
|
|
||||||
if len(tokens) > 1:
|
|
||||||
params["optionalWords"] = " ".join(tokens[1:])
|
|
||||||
|
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
|
url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
|
||||||
@@ -126,56 +117,28 @@ def search_hackernews(
|
|||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
_WORD_BOUNDARY_RE_CACHE: Dict[str, "re.Pattern[str]"] = {}
|
|
||||||
|
|
||||||
|
|
||||||
def _flatten_query_for_algolia(text: str) -> str:
|
|
||||||
"""Normalise query for Algolia + post-filter comparison.
|
|
||||||
|
|
||||||
Multi-keyword theme queries frequently contain commas (delimiters) or
|
|
||||||
hyphens (compound terms like ``ts-bun-node``); both tokenize awkwardly.
|
|
||||||
Flatten them to spaces and collapse runs of whitespace so the search
|
|
||||||
parameter and the post-filter operate on the same shape.
|
|
||||||
"""
|
|
||||||
return " ".join(text.replace(",", " ").replace("-", " ").split())
|
|
||||||
|
|
||||||
|
|
||||||
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
|
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
|
||||||
"""Check if any query token appears as a whole word in the title.
|
"""Check if the query term appears in the title content, not just an HN prefix or author.
|
||||||
|
|
||||||
Returns True when the query is empty (no filter), or when at least one
|
Returns True if the query (or any multi-word token) appears in the title
|
||||||
query token matches as a whole word in the title after stripping
|
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
|
||||||
"Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes.
|
and ignoring the author name. Returns True when query is empty (no filter).
|
||||||
|
|
||||||
We previously required *every* token to appear (all-words), which killed
|
|
||||||
every Algolia hit on multi-keyword themes like "claude, personal agents,
|
|
||||||
agentic infra" because real HN titles never contain all five tokens
|
|
||||||
verbatim. Relaxing to any-word matches Algolia's `optionalWords` behaviour
|
|
||||||
in `search_hackernews`. Token-overlap relevance scoring at parse time
|
|
||||||
demotes hits where only one weak token matched, so the loosened gate
|
|
||||||
won't surface noise to the top of the ranking.
|
|
||||||
|
|
||||||
Word-boundary matching (rather than naive substring) prevents short
|
|
||||||
tokens like ``ai`` or ``ts`` from matching unrelated words like
|
|
||||||
``email`` or ``artists``.
|
|
||||||
"""
|
"""
|
||||||
if not query:
|
if not query:
|
||||||
return True
|
return True
|
||||||
stripped = _HN_PREFIXES.sub("", title).strip()
|
stripped = _HN_PREFIXES.sub("", title).strip()
|
||||||
|
# Also check that the match isn't solely in the author's username
|
||||||
check_text = stripped.lower()
|
check_text = stripped.lower()
|
||||||
# Normalise the query the same way search_hackernews does so post-filter
|
query_lower = query.lower()
|
||||||
# tokens line up with what Algolia actually saw.
|
# Check each word of the query independently; all must appear somewhere
|
||||||
query_words = [w for w in _flatten_query_for_algolia(query.lower()).split() if w]
|
# in the stripped title (not just the prefix).
|
||||||
if not query_words:
|
query_words = query_lower.split()
|
||||||
return True
|
|
||||||
for word in query_words:
|
for word in query_words:
|
||||||
pattern = _WORD_BOUNDARY_RE_CACHE.get(word)
|
if word in check_text:
|
||||||
if pattern is None:
|
continue
|
||||||
pattern = re.compile(rf"\b{re.escape(word)}\b")
|
# Word not found in stripped title — reject
|
||||||
_WORD_BOUNDARY_RE_CACHE[word] = pattern
|
return False
|
||||||
if pattern.search(check_text):
|
return True
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||||
@@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import socket
|
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib.error
|
import urllib.error
|
||||||
@@ -23,19 +22,9 @@ def log(msg: str):
|
|||||||
MAX_RETRIES = 5
|
MAX_RETRIES = 5
|
||||||
MAX_429_RETRIES = 2
|
MAX_429_RETRIES = 2
|
||||||
RETRY_DELAY = 2.0
|
RETRY_DELAY = 2.0
|
||||||
# DNS resolution failures (gaierror) are transient — typically resolved by a
|
|
||||||
# brief backoff and retry. Use a dedicated minimum attempt count + exponential
|
|
||||||
# delays (1s, 2s, 4s) so callers that pass a small `retries` value still get a
|
|
||||||
# meaningful chance to recover from a transient resolution failure.
|
|
||||||
MIN_DNS_RETRIES = 3
|
|
||||||
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
||||||
|
|
||||||
|
|
||||||
def _is_dns_failure(err: urllib.error.URLError) -> bool:
|
|
||||||
"""Return True if a URLError was caused by DNS resolution (gaierror)."""
|
|
||||||
return isinstance(getattr(err, "reason", None), socket.gaierror)
|
|
||||||
|
|
||||||
|
|
||||||
class HTTPError(Exception):
|
class HTTPError(Exception):
|
||||||
"""HTTP request error with status code."""
|
"""HTTP request error with status code."""
|
||||||
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
||||||
@@ -96,13 +85,7 @@ def request(
|
|||||||
|
|
||||||
last_error = None
|
last_error = None
|
||||||
rate_limit_count = 0
|
rate_limit_count = 0
|
||||||
# DNS failures get a dedicated minimum attempt count + exponential backoff.
|
for attempt in range(retries):
|
||||||
# `effective_retries` is the actual loop bound; we expand it on the first
|
|
||||||
# gaierror if the caller passed a smaller `retries` value than MIN_DNS_RETRIES.
|
|
||||||
effective_retries = retries
|
|
||||||
dns_attempts = 0
|
|
||||||
attempt = 0
|
|
||||||
while attempt < effective_retries:
|
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||||
body = response.read().decode('utf-8')
|
body = response.read().decode('utf-8')
|
||||||
@@ -132,8 +115,6 @@ def request(
|
|||||||
if rate_limit_count >= max_429_retries:
|
if rate_limit_count >= max_429_retries:
|
||||||
raise last_error
|
raise last_error
|
||||||
|
|
||||||
# HTTP errors respect the caller's original `retries`; only DNS
|
|
||||||
# failures get the widened `effective_retries` budget.
|
|
||||||
if attempt < retries - 1:
|
if attempt < retries - 1:
|
||||||
if e.code == 429:
|
if e.code == 429:
|
||||||
# Respect Retry-After header, fall back to exponential backoff
|
# Respect Retry-After header, fall back to exponential backoff
|
||||||
@@ -149,43 +130,11 @@ def request(
|
|||||||
else:
|
else:
|
||||||
delay = RETRY_DELAY * (2 ** attempt)
|
delay = RETRY_DELAY * (2 ** attempt)
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
else:
|
|
||||||
# Caller's original retry budget exhausted; an earlier DNS
|
|
||||||
# failure may have widened `effective_retries`, but that
|
|
||||||
# widening is DNS-only — don't grant extra HTTP attempts.
|
|
||||||
break
|
|
||||||
except urllib.error.URLError as e:
|
except urllib.error.URLError as e:
|
||||||
log(f"URL Error: {e.reason}")
|
log(f"URL Error: {e.reason}")
|
||||||
last_error = HTTPError(f"URL Error: {e.reason}")
|
last_error = HTTPError(f"URL Error: {e.reason}")
|
||||||
if _is_dns_failure(e):
|
if attempt < retries - 1:
|
||||||
# DNS resolution failures are transient; expand the retry budget
|
|
||||||
# to MIN_DNS_RETRIES if the caller passed fewer, and use
|
|
||||||
# exponential backoff (1s, 2s, 4s, ...) instead of the linear
|
|
||||||
# default. Counts DNS attempts separately so other URLError
|
|
||||||
# causes don't bypass the regular retry budget.
|
|
||||||
dns_attempts += 1
|
|
||||||
if effective_retries < MIN_DNS_RETRIES:
|
|
||||||
log(
|
|
||||||
f"DNS resolution failed; expanding retry budget from "
|
|
||||||
f"{effective_retries} to {MIN_DNS_RETRIES}"
|
|
||||||
)
|
|
||||||
effective_retries = MIN_DNS_RETRIES
|
|
||||||
if attempt < effective_retries - 1:
|
|
||||||
delay = 2 ** (dns_attempts - 1) # 1s, 2s, 4s, 8s, ...
|
|
||||||
log(
|
|
||||||
f"DNS resolution failure (attempt {dns_attempts}); "
|
|
||||||
f"retrying in {delay:.1f}s"
|
|
||||||
)
|
|
||||||
time.sleep(delay)
|
|
||||||
elif attempt < retries - 1:
|
|
||||||
# Non-DNS URLError (e.g. ConnectionRefused) respects the
|
|
||||||
# caller's original retry budget, not the DNS-widened bound.
|
|
||||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||||
else:
|
|
||||||
# Caller's original retry budget exhausted; an earlier DNS
|
|
||||||
# failure widening `effective_retries` does not carry over
|
|
||||||
# to non-DNS error paths.
|
|
||||||
break
|
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError as e:
|
||||||
log(f"JSON decode error: {e}")
|
log(f"JSON decode error: {e}")
|
||||||
last_error = HTTPError(f"Invalid JSON response: {e}")
|
last_error = HTTPError(f"Invalid JSON response: {e}")
|
||||||
@@ -195,13 +144,7 @@ def request(
|
|||||||
log(f"Connection error: {type(e).__name__}: {e}")
|
log(f"Connection error: {type(e).__name__}: {e}")
|
||||||
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
||||||
if attempt < retries - 1:
|
if attempt < retries - 1:
|
||||||
# Socket errors respect the caller's original retry budget.
|
|
||||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||||
else:
|
|
||||||
# Original budget exhausted; DNS widening doesn't apply here.
|
|
||||||
break
|
|
||||||
|
|
||||||
attempt += 1
|
|
||||||
|
|
||||||
if last_error:
|
if last_error:
|
||||||
raise last_error
|
raise last_error
|
||||||
@@ -7,14 +7,17 @@ Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
|
|||||||
API docs: https://scrapecreators.com/docs
|
API docs: https://scrapecreators.com/docs
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests as _requests
|
||||||
|
except ImportError:
|
||||||
|
_requests = None
|
||||||
|
|
||||||
from . import dates, http, log
|
from . import dates, http, log
|
||||||
from .relevance import token_overlap_relevance as _compute_relevance
|
|
||||||
|
|
||||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||||
|
|
||||||
@@ -28,42 +31,7 @@ DEPTH_CONFIG = {
|
|||||||
# Max words to keep from each caption
|
# Max words to keep from each caption
|
||||||
CAPTION_MAX_WORDS = 500
|
CAPTION_MAX_WORDS = 500
|
||||||
|
|
||||||
# Default transcript fetch timeout (seconds). SC's
|
from .relevance import token_overlap_relevance as _compute_relevance
|
||||||
# /v2/instagram/media/transcript regularly takes >15s on real workloads,
|
|
||||||
# so the default is generous; override via LAST30DAYS_TRANSCRIPT_TIMEOUT.
|
|
||||||
DEFAULT_TRANSCRIPT_TIMEOUT = 30
|
|
||||||
|
|
||||||
|
|
||||||
def _resolve_transcript_timeout(
|
|
||||||
timeout: Optional[float] = None,
|
|
||||||
config: Optional[Dict[str, Any]] = None,
|
|
||||||
) -> float:
|
|
||||||
"""Resolve the IG transcript-fetch timeout.
|
|
||||||
|
|
||||||
Priority (highest wins):
|
|
||||||
1. Explicit ``timeout`` kwarg
|
|
||||||
2. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in os.environ
|
|
||||||
3. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in caller-supplied config dict
|
|
||||||
4. ``DEFAULT_TRANSCRIPT_TIMEOUT`` (30s)
|
|
||||||
|
|
||||||
Mirrors the ``os.environ.get(X) or config.get(X)`` pattern used for
|
|
||||||
LAST30DAYS_STORE in last30days.py so the env var works whether it's
|
|
||||||
shell-exported or set in ~/.config/last30days/.env.
|
|
||||||
"""
|
|
||||||
if timeout is not None:
|
|
||||||
try:
|
|
||||||
return float(timeout)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
pass
|
|
||||||
raw = os.environ.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
|
||||||
if not raw and config:
|
|
||||||
raw = config.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
|
||||||
if raw:
|
|
||||||
try:
|
|
||||||
return float(raw)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
pass
|
|
||||||
return float(DEFAULT_TRANSCRIPT_TIMEOUT)
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_core_subject(topic: str) -> str:
|
def _extract_core_subject(topic: str) -> str:
|
||||||
@@ -81,17 +49,6 @@ def _extract_core_subject(topic: str) -> str:
|
|||||||
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
||||||
|
|
||||||
|
|
||||||
def _to_hashtag_form(query: str) -> str:
|
|
||||||
"""Collapse a multi-word query to hashtag form (no spaces, lowercase).
|
|
||||||
|
|
||||||
SC's /v2/instagram/reels/search wraps Google Search and is documented
|
|
||||||
to be flaky on multi-token queries. Single-token queries map to a
|
|
||||||
hashtag page lookup which is the stable path. Used as a 500-retry
|
|
||||||
fallback before the request bubbles up as a silent failure.
|
|
||||||
"""
|
|
||||||
return ''.join(query.split()).lower()
|
|
||||||
|
|
||||||
|
|
||||||
def _infer_query_intent(topic: str) -> str:
|
def _infer_query_intent(topic: str) -> str:
|
||||||
"""Tiny local intent classifier for Instagram query expansion."""
|
"""Tiny local intent classifier for Instagram query expansion."""
|
||||||
text = topic.lower().strip()
|
text = topic.lower().strip()
|
||||||
@@ -279,17 +236,30 @@ def _user_reels(
|
|||||||
"""
|
"""
|
||||||
_log(f"User reels: @{handle}")
|
_log(f"User reels: @{handle}")
|
||||||
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
|
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
try:
|
||||||
reels_url,
|
from urllib.parse import urlencode
|
||||||
params={"handle": handle},
|
params = urlencode({"handle": handle})
|
||||||
headers=http.scrapecreators_headers(token),
|
url = f"{reels_url}?{params}"
|
||||||
timeout=30,
|
headers = http.scrapecreators_headers(token)
|
||||||
retries=2,
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
)
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"User reels error for @{handle}: {e}")
|
_log(f"User reels error (urllib) for @{handle}: {e}")
|
||||||
return []
|
return []
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
reels_url,
|
||||||
|
params={"handle": handle},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"User reels error for @{handle}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
|
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
|
||||||
_log(f" -> {len(raw_items)} reels from @{handle}")
|
_log(f" -> {len(raw_items)} reels from @{handle}")
|
||||||
@@ -323,37 +293,31 @@ def search_instagram(
|
|||||||
|
|
||||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||||
|
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
_log("requests library not installed, falling back to urllib")
|
||||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
try:
|
||||||
params={"query": core_topic},
|
from urllib.parse import urlencode
|
||||||
headers=http.scrapecreators_headers(token),
|
params = urlencode({"query": core_topic})
|
||||||
timeout=30,
|
url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
|
||||||
retries=2,
|
headers = http.scrapecreators_headers(token)
|
||||||
)
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
except http.HTTPError as e:
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
# SC's v2 reels search wraps Google Search and 500s frequently on
|
except Exception as e:
|
||||||
# multi-token queries. Single tokens hit the stable hashtag-page
|
_log(f"ScrapeCreators error (urllib): {e}")
|
||||||
# path. Retry once with hashtag form before bubbling up.
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
if getattr(e, "status_code", None) == 500 and ' ' in core_topic:
|
else:
|
||||||
_log(f"IG search 500 on '{core_topic}', retrying with hashtag form")
|
try:
|
||||||
try:
|
resp = _requests.get(
|
||||||
data = http.get(
|
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
params={"query": core_topic},
|
||||||
params={"query": _to_hashtag_form(core_topic)},
|
headers=http.scrapecreators_headers(token),
|
||||||
headers=http.scrapecreators_headers(token),
|
timeout=30,
|
||||||
timeout=30,
|
)
|
||||||
retries=2,
|
resp.raise_for_status()
|
||||||
)
|
data = resp.json()
|
||||||
except Exception as retry_e:
|
except Exception as e:
|
||||||
_log(f"IG search retry failed: {retry_e}")
|
|
||||||
return {"items": [], "error": f"{type(retry_e).__name__}: {retry_e}"}
|
|
||||||
else:
|
|
||||||
_log(f"ScrapeCreators error: {e}")
|
_log(f"ScrapeCreators error: {e}")
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
except Exception as e:
|
|
||||||
_log(f"ScrapeCreators error: {e}")
|
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
|
||||||
|
|
||||||
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
||||||
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
||||||
@@ -385,8 +349,6 @@ def fetch_captions(
|
|||||||
video_items: List[Dict[str, Any]],
|
video_items: List[Dict[str, Any]],
|
||||||
token: str,
|
token: str,
|
||||||
depth: str = "default",
|
depth: str = "default",
|
||||||
timeout: Optional[float] = None,
|
|
||||||
config: Optional[Dict[str, Any]] = None,
|
|
||||||
) -> Dict[str, str]:
|
) -> Dict[str, str]:
|
||||||
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
||||||
|
|
||||||
@@ -398,21 +360,14 @@ def fetch_captions(
|
|||||||
video_items: Items from search_instagram()
|
video_items: Items from search_instagram()
|
||||||
token: ScrapeCreators API key
|
token: ScrapeCreators API key
|
||||||
depth: Depth level for caption limit
|
depth: Depth level for caption limit
|
||||||
timeout: Optional per-request transcript timeout in seconds. When
|
|
||||||
None, resolves from LAST30DAYS_TRANSCRIPT_TIMEOUT (env or
|
|
||||||
config), defaulting to DEFAULT_TRANSCRIPT_TIMEOUT (30s).
|
|
||||||
config: Optional config dict (from env.get_config()) used as a
|
|
||||||
fallback source for LAST30DAYS_TRANSCRIPT_TIMEOUT when the
|
|
||||||
value is not exported in os.environ.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict mapping video_id -> caption text (truncated to 500 words)
|
Dict mapping video_id -> caption text (truncated to 500 words)
|
||||||
"""
|
"""
|
||||||
depth_cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||||
max_captions = depth_cfg["max_captions"]
|
max_captions = config["max_captions"]
|
||||||
transcript_timeout = _resolve_transcript_timeout(timeout, config)
|
|
||||||
|
|
||||||
if not video_items or not token:
|
if not video_items or not token or not _requests:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
top_items = video_items[:max_captions]
|
top_items = video_items[:max_captions]
|
||||||
@@ -437,24 +392,26 @@ def fetch_captions(
|
|||||||
if not url:
|
if not url:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
data = http.get(
|
resp = _requests.get(
|
||||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||||
params={"url": url},
|
params={"url": url},
|
||||||
headers=http.scrapecreators_headers(token),
|
headers=http.scrapecreators_headers(token),
|
||||||
timeout=transcript_timeout,
|
timeout=15,
|
||||||
retries=1,
|
|
||||||
)
|
)
|
||||||
transcripts = data.get("transcripts") or []
|
if resp.status_code == 200:
|
||||||
if transcripts and isinstance(transcripts, list):
|
data = resp.json()
|
||||||
transcript_text = " ".join(
|
transcripts = data.get("transcripts") or []
|
||||||
t.get("text", "") for t in transcripts
|
if transcripts and isinstance(transcripts, list):
|
||||||
if isinstance(t, dict) and t.get("text")
|
# Combine all transcript segments
|
||||||
)
|
transcript_text = " ".join(
|
||||||
if transcript_text:
|
t.get("text", "") for t in transcripts
|
||||||
words = transcript_text.split()
|
if isinstance(t, dict) and t.get("text")
|
||||||
if len(words) > CAPTION_MAX_WORDS:
|
)
|
||||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
if transcript_text:
|
||||||
captions[vid] = transcript_text
|
words = transcript_text.split()
|
||||||
|
if len(words) > CAPTION_MAX_WORDS:
|
||||||
|
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||||
|
captions[vid] = transcript_text
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Transcript fetch failed for {vid}: {e}")
|
_log(f"Transcript fetch failed for {vid}: {e}")
|
||||||
|
|
||||||
@@ -49,7 +49,6 @@ def normalize_source_items(
|
|||||||
"xquik": _normalize_x,
|
"xquik": _normalize_x,
|
||||||
"pinterest": _normalize_pinterest,
|
"pinterest": _normalize_pinterest,
|
||||||
"polymarket": _normalize_polymarket,
|
"polymarket": _normalize_polymarket,
|
||||||
"digg": _normalize_digg,
|
|
||||||
"grounding": _normalize_grounding,
|
"grounding": _normalize_grounding,
|
||||||
"xiaohongshu": _normalize_grounding,
|
"xiaohongshu": _normalize_grounding,
|
||||||
"github": _normalize_github,
|
"github": _normalize_github,
|
||||||
@@ -111,19 +110,6 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
|
|||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
def _join_comment_excerpts(
|
|
||||||
top_comments: list[Any],
|
|
||||||
key: str,
|
|
||||||
limit: int = 3,
|
|
||||||
) -> str:
|
|
||||||
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
|
|
||||||
return " ".join(
|
|
||||||
str(comment.get(key) or "").strip()
|
|
||||||
for comment in top_comments[:limit]
|
|
||||||
if isinstance(comment, dict)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _domain_from_url(url: str) -> str | None:
|
def _domain_from_url(url: str) -> str | None:
|
||||||
if not url:
|
if not url:
|
||||||
return None
|
return None
|
||||||
@@ -183,7 +169,11 @@ def _normalize_reddit(
|
|||||||
to_date: str,
|
to_date: str,
|
||||||
) -> schema.SourceItem:
|
) -> schema.SourceItem:
|
||||||
top_comments = item.get("top_comments") or []
|
top_comments = item.get("top_comments") or []
|
||||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
comment_text = " ".join(
|
||||||
|
str(comment.get("excerpt") or "").strip()
|
||||||
|
for comment in top_comments[:3]
|
||||||
|
if isinstance(comment, dict)
|
||||||
|
)
|
||||||
body = "\n".join(
|
body = "\n".join(
|
||||||
part
|
part
|
||||||
for part in [
|
for part in [
|
||||||
@@ -251,11 +241,6 @@ def _normalize_youtube(
|
|||||||
metadata: dict[str, Any] = {}
|
metadata: dict[str, Any] = {}
|
||||||
if highlights:
|
if highlights:
|
||||||
metadata["transcript_highlights"] = highlights
|
metadata["transcript_highlights"] = highlights
|
||||||
if item.get("captions_disabled"):
|
|
||||||
# Surfaced for quality_nudge: uploader disabled captions, so this
|
|
||||||
# video should be subtracted from the degraded-transcript-ratio
|
|
||||||
# denominator (it was never going to produce a transcript).
|
|
||||||
metadata["captions_disabled"] = True
|
|
||||||
metadata["top_comments"] = _remap_comments(
|
metadata["top_comments"] = _remap_comments(
|
||||||
item.get("top_comments") or [],
|
item.get("top_comments") or [],
|
||||||
score_keys=("score", "likes"),
|
score_keys=("score", "likes"),
|
||||||
@@ -353,7 +338,11 @@ def _normalize_hackernews(
|
|||||||
to_date: str,
|
to_date: str,
|
||||||
) -> schema.SourceItem:
|
) -> schema.SourceItem:
|
||||||
top_comments = item.get("top_comments") or []
|
top_comments = item.get("top_comments") or []
|
||||||
comment_text = _join_comment_excerpts(top_comments, "text")
|
comment_text = " ".join(
|
||||||
|
str(comment.get("text") or "").strip()
|
||||||
|
for comment in top_comments[:3]
|
||||||
|
if isinstance(comment, dict)
|
||||||
|
)
|
||||||
title = str(item.get("title") or "").strip()
|
title = str(item.get("title") or "").strip()
|
||||||
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
||||||
return _source_item(
|
return _source_item(
|
||||||
@@ -405,53 +394,6 @@ def _normalize_microblog(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _normalize_digg(
|
|
||||||
source: str,
|
|
||||||
item: dict[str, Any],
|
|
||||||
index: int,
|
|
||||||
from_date: str,
|
|
||||||
to_date: str,
|
|
||||||
) -> schema.SourceItem:
|
|
||||||
"""Normalizer for Digg AI 1000 clusters.
|
|
||||||
|
|
||||||
Each cluster is one item. The TLDR carries the most useful body for
|
|
||||||
rerank and synthesis. Top-ranked X posts attached at search time are
|
|
||||||
passed through under metadata['posts'] so render can emit them as
|
|
||||||
inline 'via Digg' quotes.
|
|
||||||
"""
|
|
||||||
title = str(item.get("title") or "").strip()
|
|
||||||
tldr = str(item.get("tldr") or "").strip()
|
|
||||||
body = "\n\n".join(part for part in [title, tldr] if part)
|
|
||||||
posts = item.get("posts") or []
|
|
||||||
if not isinstance(posts, list):
|
|
||||||
posts = []
|
|
||||||
cluster_url_id = str(item.get("id") or f"DG{index + 1}")
|
|
||||||
return _source_item(
|
|
||||||
item_id=cluster_url_id,
|
|
||||||
source=source,
|
|
||||||
title=title or f"Digg cluster {index + 1}",
|
|
||||||
body=body,
|
|
||||||
url=str(item.get("url") or f"https://di.gg/ai/{cluster_url_id}"),
|
|
||||||
author="",
|
|
||||||
container="Digg",
|
|
||||||
published_at=item.get("date"),
|
|
||||||
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
|
|
||||||
engagement=item.get("engagement") or {},
|
|
||||||
relevance_hint=item.get("relevance", 0.5),
|
|
||||||
why_relevant=str(item.get("why_relevant") or ""),
|
|
||||||
snippet=tldr[:400],
|
|
||||||
metadata={
|
|
||||||
"clusterUrlId": cluster_url_id,
|
|
||||||
"tldr": tldr,
|
|
||||||
"rank": (item.get("engagement") or {}).get("rank"),
|
|
||||||
"uniqueAuthors": (item.get("engagement") or {}).get("uniqueAuthors"),
|
|
||||||
"postCount": (item.get("engagement") or {}).get("postCount"),
|
|
||||||
"firstPostAge": item.get("first_post_age"),
|
|
||||||
"posts": posts,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_polymarket(
|
def _normalize_polymarket(
|
||||||
source: str,
|
source: str,
|
||||||
item: dict[str, Any],
|
item: dict[str, Any],
|
||||||
@@ -499,7 +441,11 @@ def _normalize_github(
|
|||||||
title = str(item.get("title") or "").strip()
|
title = str(item.get("title") or "").strip()
|
||||||
snippet_text = str(item.get("snippet") or "").strip()
|
snippet_text = str(item.get("snippet") or "").strip()
|
||||||
top_comments = item.get("metadata", {}).get("top_comments") or []
|
top_comments = item.get("metadata", {}).get("top_comments") or []
|
||||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
comment_text = " ".join(
|
||||||
|
str(comment.get("excerpt") or "").strip()
|
||||||
|
for comment in top_comments[:3]
|
||||||
|
if isinstance(comment, dict)
|
||||||
|
)
|
||||||
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
||||||
metadata = item.get("metadata") or {}
|
metadata = item.get("metadata") or {}
|
||||||
return _source_item(
|
return _source_item(
|
||||||
@@ -11,6 +11,11 @@ import re
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests as _requests
|
||||||
|
except ImportError:
|
||||||
|
_requests = None
|
||||||
|
|
||||||
from . import dates, http, log
|
from . import dates, http, log
|
||||||
|
|
||||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
|
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
|
||||||
@@ -135,17 +140,31 @@ def search_pinterest(
|
|||||||
|
|
||||||
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||||
|
|
||||||
try:
|
if not _requests:
|
||||||
data = http.get(
|
_log("requests library not installed, falling back to urllib")
|
||||||
f"{SCRAPECREATORS_BASE}/search",
|
try:
|
||||||
params={"keyword": core_topic},
|
from urllib.parse import urlencode
|
||||||
headers=http.scrapecreators_headers(token),
|
params = urlencode({"keyword": core_topic})
|
||||||
timeout=30,
|
url = f"{SCRAPECREATORS_BASE}/search?{params}"
|
||||||
retries=2,
|
headers = http.scrapecreators_headers(token)
|
||||||
)
|
headers["User-Agent"] = http.USER_AGENT
|
||||||
except Exception as e:
|
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||||
_log(f"ScrapeCreators error: {e}")
|
except Exception as e:
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
_log(f"ScrapeCreators error (urllib): {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
resp = _requests.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/search",
|
||||||
|
params={"keyword": core_topic},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"ScrapeCreators error: {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
|
|
||||||
# Extract items from response - try common SC response shapes
|
# Extract items from response - try common SC response shapes
|
||||||
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
|
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
|
||||||
@@ -15,7 +15,6 @@ from . import (
|
|||||||
bluesky,
|
bluesky,
|
||||||
dates,
|
dates,
|
||||||
dedupe,
|
dedupe,
|
||||||
digg,
|
|
||||||
entity_extract,
|
entity_extract,
|
||||||
env,
|
env,
|
||||||
github,
|
github,
|
||||||
@@ -31,7 +30,6 @@ from . import (
|
|||||||
query,
|
query,
|
||||||
reddit,
|
reddit,
|
||||||
reddit_public,
|
reddit_public,
|
||||||
relevance,
|
|
||||||
rerank,
|
rerank,
|
||||||
schema,
|
schema,
|
||||||
signals,
|
signals,
|
||||||
@@ -79,10 +77,7 @@ MOCK_AVAILABLE_SOURCES = [
|
|||||||
"xiaohongshu",
|
"xiaohongshu",
|
||||||
"github",
|
"github",
|
||||||
"perplexity",
|
"perplexity",
|
||||||
"threads",
|
|
||||||
"pinterest",
|
|
||||||
"xquik",
|
"xquik",
|
||||||
"digg",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -110,8 +105,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
|||||||
available.extend(["hackernews", "polymarket"])
|
available.extend(["hackernews", "polymarket"])
|
||||||
if config.get("GITHUB_TOKEN") or which("gh"):
|
if config.get("GITHUB_TOKEN") or which("gh"):
|
||||||
available.append("github")
|
available.append("github")
|
||||||
if which("digg-pp-cli"):
|
|
||||||
available.append("digg")
|
|
||||||
if env.is_bluesky_available(config):
|
if env.is_bluesky_available(config):
|
||||||
available.append("bluesky")
|
available.append("bluesky")
|
||||||
if env.is_truthsocial_available(config):
|
if env.is_truthsocial_available(config):
|
||||||
@@ -120,9 +113,7 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
|||||||
available.append("grounding")
|
available.append("grounding")
|
||||||
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
|
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
|
||||||
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
|
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
|
||||||
if config.get("OPENROUTER_API_KEY") and (
|
if config.get("OPENROUTER_API_KEY") and "perplexity" in include_sources:
|
||||||
"perplexity" in include_sources or (requested_sources and "perplexity" in requested_sources)
|
|
||||||
):
|
|
||||||
available.append("perplexity")
|
available.append("perplexity")
|
||||||
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
|
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
|
||||||
available.append("xiaohongshu")
|
available.append("xiaohongshu")
|
||||||
@@ -132,9 +123,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
|||||||
available.append("pinterest")
|
available.append("pinterest")
|
||||||
if env.is_xquik_available(config):
|
if env.is_xquik_available(config):
|
||||||
available.append("xquik")
|
available.append("xquik")
|
||||||
exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()}
|
|
||||||
if exclude:
|
|
||||||
available = [s for s in available if s not in exclude]
|
|
||||||
return available
|
return available
|
||||||
|
|
||||||
|
|
||||||
@@ -207,7 +195,7 @@ def run(
|
|||||||
available = [source for source in available if source in requested_sources]
|
available = [source for source in available if source in requested_sources]
|
||||||
if web_backend == "none":
|
if web_backend == "none":
|
||||||
available = [s for s in available if s != "grounding"]
|
available = [s for s in available if s != "grounding"]
|
||||||
elif web_backend in ("brave", "exa", "serper", "parallel") and "grounding" not in available:
|
elif web_backend in ("brave", "exa", "serper") and "grounding" not in available:
|
||||||
available.append("grounding")
|
available.append("grounding")
|
||||||
if not available:
|
if not available:
|
||||||
raise RuntimeError("No sources are available for this run.")
|
raise RuntimeError("No sources are available for this run.")
|
||||||
@@ -512,12 +500,11 @@ def _normalize_score_dedupe(
|
|||||||
source, raw_items, from_date, to_date,
|
source, raw_items, from_date, to_date,
|
||||||
freshness_mode=freshness_mode,
|
freshness_mode=freshness_mode,
|
||||||
)
|
)
|
||||||
prepared_query = relevance.PreparedQuery(ranking_query)
|
normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
|
||||||
normalized = signals.annotate_stream(normalized, prepared_query, freshness_mode)
|
|
||||||
normalized = signals.prune_low_relevance(normalized)
|
normalized = signals.prune_low_relevance(normalized)
|
||||||
normalized = dedupe.dedupe_items(normalized)
|
normalized = dedupe.dedupe_items(normalized)
|
||||||
for item in normalized:
|
for item in normalized:
|
||||||
item.snippet = snippet.extract_best_snippet(item, prepared_query)
|
item.snippet = snippet.extract_best_snippet(item, ranking_query)
|
||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
@@ -542,12 +529,6 @@ def _finalize_items_by_source(
|
|||||||
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
|
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
|
||||||
if keywords:
|
if keywords:
|
||||||
items = polymarket.filter_items_against_keywords(items, keywords)
|
items = polymarket.filter_items_against_keywords(items, keywords)
|
||||||
if source == "digg" and items:
|
|
||||||
# Pull top-ranked X posts only for the survivors that will appear
|
|
||||||
# in the brief. Spending the enrichment budget here (rather than
|
|
||||||
# at retrieval time) keeps the inline 'via Digg' quotes
|
|
||||||
# paired with the clusters dedupe actually kept.
|
|
||||||
digg.enrich_source_items(items, top_k=3)
|
|
||||||
finalized[source] = items
|
finalized[source] = items
|
||||||
return finalized
|
return finalized
|
||||||
|
|
||||||
@@ -983,13 +964,6 @@ def _retrieve_stream(
|
|||||||
if source == "hackernews":
|
if source == "hackernews":
|
||||||
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
||||||
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
||||||
if source == "digg":
|
|
||||||
result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
|
|
||||||
items = digg.parse_digg_response(result, query=subquery.search_query)
|
|
||||||
# Enrichment with attached X posts is deferred to
|
|
||||||
# _finalize_items_by_source so it runs on the items that actually
|
|
||||||
# survive dedupe rather than on top-K of the raw fanout.
|
|
||||||
return items, {}
|
|
||||||
if source == "bluesky":
|
if source == "bluesky":
|
||||||
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
||||||
return bluesky.parse_bluesky_response(result), {}
|
return bluesky.parse_bluesky_response(result), {}
|
||||||
@@ -1082,45 +1056,6 @@ def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[d
|
|||||||
"why_relevant": "Brave web search",
|
"why_relevant": "Brave web search",
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"digg": [
|
|
||||||
{
|
|
||||||
"id": "mock1abc",
|
|
||||||
"title": f"Digg cluster about {subquery.search_query}",
|
|
||||||
"url": "https://di.gg/ai/mock1abc",
|
|
||||||
"tldr": f"Curated cluster summarizing recent {subquery.search_query} discussion across the AI 1000.",
|
|
||||||
"author": "",
|
|
||||||
"date": dates.get_date_range(3)[0],
|
|
||||||
"engagement": {"postCount": 8, "uniqueAuthors": 5, "rank": 2, "rank_score": 49.0},
|
|
||||||
"first_post_age": "3d",
|
|
||||||
"posts": [
|
|
||||||
{
|
|
||||||
"username": "exampledev",
|
|
||||||
"display_name": "Example Dev",
|
|
||||||
"category": "Engineer",
|
|
||||||
"rank": 142,
|
|
||||||
"body": f"Quote from the AI 1000 about {subquery.search_query}.",
|
|
||||||
"post_type": "tweet",
|
|
||||||
"x_url": "https://x.com/exampledev/status/1",
|
|
||||||
"posted_at": dates.get_date_range(3)[0],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
"relevance": 0.84,
|
|
||||||
"why_relevant": "Mock Digg cluster",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "mock2def",
|
|
||||||
"title": f"Second Digg cluster on {subquery.search_query}",
|
|
||||||
"url": "https://di.gg/ai/mock2def",
|
|
||||||
"tldr": f"Another angle on {subquery.search_query}.",
|
|
||||||
"author": "",
|
|
||||||
"date": dates.get_date_range(8)[0],
|
|
||||||
"engagement": {"postCount": 3, "uniqueAuthors": 2, "rank": 18, "rank_score": 33.0},
|
|
||||||
"first_post_age": "8d",
|
|
||||||
"posts": [],
|
|
||||||
"relevance": 0.71,
|
|
||||||
"why_relevant": "Mock Digg cluster",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
}
|
||||||
if source == "grounding":
|
if source == "grounding":
|
||||||
return payloads.get(source, []), {
|
return payloads.get(source, []), {
|
||||||
@@ -19,14 +19,14 @@ ALLOWED_INTENTS = {
|
|||||||
}
|
}
|
||||||
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
|
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
|
||||||
QUICK_SOURCE_PRIORITY = {
|
QUICK_SOURCE_PRIORITY = {
|
||||||
"factual": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||||
"product": ["youtube", "reddit", "x", "xquik", "tiktok"],
|
"product": ["youtube", "reddit", "x", "tiktok"],
|
||||||
"concept": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
"concept": ["hackernews", "reddit", "x", "youtube"],
|
||||||
"opinion": ["reddit", "x", "xquik", "youtube", "hackernews"],
|
"opinion": ["reddit", "x", "youtube", "hackernews"],
|
||||||
"how_to": ["youtube", "reddit", "x", "xquik", "hackernews"],
|
"how_to": ["youtube", "reddit", "x", "hackernews"],
|
||||||
"comparison": ["reddit", "x", "xquik", "hackernews", "youtube"],
|
"comparison": ["reddit", "x", "hackernews", "youtube"],
|
||||||
"breaking_news": ["x", "xquik", "reddit", "hackernews", "youtube", "polymarket"],
|
"breaking_news": ["x", "reddit", "hackernews", "youtube", "polymarket"],
|
||||||
"prediction": ["polymarket", "x", "xquik", "hackernews", "reddit", "youtube"],
|
"prediction": ["polymarket", "x", "hackernews", "reddit", "youtube"],
|
||||||
}
|
}
|
||||||
SOURCE_PRIORITY = {
|
SOURCE_PRIORITY = {
|
||||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||||
@@ -60,7 +60,6 @@ INTENT_SOURCE_EXCLUSIONS: dict[str, set[str]] = {
|
|||||||
SOURCE_CAPABILITIES = {
|
SOURCE_CAPABILITIES = {
|
||||||
"reddit": {"discussion", "social"},
|
"reddit": {"discussion", "social"},
|
||||||
"x": {"discussion", "social"},
|
"x": {"discussion", "social"},
|
||||||
"xquik": {"discussion", "social"},
|
|
||||||
"youtube": {"video", "video_longform", "discussion"},
|
"youtube": {"video", "video_longform", "discussion"},
|
||||||
"tiktok": {"video", "video_shortform", "social"},
|
"tiktok": {"video", "video_shortform", "social"},
|
||||||
"instagram": {"video", "video_shortform", "social"},
|
"instagram": {"video", "video_shortform", "social"},
|
||||||
@@ -68,7 +67,6 @@ SOURCE_CAPABILITIES = {
|
|||||||
"bluesky": {"discussion", "social"},
|
"bluesky": {"discussion", "social"},
|
||||||
"truthsocial": {"discussion", "social"},
|
"truthsocial": {"discussion", "social"},
|
||||||
"polymarket": {"market"},
|
"polymarket": {"market"},
|
||||||
"digg": {"discussion", "social", "link"},
|
|
||||||
"xiaohongshu": {"video", "video_shortform", "social"},
|
"xiaohongshu": {"video", "video_shortform", "social"},
|
||||||
"github": {"discussion", "link"},
|
"github": {"discussion", "link"},
|
||||||
"grounding": {"web", "reference", "link"},
|
"grounding": {"web", "reference", "link"},
|
||||||
@@ -9,7 +9,7 @@ from typing import Any
|
|||||||
|
|
||||||
from . import env, http, schema
|
from . import env, http, schema
|
||||||
|
|
||||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite"
|
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite-preview"
|
||||||
GEMINI_PRO = "gemini-3.1-pro-preview"
|
GEMINI_PRO = "gemini-3.1-pro-preview"
|
||||||
OPENAI_DEFAULT = "gpt-5.4-nano"
|
OPENAI_DEFAULT = "gpt-5.4-nano"
|
||||||
XAI_DEFAULT = "grok-4-1-fast"
|
XAI_DEFAULT = "grok-4-1-fast"
|
||||||
@@ -19,11 +19,7 @@ OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
|||||||
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
|
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
|
||||||
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
|
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
|
||||||
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||||
# OpenRouter routes the Gemini Flash Lite tier as the -preview slug; that is the
|
OPENROUTER_DEFAULT = "google/gemini-flash-2.0"
|
||||||
# stable form on that routing layer even though native Gemini's GEMINI_FLASH_LITE
|
|
||||||
# constant is suffix-free. If GEMINI_FLASH_LITE moves to a non-preview stable ID,
|
|
||||||
# double-check that OpenRouter's slug still maps to the same upstream model.
|
|
||||||
OPENROUTER_DEFAULT = "google/gemini-3.1-flash-lite-preview"
|
|
||||||
|
|
||||||
|
|
||||||
class ReasoningClient:
|
class ReasoningClient:
|
||||||
@@ -97,6 +93,13 @@ class GeminiClient(ReasoningClient):
|
|||||||
)
|
)
|
||||||
return extract_gemini_text(payload)
|
return extract_gemini_text(payload)
|
||||||
|
|
||||||
|
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
|
||||||
|
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
|
||||||
|
|
||||||
|
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
|
||||||
|
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
|
||||||
|
|
||||||
|
|
||||||
class OpenAIClient(ReasoningClient):
|
class OpenAIClient(ReasoningClient):
|
||||||
name = "openai"
|
name = "openai"
|
||||||
|
|
||||||
@@ -236,8 +239,8 @@ def _resolve_model_pins(config: dict[str, Any], depth: str, provider_name: str)
|
|||||||
rerank_model = config.get("LAST30DAYS_RERANK_MODEL") or default_rerank
|
rerank_model = config.get("LAST30DAYS_RERANK_MODEL") or default_rerank
|
||||||
|
|
||||||
if provider_name == "gemini":
|
if provider_name == "gemini":
|
||||||
_require_gemini_31(planner_model, role="planner")
|
_require_gemini_31_preview(planner_model, role="planner")
|
||||||
_require_gemini_31(rerank_model, role="rerank")
|
_require_gemini_31_preview(rerank_model, role="rerank")
|
||||||
|
|
||||||
return planner_model, rerank_model
|
return planner_model, rerank_model
|
||||||
|
|
||||||
@@ -348,11 +351,11 @@ def _resolve_x_backend(config: dict[str, Any]) -> str | None:
|
|||||||
return env.get_x_source(config)
|
return env.get_x_source(config)
|
||||||
|
|
||||||
|
|
||||||
def _require_gemini_31(model: str, *, role: str) -> None:
|
def _require_gemini_31_preview(model: str, *, role: str) -> None:
|
||||||
if model.startswith("gemini-3.1-"):
|
if model.startswith("gemini-3.1-") and model.endswith("-preview"):
|
||||||
return
|
return
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"{role} must use a Gemini 3.1 model. Got: {model}"
|
f"{role} must use a Gemini 3.1 preview model. Got: {model}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
"""Post-research quality score and upgrade nudge.
|
||||||
|
|
||||||
|
Computes a quality score based on 5 core sources and builds
|
||||||
|
a nudge message describing what the user missed and how to fix it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
|
||||||
|
# The 5 core sources
|
||||||
|
CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit"]
|
||||||
|
|
||||||
|
# Labels for display
|
||||||
|
SOURCE_LABELS = {
|
||||||
|
"hn": "Hacker News",
|
||||||
|
"polymarket": "Polymarket",
|
||||||
|
"x": "X/Twitter",
|
||||||
|
"youtube": "YouTube",
|
||||||
|
"reddit": "Reddit",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_x_active(config: dict, research_results: dict) -> bool:
|
||||||
|
"""Check if X source is active (has credentials AND didn't error)."""
|
||||||
|
has_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||||
|
if not has_creds:
|
||||||
|
return False
|
||||||
|
# If X errored this run, it's configured but broken
|
||||||
|
if research_results.get("x_error"):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _is_youtube_active(config: dict, research_results: dict) -> bool:
|
||||||
|
"""Check if YouTube source is active (yt-dlp installed)."""
|
||||||
|
try:
|
||||||
|
from . import youtube_yt
|
||||||
|
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||||
|
except Exception:
|
||||||
|
has_ytdlp = False
|
||||||
|
if not has_ytdlp:
|
||||||
|
return False
|
||||||
|
if research_results.get("youtube_error"):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||||
|
"""Compute research quality score based on 5 core sources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
config: Configuration dict from env.get_config()
|
||||||
|
research_results: Dict with keys like x_error, youtube_error,
|
||||||
|
reddit_error reflecting what happened this run.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
"score_pct": 40-100,
|
||||||
|
"core_active": ["hn", "polymarket", ...],
|
||||||
|
"core_missing": ["x", "youtube"],
|
||||||
|
"core_errored": [], # configured but errored
|
||||||
|
"nudge_text": "..." or None if 100%
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
core_active: List[str] = []
|
||||||
|
core_missing: List[str] = []
|
||||||
|
core_errored: List[str] = []
|
||||||
|
|
||||||
|
# HN, Polymarket, and Reddit are always active
|
||||||
|
core_active.append("hn")
|
||||||
|
core_active.append("polymarket")
|
||||||
|
core_active.append("reddit")
|
||||||
|
|
||||||
|
# X
|
||||||
|
has_x_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||||
|
if _is_x_active(config, research_results):
|
||||||
|
core_active.append("x")
|
||||||
|
else:
|
||||||
|
core_missing.append("x")
|
||||||
|
if has_x_creds and research_results.get("x_error"):
|
||||||
|
core_errored.append("x")
|
||||||
|
|
||||||
|
# YouTube
|
||||||
|
yt_active = _is_youtube_active(config, research_results)
|
||||||
|
if yt_active:
|
||||||
|
core_active.append("youtube")
|
||||||
|
else:
|
||||||
|
core_missing.append("youtube")
|
||||||
|
# Check if configured but errored (yt-dlp installed but failed this run)
|
||||||
|
try:
|
||||||
|
from . import youtube_yt
|
||||||
|
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||||
|
except Exception:
|
||||||
|
has_ytdlp = False
|
||||||
|
if has_ytdlp and research_results.get("youtube_error"):
|
||||||
|
core_errored.append("youtube")
|
||||||
|
|
||||||
|
score_pct = int(len(core_active) / 5 * 100)
|
||||||
|
|
||||||
|
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||||
|
active_sources = research_results.get("active_sources") or []
|
||||||
|
nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"score_pct": score_pct,
|
||||||
|
"core_active": core_active,
|
||||||
|
"core_missing": core_missing,
|
||||||
|
"core_errored": core_errored,
|
||||||
|
"nudge_text": nudge_text,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str:
|
||||||
|
"""Build human-readable nudge text describing what was missed.
|
||||||
|
|
||||||
|
Prioritizes free suggestions. Optionally mentions bonus sources
|
||||||
|
(TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured.
|
||||||
|
"""
|
||||||
|
lines: List[str] = []
|
||||||
|
|
||||||
|
# Describe what was missed
|
||||||
|
missed_parts: List[str] = []
|
||||||
|
for src in core_missing:
|
||||||
|
label = SOURCE_LABELS[src]
|
||||||
|
if src in core_errored:
|
||||||
|
missed_parts.append(f"{label} (errored this run)")
|
||||||
|
else:
|
||||||
|
missed_parts.append(label)
|
||||||
|
|
||||||
|
active_count = 5 - len(core_missing)
|
||||||
|
lines.append(f"Research quality: {active_count}/5 core sources.")
|
||||||
|
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# Free suggestions
|
||||||
|
free_suggestions: List[str] = []
|
||||||
|
|
||||||
|
if "x" in core_missing:
|
||||||
|
if "x" in core_errored:
|
||||||
|
free_suggestions.append(
|
||||||
|
"X/Twitter errored - log into x.com in your browser, then re-run."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
free_suggestions.append(
|
||||||
|
"X/Twitter: real-time posts with likes and reposts - the fastest "
|
||||||
|
"signal for breaking topics. Two options: log into x.com in your "
|
||||||
|
"browser and re-run (cookies detected automatically), or add "
|
||||||
|
"XAI_API_KEY to your .env (no browser access, get key at api.x.ai)."
|
||||||
|
)
|
||||||
|
|
||||||
|
if "youtube" in core_missing:
|
||||||
|
if "youtube" in core_errored:
|
||||||
|
free_suggestions.append(
|
||||||
|
"YouTube errored - update yt-dlp: brew upgrade yt-dlp"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
free_suggestions.append(
|
||||||
|
"YouTube: video transcripts with key moments - often the deepest "
|
||||||
|
"explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Mention bonus opt-in sources when SC key is present
|
||||||
|
if has_sc:
|
||||||
|
bonus_hints = []
|
||||||
|
if "threads" not in (active_sources or []):
|
||||||
|
bonus_hints.append("Threads")
|
||||||
|
if "pinterest" not in (active_sources or []):
|
||||||
|
bonus_hints.append("Pinterest")
|
||||||
|
if bonus_hints:
|
||||||
|
free_suggestions.append(
|
||||||
|
f"Your SC key also powers {', '.join(bonus_hints)} and YouTube comments. "
|
||||||
|
"Add them to INCLUDE_SOURCES in your .env to enable."
|
||||||
|
)
|
||||||
|
|
||||||
|
if free_suggestions:
|
||||||
|
lines.append("Free fixes:")
|
||||||
|
for s in free_suggestions:
|
||||||
|
lines.append(f" - {s}")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# Bonus sources mention (non-blocking)
|
||||||
|
if not has_sc:
|
||||||
|
lines.append(
|
||||||
|
"Bonus: TikTok and Instagram are available with a free "
|
||||||
|
"ScrapeCreators key at scrapecreators.com (no affiliation)."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
lines.append("last30days has no affiliation with any API provider.")
|
||||||
|
|
||||||
|
return "\n".join(lines)
|
||||||
@@ -334,7 +334,7 @@ def _global_search(
|
|||||||
)
|
)
|
||||||
return data.get("posts", data.get("data", []))
|
return data.get("posts", data.get("data", []))
|
||||||
except http.HTTPError as e:
|
except http.HTTPError as e:
|
||||||
if e.status_code in (401, 402, 403):
|
if e.status_code in (401, 403):
|
||||||
raise
|
raise
|
||||||
_log(f"Global search error: {e}")
|
_log(f"Global search error: {e}")
|
||||||
return []
|
return []
|
||||||
@@ -376,11 +376,6 @@ def _subreddit_search(
|
|||||||
retries=2,
|
retries=2,
|
||||||
)
|
)
|
||||||
return data.get("posts", data.get("data", []))
|
return data.get("posts", data.get("data", []))
|
||||||
except http.HTTPError as e:
|
|
||||||
if e.status_code in (401, 402, 403):
|
|
||||||
raise
|
|
||||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||||
return []
|
return []
|
||||||
@@ -408,11 +403,6 @@ def fetch_post_comments(
|
|||||||
retries=2,
|
retries=2,
|
||||||
)
|
)
|
||||||
return data.get("comments", data.get("data", []))
|
return data.get("comments", data.get("data", []))
|
||||||
except http.HTTPError as e:
|
|
||||||
if e.status_code in (401, 402, 403):
|
|
||||||
raise
|
|
||||||
_log(f"Comment fetch error: {e}")
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Comment fetch error: {e}")
|
_log(f"Comment fetch error: {e}")
|
||||||
return []
|
return []
|
||||||