Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 61d46b54ee | |||
| e7b7e61237 | |||
| a547a0a948 | |||
| 8ea048b988 | |||
| 1b23a3e900 | |||
| 35f12cb9ea | |||
| a1afbce84c | |||
| 15781bfcc0 |
@@ -11,7 +11,7 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.3.0",
|
||||
"version": "3.2.3",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.3.0",
|
||||
"version": "3.2.3",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
|
||||
@@ -16,7 +16,7 @@ body:
|
||||
label: Steps to Reproduce
|
||||
description: How can we reproduce this?
|
||||
placeholder: |
|
||||
1. Run `python3 skills/last30days/scripts/last30days.py "topic" --emit=compact`
|
||||
1. Run `python3 scripts/last30days.py "topic" --emit compact`
|
||||
2. ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
@@ -9,7 +9,10 @@ permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build-and-release:
|
||||
# Build the existing .skill artifact (Claude Code / Codex / Cursor install
|
||||
# surface). Unchanged from prior versions; just isolated into its own job
|
||||
# so the .mcpb matrix can run in parallel.
|
||||
build-skill:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -22,10 +25,101 @@ jobs:
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Upload skill artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: last30days-skill
|
||||
path: dist/last30days.skill
|
||||
|
||||
# Cross-compile the Go MCP server for each Claude Desktop platform and
|
||||
# package each as a .mcpb. printing-press bundle handles the manifest +
|
||||
# zip layout; we only supply the pre-built binary via --skip-build.
|
||||
build-mcpb:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- goos: darwin
|
||||
goarch: arm64
|
||||
platform: darwin/arm64
|
||||
- goos: darwin
|
||||
goarch: amd64
|
||||
platform: darwin/amd64
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
platform: linux/amd64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: stable
|
||||
|
||||
- name: Install printing-press
|
||||
# Pin to a known-good PP release so the bundle command's behavior
|
||||
# is deterministic across our tags. Bump deliberately when adopting
|
||||
# a newer PP version. GOSUMDB=off skips the sumdb 404 some
|
||||
# private-namespaced go install calls hit even when the repo is
|
||||
# public; harmless here because the module path is fully qualified.
|
||||
env:
|
||||
GOPRIVATE: github.com/mvanhorn/*
|
||||
GOSUMDB: "off"
|
||||
run: go install github.com/mvanhorn/cli-printing-press/v4/cmd/printing-press@v4.8.0
|
||||
|
||||
- name: Sync engine into vendored/
|
||||
run: bash mcp/scripts/sync-engine.sh
|
||||
|
||||
- name: Build MCP binary
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: "0"
|
||||
run: |
|
||||
mkdir -p mcp/build
|
||||
go -C mcp build \
|
||||
-ldflags "-X main.Version=${{ github.ref_name }}" \
|
||||
-o build/last30days-pp-mcp \
|
||||
./cmd/last30days-pp-mcp
|
||||
|
||||
- name: Bundle .mcpb
|
||||
# printing-press bundle reads manifest.json from the cli dir and
|
||||
# rewrites the binary into bin/<entry_point> inside the zip. The
|
||||
# --platform tag drives the output filename suffix; the binary
|
||||
# itself is whatever we just cross-compiled.
|
||||
run: |
|
||||
printing-press bundle mcp \
|
||||
--skip-build \
|
||||
--binary mcp/build/last30days-pp-mcp \
|
||||
--platform ${{ matrix.platform }} \
|
||||
--output mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
|
||||
- name: Upload .mcpb artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: mcpb-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: mcp/build/last30days-pp-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.mcpb
|
||||
|
||||
# Gather every platform artifact and attach to one GitHub release.
|
||||
# release-notes generation reads commits since the prior tag.
|
||||
release:
|
||||
needs: [build-skill, build-mcpb]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
|
||||
- name: Create GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: dist/last30days.skill
|
||||
files: |
|
||||
dist/last30days.skill
|
||||
dist/last30days-pp-mcp-*.mcpb
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
name: Security
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
name: Dependency audit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Export locked dependency set
|
||||
run: |
|
||||
uv export \
|
||||
--locked \
|
||||
--all-groups \
|
||||
--no-hashes \
|
||||
--format requirements.txt \
|
||||
--output-file /tmp/last30days-requirements.txt
|
||||
|
||||
# Advisory-first: visibility before enforcement. This repo handles API keys,
|
||||
# cookies, browser tokens, and local env files, so dependency CVEs should be
|
||||
# visible in CI logs even before the project has a clean blocking baseline.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
- name: Run pip-audit against locked dependencies
|
||||
continue-on-error: true
|
||||
run: uvx --python 3.12 pip-audit -r /tmp/last30days-requirements.txt --progress-spinner=off
|
||||
|
||||
secret-scan:
|
||||
name: Secret scan
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout full history for diff-aware scanning
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Advisory-first: this reports verified secrets in pull requests and pushes to
|
||||
# main, but does not block merges until maintainers confirm a clean baseline.
|
||||
# The TruffleHog action automatically scans the PR range for pull_request
|
||||
# events and the pushed commit range for push events.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
# Contributor policy: never commit real secrets in fixtures, tests, docs, or
|
||||
# examples; use obvious dummy values and env-based auth patterns instead.
|
||||
- name: Run TruffleHog OSS secret scan
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
uses: trufflesecurity/trufflehog@v3.95.2
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: ./
|
||||
version: v3.95.2
|
||||
extra_args: --only-verified
|
||||
@@ -10,7 +10,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
plugin-contract:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -22,5 +22,5 @@ jobs:
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run test suite
|
||||
run: uv run pytest
|
||||
- name: Run plugin contract tests
|
||||
run: uv run pytest tests/test_plugin_contract.py tests/test_version_consistency.py
|
||||
|
||||
+9
-4
@@ -26,9 +26,14 @@ htmlcov/
|
||||
# build artifact from scripts/build-skill.sh
|
||||
/dist/
|
||||
|
||||
# Go MCP bundle build outputs - source of truth for vendored/ stays under
|
||||
# skills/last30days/scripts/; build/ holds cross-compiled binaries + .mcpb files.
|
||||
# vendored/ lives inside the engine package because //go:embed cannot reach
|
||||
# outside its own package directory; the .gitkeep anchor stays tracked so
|
||||
# the embed pattern always finds a match even before sync-engine runs.
|
||||
/mcp/internal/engine/vendored/*
|
||||
!/mcp/internal/engine/vendored/.gitkeep
|
||||
/mcp/build/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
.context/
|
||||
|
||||
/work
|
||||
/print
|
||||
|
||||
@@ -1,54 +1 @@
|
||||
# last30days Skill
|
||||
|
||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
||||
|
||||
## Orientation
|
||||
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
||||
- Feature design starts from the slash-command UX. A new engine flag with no SKILL.md integration is incomplete — the model invoking the skill won't know the flag exists.
|
||||
- README and PR examples show `/last30days <topic>` first. Direct CLI invocation (`python3 scripts/last30days.py ...`) is a fallback for scripting, cron, and dev-time engine testing; label it as such, never as the primary path.
|
||||
- Slash commands don't pass shell mechanics through. `/last30days OpenClaw --emit=html | pbcopy` is invalid in any harness — either use the slash form (no flags or pipes; let the model translate user intent into engine flags) or use the direct CLI form (full `python3 ...` with explicit flags and a real shell).
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Security hygiene
|
||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
||||
- Use the env-based auth patterns in `skills/last30days/scripts/lib/env.py`; tests and fixtures must use obvious dummy values only.
|
||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
||||
|
||||
## Maintaining CONFIGURATION.md
|
||||
|
||||
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
|
||||
|
||||
Update `CONFIGURATION.md` when:
|
||||
|
||||
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
|
||||
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
|
||||
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Hermes…)
|
||||
- adding a new optional source that requires its own credential
|
||||
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
|
||||
|
||||
Keep the existing structure organized by how often each layer is touched: per-run flags → env vars / `.env` → optional trend-monitoring stack → per-client patterns. Add new content into the right section rather than appending at the end.
|
||||
|
||||
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
@CLAUDE.md
|
||||
+7
-141
@@ -7,152 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [3.3.0] - 2026-05-17
|
||||
|
||||
A week-long shipping cycle: ~75 PRs merged plus 7 community fixes salvaged through PR triage. Big themes: install story modernized for the multi-harness world (Claude Code, Codex, Cursor, Gemini CLI, Copilot, Windsurf, and 50+ Agent Skills hosts), new emit and source modes, and a substantial reliability sweep across Reddit, X, Windows, YouTube, and the planner.
|
||||
|
||||
### Added
|
||||
|
||||
**Emit modes and sources**
|
||||
|
||||
- `--emit=html` for shareable, print-friendly HTML research briefs ([#332](https://github.com/mvanhorn/last30days-skill/pull/332)).
|
||||
- **Digg AI 1000 source**, auto-enabled when `digg-pp-cli` is on PATH ([#370](https://github.com/mvanhorn/last30days-skill/pull/370)). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief.
|
||||
|
||||
**Configuration knobs**
|
||||
|
||||
- `EXCLUDE_SOURCES` env var — the inverse of `INCLUDE_SOURCES`, honored in source count and pipeline filter ([#399](https://github.com/mvanhorn/last30days-skill/pull/399)).
|
||||
- `LAST30DAYS_YOUTUBE_SSH_HOST` — opt-in SSH routing for `yt-dlp` through a residential-IP host, for users on datacenter VPS hit by YouTube's bot-wall ([#376](https://github.com/mvanhorn/last30days-skill/pull/376)). Host validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection. Transcript path unchanged (uses HTTP fallback).
|
||||
- macOS Keychain as a credential source — reads from the system keychain when env vars and config files aren't set ([#407](https://github.com/mvanhorn/last30days-skill/pull/407)).
|
||||
- Configuration enablement: env-var defaults and source-resilience patterns across the config layer ([#344](https://github.com/mvanhorn/last30days-skill/pull/344)).
|
||||
|
||||
**Pipeline and storage**
|
||||
|
||||
- Reddit URL auto-enrichment from web search via the public JSON API ([#366](https://github.com/mvanhorn/last30days-skill/pull/366)).
|
||||
- Per-run finding sightings recorded in the SQLite store ([#373](https://github.com/mvanhorn/last30days-skill/pull/373)).
|
||||
- Brave browser support for X/Twitter cookie extraction ([#320](https://github.com/mvanhorn/last30days-skill/pull/320)).
|
||||
|
||||
**Tests and CI**
|
||||
|
||||
- Full pytest suite restored to CI; 13 rotted tests repaired ([#416](https://github.com/mvanhorn/last30days-skill/pull/416)).
|
||||
- `greptile.json` added with `triggerOnUpdates` + `statusCheck` ([#418](https://github.com/mvanhorn/last30days-skill/pull/418)).
|
||||
- Advisory security workflow ([#368](https://github.com/mvanhorn/last30days-skill/pull/368)).
|
||||
- Parallel grounding backend test coverage ([#355](https://github.com/mvanhorn/last30days-skill/pull/355)).
|
||||
|
||||
**Docs**
|
||||
|
||||
- New `CONFIGURATION.md` with README pointers ([#339](https://github.com/mvanhorn/last30days-skill/pull/339)).
|
||||
- `docs/solutions/` learning capture for release-time consistency-test cascades ([#413](https://github.com/mvanhorn/last30days-skill/pull/413)) and the eval-not-in-CI design decision ([#417](https://github.com/mvanhorn/last30days-skill/pull/417)).
|
||||
|
||||
### Changed
|
||||
|
||||
**Install story modernized**
|
||||
- Rename "Digg AI 1000" to just "Digg" in user-facing output (footer line, source label, inline-quote suffix, why_relevant, container attribution). Internal references to the upstream Digg AI 1000 product remain in code comments and docstrings.
|
||||
- Bump `POSTS_PER_CLUSTER` from 3 to 5 and the render-side display limit from 2 to 3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous 3/2 caps routinely truncated cluster context (e.g. dropped a Jason Calacanis quote tweet on a `cli-printing-press` run).
|
||||
- Rewrite SKILL.md path resolution. STEP 0 narrows from a global canonical-path enforcement to a Claude-Code-marketplaces-only stale-clone guard. Step 1 SKILL_ROOT resolver walks a single precedence list (Claude plugin cache, then `~/.codex/skills/`, `~/.agents/skills/`, repo checkout, `./.skills/last30days` for `npx skills add`, CWD, Gemini). Adds SKILL.md frontmatter fallback to `render.py::_skill_version` so the badge no longer prints `v?` on installs that don't include `.claude-plugin/plugin.json`.
|
||||
|
||||
- `npx skills add` is now the canonical install path for every harness ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). README and SKILL.md flipped to recommend `npx skills add . -g -y` over per-harness manual instructions. Surfaces Gemini CLI, Copilot, Windsurf, and 50+ other Agent Skills hosts that the install pattern reaches.
|
||||
- README dropped the Gemini CLI native-extension install path (now covered by `npx skills add`).
|
||||
- `hooks.json` made polyglot for Gemini CLI + Claude Code compatibility ([#318](https://github.com/mvanhorn/last30days-skill/pull/318)).
|
||||
|
||||
**Skill semantics and multi-harness reframe**
|
||||
|
||||
- `AGENTS.md` is now canonical; `CLAUDE.md` points at it ([#410](https://github.com/mvanhorn/last30days-skill/pull/410)). Reframes the project as a multi-harness Agent Skills package rather than a Claude-Code-specific tool.
|
||||
- SKILL.md path resolution rewritten: STEP 0 narrows to a Claude-Code-marketplaces-only stale-clone guard; Step 1 walks a single `SKILL_DIR` substitution pattern ([#400](https://github.com/mvanhorn/last30days-skill/pull/400), [#409](https://github.com/mvanhorn/last30days-skill/pull/409)). Removes ~80 lines of bash and fixes a real spec-vs-engine divergence where the previous resolver could pick a different install than the SKILL.md the model loaded from.
|
||||
- SKILL.md version regex consolidated into `lib/skill_meta.py` ([#412](https://github.com/mvanhorn/last30days-skill/pull/412)).
|
||||
- `--plan` / `--competitors-plan` invocation templates switched from inline single-quoted JSON to heredoc-written tmpfiles ([#404](https://github.com/mvanhorn/last30days-skill/pull/404), fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403)). Apostrophes in resolved context strings ("McDonald's", "people's choice") no longer break shell parsing.
|
||||
- `POSTS_PER_CLUSTER` raised 3→5 and render-side display limit 2→3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous caps routinely truncated cluster context.
|
||||
- Digg AI 1000 renamed to "Digg" in user-facing output ([#372](https://github.com/mvanhorn/last30days-skill/pull/372)) — footer line, source label, inline-quote suffix, why_relevant, container attribution. Internal references retain the upstream product name.
|
||||
- GitHub repo resolution canonicalized for ambiguous product comparisons ([#302](https://github.com/mvanhorn/last30days-skill/pull/302)).
|
||||
|
||||
**Dependencies and tooling**
|
||||
|
||||
- Dropped `requests` runtime dependency. All providers route through stdlib `urllib` via the `lib/http` wrapper ([#393](https://github.com/mvanhorn/last30days-skill/pull/393)).
|
||||
- Migrated to `gemini-3.1-flash-lite` GA model ([#378](https://github.com/mvanhorn/last30days-skill/pull/378)).
|
||||
- Aligned Codex/Claude plugin manifests + added Codex `AGENTS.md` ([#321](https://github.com/mvanhorn/last30days-skill/pull/321)).
|
||||
- pytest dev dep bumped 9.0.2 → 9.0.3 ([#414](https://github.com/mvanhorn/last30days-skill/pull/414)).
|
||||
- Switch SKILL.md's `--plan` and `--competitors-plan` invocation templates from inline single-quoted JSON to heredoc-written tmpfiles. Apostrophes in resolved context strings ("McDonald's", "people's choice", "developer's") previously closed the outer single-quote and broke shell parsing before the engine started — observed in a Codex run during PR #400 testing. The engine's `parse_plan()` / `parse_competitors_plan()` already supported file paths (via `os.path.isfile()` probe); only the template prose changed. Fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403).
|
||||
|
||||
### Removed
|
||||
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1 ([#400](https://github.com/mvanhorn/last30days-skill/pull/400)). Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`** — maintainer dev-deploy script ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). Replaced by `npx skills add . -g -y` (live-symlink into every detected harness's skill dir — better than sync.sh's copy model since edits propagate live). Hermes uses `hermes skills install mvanhorn/last30days-skill --force`; OpenClaw uses `clawhub install last30days-official`.
|
||||
- Orphaned `SPEC.md` and `TASKS.md` ([#419](https://github.com/mvanhorn/last30days-skill/pull/419)).
|
||||
|
||||
### Fixed
|
||||
|
||||
**Reddit**
|
||||
|
||||
- `lstrip("r/")` mangled subreddits starting with `r` (`r/robotics` → `obotics`, `r/ruby` → `uby`); replaced with `removeprefix("r/")` at 4 sites (Alex Key, salvaged from #288).
|
||||
- Browser-like User-Agent + `Accept-Language`/`Accept-Encoding`/`Connection` headers + gzip decompression to fix `urllib` 403s on Reddit's public JSON endpoint (Franco Carballar, salvaged from #199).
|
||||
- HTTP 402 re-raised across all three ScrapeCreators paths (`_global_search`, `_subreddit_search`, `fetch_post_comments`) so the OpenAI/public-JSON fallback chain triggers when credits are exhausted (Jonathan Oppenheim, salvaged from #170).
|
||||
|
||||
**Authentication and credentials**
|
||||
|
||||
- Restored multi-key rotation for `SCRAPECREATORS_API_KEY` accidentally dropped in v3.0.6 (Eric Oberhofer, salvaged from #287). Comma-separated keys round-robin via `random.choice` per run.
|
||||
|
||||
**Windows compatibility**
|
||||
|
||||
- `os.killpg` in `_cleanup_children()` guarded with `hasattr(os, "killpg")`, falls back to `os.kill(SIGTERM)` (gujishh, salvaged from #226).
|
||||
- POSIX-style secret-permission warning skipped on Windows ([#357](https://github.com/mvanhorn/last30days-skill/pull/357)).
|
||||
- Render uses forward slashes in save-path footer for Windows ([#338](https://github.com/mvanhorn/last30days-skill/pull/338)).
|
||||
|
||||
**xAI / X / xurl**
|
||||
|
||||
- `parse_x_response` now raises `http.HTTPError` on empty output, missing JSON, or decode failure — surfaces in `errors_by_source` instead of silently returning an empty result list (Kaustav Mishra, salvaged from #155).
|
||||
- `xurl` treats `PermissionError` from PATH lookup as unavailable ([#322](https://github.com/mvanhorn/last30days-skill/pull/322)).
|
||||
|
||||
**YouTube**
|
||||
|
||||
- SC YouTube + multi-token HN searches unblocked ([#388](https://github.com/mvanhorn/last30days-skill/pull/388)).
|
||||
- Transcript-fetch ratio surfaced + degraded-run nudge for stale `yt-dlp` ([#340](https://github.com/mvanhorn/last30days-skill/pull/340)).
|
||||
|
||||
**bird_x / HTTP**
|
||||
|
||||
- Subprocess retry on non-JSON stdout to handle X anti-bot HTML interstitials ([#383](https://github.com/mvanhorn/last30days-skill/pull/383)).
|
||||
- HTTP retry budget expanded + exponential backoff on DNS resolution failure ([#382](https://github.com/mvanhorn/last30days-skill/pull/382)).
|
||||
- Parallel AI search aligned with current API schema ([#341](https://github.com/mvanhorn/last30days-skill/pull/341)).
|
||||
- Parallel web backend routed through grounding ([#354](https://github.com/mvanhorn/last30days-skill/pull/354)).
|
||||
|
||||
**Planner and sources**
|
||||
|
||||
- `xquik` registered in `SOURCE_CAPABILITIES` ([#336](https://github.com/mvanhorn/last30days-skill/pull/336), fixes [#319](https://github.com/mvanhorn/last30days-skill/issues/319)).
|
||||
- Honor explicit optional source requests ([#356](https://github.com/mvanhorn/last30days-skill/pull/356)).
|
||||
- ScrapeCreators source-gating aligned between code and docs ([#415](https://github.com/mvanhorn/last30days-skill/pull/415)).
|
||||
- OpenClaw works without ScrapeCreators key ([#392](https://github.com/mvanhorn/last30days-skill/pull/392), by @thinkun).
|
||||
|
||||
**Render, version display, hosting paths**
|
||||
|
||||
- Hardcoded `v3.0.0` in render replaced with dynamic `_skill_version()` ([#365](https://github.com/mvanhorn/last30days-skill/pull/365)).
|
||||
- Comparison HTML artifacts saved correctly ([#389](https://github.com/mvanhorn/last30days-skill/pull/389)).
|
||||
- `OPENROUTER_DEFAULT` model ID corrected ([#323](https://github.com/mvanhorn/last30days-skill/pull/323)).
|
||||
- OpenClaw poll-timing initialized once ([#358](https://github.com/mvanhorn/last30days-skill/pull/358)).
|
||||
- Prefer sandboxed Safari cookie path ([#343](https://github.com/mvanhorn/last30days-skill/pull/343)).
|
||||
- Preserve clean mode for last-run state ([#334](https://github.com/mvanhorn/last30days-skill/pull/334)).
|
||||
- Replaced hardcoded `/Users/mvanhorn/...` paths in `test-v1-vs-v2.sh` with portable env-var overrides (Dave Morin, salvaged from #297).
|
||||
|
||||
**Hooks**
|
||||
|
||||
- `check-config.sh` path-quoting fix for paths with spaces ([#337](https://github.com/mvanhorn/last30days-skill/pull/337)).
|
||||
- Replaced unsafe `eval` with `declare` in `check-config.sh` ([#364](https://github.com/mvanhorn/last30days-skill/pull/364)).
|
||||
|
||||
**Sync and version metadata**
|
||||
|
||||
- `sync.sh` pointed at this repo's plugin cache, not the private repo's ([#402](https://github.com/mvanhorn/last30days-skill/pull/402)).
|
||||
- Sync cache target bumped to 3.2.1 to match SKILL.md ([#397](https://github.com/mvanhorn/last30days-skill/pull/397)).
|
||||
- ScrapeCreators free-tier credit count corrected to 100 in docs ([#369](https://github.com/mvanhorn/last30days-skill/pull/369), fixes [#367](https://github.com/mvanhorn/last30days-skill/issues/367)).
|
||||
- Gemini extension version synced ([#349](https://github.com/mvanhorn/last30days-skill/pull/349)).
|
||||
- Various stale path/link fixes ([#345](https://github.com/mvanhorn/last30days-skill/pull/345), [#346](https://github.com/mvanhorn/last30days-skill/pull/346), [#347](https://github.com/mvanhorn/last30days-skill/pull/347), [#348](https://github.com/mvanhorn/last30days-skill/pull/348), [#351](https://github.com/mvanhorn/last30days-skill/pull/351)).
|
||||
|
||||
### Contributors
|
||||
|
||||
First-time contributors whose fixes shipped in this release (most via PR triage salvage — fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
||||
|
||||
- Dave Morin — portable test-harness paths
|
||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
||||
- Eric Oberhofer — multi-key rotation restored
|
||||
- gujishh — Windows process cleanup
|
||||
- Franco Carballar — Reddit browser-like headers
|
||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
||||
- Kaustav Mishra — xAI error surfacing
|
||||
- [@thinkun](https://github.com/thinkun) ([#363](https://github.com/mvanhorn/last30days-skill/pull/363)) — OpenClaw ScrapeCreators-key-optional fix
|
||||
|
||||
Full PR list at [github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0](https://github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0).
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1. Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`.** The maintainer dev-deploy script is gone. Every job it did has a better replacement: `npx skills add . -g -y` symlinks the working tree into every detected harness's skill dir (better than sync.sh's copy model — edits propagate live), `hermes skills install mvanhorn/last30days-skill --force` handles Hermes, `clawhub install last30days-official` handles OpenClaw, and the Claude marketplace cache target was a "test against the official install path" hack we shouldn't have been recommending in the first place. The `test_sync_cache_path_uses_skill_version` test was dropped along with it. CLAUDE.md, HERMES_SETUP.md, the PR template, and a render.py docstring were updated to drop references; CHANGELOG and historical docs (release notes, plan files) keep their existing mentions as accurate history.
|
||||
|
||||
## [3.2.0] - 2026-05-09
|
||||
|
||||
@@ -179,7 +45,7 @@ Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`,
|
||||
### Fixed
|
||||
|
||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||
- **Broken README link.** The README's "source of truth" link pointed at root `SKILL.md`, which is no longer maintained after the plugin-layout restructure. Fixed to point at `skills/last30days/SKILL.md`.
|
||||
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
|
||||
|
||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||
|
||||
|
||||
@@ -1 +1,25 @@
|
||||
@AGENTS.md
|
||||
# last30days Skill
|
||||
|
||||
Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||
Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
||||
```
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
# Concepts
|
||||
|
||||
Shared vocabulary for `last30days-skill`. Terms here have a precise project-specific meaning — distinct enough from their general technical sense that a new contributor would need them defined to follow conversations, PR descriptions, or the SKILL.md contract.
|
||||
|
||||
## The package
|
||||
|
||||
### Skill
|
||||
|
||||
A self-contained agent-instructions package consisting of a `SKILL.md` prose contract plus a sibling `scripts/` directory containing the executable code the SKILL.md invokes. The package conforms to the [Agent Skills](https://agentskills.io) open format and installs across every major harness (Claude Code, Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ others) via `npx skills add`, harness-native plugin installers, or per-harness skill directories. A Skill is the unit of distribution; the Skill is the product.
|
||||
|
||||
### Engine
|
||||
|
||||
The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do the actual research work. The Engine and SKILL.md have a contract: SKILL.md tells the model which flags to pass (`--plan`, `--competitors-plan`, `--x-handle`, `--subreddits`, `--emit=compact`, etc.), and the Engine produces a specific output shape (badge line, ranked evidence clusters, emoji-tree footer) that the model is contractually required to pass through. The Engine is implementation; the SKILL.md prose is the agent-facing surface.
|
||||
|
||||
### Harness
|
||||
|
||||
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
||||
|
||||
## Distribution
|
||||
|
||||
### Beta channel
|
||||
|
||||
A parallel install of the Skill, sourced from the private `mvanhorn/last30days-skill-private` repo and installed as `/last30days-beta` rather than `/last30days`. The Beta channel exists so experimental changes can be tested by real users before they ship to the public `/last30days`. Promotion from Beta to public happens via a review PR against this (public) repo — Beta-only changes never ship to public without that PR. The Beta channel workflow guide lives in `BETA.md` in the private repo.
|
||||
@@ -1,268 +0,0 @@
|
||||
# Configuration
|
||||
|
||||
Everything you can tune in `/last30days` without editing the engine source.
|
||||
Three layers, in order of how often you'll touch them:
|
||||
|
||||
1. **Per-run flags** - what you pass on the command line.
|
||||
2. **Environment variables and `.env`** - what's enabled across all runs.
|
||||
3. **Optional trend-monitoring stack** - SQLite store, watchlist, briefings.
|
||||
|
||||
Per-client patterns and the experimental beta channel are at the bottom.
|
||||
|
||||
> Skip ahead: [Where output is saved](#where-output-is-saved) - [API keys](#api-keys-env) - [Reasoning provider](#reasoning-provider-priority) - [Web search backend](#web-search-backend-priority) - [Trend monitoring](#trend-monitoring-store--watchlist--briefings) - [Per-client patterns](#per-client-patterns) - [Beta channel](#beta-channel)
|
||||
|
||||
## Why this document exists
|
||||
|
||||
This is a focused **configuration reference** maintained alongside the engine. The runtime contract (the voice rules, the planner protocol, the LAWs the synthesizing model follows) lives in [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md) - that file is authoritative when the two ever differ. This file's job is narrower: surface every knob a user or operator can turn, in one place, kept current with the code so client-facing setups stay reliable. New configuration knobs added to the engine should be reflected here in the same PR.
|
||||
|
||||
---
|
||||
|
||||
## Where output is saved
|
||||
|
||||
| Platform | Default path | Override |
|
||||
|---|---|---|
|
||||
| Linux / macOS | `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` | set `LAST30DAYS_MEMORY_DIR=/path` |
|
||||
| Windows | `LAST30DAYS_MEMORY_DIR` defaults to `C:\Users\<you>\Documents\Last30Days\` | set `LAST30DAYS_MEMORY_DIR=C:\path` |
|
||||
|
||||
Each run produces one file per topic, slug-named:
|
||||
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
|
||||
|
||||
**Per-run overrides:**
|
||||
- `--save-dir <path>` - one-off output location.
|
||||
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
||||
|
||||
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
||||
|
||||
---
|
||||
|
||||
## API keys (`.env`)
|
||||
|
||||
The skill reads keys from a `.env` file. Two locations are supported, in priority order:
|
||||
|
||||
1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present.
|
||||
2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback.
|
||||
|
||||
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
||||
|
||||
The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case.
|
||||
|
||||
**Source-by-source** - what each key unlocks:
|
||||
|
||||
| Source | Key(s) | Required for | Free tier |
|
||||
|---|---|---|---|
|
||||
| Reddit (public) | none | always on | yes |
|
||||
| Hacker News | none | always on | yes |
|
||||
| Polymarket | none | always on | yes |
|
||||
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
|
||||
| YouTube | `yt-dlp` CLI installed | always on if `yt-dlp` present | yes |
|
||||
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; xAI / ScrapeCreators = paid |
|
||||
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
|
||||
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
|
||||
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
|
||||
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
|
||||
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
|
||||
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
|
||||
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
|
||||
| Perplexity Deep Research | `OPENROUTER_API_KEY` | `--deep-research` flag (~$0.90/query) | no |
|
||||
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
|
||||
|
||||
**Example `.env` skeleton** (placeholders only - replace with your own values):
|
||||
|
||||
```bash
|
||||
# Reasoning + planning (one provider; see priority below)
|
||||
GOOGLE_API_KEY=<your-gemini-key>
|
||||
|
||||
# Web search backend (one is enough; Brave is the cheapest)
|
||||
BRAVE_API_KEY=<your-brave-key>
|
||||
|
||||
# Optional sources
|
||||
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
|
||||
# X authentication (one option only)
|
||||
XAI_API_KEY=<your-xai-key>
|
||||
# OR cookie-jar (no key needed; logs in via your browser session)
|
||||
# FROM_BROWSER=firefox
|
||||
|
||||
# Bluesky
|
||||
BSKY_HANDLE=<your-handle>.bsky.social
|
||||
BSKY_APP_PASSWORD=<your-app-password>
|
||||
```
|
||||
|
||||
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
||||
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search.
|
||||
|
||||
### Bluesky app-password format and search host
|
||||
|
||||
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
|
||||
|
||||
The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`:
|
||||
|
||||
```bash
|
||||
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Reasoning provider priority
|
||||
|
||||
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
||||
|
||||
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
||||
2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`)
|
||||
3. **xAI** - `XAI_API_KEY`
|
||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (also unlocks `--deep-research`)
|
||||
5. **Local / deterministic** - always available, lowest quality
|
||||
|
||||
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
|
||||
|
||||
---
|
||||
|
||||
## Web search backend priority
|
||||
|
||||
Used by `--auto-resolve` (when WebSearch isn't available from the host) and Step 2 supplements. Auto-detect priority (override per-run with `--web-backend=<name>`):
|
||||
|
||||
1. **Brave** - `BRAVE_API_KEY`
|
||||
2. **Exa** - `EXA_API_KEY`
|
||||
3. **Serper** - `SERPER_API_KEY`
|
||||
4. **Parallel** - `PARALLEL_API_KEY`
|
||||
5. **Host's native WebSearch** - Claude Code, Codex, Gemini all have one built in
|
||||
|
||||
Visible quality difference between hosts with vs without a configured backend. If your client setup produces thinner results than yours, this is usually why.
|
||||
|
||||
---
|
||||
|
||||
## Trend monitoring (`--store` + watchlist + briefings)
|
||||
|
||||
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
|
||||
|
||||
### `--store` flag
|
||||
|
||||
Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate.
|
||||
|
||||
**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`.
|
||||
|
||||
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
|
||||
|
||||
### `watchlist.py` - recurring topics
|
||||
|
||||
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
|
||||
|
||||
Two-step flow (the watchlist holds the topic; an external scheduler invokes the run):
|
||||
|
||||
```bash
|
||||
# 1. Add the topic to the watchlist
|
||||
# Default schedule daily 8am; --weekly switches to Mondays 8am
|
||||
python3 scripts/watchlist.py add "british airways middle east" --weekly
|
||||
|
||||
# 2. Configure delivery and budget (optional)
|
||||
python3 scripts/watchlist.py config delivery "https://hooks.slack.com/services/..."
|
||||
python3 scripts/watchlist.py config budget 5.00
|
||||
|
||||
# 3. Trigger via cron / Task Scheduler / GitHub Actions
|
||||
python3 scripts/watchlist.py run-one "british airways middle east"
|
||||
# or run every enabled topic, gated by daily_budget
|
||||
python3 scripts/watchlist.py run-all
|
||||
```
|
||||
|
||||
The schedule field stored on each topic is metadata - the actual cron / Task Scheduler invocation is your responsibility. Watchlist runs hardcode `--quick` and `--lookback-days 90` when spawning the underlying engine.
|
||||
|
||||
### `briefing.py` - daily / weekly digests
|
||||
|
||||
[`scripts/briefing.py`](skills/last30days/scripts/briefing.py) reads the SQLite store and emits structured data the agent then synthesizes into prose. Modes: `generate` (daily), `generate --weekly`, `show [--date DATE]` (display a saved briefing). Briefs save to `~/.local/share/last30days/briefs/`.
|
||||
|
||||
### Recommended cadence pattern
|
||||
|
||||
| Step | Cadence | Command |
|
||||
|---|---|---|
|
||||
| Baseline | one-time per topic | `/last30days "<topic>" --days=30 --store` |
|
||||
| Add to watchlist | one-time per topic | `python3 scripts/watchlist.py add "<topic>" --weekly` |
|
||||
| Recurring run | daily or weekly (external scheduler) | `python3 scripts/watchlist.py run-all` |
|
||||
| Digest | weekly | `python3 scripts/briefing.py generate --weekly` |
|
||||
|
||||
---
|
||||
|
||||
## Per-client patterns
|
||||
|
||||
The skill is built to flex around different client environments. Four patterns that compose well:
|
||||
|
||||
### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders)
|
||||
|
||||
The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
||||
SCRAPECREATORS_API_KEY=<acme-scoped-key-or-shared>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
||||
```
|
||||
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no flags or wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
|
||||
### 2. Per-client save dir + suffix wrapper
|
||||
|
||||
For workflows where you don't `cd` into a client folder (running from anywhere, scripted batches), a tiny shell function isolates each client's research without engine changes.
|
||||
|
||||
PowerShell example:
|
||||
|
||||
```powershell
|
||||
function Run-L30D-Client {
|
||||
param([string]$ClientSlug, [Parameter(ValueFromRemainingArguments=$true)]$Args)
|
||||
$env:LAST30DAYS_MEMORY_DIR = "C:\Users\$env:USERNAME\Clients\$ClientSlug\Research\Last30Days"
|
||||
/last30days @Args --save-suffix=$ClientSlug
|
||||
}
|
||||
# Usage: Run-L30D-Client acme "british airways middle east"
|
||||
```
|
||||
|
||||
Bash example:
|
||||
|
||||
```bash
|
||||
l30d-client() {
|
||||
local client=$1; shift
|
||||
LAST30DAYS_MEMORY_DIR="$HOME/Clients/$client/Research/Last30Days" \
|
||||
/last30days "$@" --save-suffix="$client"
|
||||
}
|
||||
# Usage: l30d-client acme "british airways middle east"
|
||||
```
|
||||
|
||||
### 3. Custom category-peer subreddits
|
||||
|
||||
[`scripts/lib/categories.py`](skills/last30days/scripts/lib/categories.py) holds a table of `(category_id, trigger_keywords, peer_subreddits)`. If a client lives in a vertical that isn't covered (legal-tech, real-estate-tech, B2B HR SaaS), add a row. Pure data, no logic.
|
||||
|
||||
Section 2a of `SKILL.md` documents the merging rule the skill applies when your topic matches a category.
|
||||
|
||||
### 4. Pre-built `--competitors-plan` JSON
|
||||
|
||||
For competitor-vs-comparisons that recur, a pre-written JSON skeleton per client industry saves real time:
|
||||
|
||||
```json
|
||||
{
|
||||
"Competitor B": {
|
||||
"x_handle": "competitor_b_handle",
|
||||
"subreddits": ["sub1", "sub2"],
|
||||
"github_user": "competitor-b-org",
|
||||
"context": "Founded 2019, focused on ..."
|
||||
},
|
||||
"Competitor C": { ... }
|
||||
}
|
||||
```
|
||||
|
||||
Pass as `--competitors-plan @client/competitors-plan.json` (or as a string). See `SKILL.md` section "If QUERY_TYPE = COMPARISON" for the full schema.
|
||||
|
||||
---
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental customizations live on a private companion repo (`mvanhorn/last30days-skill-private`) installed as `/last30days-beta`. Never ship beta-only changes to the public marketplace without a review PR against the public repo. Workflow guide: `BETA.md` in the private repo.
|
||||
|
||||
This is the right home for client-specific changes you don't intend to upstream - custom category rows, internal subreddit lists, per-vertical plan templates.
|
||||
|
||||
---
|
||||
|
||||
## Cross-references
|
||||
|
||||
- The CLI flag surface: `python3 scripts/last30days.py --help`
|
||||
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
|
||||
- Engine spec (some sections stale; SKILL.md wins on conflicts): [`SPEC.md`](SPEC.md)
|
||||
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
|
||||
+1
-1
@@ -23,7 +23,7 @@ v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos
|
||||
### @thinkun
|
||||
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
||||
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
||||
> Thinker, technologist, AI expert, music-tinkerer. Founder of [Thinkun](https://thinkun.com). [@thinkun on GitHub](https://github.com/thinkun) · [@unthink on X](https://x.com/unthink)
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @thomasmktong
|
||||
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ On first run, the skill will guide you through setup:
|
||||
|
||||
2. **Optional: ScrapeCreators**
|
||||
- Adds TikTok, Instagram, Reddit backup
|
||||
- 100 free credits (no expiration)
|
||||
- 10,000 free API calls
|
||||
- Sign up at scrapecreators.com
|
||||
|
||||
3. **Optional: API Keys**
|
||||
|
||||
@@ -12,12 +12,11 @@
|
||||
|
||||
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
||||
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [SKILL.md](SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
|
||||
**Claude Code (recommended — auto-updates via marketplace):**
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
||||
@@ -153,10 +152,8 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
|
||||
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
||||
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
||||
- **TikTok, Instagram, Threads.** All three activate automatically once `SCRAPECREATORS_API_KEY` is set — same key, same per-call cost. Suppress any of them with `EXCLUDE_SOURCES=tiktok,instagram,threads` (any comma-separated subset).
|
||||
- **Pinterest.** Per-query opt-in (visual pins, narrow utility): the model passes `--search=pinterest` for the runs that need it. Requires `SCRAPECREATORS_API_KEY`.
|
||||
- **YouTube + TikTok comments.** Persistent opt-in via `INCLUDE_SOURCES=youtube_comments,tiktok_comments` because each video pulls N extra ScrapeCreators calls on top of the base search. Surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` and `INCLUDE_SOURCES=perplexity` (it's a separate paid API — opt-in keeps you from being surprise-billed).
|
||||
- **Threads, Pinterest, YouTube + TikTok comments.** Opt-in sources via ScrapeCreators. Set `INCLUDE_SOURCES=tiktok,instagram` and add threads, pinterest, youtube_comments, tiktok_comments for more. `youtube_comments` and `tiktok_comments` surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` to unlock.
|
||||
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
||||
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
||||
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
||||
@@ -173,6 +170,7 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
||||
| **Claude Desktop** | [Download the `.mcpb` for your platform](https://github.com/mvanhorn/last30days-skill/releases/latest) and drag into Settings > Extensions | Re-download and drag the new bundle in |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recommended)
|
||||
@@ -232,6 +230,24 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
||||
|
||||
### Claude Desktop
|
||||
|
||||
Claude Desktop installs `/last30days` as an MCP server via a `.mcpb` bundle (a one-click Model Context Protocol package).
|
||||
|
||||
1. Go to the [latest release](https://github.com/mvanhorn/last30days-skill/releases/latest) and download the `.mcpb` for your platform:
|
||||
- macOS Apple Silicon: `last30days-pp-mcp-darwin-arm64.mcpb`
|
||||
- macOS Intel: `last30days-pp-mcp-darwin-amd64.mcpb`
|
||||
- Linux x86_64: `last30days-pp-mcp-linux-amd64.mcpb`
|
||||
2. Open Claude Desktop, go to Settings > Extensions, and drag the file in.
|
||||
3. When prompted, paste API keys for the sources you want to enable. Every field is optional — the engine degrades to web-only mode if you skip them all. Keys are stored in your OS keychain.
|
||||
4. Restart Claude Desktop. Ask Claude to "research Peter Steinberger" or any topic and it will call the `research` tool.
|
||||
|
||||
**Host requirement:** Python 3.12+ on PATH. The bundle ships the engine source but uses your local Python interpreter. Install from [python.org](https://www.python.org/downloads/) on Windows; macOS and most Linux distros ship a compatible version.
|
||||
|
||||
**Keys don't sync with the Code skill.** Claude Desktop and Claude Code maintain separate credential stores by design. If you already configured `~/.config/last30days/.env` for the Code skill, you'll re-enter the same keys here once.
|
||||
|
||||
Windows support is deferred until per-platform manifest entry points are sorted out; track in a follow-up issue.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
```bash
|
||||
@@ -259,40 +275,10 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
| X / Twitter | Log into x.com in any browser | Free |
|
||||
| YouTube | `brew install yt-dlp` | Free |
|
||||
| Bluesky | App password from bsky.app | Free |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls |
|
||||
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
||||
| Web search | Brave Search key | 2,000 free queries/month |
|
||||
|
||||
### macOS Keychain (optional)
|
||||
|
||||
On macOS you can store keys in the system Keychain instead of a `.env` file. The skill picks them up automatically as the lowest-priority source — `.env` files and process environment still win on collision.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
||||
|
||||
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
|
||||
|
||||
## Configuration
|
||||
|
||||
Two things you'll likely want to know on day one:
|
||||
|
||||
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each run produces `<slug>-raw[-suffix].md`.
|
||||
|
||||
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# last30days Skill Specification
|
||||
|
||||
## Overview
|
||||
|
||||
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. OpenAI auth can come from `OPENAI_API_KEY` or Codex login credentials.
|
||||
|
||||
The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability.
|
||||
|
||||
## Architecture
|
||||
|
||||
The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`:
|
||||
|
||||
- **env.py**: Load API keys from `~/.config/last30days/.env` and Codex auth from `~/.codex/auth.json`
|
||||
- **dates.py**: Date range calculation and confidence scoring
|
||||
- **cache.py**: 24-hour TTL caching keyed by topic + date range
|
||||
- **http.py**: stdlib-only HTTP client with retry logic
|
||||
- **models.py**: Auto-selection of OpenAI/xAI models with 7-day caching
|
||||
- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit
|
||||
- **xai_x.py**: xAI Responses API + x_search for X
|
||||
- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics
|
||||
- **hackernews.py**: Hacker News search via Algolia API (free, no auth)
|
||||
- **polymarket.py**: Polymarket prediction market search via Gamma API (free, no auth)
|
||||
- **normalize.py**: Convert raw API responses to canonical schema
|
||||
- **score.py**: Compute popularity-aware scores (relevance + recency + engagement)
|
||||
- **dedupe.py**: Near-duplicate detection via text similarity
|
||||
- **render.py**: Generate markdown and JSON outputs
|
||||
- **schema.py**: Type definitions and validation
|
||||
|
||||
## Embedding in Other Skills
|
||||
|
||||
Other skills can import the research context in several ways:
|
||||
|
||||
### Inline Context Injection
|
||||
```markdown
|
||||
## Recent Research Context
|
||||
!python3 ~/.claude/skills/last30days/scripts/last30days.py "your topic" --emit=context
|
||||
```
|
||||
|
||||
### Read from File
|
||||
```markdown
|
||||
## Research Context
|
||||
!cat ~/.local/share/last30days/out/last30days.context.md
|
||||
```
|
||||
|
||||
### Get Path for Dynamic Loading
|
||||
```bash
|
||||
CONTEXT_PATH=$(python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=path)
|
||||
cat "$CONTEXT_PATH"
|
||||
```
|
||||
|
||||
### JSON for Programmatic Use
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=json > research.json
|
||||
```
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py <topic> [options]
|
||||
|
||||
Options:
|
||||
--refresh Bypass cache and fetch fresh data
|
||||
--mock Use fixtures instead of real API calls
|
||||
--emit=MODE Output mode: compact|json|md|context|path (default: compact)
|
||||
--sources=MODE Source selection: auto|reddit|x|both (default: auto)
|
||||
```
|
||||
|
||||
## Output Files
|
||||
|
||||
All outputs are written to `~/.local/share/last30days/out/`:
|
||||
|
||||
- `report.md` - Human-readable full report
|
||||
- `report.json` - Normalized data with scores
|
||||
- `last30days.context.md` - Compact reusable snippet for other skills
|
||||
- `raw_openai.json` - Raw OpenAI API response
|
||||
- `raw_xai.json` - Raw xAI API response
|
||||
- `raw_reddit_threads_enriched.json` - Enriched Reddit thread data
|
||||
@@ -0,0 +1,47 @@
|
||||
# last30days Implementation Tasks
|
||||
|
||||
## Setup & Configuration
|
||||
- [x] Create directory structure
|
||||
- [x] Write SPEC.md
|
||||
- [x] Write TASKS.md
|
||||
- [x] Write SKILL.md with proper frontmatter
|
||||
|
||||
## Core Library Modules
|
||||
- [x] scripts/lib/env.py - Environment and API key loading
|
||||
- [x] scripts/lib/dates.py - Date range and confidence utilities
|
||||
- [x] scripts/lib/cache.py - TTL-based caching
|
||||
- [x] scripts/lib/http.py - HTTP client with retry
|
||||
- [x] scripts/lib/models.py - Auto model selection
|
||||
- [x] scripts/lib/schema.py - Data structures
|
||||
- [x] scripts/lib/openai_reddit.py - OpenAI Responses API
|
||||
- [x] scripts/lib/xai_x.py - xAI Responses API
|
||||
- [x] scripts/lib/reddit_enrich.py - Reddit thread JSON fetcher
|
||||
- [x] scripts/lib/normalize.py - Schema normalization
|
||||
- [x] scripts/lib/score.py - Popularity scoring
|
||||
- [x] scripts/lib/dedupe.py - Near-duplicate detection
|
||||
- [x] scripts/lib/render.py - Output rendering
|
||||
|
||||
## Main Script
|
||||
- [x] scripts/last30days.py - CLI orchestrator
|
||||
|
||||
## Fixtures
|
||||
- [x] fixtures/openai_sample.json
|
||||
- [x] fixtures/xai_sample.json
|
||||
- [x] fixtures/reddit_thread_sample.json
|
||||
- [x] fixtures/models_openai_sample.json
|
||||
- [x] fixtures/models_xai_sample.json
|
||||
|
||||
## Tests
|
||||
- [x] tests/test_dates.py
|
||||
- [x] tests/test_cache.py
|
||||
- [x] tests/test_models.py
|
||||
- [x] tests/test_score.py
|
||||
- [x] tests/test_dedupe.py
|
||||
- [x] tests/test_normalize.py
|
||||
- [x] tests/test_render.py
|
||||
|
||||
## Validation
|
||||
- [x] Run tests in mock mode
|
||||
- [x] Demo --emit=compact
|
||||
- [x] Demo --emit=context
|
||||
- [x] Verify file tree
|
||||
+11
-12
@@ -142,7 +142,7 @@ The repo vendors a search-only subset of Bird's Twitter GraphQL client and shell
|
||||
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
||||
| Replies/quotes | Real | Real |
|
||||
| Author handle | Real | Real |
|
||||
| Relevance score | Default 0.7 (re-ranked by relevance.py) | AI-assessed 0.0-1.0 |
|
||||
| Relevance score | Default 0.7 (re-ranked by score.py) | AI-assessed 0.0-1.0 |
|
||||
|
||||
### Depth settings
|
||||
|
||||
@@ -183,14 +183,13 @@ After both searches complete:
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `skills/last30days/scripts/last30days.py` | Main CLI entry point |
|
||||
| `skills/last30days/scripts/lib/pipeline.py` | Multi-source retrieval orchestration |
|
||||
| `skills/last30days/scripts/lib/reddit_public.py` | Reddit public JSON search |
|
||||
| `skills/last30days/scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `skills/last30days/scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `skills/last30days/scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `skills/last30days/scripts/lib/providers.py` | Reasoning provider and model selection |
|
||||
| `skills/last30days/scripts/lib/env.py` | API key loading, source detection |
|
||||
| `skills/last30days/scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `skills/last30days/scripts/lib/relevance.py` | Query matching and relevance scoring |
|
||||
| `skills/last30days/scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
| `scripts/last30days.py` | Main orchestrator, concurrent execution |
|
||||
| `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API |
|
||||
| `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `scripts/lib/models.py` | Auto-select best available model |
|
||||
| `scripts/lib/env.py` | API key loading, source detection |
|
||||
| `scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `scripts/lib/score.py` | Relevance scoring |
|
||||
| `scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
||||
type: feat
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
||||
type: fix
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
||||
type: feat
|
||||
status: active
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
---
|
||||
|
||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
||||
|
||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
||||
type: fix
|
||||
status: active
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Search Quality Eval
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
|
||||
What it does:
|
||||
|
||||
@@ -18,13 +18,13 @@ What it does:
|
||||
Recommended usage:
|
||||
|
||||
```bash
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py
|
||||
uv run python scripts/evaluate_search_quality.py
|
||||
```
|
||||
|
||||
Useful flags:
|
||||
|
||||
```bash
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py \
|
||||
uv run python scripts/evaluate_search_quality.py \
|
||||
--baseline-rev origin/main \
|
||||
--candidate-rev HEAD \
|
||||
--no-default-topics \
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
title: Search-quality eval is manual by default, not a CI gate on every PR
|
||||
date: 2026-05-10
|
||||
category: docs/solutions/architecture
|
||||
module: skills/last30days/scripts/evaluate_search_quality.py
|
||||
problem_type: design_decision
|
||||
component: ci_policy
|
||||
severity: low
|
||||
applies_when:
|
||||
- a contributor proposes wiring search-quality eval into PR CI
|
||||
- a change affects retrieval, ranking, grounding, or synthesis quality and a reviewer asks "why aren't we testing this in CI?"
|
||||
- someone is deciding whether a new evaluator-style script belongs in the default CI workflow
|
||||
related_components:
|
||||
- search_quality_evaluation
|
||||
- ci_workflow
|
||||
- llm_judging
|
||||
tags:
|
||||
- ci-policy
|
||||
- eval
|
||||
- design-decision
|
||||
- cost-vs-signal
|
||||
- non-determinism
|
||||
- manual-gates
|
||||
---
|
||||
|
||||
# Search-quality eval is manual by default, not a CI gate on every PR
|
||||
|
||||
## Context
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` compares a baseline revision against a candidate revision across a fixed pool of reviewer topics. It produces two flavors of metrics: deterministic overlap (Jaccard, retention) and LLM-judged quality scores. The natural impulse on seeing an evaluator script is to wire it into CI on every PR — "regression catcher, run it automatically." We deliberately don't.
|
||||
|
||||
Three properties of this particular evaluator make CI-on-every-PR the wrong default:
|
||||
|
||||
1. **Live API access.** The candidate revision typically needs the engine to actually run, which means real ScrapeCreators calls, real reddit fetches, real YouTube searches. CI runs would either need production credentials or a record/replay fixture set that drifts almost immediately as external APIs change shape.
|
||||
|
||||
2. **Cost and latency.** A full eval pass runs the pipeline N times across reviewer topics. Multiplied by every PR (including doc-only PRs), the spend is meaningful and the wall-clock pushes CI from ~30s to many minutes.
|
||||
|
||||
3. **Non-determinism in the judging path.** The LLM-judged metrics are valuable for review but depend on judge-model behavior on a given day. A flaky eval that fails 1 PR in 20 because the judge re-scored an item differently is a worse CI signal than no eval at all — it teaches contributors to retry rather than read the result.
|
||||
|
||||
The deterministic overlap metrics are useful regression signals but they are not the same as user-facing correctness. A change that improves overlap can degrade synthesis quality; a change that drops overlap can be a deliberate improvement. So even the deterministic side isn't safe to auto-fail on.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Keep search-quality eval available, just not automatic
|
||||
|
||||
The script stays runnable by maintainers and contributors. The pattern is:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_PYTHON=python3.13 \
|
||||
python3 skills/last30days/scripts/evaluate_search_quality.py \
|
||||
--baseline main --candidate HEAD
|
||||
```
|
||||
|
||||
Reviewers can request a manual eval run when a PR is in the retrieval/ranking/synthesis path and the risk warrants it. Contributors can run it locally before submitting if they want signal upfront.
|
||||
|
||||
### 2. Standard PR CI gates remain deterministic and contract-shaped
|
||||
|
||||
`pytest` (offline-safe), plugin-contract checks, version-consistency contracts, ruff/lint. Anything that returns the same answer twice for the same input. Quality-of-output assessment lives outside that loop.
|
||||
|
||||
### 3. The middle ground is `workflow_dispatch`, not auto-PR-gating
|
||||
|
||||
If maintainers want a GitHub-triggered eval that doesn't make every PR pay the live-API cost, the right shape is a manually-dispatched workflow (or a label-triggered one) — not a `pull_request:` workflow that runs unconditionally. That keeps the cost knob in human hands.
|
||||
|
||||
### 4. Revisit if the eval can ever be made offline-deterministic
|
||||
|
||||
The blocker is the live-API + non-determinism combination. If a future iteration of the script can compute meaningful Jaccard/retention metrics against static fixtures (no live API calls, no LLM judging), the decision flips and it becomes a candidate for default CI. The decision below tracks that condition; revisit when it's met.
|
||||
|
||||
## What this means in practice
|
||||
|
||||
- Don't merge PRs that wire `evaluate_search_quality.py` into the default `validate.yml` workflow.
|
||||
- Do merge PRs that add `workflow_dispatch` triggers or label-gated runs.
|
||||
- When reviewing a retrieval/ranking change, request a manual eval if the diff suggests it could regress quality — don't expect CI to catch it.
|
||||
|
||||
## Links
|
||||
|
||||
- `skills/last30days/scripts/evaluate_search_quality.py` — the evaluator script
|
||||
- `docs/search-quality-eval.md` — user-facing usage documentation
|
||||
- `.github/workflows/validate.yml` — the default CI workflow (deterministic gates only)
|
||||
|
||||
---
|
||||
|
||||
*Adapted from a draft ADR proposed by @hnshah in [#374](https://github.com/mvanhorn/last30days-skill/pull/374), restructured into the `docs/solutions/` convention. The original ADR text correctly identified the constraint; this version adds the "why workflow_dispatch is the middle ground" framing and the revisit-condition.*
|
||||
@@ -1,219 +0,0 @@
|
||||
---
|
||||
title: Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
date: 2026-05-16
|
||||
category: docs/solutions/workflow-issues
|
||||
module: ci-release-engineering
|
||||
problem_type: workflow_issue
|
||||
component: testing_framework
|
||||
severity: high
|
||||
applies_when:
|
||||
- a test asserts consistency between two release-time artifacts (e.g., SKILL.md version and a hardcoded pin in a shell script)
|
||||
- one artifact is updated as part of a version bump and the other requires a manual lockstep update
|
||||
- multiple long-lived PRs are open simultaneously against the same base branch
|
||||
symptoms:
|
||||
- every open PR's CI fails after a version bump even though the PRs are unrelated to versioning
|
||||
- the failing test references a stale hardcoded value that was not updated alongside the bumped version
|
||||
- PR authors must rebase and manually fix an artifact they did not touch
|
||||
root_cause: missing_workflow_step
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- development_workflow
|
||||
- documentation
|
||||
tags:
|
||||
- ci
|
||||
- release-engineering
|
||||
- consistency-test
|
||||
- version-pin
|
||||
- cascade-failure
|
||||
- test-design
|
||||
- workflow
|
||||
---
|
||||
|
||||
# Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
|
||||
## Context
|
||||
|
||||
A `tests/test_version_consistency.py::test_sync_cache_path_uses_skill_version` test was added to enforce that the version string embedded in `skills/last30days/scripts/sync.sh` (a hardcoded plugin-cache path segment) matched the version frontmatter in `skills/last30days/SKILL.md`. The intention was sound: the cache path had to stay in lockstep with the skill version or the sync would silently pull stale files.
|
||||
|
||||
The test worked as designed until a release shipped. At that point it turned into a cascade-failure machine:
|
||||
|
||||
1. A release PR bumps `SKILL.md` version (e.g., 3.2.0 → 3.2.1) **and** bumps the `sync.sh` pin. That PR's CI is green.
|
||||
2. The release PR merges to `main`.
|
||||
3. Every PR that was open at merge time was branched from pre-release `main`. Those PRs have `SKILL.md` 3.2.1 (inherited via merge-base with `main`) but their branch never touched `sync.sh`.
|
||||
4. CI for those PRs runs the consistency test against the new `main` — `SKILL.md` says 3.2.1, `sync.sh` still says 3.2.0 — and fails.
|
||||
5. All open PRs are now red simultaneously, with a failure that has nothing to do with their changes.
|
||||
|
||||
This affected at least five PRs during the 2026-05-13 to 2026-05-15 window: PR #400 (caught during rebase, required a manual pin bump), PRs #390 and #392 (OpenClaw `SCRAPECREATORS_API_KEY` fix, both stalled for the same stale-pin reason), and at least two others. A follow-up hotfix PR (#397 — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`) was required just to unblock the queue.
|
||||
|
||||
The permanent fix was PR #405: delete `sync.sh` entirely (the install workflow made it redundant) and drop `test_sync_cache_path_uses_skill_version`. Once both were gone, no version-consistency cascade was possible.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Don't write consistency tests that read two files and assert one matches a substring derived from the other
|
||||
|
||||
This pattern looks safe but is not:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # reads SKILL.md
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # asserts sync.sh contains that string
|
||||
)
|
||||
```
|
||||
|
||||
It encodes the assumption that both files are always updated together, in the same commit, on the same branch. That assumption breaks the moment two files have independent lifecycle owners — a versioned manifest and a deployment script are archetypal examples.
|
||||
|
||||
### 2. If the values genuinely need to stay in sync, derive one from the other at runtime
|
||||
|
||||
Remove the hardcoded pin from `sync.sh` and compute it:
|
||||
|
||||
```bash
|
||||
# sync.sh — derive version from SKILL.md at runtime, no pin to maintain
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "$(dirname "$0")/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
CACHE_PATH="last30days-skill/last30days/${SKILL_VERSION}"
|
||||
```
|
||||
|
||||
Now there is only one source of truth (`SKILL.md`). The test that asserted they matched becomes vacuous and should be deleted. If `SKILL.md` is wrong, the sync itself will fail loudly — which is better feedback than a CI gate on a different PR.
|
||||
|
||||
### 3. If two values must stay independent for legitimate reasons, update them together and make the test self-skip if either source is missing
|
||||
|
||||
If separate versioning is genuinely required (e.g., SKILL.md versions for harness consumers, sync.sh versions a private artifact store with its own cadence), update both in the same PR — never staggered — and write the test to self-skip rather than error when either file is absent:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_sh = SKILL_ROOT / "scripts" / "sync.sh"
|
||||
if not sync_sh.exists():
|
||||
self.skipTest("sync.sh not present; skipping pin consistency check")
|
||||
sync_text = sync_sh.read_text(encoding="utf-8")
|
||||
version = _skill_version()
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text,
|
||||
)
|
||||
```
|
||||
|
||||
Self-skipping means deleting the file is a non-event in CI — no cascading red, no hotfix PR to the queue.
|
||||
|
||||
### 4. Run consistency tests against the merge-base diff, not main
|
||||
|
||||
If you keep a two-file consistency test, scope it so it only fails when the PR itself modifies one of the two files but not the other. A GitHub Actions step can do this:
|
||||
|
||||
```yaml
|
||||
- name: Check sync.sh version pin consistency
|
||||
run: |
|
||||
BASE=$(git merge-base HEAD origin/main)
|
||||
SKILL_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'SKILL\.md' || true)
|
||||
SYNC_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'sync\.sh' || true)
|
||||
if [ "$SKILL_CHANGED" -gt 0 ] && [ "$SYNC_CHANGED" -eq 0 ]; then
|
||||
echo "SKILL.md version bumped but sync.sh pin was not updated"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
This only fires when your PR touched `SKILL.md` and left `sync.sh` alone — never because a release merged to `main` after you branched.
|
||||
|
||||
### 5. Ask whether you actually need this test
|
||||
|
||||
If the values are wrong, downstream tooling will fail loudly: the sync will fetch the wrong artifact, installs will break, or the harness will reject the version. A test that exists only to catch a human-bookkeeping error at release time adds cascade-fail risk without offering a meaningfully earlier signal. Weigh that cost before adding any two-file consistency gate.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The damage from a stale-pin consistency test is asymmetric. It:
|
||||
|
||||
- Fails on every open PR simultaneously the moment a release lands on `main` — not just the PR that forgot to update the pin.
|
||||
- Produces a failure message that points at a line in a test file with no obvious relationship to the PR's actual changes.
|
||||
- Requires either a hotfix PR (touching a file the failing PRs have no business touching) or a manual rebase of every affected branch.
|
||||
- Blocks work that has already been reviewed and approved.
|
||||
|
||||
In this repo the effect was measurable: at least five PRs stalled across a two-day window, one hotfix PR was shipped just to unblock the queue, and multiple authors spent time debugging a failure completely unrelated to their changes.
|
||||
|
||||
The broader principle is that tests which gate on *bookkeeping consistency between files* impose their maintenance cost on every contributor, every time, even when those contributors did nothing wrong. That cost compounds with team size and release cadence.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Apply this guidance whenever you find yourself:
|
||||
|
||||
- Writing a test that reads two files and asserts that a string in one matches a value derived from the other.
|
||||
- Adding a CI step labeled "consistency check," "sync check," or "pin check" where the check compares a hardcoded value against a computed one from a separate file.
|
||||
- Working in a repo where a versioned manifest (e.g., `SKILL.md`, `package.json`, `pyproject.toml`) and a deployment artifact (e.g., a shell script, a Dockerfile, a Helm values file) are both maintained by hand.
|
||||
- Reviewing a PR that touches only one of two "paired" files and fails a consistency test for the other.
|
||||
|
||||
It does *not* apply to tests that read a single source of truth and validate its internal structure (e.g., asserting that `SKILL.md`'s frontmatter version is double-quoted, or that `package.json`'s `version` field is a valid semver string). Those tests have one file and one assertion; they cannot cascade across branches.
|
||||
|
||||
## Examples
|
||||
|
||||
### Before — the pattern that caused the cascade
|
||||
|
||||
Original `tests/test_version_consistency.py` (deleted in commit `9fb19ea`):
|
||||
|
||||
```python
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_ROOT = ROOT / "skills" / "last30days"
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
|
||||
if not match:
|
||||
raise AssertionError("SKILL.md version frontmatter not found")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
class TestVersionConsistency(unittest.TestCase):
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # source 1: SKILL.md frontmatter
|
||||
self.assertIn( # assertion: sync.sh must contain
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # source 2: hardcoded string in sync.sh
|
||||
)
|
||||
```
|
||||
|
||||
`sync.sh` contained a line like:
|
||||
|
||||
```bash
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/3.2.0"
|
||||
```
|
||||
|
||||
When SKILL.md bumped to `3.2.1` in a release PR, `sync.sh` was updated in the same PR and CI stayed green. But every PR branched before that release still had `sync.sh` at `3.2.0`. Their CI failed immediately, with an assertion error pointing at the test, not at the release PR.
|
||||
|
||||
### After — what we did: delete both
|
||||
|
||||
PR #405 deleted `sync.sh` (the install workflow replaced it) and dropped `test_sync_cache_path_uses_skill_version` in the same change. No consistency gate, no pin to maintain, no cascade possible.
|
||||
|
||||
### After — what we could have done instead: derive at runtime
|
||||
|
||||
If `sync.sh` had still been needed, the right fix would have been to remove the hardcoded version from the script and derive it from `SKILL.md`:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# sync.sh — no hardcoded version; reads SKILL.md as single source of truth
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "${SCRIPT_DIR}/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
|
||||
if [ -z "$SKILL_VERSION" ]; then
|
||||
echo "error: could not parse version from SKILL.md" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/${SKILL_VERSION}"
|
||||
# ... rest of sync logic
|
||||
```
|
||||
|
||||
With this in place, `test_sync_cache_path_uses_skill_version` has no reason to exist — there is nothing to assert. Delete it. If the version parsing breaks, `sync.sh` itself exits non-zero with a clear message.
|
||||
|
||||
## Related
|
||||
|
||||
- **PR #397** (merged) — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`. The hotfix that unblocked the cascade temporarily by bumping the pin.
|
||||
- **PR #400** (merged) — caught the same cascade during rebase; had to bump the pin to clear CI.
|
||||
- **PR #390** (closed) and **PR #392** (rebased + merged) — OpenClaw `SCRAPECREATORS_API_KEY` fix; both blocked by the cascade until rebased onto post-#405 main.
|
||||
- **PR #405** (merged) — the permanent fix: deleted `sync.sh` + `test_sync_cache_path_uses_skill_version` together.
|
||||
- **PR #412** (merged) — adjacent work that consolidated SKILL.md version parsing into `lib/skill_meta.py`, reducing future drift risk by giving the version field one canonical reader.
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"version": "3.2.4",
|
||||
"version": "3.0.5",
|
||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||
"settings": [
|
||||
{
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"triggerOnUpdates": true,
|
||||
"statusCheck": true
|
||||
}
|
||||
+2
-1
@@ -6,7 +6,8 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash \"${CLAUDE_PLUGIN_ROOT:-${extensionPath:-.}}/hooks/scripts/check-config.sh\""
|
||||
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-config.sh",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -33,13 +33,8 @@ load_env_vars() {
|
||||
[[ -z "$key" ]] && continue
|
||||
key=$(echo "$key" | xargs)
|
||||
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||
# Strip inline comments (# preceded by whitespace) to prevent
|
||||
# command substitution in backtick-containing comments
|
||||
value="${value%%[[:space:]]#*}"
|
||||
if [[ -n "$key" && -n "$value" ]]; then
|
||||
# printf -v writes via assignment semantics (global from inside a
|
||||
# function), works on macOS's /bin/bash 3.2 — `declare -g` is 4.2+.
|
||||
printf -v "ENV_${key}" '%s' "$value"
|
||||
eval "ENV_${key}=\"${value}\""
|
||||
fi
|
||||
done < "$file"
|
||||
fi
|
||||
@@ -63,53 +58,14 @@ fi
|
||||
# Check SETUP_COMPLETE (from file or env)
|
||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||
|
||||
# Compute last-run summary line (if last-run.json exists)
|
||||
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
||||
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
||||
LAST_RUN_FILE="$LAST30DAYS_CONFIG_DIR/last-run.json"
|
||||
else
|
||||
LAST_RUN_FILE=""
|
||||
fi
|
||||
else
|
||||
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
|
||||
fi
|
||||
LAST_RUN_LINE=""
|
||||
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
|
||||
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 - <<'PY' 2>/dev/null || true
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
|
||||
path = os.environ["LAST_RUN_FILE"]
|
||||
try:
|
||||
with open(path) as fh:
|
||||
d = json.load(fh)
|
||||
topic = (d.get("topic") or "?")[:60]
|
||||
ts = d.get("timestamp", "")
|
||||
dt = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
delta = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
|
||||
if delta < 60: ago = f"{int(delta)}s ago"
|
||||
elif delta < 3600: ago = f"{int(delta//60)}m ago"
|
||||
elif delta < 86400: ago = f"{int(delta//3600)}h ago"
|
||||
else: ago = f"{int(delta//86400)}d ago"
|
||||
total = d.get("total", 0)
|
||||
print(f" Last run: \"{topic}\" · {ago} · {total} results")
|
||||
except Exception:
|
||||
pass
|
||||
PY
|
||||
)
|
||||
fi
|
||||
|
||||
# If setup has never been run, show welcome message for new users
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||
cat <<'EOF'
|
||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||
Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).
|
||||
|
||||
Reddit, Hacker News, and Polymarket work out of the box.
|
||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||
EOF
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -141,33 +97,16 @@ if [[ -n "$HAS_BSKY" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
|
||||
# Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer
|
||||
# whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing.
|
||||
SC_ADD=3
|
||||
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
|
||||
EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \
|
||||
| sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//')
|
||||
if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||
fi
|
||||
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Fully configured — compact ready message
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
else
|
||||
# Setup done but missing ScrapeCreators — recommend it
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
||||
echo " 10,000 free API calls, no credit card — scrapecreators.com"
|
||||
echo " last30days has no affiliation with any API provider."
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# Mirror of skills/last30days/scripts/, populated by scripts/sync-engine.sh.
|
||||
# Source of truth lives in the Python skill; never commit the mirror.
|
||||
# Lives inside internal/engine/ because //go:embed cannot reach outside
|
||||
# its own package directory.
|
||||
internal/engine/vendored/*
|
||||
!internal/engine/vendored/.gitkeep
|
||||
|
||||
# Local build output: cross-compiled binaries and packaged .mcpb files.
|
||||
build/
|
||||
# Anchor to the mcp/ root so the cmd/last30days-pp-mcp/ package directory
|
||||
# is not also excluded (subdirs with the same name would otherwise match).
|
||||
/last30days-pp-mcp
|
||||
@@ -0,0 +1,36 @@
|
||||
# last30days-pp-mcp
|
||||
|
||||
Go MCP server that wraps the last30days Python engine for Claude Desktop. Packaged as a `.mcpb` bundle (drag-drop install into Claude Desktop).
|
||||
|
||||
The MCP server exposes a single `research` tool that mirrors the `/last30days <topic>` slash command available in Claude Code. At runtime the binary extracts the vendored Python engine into a per-user cache and shells out to `python3` to produce the synthesis input Claude renders.
|
||||
|
||||
## Architecture
|
||||
|
||||
- `cmd/last30days-pp-mcp/` - server entry point
|
||||
- `internal/engine/` - `embed.FS` of the Python engine + cache extractor + subprocess wrapper
|
||||
- `internal/tools/` - MCP tool handlers (currently `research`)
|
||||
- `internal/engine/vendored/` - mirror of `skills/last30days/scripts/`, generated by `scripts/sync-engine.sh` (gitignored). Lives inside the engine package because `//go:embed` cannot reach files outside its own package directory.
|
||||
- `manifest.json` - MCPB v0.3 manifest consumed by Claude Desktop and `printing-press bundle`
|
||||
|
||||
## Local build
|
||||
|
||||
```bash
|
||||
# Mirror the Python engine into vendored/.
|
||||
bash scripts/sync-engine.sh
|
||||
|
||||
# Build for the current host.
|
||||
go build -ldflags "-X main.Version=dev" -o build/last30days-pp-mcp ./cmd/last30days-pp-mcp
|
||||
|
||||
# Package as a .mcpb (requires the printing-press binary on PATH).
|
||||
printing-press bundle . --skip-build --binary build/last30days-pp-mcp
|
||||
```
|
||||
|
||||
The output `.mcpb` lands at `build/last30days-pp-mcp-<os>-<arch>.mcpb`. Drag it into Claude Desktop's Extensions panel to install.
|
||||
|
||||
## Runtime requirements
|
||||
|
||||
End users need Python 3.12+ on PATH. The bundle ships the engine source but relies on the host interpreter.
|
||||
|
||||
## Versioning
|
||||
|
||||
The MCPB `manifest.json` version is hand-bumped in the same PR that ships engine changes worth releasing. Release CI stamps the Go binary's `main.Version` from the tag.
|
||||
@@ -0,0 +1,39 @@
|
||||
// Package main is the entry point for the last30days MCP server bundled
|
||||
// as a .mcpb for Claude Desktop. The server registers a single research
|
||||
// tool (see internal/tools) and serves it over stdio. See mcp/README.md
|
||||
// for build and packaging instructions.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/tools"
|
||||
)
|
||||
|
||||
// Version is stamped at build time via -ldflags "-X main.Version=<tag>".
|
||||
// It namespaces the per-user cache directory in internal/engine so multiple
|
||||
// installed versions can coexist without clobbering each other.
|
||||
var Version = "dev"
|
||||
|
||||
const (
|
||||
serverName = "last30days"
|
||||
serverVersion = "1"
|
||||
)
|
||||
|
||||
func main() {
|
||||
s := server.NewMCPServer(
|
||||
serverName,
|
||||
serverVersion,
|
||||
server.WithToolCapabilities(false),
|
||||
)
|
||||
|
||||
tools.Register(s, tools.Config{Version: Version})
|
||||
|
||||
if err := server.ServeStdio(s); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "last30days-pp-mcp: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
module github.com/mvanhorn/last30days-skill/mcp
|
||||
|
||||
go 1.25.5
|
||||
|
||||
require github.com/mark3labs/mcp-go v0.54.0
|
||||
|
||||
require (
|
||||
github.com/google/jsonschema-go v0.4.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/spf13/cast v1.7.1 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
|
||||
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mark3labs/mcp-go v0.54.0 h1:PZhQvd+5xrT43cUoiaKn/hDcvLUhcLc1twSEKYPTcTA=
|
||||
github.com/mark3labs/mcp-go v0.54.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
|
||||
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,26 @@
|
||||
// Package engine wraps the vendored Python last30days engine. The engine
|
||||
// is embedded at build time via //go:embed and extracted into a per-user
|
||||
// cache directory on first use, then invoked through python3 in a
|
||||
// subprocess. Consumers should call EnsureUserCache to materialize the
|
||||
// engine and Run to execute it.
|
||||
package engine
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// EngineSourceDir is the embed root inside the binary. scripts/sync-engine.sh
|
||||
// mirrors skills/last30days/scripts/ into this directory before each build.
|
||||
// The all: prefix preserves files starting with "." or "_" so the .gitkeep
|
||||
// anchor file survives - without it the embed would error before sync runs.
|
||||
//
|
||||
//go:embed all:vendored
|
||||
var vendored embed.FS
|
||||
|
||||
// EngineFS returns the embedded engine as a filesystem rooted at the
|
||||
// vendored/ directory contents (so callers see "last30days.py" at the
|
||||
// root, not "vendored/last30days.py").
|
||||
func EngineFS() (fs.FS, error) {
|
||||
return fs.Sub(vendored, "vendored")
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// SentinelFilename names the file Ensure writes inside the cache directory
|
||||
// after a successful extraction. Its contents are compared to the requested
|
||||
// version; a match short-circuits re-extraction on subsequent calls.
|
||||
const SentinelFilename = ".version"
|
||||
|
||||
// cacheSubdir namespaces our cache under the OS user cache directory so
|
||||
// multiple printing-press-style bundles can coexist.
|
||||
const cacheSubdir = "last30days-pp-mcp"
|
||||
|
||||
// CacheEnvOverride lets users redirect the cache directory when the default
|
||||
// OS cache location is read-only (locked-down corp images, ephemeral CI
|
||||
// containers). Pointed at by extract errors via the documented escape hatch.
|
||||
const CacheEnvOverride = "LAST30DAYS_CACHE_DIR"
|
||||
|
||||
// Ensure extracts src into baseDir/last30days-pp-mcp/<version> and returns
|
||||
// the cache path. If the sentinel file already records the same version the
|
||||
// directory is reused without rewriting. version must be non-empty so the
|
||||
// cache layout always namespaces by version.
|
||||
//
|
||||
// Extraction writes to a sibling .tmp directory and renames it on success
|
||||
// so a partial extraction can never be mistaken for a complete one. Concurrent
|
||||
// callers within the same process serialize behind a per-cache-dir sync.Once
|
||||
// so the rename happens exactly once.
|
||||
func Ensure(src fs.FS, baseDir, version string) (string, error) {
|
||||
if version == "" {
|
||||
return "", errors.New("engine: version is required")
|
||||
}
|
||||
cacheDir := filepath.Join(baseDir, cacheSubdir, version)
|
||||
|
||||
once := getOnce(cacheDir)
|
||||
var extractErr error
|
||||
once.Do(func() {
|
||||
extractErr = ensureLocked(src, cacheDir, version)
|
||||
})
|
||||
if extractErr != nil {
|
||||
// Reset the sync.Once so a follow-up call can retry rather than
|
||||
// permanently caching the error. Retry is the right default when
|
||||
// the failure is transient (e.g., disk full, parent dir restored).
|
||||
resetOnce(cacheDir)
|
||||
return "", extractErr
|
||||
}
|
||||
return cacheDir, nil
|
||||
}
|
||||
|
||||
// EnsureUserCache wraps Ensure with the OS user cache dir (or the
|
||||
// LAST30DAYS_CACHE_DIR override) as base. Production callers use this; tests
|
||||
// use Ensure with an explicit temp dir.
|
||||
func EnsureUserCache(src fs.FS, version string) (string, error) {
|
||||
if override := os.Getenv(CacheEnvOverride); override != "" {
|
||||
return Ensure(src, override, version)
|
||||
}
|
||||
base, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("engine: resolve user cache dir (set %s to override): %w", CacheEnvOverride, err)
|
||||
}
|
||||
return Ensure(src, base, version)
|
||||
}
|
||||
|
||||
func ensureLocked(src fs.FS, cacheDir, version string) error {
|
||||
if sentinelMatches(cacheDir, version) {
|
||||
return nil
|
||||
}
|
||||
tmpDir := cacheDir + ".tmp"
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
return fmt.Errorf("engine: clean tmp cache: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(tmpDir, 0o755); err != nil {
|
||||
return fmt.Errorf("engine: create tmp cache (%s, set %s to override): %w", tmpDir, CacheEnvOverride, err)
|
||||
}
|
||||
if err := extractAll(src, tmpDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return err
|
||||
}
|
||||
sentinel := filepath.Join(tmpDir, SentinelFilename)
|
||||
if err := os.WriteFile(sentinel, []byte(version), 0o644); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: write sentinel: %w", err)
|
||||
}
|
||||
if err := os.RemoveAll(cacheDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: clean old cache: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpDir, cacheDir); err != nil {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
return fmt.Errorf("engine: promote tmp cache: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sentinelMatches(cacheDir, version string) bool {
|
||||
data, err := os.ReadFile(filepath.Join(cacheDir, SentinelFilename))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return string(data) == version
|
||||
}
|
||||
|
||||
func extractAll(src fs.FS, dst string) error {
|
||||
return fs.WalkDir(src, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
target := filepath.Join(dst, path)
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0o755)
|
||||
}
|
||||
return copyEmbeddedFile(src, path, target)
|
||||
})
|
||||
}
|
||||
|
||||
func copyEmbeddedFile(src fs.FS, srcPath, dst string) error {
|
||||
in, err := src.Open(srcPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("engine: open %s: %w", srcPath, err)
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return fmt.Errorf("engine: ensure parent of %s: %w", dst, err)
|
||||
}
|
||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("engine: create %s: %w", dst, err)
|
||||
}
|
||||
defer func() { _ = out.Close() }()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return fmt.Errorf("engine: write %s: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// onceRegistry serializes first-call extraction per cache directory so the
|
||||
// rename in ensureLocked happens exactly once across goroutines.
|
||||
var (
|
||||
onceMu sync.Mutex
|
||||
onceRegistry = map[string]*sync.Once{}
|
||||
)
|
||||
|
||||
func getOnce(cacheDir string) *sync.Once {
|
||||
onceMu.Lock()
|
||||
defer onceMu.Unlock()
|
||||
if o, ok := onceRegistry[cacheDir]; ok {
|
||||
return o
|
||||
}
|
||||
o := &sync.Once{}
|
||||
onceRegistry[cacheDir] = o
|
||||
return o
|
||||
}
|
||||
|
||||
func resetOnce(cacheDir string) {
|
||||
onceMu.Lock()
|
||||
defer onceMu.Unlock()
|
||||
delete(onceRegistry, cacheDir)
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func newTestFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("# last30days entry\n"), Mode: 0o644},
|
||||
"lib/__init__.py": &fstest.MapFile{Data: []byte(""), Mode: 0o644},
|
||||
"lib/env.py": &fstest.MapFile{Data: []byte("# env helpers\n"), Mode: 0o644},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureExtractsEngine(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
cacheDir, err := Ensure(src, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure: %v", err)
|
||||
}
|
||||
if cacheDir != filepath.Join(base, cacheSubdir, "v1") {
|
||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, filepath.Join(base, cacheSubdir, "v1"))
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
||||
mustReadFile(t, filepath.Join(cacheDir, "lib/env.py"), "# env helpers\n")
|
||||
mustReadFile(t, filepath.Join(cacheDir, SentinelFilename), "v1")
|
||||
}
|
||||
|
||||
func TestEnsureSkipsWhenSentinelMatches(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
cacheDir, err := Ensure(src, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("first Ensure: %v", err)
|
||||
}
|
||||
target := filepath.Join(cacheDir, "last30days.py")
|
||||
info1, err := os.Stat(target)
|
||||
if err != nil {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
|
||||
// Reset the sync.Once so a second call would re-extract if not for the
|
||||
// sentinel short-circuit. Without the reset, sync.Once would skip the
|
||||
// extraction regardless of sentinel state.
|
||||
resetOnce(cacheDir)
|
||||
|
||||
if _, err := Ensure(src, base, "v1"); err != nil {
|
||||
t.Fatalf("second Ensure: %v", err)
|
||||
}
|
||||
info2, err := os.Stat(target)
|
||||
if err != nil {
|
||||
t.Fatalf("stat second: %v", err)
|
||||
}
|
||||
if !info2.ModTime().Equal(info1.ModTime()) {
|
||||
t.Fatalf("expected file untouched on sentinel match; got mtime %v -> %v", info1.ModTime(), info2.ModTime())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureReExtractsOnVersionChange(t *testing.T) {
|
||||
v1 := fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("v1\n"), Mode: 0o644},
|
||||
}
|
||||
v2 := fstest.MapFS{
|
||||
"last30days.py": &fstest.MapFile{Data: []byte("v2\n"), Mode: 0o644},
|
||||
}
|
||||
base := t.TempDir()
|
||||
|
||||
cache1, err := Ensure(v1, base, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure v1: %v", err)
|
||||
}
|
||||
cache2, err := Ensure(v2, base, "v2")
|
||||
if err != nil {
|
||||
t.Fatalf("Ensure v2: %v", err)
|
||||
}
|
||||
if cache1 == cache2 {
|
||||
t.Fatalf("expected distinct cache dirs per version, got %q == %q", cache1, cache2)
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cache1, "last30days.py"), "v1\n")
|
||||
mustReadFile(t, filepath.Join(cache2, "last30days.py"), "v2\n")
|
||||
}
|
||||
|
||||
func TestEnsureConcurrentFirstCall(t *testing.T) {
|
||||
src := newTestFS()
|
||||
base := t.TempDir()
|
||||
|
||||
const goroutines = 10
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
results := make([]string, goroutines)
|
||||
errs := make([]error, goroutines)
|
||||
for i := 0; i < goroutines; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
results[i], errs[i] = Ensure(src, base, "v1")
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("goroutine %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
for i := 1; i < goroutines; i++ {
|
||||
if results[i] != results[0] {
|
||||
t.Fatalf("goroutine 0 saw %q, goroutine %d saw %q", results[0], i, results[i])
|
||||
}
|
||||
}
|
||||
mustReadFile(t, filepath.Join(results[0], "last30days.py"), "# last30days entry\n")
|
||||
}
|
||||
|
||||
func TestEnsureRejectsEmptyVersion(t *testing.T) {
|
||||
if _, err := Ensure(newTestFS(), t.TempDir(), ""); err == nil {
|
||||
t.Fatal("expected error for empty version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureReturnsErrorWhenCacheUnwritable(t *testing.T) {
|
||||
// Place the cache root at a path that cannot exist (a regular file).
|
||||
// MkdirAll will refuse and Ensure must surface a wrapped error.
|
||||
base := t.TempDir()
|
||||
blocker := filepath.Join(base, "blocker")
|
||||
if err := os.WriteFile(blocker, []byte("not a dir"), 0o644); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
_, err := Ensure(newTestFS(), blocker, "v1")
|
||||
if err == nil {
|
||||
t.Fatal("expected error when cache parent is not a directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureUserCacheHonorsOverride(t *testing.T) {
|
||||
override := t.TempDir()
|
||||
t.Setenv(CacheEnvOverride, override)
|
||||
|
||||
src := newTestFS()
|
||||
cacheDir, err := EnsureUserCache(src, "v1")
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureUserCache: %v", err)
|
||||
}
|
||||
want := filepath.Join(override, cacheSubdir, "v1")
|
||||
if cacheDir != want {
|
||||
t.Fatalf("cacheDir = %q, want %q", cacheDir, want)
|
||||
}
|
||||
mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
|
||||
}
|
||||
|
||||
func mustReadFile(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("%s: got %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultPythonBinary is the interpreter we look up unless RunOptions
|
||||
// overrides it. Windows installs may expose only "python"; we surface a
|
||||
// clear error in that case rather than silently picking the wrong binary.
|
||||
const DefaultPythonBinary = "python3"
|
||||
|
||||
// MinPythonVersion mirrors the engine's MIN_PYTHON constant in
|
||||
// last30days.py. Surfaced in errors so users know what they're missing.
|
||||
const MinPythonVersion = "3.12"
|
||||
|
||||
// PythonInstallURL is included in the missing-interpreter error so users
|
||||
// have a direct route from the failure to a fix.
|
||||
const PythonInstallURL = "https://www.python.org/downloads/"
|
||||
|
||||
// DefaultTimeout caps a single research subprocess. The engine's deep mode
|
||||
// can run several minutes; five minutes is a safe upper bound that still
|
||||
// fails fast when something hangs.
|
||||
const DefaultTimeout = 5 * time.Minute
|
||||
|
||||
// TimeoutEnvOverride lets operators override DefaultTimeout per install
|
||||
// (seconds, integer). Honored by Run when RunOptions.Timeout is zero.
|
||||
const TimeoutEnvOverride = "LAST30DAYS_MCP_TIMEOUT"
|
||||
|
||||
// RunOptions configures one invocation of the embedded Python engine.
|
||||
// PythonPath is exposed so tests can substitute a stub interpreter without
|
||||
// manipulating the process PATH.
|
||||
type RunOptions struct {
|
||||
PythonPath string // resolved python3 binary; empty means look up DefaultPythonBinary on PATH
|
||||
CacheDir string // engine.Ensure result; lib/ here is added to PYTHONPATH
|
||||
Args []string // arguments after last30days.py (topic, --emit=..., etc.)
|
||||
ExtraEnv []string // appended to os.Environ() for the child process
|
||||
Timeout time.Duration // zero means DefaultTimeout or TimeoutEnvOverride
|
||||
}
|
||||
|
||||
// RunResult captures the engine's full output. Stdout is what we surface to
|
||||
// the agent; Stderr is included in error messages so users can diagnose
|
||||
// engine failures without leaving Claude Desktop.
|
||||
type RunResult struct {
|
||||
Stdout []byte
|
||||
Stderr []byte
|
||||
ExitCode int
|
||||
TimedOut bool
|
||||
}
|
||||
|
||||
// Run shells out to python3 with last30days.py inside cacheDir. The child
|
||||
// receives the parent environment (so MCPB user_config env-injection
|
||||
// reaches the engine) plus ExtraEnv and a PYTHONPATH that points at the
|
||||
// cache so the engine's `from lib import ...` statements resolve.
|
||||
//
|
||||
// A missing interpreter, a non-zero exit, and a timeout each surface as
|
||||
// distinct errors so the tool handler can map them to user-facing
|
||||
// messages without re-parsing stderr.
|
||||
func Run(ctx context.Context, opts RunOptions) (*RunResult, error) {
|
||||
if opts.CacheDir == "" {
|
||||
return nil, errors.New("engine: CacheDir is required")
|
||||
}
|
||||
pythonPath, err := resolvePython(opts.PythonPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scriptPath := filepath.Join(opts.CacheDir, "last30days.py")
|
||||
if _, err := os.Stat(scriptPath); err != nil {
|
||||
return nil, fmt.Errorf("engine: last30days.py not found in cache %s: %w", opts.CacheDir, err)
|
||||
}
|
||||
|
||||
timeout := resolveTimeout(opts.Timeout)
|
||||
subCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
args := append([]string{scriptPath}, opts.Args...)
|
||||
cmd := exec.CommandContext(subCtx, pythonPath, args...)
|
||||
cmd.Env = buildEnv(opts.CacheDir, opts.ExtraEnv)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
err = cmd.Run()
|
||||
res := &RunResult{
|
||||
Stdout: stdout.Bytes(),
|
||||
Stderr: stderr.Bytes(),
|
||||
ExitCode: 0,
|
||||
TimedOut: errors.Is(subCtx.Err(), context.DeadlineExceeded),
|
||||
}
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
res.ExitCode = exitErr.ExitCode()
|
||||
if res.TimedOut {
|
||||
return res, fmt.Errorf("engine: subprocess exceeded %s timeout", timeout)
|
||||
}
|
||||
return res, fmt.Errorf("engine: subprocess exited with code %d", res.ExitCode)
|
||||
}
|
||||
return res, fmt.Errorf("engine: subprocess failed to start: %w", err)
|
||||
}
|
||||
|
||||
// resolvePython returns an absolute path to the interpreter or an error
|
||||
// naming the install URL. If the caller supplied a path we trust it - tests
|
||||
// rely on this to inject a stub. Otherwise we look up python3 on PATH.
|
||||
func resolvePython(override string) (string, error) {
|
||||
if override != "" {
|
||||
return override, nil
|
||||
}
|
||||
path, err := exec.LookPath(DefaultPythonBinary)
|
||||
if err == nil {
|
||||
return path, nil
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"engine: %s not found on PATH (need Python %s+, install from %s; current GOOS=%s)",
|
||||
DefaultPythonBinary, MinPythonVersion, PythonInstallURL, runtime.GOOS,
|
||||
)
|
||||
}
|
||||
|
||||
func resolveTimeout(explicit time.Duration) time.Duration {
|
||||
if explicit > 0 {
|
||||
return explicit
|
||||
}
|
||||
if raw := os.Getenv(TimeoutEnvOverride); raw != "" {
|
||||
if d, err := time.ParseDuration(raw); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return DefaultTimeout
|
||||
}
|
||||
|
||||
// buildEnv stitches PYTHONPATH onto os.Environ + ExtraEnv. Any pre-existing
|
||||
// PYTHONPATH in the parent environment is dropped before appending the
|
||||
// cache dir; otherwise the child sees two PYTHONPATH= entries and POSIX
|
||||
// getenv returns the first one, so the user's value wins and the engine's
|
||||
// `from lib import ...` fails with ModuleNotFoundError. The engine is
|
||||
// self-contained and does not need the user's Python module search path.
|
||||
func buildEnv(cacheDir string, extra []string) []string {
|
||||
const pyKey = "PYTHONPATH="
|
||||
parent := os.Environ()
|
||||
base := make([]string, 0, len(parent)+1+len(extra))
|
||||
for _, kv := range parent {
|
||||
if strings.HasPrefix(kv, pyKey) {
|
||||
continue
|
||||
}
|
||||
base = append(base, kv)
|
||||
}
|
||||
base = append(base, pyKey+cacheDir)
|
||||
base = append(base, extra...)
|
||||
return base
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// makeStubPython writes a shell script that simulates python3 and returns
|
||||
// its absolute path. The script honors a small env-driven protocol so each
|
||||
// test can shape its output:
|
||||
//
|
||||
// STUB_STDOUT - text printed to stdout
|
||||
// STUB_STDERR - text printed to stderr
|
||||
// STUB_EXIT_CODE - integer exit code (default 0)
|
||||
// STUB_SLEEP_SECS - sleep before exiting (for timeout tests)
|
||||
// STUB_ECHO_ENV - name of an env var; the stub prints "<NAME>=<VALUE>"
|
||||
// STUB_ECHO_ARG - integer index; the stub prints "ARG<i>=<args[i]>"
|
||||
//
|
||||
// The stub ignores its first argument (the script path), matching how a
|
||||
// real python3 invocation treats `python3 last30days.py ...`.
|
||||
func makeStubPython(t *testing.T) string {
|
||||
t.Helper()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("stub-python tests rely on POSIX shell")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "python3-stub.sh")
|
||||
script := `#!/usr/bin/env bash
|
||||
if [ -n "${STUB_SLEEP_SECS:-}" ]; then sleep "$STUB_SLEEP_SECS"; fi
|
||||
if [ -n "${STUB_STDOUT:-}" ]; then printf "%s" "$STUB_STDOUT"; fi
|
||||
if [ -n "${STUB_STDERR:-}" ]; then printf "%s" "$STUB_STDERR" >&2; fi
|
||||
if [ -n "${STUB_ECHO_ENV:-}" ]; then echo "${STUB_ECHO_ENV}=${!STUB_ECHO_ENV:-<unset>}"; fi
|
||||
if [ -n "${STUB_ECHO_ARG:-}" ]; then echo "ARG${STUB_ECHO_ARG}=${!STUB_ECHO_ARG:-<unset>}"; fi
|
||||
exit "${STUB_EXIT_CODE:-0}"
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatalf("write stub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// stageCache materializes a fake CacheDir with a no-op last30days.py so
|
||||
// the existence check in Run passes. The stub python3 ignores the script
|
||||
// contents, so the file just has to exist.
|
||||
func stageCache(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "last30days.py"), []byte("# stub\n"), 0o644); err != nil {
|
||||
t.Fatalf("stage cache: %v", err)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestRunHappyPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_STDOUT", "synthesis output\n")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
Args: []string{"my topic", "--emit=compact"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if string(res.Stdout) != "synthesis output\n" {
|
||||
t.Fatalf("stdout = %q, want %q", res.Stdout, "synthesis output\n")
|
||||
}
|
||||
if res.ExitCode != 0 {
|
||||
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
||||
}
|
||||
if res.TimedOut {
|
||||
t.Fatal("TimedOut = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForwardsEnv(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("OPENAI_API_KEY", "sk-test-value")
|
||||
t.Setenv("STUB_ECHO_ENV", "OPENAI_API_KEY")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if got := strings.TrimSpace(string(res.Stdout)); got != "OPENAI_API_KEY=sk-test-value" {
|
||||
t.Fatalf("stdout = %q, want OPENAI_API_KEY=sk-test-value", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSetsPythonPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
want := "PYTHONPATH=" + cache
|
||||
if got := strings.TrimSpace(string(res.Stdout)); got != want {
|
||||
t.Fatalf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDropsPreExistingPythonPath guards the buildEnv dedup: when the
|
||||
// parent already sets PYTHONPATH (common on dev machines and CI runners
|
||||
// that touch Python), the child must NOT see two PYTHONPATH= entries.
|
||||
// POSIX getenv returns the first match, so a duplicate from os.Environ
|
||||
// would shadow our cache-dir entry and break `from lib import ...`.
|
||||
func TestRunDropsPreExistingPythonPath(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("PYTHONPATH", "/users-stale-pythonpath")
|
||||
t.Setenv("STUB_ECHO_ENV", "PYTHONPATH")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
got := strings.TrimSpace(string(res.Stdout))
|
||||
want := "PYTHONPATH=" + cache
|
||||
if got != want {
|
||||
t.Fatalf("stdout = %q, want %q (stale parent value leaked through)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildEnvDropsAllPreExistingPythonPath(t *testing.T) {
|
||||
// Direct unit test on buildEnv to catch the case where the parent has
|
||||
// PYTHONPATH set: the returned slice must contain exactly one
|
||||
// PYTHONPATH= entry, and it must be ours.
|
||||
t.Setenv("PYTHONPATH", "/parent/one")
|
||||
cache := "/cache/dir"
|
||||
out := buildEnv(cache, []string{"EXTRA=1"})
|
||||
|
||||
var pythonPaths []string
|
||||
for _, kv := range out {
|
||||
if strings.HasPrefix(kv, "PYTHONPATH=") {
|
||||
pythonPaths = append(pythonPaths, kv)
|
||||
}
|
||||
}
|
||||
if len(pythonPaths) != 1 {
|
||||
t.Fatalf("got %d PYTHONPATH entries, want 1: %v", len(pythonPaths), pythonPaths)
|
||||
}
|
||||
if pythonPaths[0] != "PYTHONPATH="+cache {
|
||||
t.Fatalf("PYTHONPATH = %q, want %q", pythonPaths[0], "PYTHONPATH="+cache)
|
||||
}
|
||||
// Confirm ExtraEnv still rides along.
|
||||
found := false
|
||||
for _, kv := range out {
|
||||
if kv == "EXTRA=1" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("EXTRA=1 missing from buildEnv output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSurfacesExitCode(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_STDERR", "engine boom\n")
|
||||
t.Setenv("STUB_EXIT_CODE", "2")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for non-zero exit")
|
||||
}
|
||||
if res == nil {
|
||||
t.Fatal("res is nil; want populated result alongside error")
|
||||
}
|
||||
if res.ExitCode != 2 {
|
||||
t.Fatalf("ExitCode = %d, want 2", res.ExitCode)
|
||||
}
|
||||
if !strings.Contains(string(res.Stderr), "engine boom") {
|
||||
t.Fatalf("stderr did not surface engine output: %q", res.Stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTimesOut(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
cache := stageCache(t)
|
||||
t.Setenv("STUB_SLEEP_SECS", "3")
|
||||
|
||||
res, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
Timeout: 200 * time.Millisecond,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error")
|
||||
}
|
||||
if !res.TimedOut {
|
||||
t.Fatal("TimedOut = false, want true")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "timeout") {
|
||||
t.Fatalf("error %q lacks 'timeout' marker", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingPython(t *testing.T) {
|
||||
cache := stageCache(t)
|
||||
// Empty PATH guarantees the lookup fails. PythonPath stays unset so Run
|
||||
// falls through to exec.LookPath.
|
||||
t.Setenv("PATH", "")
|
||||
|
||||
_, err := Run(context.Background(), RunOptions{CacheDir: cache})
|
||||
if err == nil {
|
||||
t.Fatal("expected lookup failure with empty PATH")
|
||||
}
|
||||
if !strings.Contains(err.Error(), DefaultPythonBinary) {
|
||||
t.Fatalf("error %q does not mention %s", err, DefaultPythonBinary)
|
||||
}
|
||||
if !strings.Contains(err.Error(), PythonInstallURL) {
|
||||
t.Fatalf("error %q does not include install URL", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingScript(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
// CacheDir exists but contains no last30days.py.
|
||||
cache := t.TempDir()
|
||||
|
||||
_, err := Run(context.Background(), RunOptions{
|
||||
PythonPath: stub,
|
||||
CacheDir: cache,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error when last30days.py missing")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "last30days.py") {
|
||||
t.Fatalf("error %q does not name missing script", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsEmptyCacheDir(t *testing.T) {
|
||||
stub := makeStubPython(t)
|
||||
_, err := Run(context.Background(), RunOptions{PythonPath: stub})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty CacheDir")
|
||||
}
|
||||
if !errors.Is(err, err) || !strings.Contains(err.Error(), "CacheDir") {
|
||||
t.Fatalf("error %q does not name CacheDir", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTimeoutHonorsEnv(t *testing.T) {
|
||||
t.Setenv(TimeoutEnvOverride, "750ms")
|
||||
if got := resolveTimeout(0); got != 750*time.Millisecond {
|
||||
t.Fatalf("resolveTimeout = %v, want 750ms", got)
|
||||
}
|
||||
t.Setenv(TimeoutEnvOverride, "garbage")
|
||||
if got := resolveTimeout(0); got != DefaultTimeout {
|
||||
t.Fatalf("garbage value: got %v, want default %v", got, DefaultTimeout)
|
||||
}
|
||||
if got := resolveTimeout(time.Minute); got != time.Minute {
|
||||
t.Fatalf("explicit value not honored: got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Populated at build time by scripts/sync-engine.sh.
|
||||
Source of truth: skills/last30days/scripts/.
|
||||
@@ -0,0 +1,189 @@
|
||||
// Package manifest holds tests for mcp/manifest.json. It contains no
|
||||
// production code - the manifest itself is the artifact, and these tests
|
||||
// guard structural invariants the bundling pipeline depends on.
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// envBinding is a minimal subset of the MCPB v0.3 manifest just covering
|
||||
// the fields these tests assert on. We deliberately do not depend on the
|
||||
// printing-press internal/pipeline types (that's an internal/ package and
|
||||
// not importable across modules) - the structural invariants below are
|
||||
// what actually matter for Claude Desktop install correctness.
|
||||
type manifestShape struct {
|
||||
ManifestVersion string `json:"manifest_version"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Server struct {
|
||||
Type string `json:"type"`
|
||||
EntryPoint string `json:"entry_point"`
|
||||
MCPConfig struct {
|
||||
Command string `json:"command"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"mcp_config"`
|
||||
} `json:"server"`
|
||||
UserConfig map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Sensitive bool `json:"sensitive"`
|
||||
Required bool `json:"required"`
|
||||
} `json:"user_config"`
|
||||
Compatibility struct {
|
||||
ClaudeDesktop string `json:"claude_desktop"`
|
||||
Platforms []string `json:"platforms"`
|
||||
} `json:"compatibility"`
|
||||
}
|
||||
|
||||
// loadManifest reads mcp/manifest.json relative to this test file so the
|
||||
// test passes regardless of where `go test` is invoked from.
|
||||
func loadManifest(t *testing.T) manifestShape {
|
||||
t.Helper()
|
||||
_, thisFile, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("runtime.Caller failed")
|
||||
}
|
||||
// manifest_test.go is at mcp/internal/manifest/; manifest.json at mcp/.
|
||||
manifestPath := filepath.Join(filepath.Dir(thisFile), "..", "..", "manifest.json")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
var m manifestShape
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
t.Fatalf("parse manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestManifestRequiredFields(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
if m.ManifestVersion != "0.3" {
|
||||
t.Errorf("manifest_version = %q, want 0.3", m.ManifestVersion)
|
||||
}
|
||||
if m.Name != "last30days-pp-mcp" {
|
||||
t.Errorf("name = %q, want last30days-pp-mcp", m.Name)
|
||||
}
|
||||
if m.Version == "" {
|
||||
t.Error("version is empty")
|
||||
}
|
||||
if m.Server.Type != "binary" {
|
||||
t.Errorf("server.type = %q, want binary", m.Server.Type)
|
||||
}
|
||||
if m.Server.EntryPoint != "bin/last30days-pp-mcp" {
|
||||
t.Errorf("server.entry_point = %q, want bin/last30days-pp-mcp", m.Server.EntryPoint)
|
||||
}
|
||||
if m.Compatibility.ClaudeDesktop == "" {
|
||||
t.Error("compatibility.claude_desktop is empty")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEnvAndUserConfigCrossReference is the key invariant: every
|
||||
// ${user_config.<key>} substitution in server.mcp_config.env must point
|
||||
// at a real user_config entry, and every declared user_config must be
|
||||
// wired to an env var. A typo on either side silently disables a credential
|
||||
// at install time without the binary or Claude Desktop noticing.
|
||||
func TestEnvAndUserConfigCrossReference(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
|
||||
if len(m.Server.MCPConfig.Env) == 0 {
|
||||
t.Fatal("server.mcp_config.env is empty; expected user_config substitutions")
|
||||
}
|
||||
if len(m.UserConfig) == 0 {
|
||||
t.Fatal("user_config is empty; expected per-key declarations")
|
||||
}
|
||||
|
||||
for envName, value := range m.Server.MCPConfig.Env {
|
||||
key, ok := parseUserConfigRef(value)
|
||||
if !ok {
|
||||
t.Errorf("env[%s] = %q is not a ${user_config.<key>} reference", envName, value)
|
||||
continue
|
||||
}
|
||||
if _, declared := m.UserConfig[key]; !declared {
|
||||
t.Errorf("env[%s] references user_config[%q], which is not declared", envName, key)
|
||||
}
|
||||
// The user_config key must be the lowercased env var so Claude
|
||||
// Desktop's substitution rule matches PP's emitted shape.
|
||||
if got := strings.ToLower(envName); key != got {
|
||||
t.Errorf("env[%s] -> user_config[%q]; convention requires user_config[%q]", envName, key, got)
|
||||
}
|
||||
}
|
||||
|
||||
envValues := make(map[string]bool, len(m.Server.MCPConfig.Env))
|
||||
for _, value := range m.Server.MCPConfig.Env {
|
||||
if key, ok := parseUserConfigRef(value); ok {
|
||||
envValues[key] = true
|
||||
}
|
||||
}
|
||||
for key := range m.UserConfig {
|
||||
if !envValues[key] {
|
||||
t.Errorf("user_config[%q] is declared but never substituted into env", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserConfigShape(t *testing.T) {
|
||||
m := loadManifest(t)
|
||||
for key, slot := range m.UserConfig {
|
||||
if slot.Type != "string" {
|
||||
t.Errorf("user_config[%q].type = %q, want string", key, slot.Type)
|
||||
}
|
||||
if slot.Title == "" {
|
||||
t.Errorf("user_config[%q].title is empty", key)
|
||||
}
|
||||
if slot.Description == "" {
|
||||
t.Errorf("user_config[%q].description is empty", key)
|
||||
}
|
||||
if !slot.Sensitive {
|
||||
// API keys must be flagged sensitive so Claude Desktop masks
|
||||
// the input and prefers OS-keychain storage.
|
||||
t.Errorf("user_config[%q].sensitive = false; want true for API credentials", key)
|
||||
}
|
||||
if slot.Required {
|
||||
// The engine degrades to web-only mode without keys, so no
|
||||
// key is install-blocking.
|
||||
t.Errorf("user_config[%q].required = true; engine degrades without keys, so all keys are optional", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformsMatchShippingMatrix(t *testing.T) {
|
||||
// compatibility.platforms must list exactly what the release CI
|
||||
// actually packages. Listing a platform we don't ship would let
|
||||
// Claude Desktop start an install that has no matching binary inside
|
||||
// the bundle, producing a silent failure. The CI matrix in
|
||||
// .github/workflows/release.yml currently covers darwin (arm64 +
|
||||
// amd64) and linux/amd64; Windows is deferred.
|
||||
m := loadManifest(t)
|
||||
required := map[string]bool{"darwin": false, "linux": false}
|
||||
forbidden := map[string]bool{"win32": true}
|
||||
for _, p := range m.Compatibility.Platforms {
|
||||
if _, ok := required[p]; ok {
|
||||
required[p] = true
|
||||
}
|
||||
if forbidden[p] {
|
||||
t.Errorf("compatibility.platforms contains %q but the release matrix does not ship that platform; add it to the matrix or remove from the manifest", p)
|
||||
}
|
||||
}
|
||||
for p, found := range required {
|
||||
if !found {
|
||||
t.Errorf("compatibility.platforms missing %q", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseUserConfigRef(value string) (string, bool) {
|
||||
const prefix = "${user_config."
|
||||
const suffix = "}"
|
||||
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
|
||||
return "", false
|
||||
}
|
||||
return value[len(prefix) : len(value)-len(suffix)], true
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// Package tools owns the MCP tool surface for last30days. Today there is
|
||||
// exactly one tool, research, mirroring the /last30days <topic> slash
|
||||
// command available in Claude Code. Adding new tools means another file
|
||||
// here plus an additional s.AddTool call in Register.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
// Config carries the version string used to namespace the per-user cache.
|
||||
// main passes its ldflags-stamped Version here.
|
||||
type Config struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
// Register adds every tool this server exposes to s. The caller supplies a
|
||||
// Config so test harnesses can pin a version without touching globals.
|
||||
func Register(s *server.MCPServer, cfg Config) {
|
||||
s.AddTool(
|
||||
mcplib.NewTool("research",
|
||||
mcplib.WithDescription(
|
||||
"Research what people are actually saying about any topic in the last 30 days. "+
|
||||
"Aggregates Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web, "+
|
||||
"scored by upvotes, likes, transcripts, and real-money prediction-market odds. "+
|
||||
"Returns the engine's compact output for the model to synthesize.",
|
||||
),
|
||||
mcplib.WithString("topic", mcplib.Required(), mcplib.Description("The subject to research (a person, company, product, event, or general topic).")),
|
||||
mcplib.WithString("emit", mcplib.Description("Output shape: 'compact' (default) for inline synthesis or 'html' to save a shareable brief alongside the response.")),
|
||||
mcplib.WithBoolean("save", mcplib.Description("Persist the synthesis as a markdown report under ~/Documents/Last30Days/ (or LAST30DAYS_MEMORY_DIR if set).")),
|
||||
mcplib.WithReadOnlyHintAnnotation(true),
|
||||
mcplib.WithDestructiveHintAnnotation(false),
|
||||
mcplib.WithOpenWorldHintAnnotation(true),
|
||||
),
|
||||
makeResearchHandler(cfg),
|
||||
)
|
||||
}
|
||||
|
||||
func makeResearchHandler(cfg Config) server.ToolHandlerFunc {
|
||||
return func(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) {
|
||||
args := req.GetArguments()
|
||||
topic, err := requireString(args, "topic")
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
emit, err := emitArgument(args)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
save, err := boolArgument(args, "save")
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(err.Error()), nil
|
||||
}
|
||||
|
||||
src, err := engine.EngineFS()
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf("engine source unavailable: %v", err)), nil
|
||||
}
|
||||
cacheDir, err := engine.EnsureUserCache(src, cfg.Version)
|
||||
if err != nil {
|
||||
return mcplib.NewToolResultError(fmt.Sprintf(
|
||||
"engine extract failed: %v\nhint: set %s to a writable directory if the default cache location is locked down",
|
||||
err, engine.CacheEnvOverride,
|
||||
)), nil
|
||||
}
|
||||
|
||||
runArgs := []string{topic, "--emit=" + emit}
|
||||
if save {
|
||||
runArgs = append(runArgs, "--save")
|
||||
}
|
||||
|
||||
res, runErr := engine.Run(ctx, engine.RunOptions{
|
||||
CacheDir: cacheDir,
|
||||
Args: runArgs,
|
||||
})
|
||||
if runErr != nil {
|
||||
return mcplib.NewToolResultError(formatRunError(runErr, res)), nil
|
||||
}
|
||||
return mcplib.NewToolResultText(string(res.Stdout)), nil
|
||||
}
|
||||
}
|
||||
|
||||
func requireString(args map[string]any, name string) (string, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is required", name)
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok || strings.TrimSpace(value) == "" {
|
||||
return "", fmt.Errorf("%s must be a non-empty string", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func emitArgument(args map[string]any) (string, error) {
|
||||
raw, ok := args["emit"]
|
||||
if !ok {
|
||||
return "compact", nil
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok {
|
||||
return "", errors.New("emit must be a string")
|
||||
}
|
||||
switch value {
|
||||
case "":
|
||||
return "compact", nil
|
||||
case "compact", "html":
|
||||
return value, nil
|
||||
default:
|
||||
return "", fmt.Errorf("emit must be 'compact' or 'html', got %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
func boolArgument(args map[string]any, name string) (bool, error) {
|
||||
raw, ok := args[name]
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
value, ok := raw.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%s must be a boolean", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// formatRunError flattens engine.Run's distinct error shapes into a single
|
||||
// user-facing message that includes the relevant stderr context.
|
||||
func formatRunError(runErr error, res *engine.RunResult) string {
|
||||
var msg strings.Builder
|
||||
msg.WriteString(runErr.Error())
|
||||
if res != nil && len(res.Stderr) > 0 {
|
||||
msg.WriteString("\nengine stderr:\n")
|
||||
msg.Write(res.Stderr)
|
||||
}
|
||||
return msg.String()
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mcplib "github.com/mark3labs/mcp-go/mcp"
|
||||
|
||||
"github.com/mvanhorn/last30days-skill/mcp/internal/engine"
|
||||
)
|
||||
|
||||
func newCallToolRequest(args map[string]any) mcplib.CallToolRequest {
|
||||
var req mcplib.CallToolRequest
|
||||
req.Params.Arguments = args
|
||||
return req
|
||||
}
|
||||
|
||||
// resultText pulls text content out of a tool result so tests can assert on
|
||||
// the body Claude will see. Returns empty string when the result is nil or
|
||||
// has no text content.
|
||||
func resultText(res *mcplib.CallToolResult) string {
|
||||
if res == nil {
|
||||
return ""
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, item := range res.Content {
|
||||
if tc, ok := item.(mcplib.TextContent); ok {
|
||||
out.WriteString(tc.Text)
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestRequireStringRejectsMissingAndBlank(t *testing.T) {
|
||||
if _, err := requireString(map[string]any{}, "topic"); err == nil {
|
||||
t.Fatal("expected error for missing topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": ""}, "topic"); err == nil {
|
||||
t.Fatal("expected error for empty topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": " "}, "topic"); err == nil {
|
||||
t.Fatal("expected error for whitespace-only topic")
|
||||
}
|
||||
if _, err := requireString(map[string]any{"topic": 42}, "topic"); err == nil {
|
||||
t.Fatal("expected error for non-string topic")
|
||||
}
|
||||
v, err := requireString(map[string]any{"topic": "OpenAI"}, "topic")
|
||||
if err != nil || v != "OpenAI" {
|
||||
t.Fatalf("requireString ok = %q, %v", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitArgumentDefaultsAndValidates(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"missing defaults to compact", map[string]any{}, "compact", false},
|
||||
{"empty string defaults to compact", map[string]any{"emit": ""}, "compact", false},
|
||||
{"compact passes through", map[string]any{"emit": "compact"}, "compact", false},
|
||||
{"html passes through", map[string]any{"emit": "html"}, "html", false},
|
||||
{"invalid value rejected", map[string]any{"emit": "json"}, "", true},
|
||||
{"non-string rejected", map[string]any{"emit": 7}, "", true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := emitArgument(tc.args)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("got %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoolArgument(t *testing.T) {
|
||||
v, err := boolArgument(map[string]any{}, "save")
|
||||
if err != nil || v {
|
||||
t.Fatalf("missing: %v, %v", v, err)
|
||||
}
|
||||
v, err = boolArgument(map[string]any{"save": true}, "save")
|
||||
if err != nil || !v {
|
||||
t.Fatalf("true: %v, %v", v, err)
|
||||
}
|
||||
v, err = boolArgument(map[string]any{"save": false}, "save")
|
||||
if err != nil || v {
|
||||
t.Fatalf("false: %v, %v", v, err)
|
||||
}
|
||||
if _, err := boolArgument(map[string]any{"save": "true"}, "save"); err == nil {
|
||||
t.Fatal("expected error for string value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchHandlerValidationErrorsAreToolErrors(t *testing.T) {
|
||||
// Validation failures are returned as MCP tool errors (not Go errors)
|
||||
// so Claude sees a structured failure with a readable message rather
|
||||
// than a transport-level fault.
|
||||
handler := makeResearchHandler(Config{Version: "test"})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
wantSub string
|
||||
}{
|
||||
{"missing topic", map[string]any{}, "topic is required"},
|
||||
{"blank topic", map[string]any{"topic": " "}, "non-empty string"},
|
||||
{"invalid emit", map[string]any{"topic": "OpenAI", "emit": "json"}, "must be 'compact' or 'html'"},
|
||||
{"non-bool save", map[string]any{"topic": "OpenAI", "save": "yes"}, "save must be a boolean"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res, err := handler(context.Background(), newCallToolRequest(tc.args))
|
||||
if err != nil {
|
||||
t.Fatalf("handler should not return Go error for validation; got %v", err)
|
||||
}
|
||||
if res == nil || !res.IsError {
|
||||
t.Fatalf("expected IsError result, got %+v", res)
|
||||
}
|
||||
if !strings.Contains(resultText(res), tc.wantSub) {
|
||||
t.Fatalf("result text %q missing substring %q", resultText(res), tc.wantSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRunErrorIncludesStderr(t *testing.T) {
|
||||
res := &engine.RunResult{Stderr: []byte("engine exploded\n")}
|
||||
msg := formatRunError(errors.New("boom"), res)
|
||||
if !strings.Contains(msg, "boom") || !strings.Contains(msg, "engine exploded") {
|
||||
t.Fatalf("formatRunError missed pieces: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatRunErrorHandlesNilResult(t *testing.T) {
|
||||
msg := formatRunError(errors.New("boom"), nil)
|
||||
if msg != "boom" {
|
||||
t.Fatalf("nil result: got %q, want %q", msg, "boom")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"manifest_version": "0.3",
|
||||
"name": "last30days-pp-mcp",
|
||||
"display_name": "Last30Days",
|
||||
"version": "3.0.0",
|
||||
"description": "Research any topic across Reddit, X, YouTube, Hacker News, Polymarket, GitHub, and the web - last 30 days, scored by upvotes, likes, and real-money prediction-market odds.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mvanhorn/last30days-skill"
|
||||
},
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"research",
|
||||
"reddit",
|
||||
"twitter",
|
||||
"x",
|
||||
"youtube",
|
||||
"hacker-news",
|
||||
"polymarket",
|
||||
"github",
|
||||
"search",
|
||||
"synthesis"
|
||||
],
|
||||
"server": {
|
||||
"type": "binary",
|
||||
"entry_point": "bin/last30days-pp-mcp",
|
||||
"mcp_config": {
|
||||
"command": "${__dirname}/bin/last30days-pp-mcp",
|
||||
"args": [],
|
||||
"env": {
|
||||
"OPENAI_API_KEY": "${user_config.openai_api_key}",
|
||||
"XAI_API_KEY": "${user_config.xai_api_key}",
|
||||
"BRAVE_API_KEY": "${user_config.brave_api_key}",
|
||||
"EXA_API_KEY": "${user_config.exa_api_key}",
|
||||
"SERPER_API_KEY": "${user_config.serper_api_key}",
|
||||
"GOOGLE_API_KEY": "${user_config.google_api_key}",
|
||||
"GEMINI_API_KEY": "${user_config.gemini_api_key}",
|
||||
"GOOGLE_GENAI_API_KEY": "${user_config.google_genai_api_key}",
|
||||
"APIFY_API_TOKEN": "${user_config.apify_api_token}",
|
||||
"BSKY_APP_PASSWORD": "${user_config.bsky_app_password}",
|
||||
"PARALLEL_API_KEY": "${user_config.parallel_api_key}",
|
||||
"SCRAPECREATORS_API_KEY": "${user_config.scrapecreators_api_key}",
|
||||
"OPENROUTER_API_KEY": "${user_config.openrouter_api_key}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"user_config": {
|
||||
"openai_api_key": {
|
||||
"type": "string",
|
||||
"title": "OPENAI_API_KEY",
|
||||
"description": "OpenAI API key. Powers Reddit research via OpenAI's web_search tool. Get one at https://platform.openai.com/api-keys.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"xai_api_key": {
|
||||
"type": "string",
|
||||
"title": "XAI_API_KEY",
|
||||
"description": "xAI API key. Powers X / Twitter research via xAI's x_search tool. Get one at https://console.x.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"brave_api_key": {
|
||||
"type": "string",
|
||||
"title": "BRAVE_API_KEY",
|
||||
"description": "Brave Search API key. Used for grounded web search results. Get one at https://brave.com/search/api/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"exa_api_key": {
|
||||
"type": "string",
|
||||
"title": "EXA_API_KEY",
|
||||
"description": "Exa search API key. Alternative web search backend with semantic ranking. Get one at https://exa.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"serper_api_key": {
|
||||
"type": "string",
|
||||
"title": "SERPER_API_KEY",
|
||||
"description": "Serper API key. Google search via API. Get one at https://serper.dev/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"google_api_key": {
|
||||
"type": "string",
|
||||
"title": "GOOGLE_API_KEY",
|
||||
"description": "Google API key for YouTube transcript fetching and other Google services. Get one at https://console.cloud.google.com/apis/credentials.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"gemini_api_key": {
|
||||
"type": "string",
|
||||
"title": "GEMINI_API_KEY",
|
||||
"description": "Gemini API key. Used for synthesis fallback when other LLM providers are unavailable. Get one at https://aistudio.google.com/apikey.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"google_genai_api_key": {
|
||||
"type": "string",
|
||||
"title": "GOOGLE_GENAI_API_KEY",
|
||||
"description": "Alternative Google generative-AI API key. Same source as GEMINI_API_KEY; set whichever name your tooling expects.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"apify_api_token": {
|
||||
"type": "string",
|
||||
"title": "APIFY_API_TOKEN",
|
||||
"description": "Apify API token. Powers TikTok and Instagram Reels search via Apify actors. Get one at https://console.apify.com/account/integrations.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"bsky_app_password": {
|
||||
"type": "string",
|
||||
"title": "BSKY_APP_PASSWORD",
|
||||
"description": "Bluesky app password (not your main password). Powers AT Protocol post search. Create at https://bsky.app/settings/app-passwords.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"parallel_api_key": {
|
||||
"type": "string",
|
||||
"title": "PARALLEL_API_KEY",
|
||||
"description": "Parallel AI key. Powers parallel research runs across sources. Get one at https://parallel.ai/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"scrapecreators_api_key": {
|
||||
"type": "string",
|
||||
"title": "SCRAPECREATORS_API_KEY",
|
||||
"description": "ScrapeCreators API key. Powers creator-focused social search across TikTok, Instagram, and YouTube. Get one at https://scrapecreators.com/.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
},
|
||||
"openrouter_api_key": {
|
||||
"type": "string",
|
||||
"title": "OPENROUTER_API_KEY",
|
||||
"description": "OpenRouter API key. Alternative LLM provider gateway for synthesis. Get one at https://openrouter.ai/keys.",
|
||||
"sensitive": true,
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"compatibility": {
|
||||
"claude_desktop": ">=1.0.0",
|
||||
"platforms": [
|
||||
"darwin",
|
||||
"linux"
|
||||
]
|
||||
}
|
||||
}
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# Mirrors skills/last30days/scripts/{last30days.py,lib/} into mcp/vendored/
|
||||
# so the Go binary's embed.FS captures the engine at build time.
|
||||
#
|
||||
# Source of truth: skills/last30days/scripts/. Never edit mcp/vendored/ directly.
|
||||
# Run before `go build` locally and in CI before `printing-press bundle`.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MCP_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
REPO_ROOT="$(cd "${MCP_DIR}/.." && pwd)"
|
||||
ENGINE_SRC="${REPO_ROOT}/skills/last30days/scripts"
|
||||
# Embed path must live inside the consuming package (Go //go:embed cannot
|
||||
# reach outside its own directory tree), so vendored/ sits under engine/.
|
||||
VENDORED="${MCP_DIR}/internal/engine/vendored"
|
||||
|
||||
if [ ! -f "${ENGINE_SRC}/last30days.py" ]; then
|
||||
echo "sync-engine: ${ENGINE_SRC}/last30days.py not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${VENDORED}"
|
||||
# Clear stale content while keeping the .gitkeep that anchors the embed path.
|
||||
find "${VENDORED}" -mindepth 1 -not -name ".gitkeep" -delete
|
||||
|
||||
# Copy the entry script and the lib/ tree (modules + lib/vendor/).
|
||||
cp "${ENGINE_SRC}/last30days.py" "${VENDORED}/last30days.py"
|
||||
cp -R "${ENGINE_SRC}/lib" "${VENDORED}/lib"
|
||||
|
||||
# Strip caches so the embed.FS stays deterministic.
|
||||
find "${VENDORED}" -type d -name "__pycache__" -prune -exec rm -rf {} +
|
||||
find "${VENDORED}" -type f -name "*.pyc" -delete
|
||||
|
||||
echo "sync-engine: vendored engine at ${VENDORED}"
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.3.0"
|
||||
version = "3.2.3"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -8,7 +8,7 @@ dependencies = []
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.0.3,<10",
|
||||
"pytest>=9,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
]
|
||||
|
||||
|
||||
+43
-47
@@ -1,64 +1,52 @@
|
||||
## v3.3.0 — install everywhere, ship the reliability sweep
|
||||
|
||||
The AI world reinvents itself every month. This skill keeps you current.
|
||||
|
||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, Digg, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||
|
||||
## What's new in v3.3.0
|
||||
## v3 is the intelligent search release
|
||||
|
||||
### Install everywhere with one command
|
||||
v3 is a ground-up engine rewrite by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
|
||||
`npx skills add mvanhorn/last30days-skill -g -y` is now the canonical install path for **every harness** — Claude Code, OpenAI Codex CLI, Cursor, Gemini CLI, GitHub Copilot, Windsurf, and 50+ other Agent Skills hosts. The skill auto-detects each harness's skills directory and symlinks in place, so edits propagate live. No more per-harness manual paths in the README.
|
||||
Type "OpenClaw" and v3 resolves @steipete, r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags before a single API call fires. Type "Peter Steinberger" and it resolves his X handle and GitHub profile, switches to person mode, and shows what he shipped this month at 85% merge rate across 22 PRs. None of that was on Google.
|
||||
|
||||
### New emit mode: `--emit=html`
|
||||
## Headline features
|
||||
|
||||
Shareable, print-friendly HTML briefs. Drop the file in Slack, mail it to a stakeholder, or print it for the meeting. Same data as compact mode, structured for human reading.
|
||||
### Intelligent pre-research
|
||||
|
||||
### New source: Digg
|
||||
The killer feature. A new Python pre-research brain resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags, all resolved before a single API call.
|
||||
|
||||
Digg surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes directly into the brief. Auto-enabled when `digg-pp-cli` is on PATH. Footer line: `⛏️ Digg: N clusters │ K posts │ M authors`. No X auth required for the inline quotes.
|
||||
### Best Takes
|
||||
|
||||
### YouTube residential-IP routing (`LAST30DAYS_YOUTUBE_SSH_HOST`)
|
||||
A second LLM judge scores every result for humor, wit, and virality alongside relevance. Every brief now ends with a Best Takes section surfacing the cleverest one-liners and most viral quotes. The Reddit and X people are funny, and the old engine buried their best stuff.
|
||||
|
||||
Running on a datacenter VPS (Hetzner, DigitalOcean, AWS, etc.)? YouTube's bot-wall fingerprints datacenter IP ranges before any cookie check. Set `LAST30DAYS_YOUTUBE_SSH_HOST=<ssh-alias>` and yt-dlp runs over SSH against a residential-IP host instead. One env var, no proxy service required.
|
||||
### Cross-source cluster merging
|
||||
|
||||
### macOS Keychain credential source
|
||||
When the same story hits Reddit, X, and YouTube, v3 merges them into one cluster instead of three duplicates. Entity-based overlap detection catches matches even when the titles use different words.
|
||||
|
||||
When env vars and config files aren't set, the engine now reads credentials from the macOS Keychain. Stores secrets where macOS expects them; nothing on disk in plaintext.
|
||||
### Single-pass comparisons
|
||||
|
||||
### `EXCLUDE_SOURCES` env var
|
||||
"X vs Y" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides at once. Same depth, 3 minutes.
|
||||
|
||||
The inverse of `INCLUDE_SOURCES`. Useful for "everything except TikTok" or "everything except the slow ones."
|
||||
### GitHub person-mode and project-mode
|
||||
|
||||
## Reliability sweep
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. PR velocity, top repos by stars, release notes for what shipped this month, woven into the narrative alongside X posts and Reddit threads.
|
||||
|
||||
This release closes a long tail of platform-specific issues that have been accumulating:
|
||||
When the topic is a project, it pulls live star counts, READMEs, releases, and top issues from the GitHub API. No stale blog posts.
|
||||
|
||||
- **Reddit**: subreddits starting with `r` no longer get mangled by `lstrip("r/")`. Browser-like headers + gzip handling fix urllib 403s on the public JSON endpoint. HTTP 402 now triggers the OpenAI/public-JSON fallback chain when ScrapeCreators credits are exhausted.
|
||||
- **xAI**: empty or malformed responses now surface in `errors_by_source` instead of silently returning zero results.
|
||||
- **Windows**: process cleanup no longer crashes on `os.killpg`. POSIX-style secret-permission warnings skipped. Save-path footer uses forward slashes.
|
||||
- **Auth**: comma-separated `SCRAPECREATORS_API_KEY=key1,key2` rotation restored (accidentally dropped in v3.0.6).
|
||||
- **YouTube + HN**: SC YouTube + multi-token HN searches unblocked. Transcript-fetch ratio surfaced.
|
||||
- **HTTP**: retry budget expanded with exponential backoff on DNS failure. Parallel AI search aligned with current API schema.
|
||||
- **OpenClaw**: now works without a ScrapeCreators key. Poll-timing initialized once.
|
||||
### ELI5 mode
|
||||
|
||||
## Multi-harness reframe
|
||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations, just clearer. Say "eli5 off" to go back.
|
||||
|
||||
`AGENTS.md` is now the canonical project doc; `CLAUDE.md` points at it. The skill is positioned as a multi-harness Agent Skills package, not a Claude-Code-specific tool. SKILL.md's path resolution rewrote `SKILL_ROOT` → `SKILL_DIR`, removing ~80 lines of bash and fixing a real spec-vs-engine divergence bug.
|
||||
### 13+ sources
|
||||
|
||||
## Breaking change
|
||||
v3 adds Threads, Pinterest, Perplexity, Bluesky, and Parallel AI grounding to the existing Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and Web lineup. Perplexity Deep Research (`--deep-research`) gives you 50+ citation reports for serious investigation.
|
||||
|
||||
**`.codex-plugin/plugin.json` removed.** Codex native-plugin users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`. The `npx skills add` path now reaches every harness uniformly.
|
||||
### Per-author cap and entity disambiguation
|
||||
|
||||
Max 3 items per author prevents single-voice dominance. Synthesis trusts resolved handles over fuzzy keyword matches.
|
||||
|
||||
## Install
|
||||
|
||||
Any harness (recommended):
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -y
|
||||
```
|
||||
|
||||
Claude Code marketplace:
|
||||
Claude Code:
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
@@ -70,21 +58,29 @@ OpenClaw:
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`.
|
||||
|
||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
## Contributors
|
||||
## v3 Community
|
||||
|
||||
First-time contributors whose fixes shipped in v3.3.0 (most via PR triage salvage — the fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
||||
v3 was shaped by community contributors whose PRs and issues inspired core features. Their code wasn't merged directly (v3 was a ground-up rewrite), but their ideas drove what shipped.
|
||||
|
||||
- Dave Morin — portable test-harness paths
|
||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
||||
- Eric Oberhofer — multi-key rotation restored
|
||||
- gujishh — Windows process cleanup
|
||||
- Franco Carballar — Reddit browser-like headers
|
||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
||||
- Kaustav Mishra — xAI error surfacing
|
||||
- [@thinkun](https://github.com/thinkun) — OpenClaw ScrapeCreators-key-optional fix
|
||||
Thanks to @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, and @hnshah. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list.
|
||||
|
||||
Plus every contributor who shipped one of the ~75 PRs merged this cycle. See [CHANGELOG.md](CHANGELOG.md) under `[3.3.0]` for the full PR list and `git log v3.2.0..v3.3.0` for the complete commit graph.
|
||||
Contributors who shaped the release itself:
|
||||
|
||||
- @Jah-yee (#153) surfaced the need for a real Codex CLI integration, which shipped in #219
|
||||
- @Cody-Coyote (#204) reported the marketplace validation bug that needed fixing before v3 could ship cleanly
|
||||
- @dannyshmueli pushed for v3 and Codex family support publicly on X
|
||||
|
||||
Full Added / Changed / Fixed detail lives in [CHANGELOG.md](CHANGELOG.md) under `[3.0.0]`.
|
||||
|
||||
## Earlier contributors
|
||||
|
||||
From the v1 and v2 lineage:
|
||||
|
||||
- [@galligan](https://github.com/galligan) for marketplace plugin inspiration
|
||||
- [@hutchins](https://x.com/hutchins) for pushing the YouTube feature
|
||||
|
||||
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
||||
|
||||
+69
-42
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: last30days
|
||||
version: "3.3.0"
|
||||
version: "3.2.3"
|
||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
@@ -13,9 +13,9 @@ metadata:
|
||||
openclaw:
|
||||
emoji: "📰"
|
||||
requires:
|
||||
env: []
|
||||
optionalEnv:
|
||||
env:
|
||||
- SCRAPECREATORS_API_KEY
|
||||
optionalEnv:
|
||||
- OPENAI_API_KEY
|
||||
- XAI_API_KEY
|
||||
- OPENROUTER_API_KEY
|
||||
@@ -97,7 +97,7 @@ You are inside the `/last30days` SKILL. This is a specific research tool with a
|
||||
|
||||
**How v3.0.7 fixes it:** three structural anchors.
|
||||
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
||||
2. **The SKILL_DIR substitution** in the engine Bash calls uses the directory of the SKILL.md the model just Read — no resolver list, no precedence walk. Whichever install the harness loaded SKILL.md from is the install whose engine runs. Aligns spec-with-code and works for any harness without enumerating its install path.
|
||||
2. **The SKILL_ROOT resolver** in the engine Bash calls walks a precedence list of known install locations and picks the highest-versioned freshest copy, never `~/.openclaw/` or other stale copies.
|
||||
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
||||
|
||||
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
||||
@@ -114,7 +114,7 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
```
|
||||
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_DIR/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_DIR/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_ROOT/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
|
||||
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
||||
|
||||
@@ -243,7 +243,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
||||
|
||||
---
|
||||
|
||||
# last30days v3.3.0: Research Any Topic from the Last 30 Days
|
||||
# last30days v3.2.3: Research Any Topic from the Last 30 Days
|
||||
|
||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||
|
||||
@@ -330,11 +330,12 @@ Common patterns:
|
||||
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg
|
||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
||||
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
||||
- If SCRAPECREATORS_API_KEY is set: add TikTok, Instagram, Threads (suppress any of these via EXCLUDE_SOURCES)
|
||||
- If SCRAPECREATORS_API_KEY is set and the user explicitly requested pinterest for this query (e.g. via `--search=pinterest`): add Pinterest
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains threads: add Threads
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest
|
||||
- If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky
|
||||
- If OPENROUTER_API_KEY is set and INCLUDE_SOURCES contains perplexity: add Perplexity
|
||||
- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying
|
||||
- If OPENROUTER_API_KEY is set: add Perplexity
|
||||
|
||||
Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma):
|
||||
|
||||
@@ -591,21 +592,27 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe
|
||||
|
||||
**Invocation:**
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
exit 1
|
||||
# Comparison mode skips Step 1, so resolve SKILL_ROOT inline here (same precedence
|
||||
# walk as Step 1 — keep the two in sync if you edit either).
|
||||
SKILL_ROOT=""
|
||||
CLAUDE_PLUGIN_ROOT="$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)"
|
||||
if [ -n "$CLAUDE_PLUGIN_ROOT" ]; then
|
||||
if [ -f "$CLAUDE_PLUGIN_ROOT/skills/last30days/scripts/last30days.py" ]; then
|
||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT/skills/last30days"
|
||||
elif [ -f "$CLAUDE_PLUGIN_ROOT/scripts/last30days.py" ]; then
|
||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT"
|
||||
fi
|
||||
fi
|
||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
for dir in \
|
||||
"$HOME/.codex/skills/last30days" \
|
||||
"$HOME/.agents/skills/last30days" \
|
||||
"./skills/last30days" \
|
||||
"./.skills/last30days" \
|
||||
"." \
|
||||
"${GEMINI_EXTENSION_DIR:-}"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
fi
|
||||
|
||||
# Write the per-entity plan to a tmpfile and pass the path to the engine.
|
||||
@@ -624,7 +631,7 @@ cat > "$COMPETITORS_PLAN_FILE" <<'PLAN_EOF'
|
||||
}
|
||||
PLAN_EOF
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||
--emit=compact \
|
||||
--save-dir="${LAST30DAYS_MEMORY_DIR}" \
|
||||
--save-suffix=v3 \
|
||||
@@ -909,24 +916,44 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
||||
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.0/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
# Resolve SKILL_ROOT by walking a precedence list of known install locations.
|
||||
# Claude Code plugin cache wins when present (highest version dir picked on upgrade),
|
||||
# then common per-harness skill dirs, then a repo checkout.
|
||||
SKILL_ROOT=""
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
# 1. Claude Code plugin cache (versioned, sort -V picks freshest). Two cache layouts ship in the wild:
|
||||
# nested ({cache}/{version}/skills/last30days/scripts/...) and flat ({cache}/{version}/scripts/...).
|
||||
# `find` (not `ls + glob`) because zsh errors on globs that match nothing, leaking
|
||||
# noisy "no matches found" stderr in Codex/zsh sessions even with 2>/dev/null.
|
||||
CLAUDE_PLUGIN_ROOT="$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)"
|
||||
if [ -n "$CLAUDE_PLUGIN_ROOT" ]; then
|
||||
if [ -f "$CLAUDE_PLUGIN_ROOT/skills/last30days/scripts/last30days.py" ]; then
|
||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT/skills/last30days"
|
||||
elif [ -f "$CLAUDE_PLUGIN_ROOT/scripts/last30days.py" ]; then
|
||||
SKILL_ROOT="$CLAUDE_PLUGIN_ROOT"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Common per-harness skill dirs and repo checkout (npx skills, Codex, Agents, Gemini, etc).
|
||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
for dir in \
|
||||
"$HOME/.codex/skills/last30days" \
|
||||
"$HOME/.agents/skills/last30days" \
|
||||
"./skills/last30days" \
|
||||
"./.skills/last30days" \
|
||||
"." \
|
||||
"${GEMINI_EXTENSION_DIR:-}"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
echo "ERROR: Could not find scripts/last30days.py in any known install location" >&2
|
||||
echo "Searched: ~/.claude/plugins/cache/, ~/.codex/skills/, ~/.agents/skills/, ./skills/last30days, ./.skills/last30days, ." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||
```
|
||||
|
||||
**If you ran Steps 0.55 and 0.75 (agent planning), pass the plan via a tmpfile and add the targeting flags:**
|
||||
@@ -1688,7 +1715,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 100 free credits)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 10,000 free API calls)
|
||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||
- Stores research findings in local SQLite database (watchlist mode only)
|
||||
@@ -1701,7 +1728,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Does not log, cache, or write API keys to output files
|
||||
- Does not send data to any endpoint not listed above
|
||||
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (100 free credits one-time, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (10,000 free API calls, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
||||
|
||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||
|
||||
@@ -20,7 +20,6 @@ sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from lib import env as envlib
|
||||
from lib import schema
|
||||
from lib.providers import GEMINI_FLASH_LITE
|
||||
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -44,7 +43,7 @@ def _load_default_topics() -> list[tuple[str, str]]:
|
||||
|
||||
DEFAULT_TOPICS = _load_default_topics()
|
||||
DEFAULT_SEARCH = ""
|
||||
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
|
||||
DEFAULT_JUDGE_MODEL = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# ruff: noqa: E402
|
||||
"""last30days CLI."""
|
||||
"""last30days v3.0.0 CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -63,10 +62,7 @@ def _cleanup_children() -> None:
|
||||
pids = list(_child_pids)
|
||||
for pid in pids:
|
||||
try:
|
||||
if hasattr(os, "killpg"):
|
||||
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||
else:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
continue
|
||||
|
||||
@@ -101,13 +97,11 @@ def save_output(
|
||||
save_dir: str,
|
||||
suffix: str = "",
|
||||
synthesis_md: str | None = None,
|
||||
topic_override: str | None = None,
|
||||
rendered_content: str | None = None,
|
||||
) -> Path:
|
||||
from datetime import datetime
|
||||
path = Path(save_dir).expanduser().resolve()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
slug = slugify(topic_override or report.topic)
|
||||
slug = slugify(report.topic)
|
||||
extension = "json" if emit == "json" else "html" if emit == "html" else "md"
|
||||
raw_label = "raw-html" if emit == "html" else "raw"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
@@ -116,9 +110,7 @@ def save_output(
|
||||
out_path = path / f"{slug}-{raw_label}{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Markdown saves keep the complete debug artifact. JSON and HTML preserve
|
||||
# their requested wire format so file extensions match their content.
|
||||
if rendered_content is not None:
|
||||
content = rendered_content
|
||||
elif emit in {"json", "html"}:
|
||||
if emit in {"json", "html"}:
|
||||
content = emit_output(report, emit, synthesis_md=synthesis_md)
|
||||
else:
|
||||
content = render.render_full(report)
|
||||
@@ -179,10 +171,6 @@ def emit_comparison_output(
|
||||
raise SystemExit(f"Unsupported emit mode: {emit}")
|
||||
|
||||
|
||||
def comparison_topic(entity_reports: list[tuple[str, schema.Report]]) -> str:
|
||||
return " vs ".join(label for label, _ in entity_reports)
|
||||
|
||||
|
||||
def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str) -> str:
|
||||
"""Compute the user-friendly save path string that will be shown in the footer.
|
||||
|
||||
@@ -199,9 +187,9 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
||||
try:
|
||||
home = _Path.home().resolve()
|
||||
relative = raw.relative_to(home)
|
||||
return f"~/{relative.as_posix()}"
|
||||
return f"~/{relative}"
|
||||
except ValueError:
|
||||
return raw.as_posix()
|
||||
return str(raw)
|
||||
|
||||
|
||||
def read_synthesis_file(path: str) -> str:
|
||||
@@ -391,7 +379,7 @@ def subrun_kwargs_for(
|
||||
|
||||
subreddits = _choose("subreddits", "subreddits")
|
||||
if isinstance(subreddits, list):
|
||||
subreddits = [s.strip().removeprefix("r/") for s in subreddits if s.strip()] or None
|
||||
subreddits = [s.strip().lstrip("r/") for s in subreddits if s.strip()] or None
|
||||
|
||||
x_related = plan_entry.get("x_related")
|
||||
if isinstance(x_related, list):
|
||||
@@ -535,24 +523,6 @@ def _show_runtime_ui(
|
||||
progress.show_promo(promo, diag=diag)
|
||||
|
||||
|
||||
def _write_last_run(topic: str, report: "schema.Report") -> None:
|
||||
try:
|
||||
if env.CONFIG_DIR is None:
|
||||
return
|
||||
target = env.CONFIG_DIR
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
counts = {source: len(items) for source, items in report.items_by_source.items()}
|
||||
payload = {
|
||||
"topic": topic,
|
||||
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"sources": counts,
|
||||
"total": sum(counts.values()),
|
||||
}
|
||||
(target / "last-run.json").write_text(json.dumps(payload, indent=2))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
||||
@@ -563,13 +533,6 @@ def main() -> int:
|
||||
|
||||
config = env.get_config()
|
||||
|
||||
# Surface SSH-routing config as an env var so library modules (e.g.
|
||||
# youtube_yt) can read it without taking a config dependency. This
|
||||
# routes yt-dlp through `ssh <host>` to bypass YouTube's bot-wall on
|
||||
# datacenter IPs (see lib/youtube_yt.py for details).
|
||||
if config.get("LAST30DAYS_YOUTUBE_SSH_HOST") and "LAST30DAYS_YOUTUBE_SSH_HOST" not in os.environ:
|
||||
os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"]
|
||||
|
||||
# Handle setup subcommand
|
||||
topic = " ".join(args.topic).strip()
|
||||
if topic.lower() == "setup":
|
||||
@@ -628,7 +591,7 @@ def main() -> int:
|
||||
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
||||
try:
|
||||
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
||||
subreddits = [s.strip().removeprefix("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||
subreddits = [s.strip().lstrip("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
||||
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
||||
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
||||
@@ -647,7 +610,6 @@ def main() -> int:
|
||||
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
||||
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
||||
# without WebSearch (OpenClaw, Codex, raw CLI).
|
||||
repos_from_auto_resolve = False
|
||||
if args.auto_resolve and not external_plan:
|
||||
from lib import resolve
|
||||
resolution = resolve.auto_resolve(topic, config)
|
||||
@@ -662,9 +624,6 @@ def main() -> int:
|
||||
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
||||
if resolution.get("github_repos") and not args.github_repo:
|
||||
args.github_repo = ",".join(resolution["github_repos"])
|
||||
# auto_resolve already canonicalized via canonicalize_github_repos(cap=5);
|
||||
# mark so we don't re-canonicalize below and clobber its relevance order.
|
||||
repos_from_auto_resolve = True
|
||||
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
||||
if resolution.get("context"):
|
||||
# Inject context into external_plan metadata for the planner to use
|
||||
@@ -677,20 +636,6 @@ def main() -> int:
|
||||
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
||||
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
||||
|
||||
# Only canonicalize when repos came from a user-supplied --github-repo flag.
|
||||
# When repos_from_auto_resolve is True, auto_resolve already ran
|
||||
# canonicalize_github_repos(cap=5) and ranked by relevance; re-running here
|
||||
# with cap=None can re-sort by topic-slug match and lose that ordering.
|
||||
if github_repos and not repos_from_auto_resolve:
|
||||
from lib import resolve as resolve_lib
|
||||
original_github_repos = github_repos[:]
|
||||
github_repos = resolve_lib.canonicalize_github_repos(topic, github_repos, cap=None)
|
||||
if github_repos != original_github_repos:
|
||||
sys.stderr.write(
|
||||
"[GitHub] Canonicalized repos: "
|
||||
f"{','.join(original_github_repos)} -> {','.join(github_repos)}\n"
|
||||
)
|
||||
|
||||
# --deep-research: auto-enable perplexity source and set deep flag
|
||||
if args.deep_research:
|
||||
if not config.get("OPENROUTER_API_KEY"):
|
||||
@@ -910,18 +855,7 @@ def main() -> int:
|
||||
report, progress, diag,
|
||||
suppress_web_promo=bool(external_plan or comp_plan),
|
||||
)
|
||||
_write_last_run(topic, report)
|
||||
# LAST30DAYS_STORE env var = persistence default-on. Read both os.environ
|
||||
# (for shell-exported users) and config (for users who set it in
|
||||
# ~/.config/last30days/.env, which env.py loads but does not propagate
|
||||
# to os.environ). Mirrors the LAST30DAYS_DEBUG / LAST30DAYS_SKIP_PREFLIGHT
|
||||
# convention; env-var or config wins, with `--store` flag still working.
|
||||
_store_env = (
|
||||
os.environ.get("LAST30DAYS_STORE")
|
||||
or config.get("LAST30DAYS_STORE")
|
||||
or ""
|
||||
).lower()
|
||||
if args.store or _store_env in ("1", "true", "yes"):
|
||||
if args.store:
|
||||
counts = persist_report(report)
|
||||
sys.stderr.write(
|
||||
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
||||
@@ -931,32 +865,7 @@ def main() -> int:
|
||||
# Show quality nudge if applicable
|
||||
try:
|
||||
from lib import quality_nudge
|
||||
# Populate transcript-fetch ratio so quality_nudge can detect the
|
||||
# degraded-YouTube failure mode (videos returned but transcripts
|
||||
# silently failed - typically a stale yt-dlp binary).
|
||||
youtube_items = report.items_by_source.get("youtube") or []
|
||||
instagram_items = report.items_by_source.get("instagram") or []
|
||||
research_results = {
|
||||
"youtube_videos_count": len(youtube_items),
|
||||
"youtube_transcripts_count": sum(
|
||||
1 for it in youtube_items
|
||||
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
||||
),
|
||||
"youtube_error": report.errors_by_source.get("youtube"),
|
||||
"x_error": report.errors_by_source.get("x"),
|
||||
# Captions-disabled videos can never produce a transcript regardless
|
||||
# of yt-dlp version; subtract them from the degraded-ratio
|
||||
# denominator so a single uploader-disabled video does not trip the
|
||||
# "stale yt-dlp" nudge.
|
||||
"youtube_captions_disabled_count": sum(
|
||||
1 for it in youtube_items if it.metadata.get("captions_disabled")
|
||||
),
|
||||
# Track Instagram returned-zero-items so quality_nudge can detect
|
||||
# the silent-failure case (SC configured but the v2 reels endpoint
|
||||
# 500'd through both the original query and the hashtag retry).
|
||||
"instagram_items_count": len(instagram_items),
|
||||
}
|
||||
quality = quality_nudge.compute_quality_score(config, research_results)
|
||||
quality = quality_nudge.compute_quality_score(config, {})
|
||||
if quality.get("nudge_text"):
|
||||
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
||||
sys.stderr.flush()
|
||||
@@ -964,15 +873,10 @@ def main() -> int:
|
||||
pass
|
||||
|
||||
fun_level = config.get("FUN_LEVEL", "medium").lower()
|
||||
# Comparison HTML is the one case where the saved file's title and content
|
||||
# have to be overridden away from the leading entity's report. Compute the
|
||||
# gate once so the footer-display and save-output paths can't disagree.
|
||||
is_comparison_html = bool(entity_reports) and args.emit == "html"
|
||||
footer_save_path = None
|
||||
if args.save_dir:
|
||||
save_topic_for_display = comparison_topic(entity_reports) if is_comparison_html else report.topic
|
||||
footer_save_path = compute_save_path_display(
|
||||
args.save_dir, save_topic_for_display, args.save_suffix or "", args.emit
|
||||
args.save_dir, report.topic, args.save_suffix or "", args.emit
|
||||
)
|
||||
|
||||
# Signal to render_compact whether pre-research flags were supplied.
|
||||
@@ -1013,8 +917,6 @@ def main() -> int:
|
||||
args.save_dir,
|
||||
suffix=args.save_suffix or "",
|
||||
synthesis_md=synthesis_md,
|
||||
topic_override=comparison_topic(entity_reports) if is_comparison_html else None,
|
||||
rendered_content=rendered if is_comparison_html else None,
|
||||
)
|
||||
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
||||
# Competitor / vs-mode: also save a per-entity raw file for each peer.
|
||||
|
||||
@@ -9,7 +9,6 @@ import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log, subproc
|
||||
@@ -18,11 +17,6 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
# How many times to retry the bird-search subprocess when stdout is non-JSON
|
||||
# (typically an HTML anti-bot interstitial from Twitter's edge).
|
||||
MAX_JSON_DECODE_RETRIES = 2
|
||||
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
|
||||
|
||||
|
||||
def _first_of(*values):
|
||||
"""Return first value that is not None."""
|
||||
@@ -154,14 +148,16 @@ def get_bird_status() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"""Invoke the vendored bird-search.mjs subprocess once.
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
Returns (result, error_dict). If error_dict is non-None, treat it as the
|
||||
final result and do not retry — those errors are terminal (timeout,
|
||||
spawn failure). If error_dict is None, the subprocess ran to completion
|
||||
and `result` is the SubprocResult; the caller decides whether to retry
|
||||
based on the result.stdout content.
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
@@ -188,9 +184,9 @@ def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
on_pid=_register,
|
||||
)
|
||||
except subproc.SubprocTimeout:
|
||||
return None, {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
except Exception as e:
|
||||
return None, {"error": str(e), "items": []}
|
||||
return {"error": str(e), "items": []}
|
||||
finally:
|
||||
if pid_holder:
|
||||
try:
|
||||
@@ -199,80 +195,22 @@ def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result, None
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||
|
||||
Retries the subprocess on JSON-decode failure (typically a Twitter
|
||||
anti-bot HTML interstitial in stdout) up to MAX_JSON_DECODE_RETRIES
|
||||
times with JSON_DECODE_RETRY_DELAY seconds between attempts. Terminal
|
||||
errors (subprocess timeout, non-zero return code) are returned
|
||||
immediately without retry.
|
||||
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds (per attempt)
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
last_decode_error: Optional[str] = None
|
||||
|
||||
for attempt in range(MAX_JSON_DECODE_RETRIES):
|
||||
result, terminal_error = _invoke_bird_subprocess(query, count, timeout)
|
||||
if terminal_error is not None:
|
||||
return terminal_error
|
||||
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError as e:
|
||||
# Twitter's edge sometimes serves an HTML anti-bot interstitial
|
||||
# in place of JSON. Tag the failure shape so it's distinguishable
|
||||
# from "no results" in logs, then retry the subprocess.
|
||||
looks_html = output.lstrip().lower().startswith(("<!doctype", "<html", "<"))
|
||||
attempt_num = attempt + 1
|
||||
log_msg = (
|
||||
f"Bird search returned non-JSON stdout "
|
||||
f"(looks_html={looks_html}, attempt {attempt_num}/{MAX_JSON_DECODE_RETRIES}, "
|
||||
f"first 80 chars: {output[:80]!r})"
|
||||
)
|
||||
last_decode_error = str(e)
|
||||
if attempt_num < MAX_JSON_DECODE_RETRIES:
|
||||
log.source_log(
|
||||
"X/bird",
|
||||
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
|
||||
)
|
||||
time.sleep(JSON_DECODE_RETRY_DELAY)
|
||||
continue
|
||||
log.source_log("X/bird", log_msg)
|
||||
return {
|
||||
"error": (
|
||||
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
|
||||
f"(likely Twitter anti-bot interstitial): {e}"
|
||||
),
|
||||
"items": [],
|
||||
}
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
# Defensive fallthrough — loop should always return above.
|
||||
return {
|
||||
"error": f"Bird search exhausted retries: {last_decode_error}",
|
||||
"items": [],
|
||||
}
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
|
||||
def search_x(
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
"""Bluesky search via AT Protocol (requires app password).
|
||||
|
||||
Uses bsky.social for auth and api.bsky.app for post search (the canonical
|
||||
authenticated AppView). The previous default `public.api.bsky.app` is the
|
||||
unauthenticated public mirror, which BunnyCDN now blocks for searchPosts
|
||||
regardless of auth header (verified 2026-05-04). Override the search host
|
||||
via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again.
|
||||
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are
|
||||
19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords.
|
||||
The createSession endpoint accepts main-account passwords too, but they're
|
||||
bad hygiene (no scope, can't revoke individually).
|
||||
Uses bsky.social for auth and public.api.bsky.app for post search.
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
@@ -23,64 +14,7 @@ from typing import Any, Dict, List, Optional
|
||||
from . import http, log
|
||||
|
||||
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
||||
_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
|
||||
|
||||
|
||||
def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""Resolve the Bluesky search URL with BSKY_SEARCH_HOST override.
|
||||
|
||||
Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or
|
||||
.env file. The project's env.py loads .env into config but not into
|
||||
os.environ, so check both — same hybrid pattern as last30days.py for
|
||||
LAST30DAYS_STORE.
|
||||
|
||||
Hardens user-supplied host values against three common mis-configurations:
|
||||
whitespace (e.g. " api.bsky.app "), embedded path components (e.g.
|
||||
"api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and
|
||||
embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these
|
||||
we log a warning and fall back to the default rather than building an
|
||||
invalid URL with an opaque downstream error.
|
||||
"""
|
||||
config = config or {}
|
||||
raw = (
|
||||
os.environ.get("BSKY_SEARCH_HOST")
|
||||
or config.get("BSKY_SEARCH_HOST")
|
||||
or _DEFAULT_BSKY_SEARCH_HOST
|
||||
)
|
||||
host = raw.strip().rstrip("/")
|
||||
# Strip embedded scheme so users who paste full URLs do not break the f-string.
|
||||
for prefix in ("https://", "http://"):
|
||||
if host.lower().startswith(prefix):
|
||||
host = host[len(prefix):]
|
||||
break
|
||||
if not host or "/" in host or " " in host:
|
||||
# Embedded path or whitespace remains — don't trust it. Default + log.
|
||||
if raw != _DEFAULT_BSKY_SEARCH_HOST:
|
||||
_log(
|
||||
f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; "
|
||||
f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}"
|
||||
)
|
||||
host = _DEFAULT_BSKY_SEARCH_HOST
|
||||
return f"https://{host}/xrpc/app.bsky.feed.searchPosts"
|
||||
|
||||
|
||||
# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric
|
||||
# with three hyphens at fixed positions).
|
||||
_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$")
|
||||
|
||||
|
||||
def _validate_app_password_format(value) -> bool:
|
||||
"""Return True if value matches Bluesky's 19-char app-password format.
|
||||
|
||||
False for non-strings (None, int, list) so callers passing config dict
|
||||
values directly don't crash. Detect-but-not-gate: the createSession
|
||||
endpoint also accepts main-account passwords, so failing this check is
|
||||
a hygiene smell, not a hard error.
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return bool(_APP_PASSWORD_RE.fullmatch(value))
|
||||
|
||||
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
|
||||
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 15,
|
||||
@@ -210,20 +144,6 @@ def search_bluesky(
|
||||
if not handle or not app_password:
|
||||
return {"posts": [], "error": "Bluesky credentials not configured"}
|
||||
|
||||
# One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password
|
||||
# form. createSession accepts main-account passwords too — but main
|
||||
# passwords have no scope (full account access), can't be revoked
|
||||
# individually, and rotating them breaks every service that holds them.
|
||||
# We warn but do not gate, matching the project's detect-don't-block
|
||||
# philosophy elsewhere.
|
||||
if not _validate_app_password_format(app_password):
|
||||
_log(
|
||||
"BSKY_APP_PASSWORD does not look like an app password "
|
||||
"(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main "
|
||||
"account password — those work but are bad hygiene. Generate "
|
||||
"an app password at https://bsky.app/settings/app-passwords"
|
||||
)
|
||||
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
@@ -235,7 +155,7 @@ def search_bluesky(
|
||||
"limit": str(min(count, 100)),
|
||||
"sort": "top",
|
||||
}
|
||||
url = f"{_resolve_search_url(config)}?{urlencode(params)}"
|
||||
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
|
||||
|
||||
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
token = _create_session(handle, app_password)
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
"""Chrome and Brave cookie extraction for macOS.
|
||||
"""Chrome cookie extraction for macOS.
|
||||
|
||||
Extracts cookies from Chromium-based browser SQLite databases using only
|
||||
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
|
||||
dependencies.
|
||||
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
|
||||
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
|
||||
|
||||
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
Chrome and Brave share the same algorithm; only the DB path and Keychain
|
||||
service name differ.
|
||||
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||
"""
|
||||
|
||||
@@ -21,11 +18,10 @@ from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cookie DB locations on macOS
|
||||
# Chrome cookie DB location on macOS
|
||||
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||
BRAVE_BASE_DIR = Path.home() / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser"
|
||||
|
||||
# Chromium v10 encryption constants (shared by Chrome and Brave)
|
||||
# Chrome v10 encryption constants
|
||||
CHROME_SALT = b"saltysalt"
|
||||
CHROME_PBKDF2_ITERATIONS = 1003
|
||||
CHROME_KEY_LENGTH = 16
|
||||
@@ -33,8 +29,8 @@ CHROME_KEY_LENGTH = 16
|
||||
CHROME_IV_HEX = "20" * 16
|
||||
|
||||
|
||||
def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||
"""Retrieve the encryption passphrase for a Chromium-based browser from macOS Keychain.
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
|
||||
|
||||
Calls `security find-generic-password` which may trigger a system dialog
|
||||
on first access.
|
||||
@@ -43,34 +39,30 @@ def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["security", "find-generic-password", "-w", "-s", service_name],
|
||||
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.info("%s Keychain access denied or browser not installed: %s", service_name, result.stderr.strip())
|
||||
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
|
||||
return None
|
||||
passphrase = result.stdout.strip()
|
||||
if not passphrase:
|
||||
logger.info("%s Keychain returned empty passphrase", service_name)
|
||||
logger.info("Chrome Keychain returned empty passphrase")
|
||||
return None
|
||||
return passphrase.encode("utf-8")
|
||||
except FileNotFoundError:
|
||||
logger.info("'security' command not found — not on macOS?")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.info("%s Keychain access timed out", service_name)
|
||||
logger.info("Chrome Keychain access timed out")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Failed to get %s encryption key: %s", service_name, e)
|
||||
logger.info("Failed to get Chrome encryption key: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
return _get_chromium_encryption_key("Chrome Safe Storage")
|
||||
|
||||
|
||||
def _derive_aes_key(passphrase: bytes) -> bytes:
|
||||
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
||||
return hashlib.pbkdf2_hmac(
|
||||
@@ -173,42 +165,36 @@ def _get_db_version(cursor: sqlite3.Cursor) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_chromium_cookies_macos(
|
||||
db_path: Path,
|
||||
keychain_service: str,
|
||||
domain: str,
|
||||
cookie_names: list[str],
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from any Chromium-based browser on macOS.
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS.
|
||||
|
||||
Copies the locked Cookies database to a temp file, reads specified cookies,
|
||||
and decrypts v10-encrypted values using the Keychain-stored key.
|
||||
|
||||
Args:
|
||||
db_path: Path to the browser's Cookies SQLite file.
|
||||
keychain_service: macOS Keychain service name (e.g. "Chrome Safe Storage").
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
|
||||
cookie_names: List of cookie names to extract
|
||||
|
||||
Returns:
|
||||
Dict mapping cookie name to decrypted value, or None on failure.
|
||||
Only includes cookies that were successfully found and decrypted.
|
||||
"""
|
||||
if not db_path.exists():
|
||||
logger.info("%s cookies database not found at %s", keychain_service, db_path)
|
||||
if not CHROME_COOKIES_DB.exists():
|
||||
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
|
||||
return None
|
||||
|
||||
passphrase = _get_chromium_encryption_key(keychain_service)
|
||||
# Get encryption key from Keychain
|
||||
passphrase = _get_chrome_encryption_key()
|
||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||
|
||||
# Copy DB to temp file (browser locks the original while running)
|
||||
# Copy DB to temp file (Chrome locks the original)
|
||||
tmp_fd = None
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(db_path), tmp_path)
|
||||
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
|
||||
except Exception as e:
|
||||
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
|
||||
logger.info("Failed to copy Chrome cookies database: %s", e)
|
||||
if tmp_path:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
@@ -225,22 +211,26 @@ def _extract_chromium_cookies_macos(
|
||||
cursor = conn.cursor()
|
||||
|
||||
db_version = _get_db_version(cursor)
|
||||
logger.debug("%s cookie DB version: %d", keychain_service, db_version)
|
||||
logger.debug("Chrome cookie DB version: %d", db_version)
|
||||
|
||||
# Build query with placeholders for cookie names
|
||||
placeholders = ",".join("?" for _ in cookie_names)
|
||||
query = (
|
||||
f"SELECT name, value, encrypted_value FROM cookies "
|
||||
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
||||
)
|
||||
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
|
||||
params = [f"%{domain}"] + list(cookie_names)
|
||||
cursor.execute(query, params)
|
||||
|
||||
results: dict[str, str] = {}
|
||||
for name, value, encrypted_value in cursor.fetchall():
|
||||
# Prefer unencrypted value if present
|
||||
if value:
|
||||
results[name] = value
|
||||
continue
|
||||
|
||||
# Handle encrypted value
|
||||
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||
if aes_key is None:
|
||||
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||
@@ -251,72 +241,25 @@ def _extract_chromium_cookies_macos(
|
||||
else:
|
||||
logger.debug("Failed to decrypt cookie %s", name)
|
||||
elif encrypted_value:
|
||||
# Unknown encryption version
|
||||
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
||||
|
||||
conn.close()
|
||||
|
||||
if not results:
|
||||
logger.info("No matching cookies found in %s for domain %s", keychain_service, domain)
|
||||
logger.info("No matching cookies found in Chrome for domain %s", domain)
|
||||
return None
|
||||
|
||||
return results
|
||||
|
||||
except sqlite3.Error as e:
|
||||
logger.info("Failed to read %s cookies database: %s", keychain_service, e)
|
||||
logger.info("Failed to read Chrome cookies database: %s", e)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Unexpected error reading %s cookies: %s", keychain_service, e)
|
||||
logger.info("Unexpected error reading Chrome cookies: %s", e)
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS."""
|
||||
return _extract_chromium_cookies_macos(
|
||||
CHROME_COOKIES_DB, "Chrome Safe Storage", domain, cookie_names
|
||||
)
|
||||
|
||||
|
||||
def _find_brave_cookies_db() -> Optional[Path]:
|
||||
"""Find Brave's Cookies database on macOS.
|
||||
|
||||
Tries the Default profile first, then scans numbered Profile directories
|
||||
by most-recently-modified. Brave creates extra profiles as "Profile 1",
|
||||
"Profile 2", etc. alongside Default; the most recently used one is the
|
||||
likeliest to hold current cookies. Lexicographic sort would visit
|
||||
"Profile 10" before "Profile 2", which can return the wrong profile.
|
||||
"""
|
||||
default = BRAVE_BASE_DIR / "Default" / "Cookies"
|
||||
if default.exists():
|
||||
return default
|
||||
|
||||
try:
|
||||
candidates = [
|
||||
child for child in BRAVE_BASE_DIR.iterdir()
|
||||
if child.is_dir() and child.name.startswith("Profile ")
|
||||
]
|
||||
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
||||
candidate = child / "Cookies"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Brave on macOS.
|
||||
|
||||
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
|
||||
path and Keychain service name differ.
|
||||
"""
|
||||
db_path = _find_brave_cookies_db()
|
||||
if db_path is None:
|
||||
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
|
||||
return None
|
||||
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Browser cookie extraction for last30days.
|
||||
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Brave, Safari)
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
|
||||
to enable zero-config authentication for services like X/Twitter.
|
||||
|
||||
Only uses Python stdlib — no external dependencies.
|
||||
@@ -255,29 +255,6 @@ def extract_chrome_cookies(
|
||||
return None
|
||||
|
||||
|
||||
def extract_brave_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Brave for the given domain and cookie names.
|
||||
|
||||
macOS only — Brave uses the same v10 AES-128-CBC encryption as Chrome,
|
||||
with a different DB path and Keychain service name ("Brave Safe Storage").
|
||||
Tries the Default profile first, then scans numbered Profile directories.
|
||||
|
||||
Returns:
|
||||
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||
"""
|
||||
if platform.system() != "Darwin":
|
||||
logger.debug("Brave cookie extraction only supported on macOS")
|
||||
return None
|
||||
try:
|
||||
from .chrome_cookies import extract_brave_cookies_macos
|
||||
return extract_brave_cookies_macos(domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("Brave cookie extraction failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def extract_safari_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
@@ -305,9 +282,9 @@ def extract_cookies(
|
||||
"""Extract cookies from the specified browser.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
'auto' tries browsers in platform-appropriate order:
|
||||
- macOS: Chrome -> Brave -> Firefox -> Safari
|
||||
- macOS: Chrome -> Firefox -> Safari
|
||||
- Linux: Firefox only
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
@@ -356,7 +333,7 @@ def extract_cookies_with_source(
|
||||
so callers can track the source.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
|
||||
@@ -367,7 +344,6 @@ def extract_cookies_with_source(
|
||||
extractors = {
|
||||
"firefox": extract_firefox_cookies,
|
||||
"chrome": extract_chrome_cookies,
|
||||
"brave": extract_brave_cookies,
|
||||
"safari": extract_safari_cookies,
|
||||
}
|
||||
|
||||
@@ -384,7 +360,7 @@ def extract_cookies_with_source(
|
||||
# Auto mode: try browsers in platform-appropriate order
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
order = ["chrome", "brave", "firefox", "safari"]
|
||||
order = ["chrome", "firefox", "safari"]
|
||||
elif system == "Linux":
|
||||
order = ["firefox"]
|
||||
else:
|
||||
|
||||
@@ -106,7 +106,7 @@ def _extract_subreddits(reddit_items: List[Dict[str, Any]]) -> List[str]:
|
||||
|
||||
for item in reddit_items:
|
||||
# Primary subreddit
|
||||
sub = item.get("subreddit", "").strip().removeprefix("r/")
|
||||
sub = item.get("subreddit", "").strip().lstrip("r/")
|
||||
if sub:
|
||||
sub_counts[sub] += 1
|
||||
|
||||
|
||||
@@ -29,23 +29,6 @@ else:
|
||||
|
||||
CODEX_AUTH_FILE = Path(os.environ.get("CODEX_AUTH_FILE", str(Path.home() / ".codex" / "auth.json")))
|
||||
|
||||
# macOS Keychain integration: items stored with this service prefix are picked
|
||||
# up automatically on Darwin as the lowest-priority credential source.
|
||||
# Example: `security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."`.
|
||||
KEYCHAIN_SERVICE_PREFIX = "last30days-"
|
||||
|
||||
# Single source of truth for which credentials the Keychain loader looks up.
|
||||
# The setup-keychain.sh helper mirrors this list and is held in sync via
|
||||
# tests/test_env_keychain.py::test_keychain_keys_match_setup_script.
|
||||
KEYCHAIN_KEYS = (
|
||||
"OPENAI_API_KEY", "XAI_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY", "SCRAPECREATORS_API_KEY", "APIFY_API_TOKEN",
|
||||
"AUTH_TOKEN", "CT0", "BSKY_HANDLE", "BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN", "BRAVE_API_KEY", "EXA_API_KEY", "SERPER_API_KEY",
|
||||
"OPENROUTER_API_KEY", "PARALLEL_API_KEY", "XQUIK_API_KEY",
|
||||
"XIAOHONGSHU_API_BASE",
|
||||
)
|
||||
|
||||
AuthSource = Literal["api_key", "codex", "none"]
|
||||
AuthStatus = Literal["ok", "missing", "expired", "missing_account_id"]
|
||||
|
||||
@@ -70,10 +53,6 @@ class OpenAIAuth:
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file has overly permissive permissions."""
|
||||
if os.name == "nt":
|
||||
# Windows reports synthesized POSIX mode bits that do not reflect NTFS ACLs.
|
||||
return
|
||||
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
# Check if group or other can read (bits 0o044)
|
||||
@@ -112,46 +91,6 @@ def load_env_file(path: Path) -> dict[str, str]:
|
||||
return env
|
||||
|
||||
|
||||
def _load_keychain(keys: list[str]) -> dict[str, str]:
|
||||
"""Load credentials from macOS Keychain (no-op on other platforms).
|
||||
|
||||
Each key is looked up as a generic password with service name
|
||||
``f"{KEYCHAIN_SERVICE_PREFIX}{key}"`` for the current user. Missing items
|
||||
and lookup failures are silent — Keychain is the lowest-priority source
|
||||
and is meant to be additive over `.env` files and process environment.
|
||||
"""
|
||||
import platform
|
||||
if platform.system() != "Darwin":
|
||||
return {}
|
||||
|
||||
import shutil
|
||||
security = shutil.which("security")
|
||||
if not security:
|
||||
return {}
|
||||
|
||||
import subprocess
|
||||
import pwd
|
||||
# USER can be unset under sudo, in Docker without --env USER, or in some CI
|
||||
# runners; fall back to the OS user record so lookups still match items
|
||||
# stored by setup-keychain.sh (which uses $USER).
|
||||
user = os.environ.get("USER") or pwd.getpwuid(os.getuid()).pw_name
|
||||
env: dict[str, str] = {}
|
||||
for key in keys:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[security, "find-generic-password",
|
||||
"-a", user,
|
||||
"-s", f"{KEYCHAIN_SERVICE_PREFIX}{key}",
|
||||
"-w"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
continue
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
env[key] = result.stdout.strip()
|
||||
return env
|
||||
|
||||
|
||||
def _decode_jwt_payload(token: str) -> dict[str, Any] | None:
|
||||
"""Decode JWT payload without verification."""
|
||||
try:
|
||||
@@ -275,7 +214,6 @@ def get_config() -> dict[str, Any]:
|
||||
1. Environment variables (os.environ)
|
||||
2. .claude/last30days.env (per-project config)
|
||||
3. ~/.config/last30days/.env (global config)
|
||||
4. macOS Keychain items prefixed ``last30days-`` (Darwin only)
|
||||
"""
|
||||
# Load from global config file
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||
@@ -284,14 +222,9 @@ def get_config() -> dict[str, Any]:
|
||||
project_env_path = _find_project_env()
|
||||
project_env = load_env_file(project_env_path) if project_env_path else {}
|
||||
|
||||
# Merge file sources: project > global
|
||||
# Merge: project overrides global
|
||||
merged_env = {**file_env, **project_env}
|
||||
|
||||
# Keychain is the lowest-priority source (Darwin only; no-op elsewhere).
|
||||
# Loaded before openai_auth so OPENAI_API_KEY can come from Keychain too.
|
||||
keychain_env = _load_keychain(list(KEYCHAIN_KEYS))
|
||||
merged_env = {**keychain_env, **merged_env}
|
||||
|
||||
openai_auth = get_openai_auth(merged_env)
|
||||
|
||||
# Build config: Codex/OpenAI auth + process.env > project .env > global .env
|
||||
@@ -314,7 +247,6 @@ def get_config() -> dict[str, Any]:
|
||||
('LAST30DAYS_RERANK_MODEL', None),
|
||||
('LAST30DAYS_X_MODEL', None),
|
||||
('LAST30DAYS_X_BACKEND', None),
|
||||
('LAST30DAYS_STORE', None),
|
||||
('OPENAI_MODEL_PIN', None),
|
||||
('XAI_MODEL_PIN', None),
|
||||
('SCRAPECREATORS_API_KEY', None),
|
||||
@@ -323,7 +255,6 @@ def get_config() -> dict[str, Any]:
|
||||
('CT0', None),
|
||||
('BSKY_HANDLE', None),
|
||||
('BSKY_APP_PASSWORD', None),
|
||||
('BSKY_SEARCH_HOST', None),
|
||||
('TRUTHSOCIAL_TOKEN', None),
|
||||
('BRAVE_API_KEY', None),
|
||||
('EXA_API_KEY', None),
|
||||
@@ -334,41 +265,16 @@ def get_config() -> dict[str, Any]:
|
||||
('FROM_BROWSER', None),
|
||||
('SETUP_COMPLETE', None),
|
||||
('INCLUDE_SOURCES', ''),
|
||||
('EXCLUDE_SOURCES', ''),
|
||||
('LAST30DAYS_YOUTUBE_SSH_HOST', None),
|
||||
('LAST30DAYS_TRANSCRIPT_TIMEOUT', None),
|
||||
]
|
||||
|
||||
for key, default in keys:
|
||||
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
||||
|
||||
# Backward-compat: ScrapeCreators' own examples and tutorials use the
|
||||
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and
|
||||
# CREATORS). Accept that form too so users who follow the vendor's docs
|
||||
# don't silently end up with has_scrapecreators=False. Canonical name
|
||||
# wins when both are set.
|
||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||
legacy = os.environ.get('SCRAPE_CREATORS_API_KEY') or merged_env.get('SCRAPE_CREATORS_API_KEY')
|
||||
if legacy:
|
||||
config['SCRAPECREATORS_API_KEY'] = legacy
|
||||
|
||||
# Multi-key rotation: comma-separated SCRAPECREATORS_API_KEY round-robins
|
||||
# via random.choice per run. Originally added in #268, accidentally dropped
|
||||
# in v3.0.6, restored here.
|
||||
sc_key_raw = config.get('SCRAPECREATORS_API_KEY') or ''
|
||||
if ',' in sc_key_raw:
|
||||
import random
|
||||
sc_keys = [k.strip() for k in sc_key_raw.split(',') if k.strip()]
|
||||
config['SCRAPECREATORS_API_KEY'] = random.choice(sc_keys) if sc_keys else ''
|
||||
|
||||
# Track which config source was used (highest-priority file source wins
|
||||
# the label; keychain is only reported when nothing else is configured).
|
||||
# Track which config source was used
|
||||
if project_env_path:
|
||||
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
||||
elif CONFIG_FILE and CONFIG_FILE.exists():
|
||||
config['_CONFIG_SOURCE'] = f'global:{CONFIG_FILE}'
|
||||
elif keychain_env:
|
||||
config['_CONFIG_SOURCE'] = 'keychain'
|
||||
else:
|
||||
config['_CONFIG_SOURCE'] = 'env_only'
|
||||
|
||||
@@ -611,12 +517,12 @@ def _parse_include_sources(config: dict[str, Any]) -> set[str]:
|
||||
def is_threads_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Threads source is available.
|
||||
|
||||
Returns True when SCRAPECREATORS_API_KEY is set. Threads runs alongside
|
||||
TikTok and Instagram as part of the SC family — same key, same per-call
|
||||
cost shape, so the same default-on rule applies. Suppress via
|
||||
EXCLUDE_SOURCES=threads.
|
||||
Requires SCRAPECREATORS_API_KEY AND 'threads' in INCLUDE_SOURCES.
|
||||
Threads is an opt-in source - it is not activated by default.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||
return False
|
||||
return 'threads' in _parse_include_sources(config)
|
||||
|
||||
|
||||
def is_instagram_available(config: dict[str, Any]) -> bool:
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
@@ -140,10 +139,7 @@ def parallel_search(
|
||||
data = http.request(
|
||||
"POST", "https://api.parallel.ai/v1/search",
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
json_data={
|
||||
"search_queries": [query],
|
||||
"advanced_settings": {"max_results": count},
|
||||
},
|
||||
json_data={"query": query, "max_results": count},
|
||||
timeout=15,
|
||||
)
|
||||
items = []
|
||||
@@ -153,7 +149,7 @@ def parallel_search(
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
raw_date = r.get("publish_date") or ""
|
||||
raw_date = r.get("published_date") or ""
|
||||
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
|
||||
if not _in_date_range(pub_date, date_range):
|
||||
continue
|
||||
@@ -162,7 +158,7 @@ def parallel_search(
|
||||
"title": r.get("title", ""),
|
||||
"url": url,
|
||||
"source_domain": _domain(url),
|
||||
"snippet": ((r.get("excerpts") or [""])[0] or "")[:500],
|
||||
"snippet": r.get("snippet", ""),
|
||||
"date": pub_date,
|
||||
"relevance": 0.8,
|
||||
"why_relevant": "Parallel AI web search",
|
||||
@@ -209,90 +205,29 @@ def web_search(
|
||||
backend = "parallel"
|
||||
else:
|
||||
return [], {}
|
||||
items: list[dict] = []
|
||||
artifact: dict = {}
|
||||
if backend == "brave":
|
||||
key = config.get("BRAVE_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
|
||||
items, artifact = brave_search(query, date_range, key)
|
||||
elif backend == "exa":
|
||||
return brave_search(query, date_range, key)
|
||||
if backend == "exa":
|
||||
key = config.get("EXA_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
|
||||
items, artifact = exa_search(query, date_range, key)
|
||||
elif backend == "serper":
|
||||
return exa_search(query, date_range, key)
|
||||
if backend == "serper":
|
||||
key = config.get("SERPER_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
|
||||
items, artifact = serper_search(query, date_range, key)
|
||||
elif backend == "parallel":
|
||||
return serper_search(query, date_range, key)
|
||||
if backend == "parallel":
|
||||
key = config.get("PARALLEL_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
|
||||
items, artifact = parallel_search(query, date_range, key)
|
||||
elif backend != "none":
|
||||
return parallel_search(query, date_range, key)
|
||||
if backend != "none":
|
||||
raise ValueError(f"Unsupported web backend: {backend!r}")
|
||||
else:
|
||||
return [], {}
|
||||
if items and not _reddit_excluded(config):
|
||||
items = _enrich_reddit_items(items)
|
||||
return items, artifact
|
||||
|
||||
|
||||
def _reddit_excluded(config: dict) -> bool:
|
||||
"""Return True when EXCLUDE_SOURCES contains 'reddit'.
|
||||
|
||||
Respects the same suppression knob the pipeline uses for source gating,
|
||||
so a user who set EXCLUDE_SOURCES=reddit doesn't get Reddit content
|
||||
smuggled back in via web-search URLs.
|
||||
"""
|
||||
raw = (config.get("EXCLUDE_SOURCES") or "").split(",")
|
||||
return any(s.strip().lower() == "reddit" for s in raw)
|
||||
|
||||
|
||||
def _enrich_reddit_items(items: list[dict]) -> list[dict]:
|
||||
"""Enrich web search results that are Reddit URLs with thread body and comments.
|
||||
|
||||
Claude Code's WebFetch blocks reddit.com, so the model can't retrieve
|
||||
Reddit content from web search results. This fetches it via the public
|
||||
JSON API (reddit.com/.../.json) which bypasses that restriction.
|
||||
|
||||
Callers should gate this with EXCLUDE_SOURCES=reddit handling (see
|
||||
`_reddit_excluded`) so a user who explicitly excluded Reddit doesn't
|
||||
get Reddit content via web-search URLs.
|
||||
"""
|
||||
from . import reddit_enrich
|
||||
from .reddit_enrich import RedditRateLimitError
|
||||
|
||||
for item in items:
|
||||
url = item.get("url", "")
|
||||
if "reddit.com" not in url or "/comments/" not in url:
|
||||
continue
|
||||
try:
|
||||
thread_data = reddit_enrich.fetch_thread_data(url, timeout=8)
|
||||
if not thread_data:
|
||||
continue
|
||||
parsed = reddit_enrich.parse_thread_data(thread_data)
|
||||
# selftext lives under parsed["submission"], not at the top level
|
||||
selftext = (parsed.get("submission") or {}).get("selftext", "")
|
||||
if selftext:
|
||||
item["snippet"] = selftext[:2000]
|
||||
comments = parsed.get("comments", [])
|
||||
top = reddit_enrich.get_top_comments(comments)
|
||||
if top:
|
||||
item["top_comments"] = [
|
||||
{"score": c.get("score", 0), "excerpt": (c.get("body") or "")[:200]}
|
||||
for c in top[:5]
|
||||
]
|
||||
item["enriched_via"] = "reddit_json_api"
|
||||
except RedditRateLimitError as exc:
|
||||
# Stop iterating to avoid flooding more 429s
|
||||
sys.stderr.write(f"[Web] Reddit rate-limited, halting enrichment: {exc}\n")
|
||||
break
|
||||
except Exception as exc:
|
||||
sys.stderr.write(f"[Web] Reddit enrichment failed for {url}: {exc}\n")
|
||||
return items
|
||||
return [], {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -88,26 +88,17 @@ def search_hackernews(
|
||||
|
||||
# Use extracted core subject instead of raw topic for cleaner Algolia matching
|
||||
core = extract_core_subject(topic)
|
||||
# Hyphens and commas tokenize awkwardly in Algolia; flatten them so themed
|
||||
# queries like "ts-bun-node" or "claude, personal agents" become plain words.
|
||||
core_flat = _flatten_query_for_algolia(core)
|
||||
_log(f"Searching for '{core_flat}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
|
||||
# Use relevance-sorted search with minimum engagement filter.
|
||||
# NOTE: restrictSearchableAttributes=title omitted intentionally — it would
|
||||
# miss Ask HN/Show HN threads where the topic appears in the body.
|
||||
params = {
|
||||
"query": core_flat,
|
||||
"query": core,
|
||||
"tags": "story",
|
||||
"numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
|
||||
"hitsPerPage": str(count),
|
||||
}
|
||||
# Algolia defaults to AND across query tokens, so a 4-5 word theme query
|
||||
# matches no stories. Mark all-but-the-first token as optional so Algolia
|
||||
# ranks by how many tokens match instead of requiring every one.
|
||||
tokens = core_flat.split()
|
||||
if len(tokens) > 1:
|
||||
params["optionalWords"] = " ".join(tokens[1:])
|
||||
|
||||
from urllib.parse import urlencode
|
||||
url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
|
||||
@@ -126,56 +117,28 @@ def search_hackernews(
|
||||
return response
|
||||
|
||||
|
||||
_WORD_BOUNDARY_RE_CACHE: Dict[str, "re.Pattern[str]"] = {}
|
||||
|
||||
|
||||
def _flatten_query_for_algolia(text: str) -> str:
|
||||
"""Normalise query for Algolia + post-filter comparison.
|
||||
|
||||
Multi-keyword theme queries frequently contain commas (delimiters) or
|
||||
hyphens (compound terms like ``ts-bun-node``); both tokenize awkwardly.
|
||||
Flatten them to spaces and collapse runs of whitespace so the search
|
||||
parameter and the post-filter operate on the same shape.
|
||||
"""
|
||||
return " ".join(text.replace(",", " ").replace("-", " ").split())
|
||||
|
||||
|
||||
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
|
||||
"""Check if any query token appears as a whole word in the title.
|
||||
"""Check if the query term appears in the title content, not just an HN prefix or author.
|
||||
|
||||
Returns True when the query is empty (no filter), or when at least one
|
||||
query token matches as a whole word in the title after stripping
|
||||
"Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes.
|
||||
|
||||
We previously required *every* token to appear (all-words), which killed
|
||||
every Algolia hit on multi-keyword themes like "claude, personal agents,
|
||||
agentic infra" because real HN titles never contain all five tokens
|
||||
verbatim. Relaxing to any-word matches Algolia's `optionalWords` behaviour
|
||||
in `search_hackernews`. Token-overlap relevance scoring at parse time
|
||||
demotes hits where only one weak token matched, so the loosened gate
|
||||
won't surface noise to the top of the ranking.
|
||||
|
||||
Word-boundary matching (rather than naive substring) prevents short
|
||||
tokens like ``ai`` or ``ts`` from matching unrelated words like
|
||||
``email`` or ``artists``.
|
||||
Returns True if the query (or any multi-word token) appears in the title
|
||||
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
|
||||
and ignoring the author name. Returns True when query is empty (no filter).
|
||||
"""
|
||||
if not query:
|
||||
return True
|
||||
stripped = _HN_PREFIXES.sub("", title).strip()
|
||||
# Also check that the match isn't solely in the author's username
|
||||
check_text = stripped.lower()
|
||||
# Normalise the query the same way search_hackernews does so post-filter
|
||||
# tokens line up with what Algolia actually saw.
|
||||
query_words = [w for w in _flatten_query_for_algolia(query.lower()).split() if w]
|
||||
if not query_words:
|
||||
return True
|
||||
query_lower = query.lower()
|
||||
# Check each word of the query independently; all must appear somewhere
|
||||
# in the stripped title (not just the prefix).
|
||||
query_words = query_lower.split()
|
||||
for word in query_words:
|
||||
pattern = _WORD_BOUNDARY_RE_CACHE.get(word)
|
||||
if pattern is None:
|
||||
pattern = re.compile(rf"\b{re.escape(word)}\b")
|
||||
_WORD_BOUNDARY_RE_CACHE[word] = pattern
|
||||
if pattern.search(check_text):
|
||||
return True
|
||||
return False
|
||||
if word in check_text:
|
||||
continue
|
||||
# Word not found in stripped title — reject
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -23,19 +22,9 @@ def log(msg: str):
|
||||
MAX_RETRIES = 5
|
||||
MAX_429_RETRIES = 2
|
||||
RETRY_DELAY = 2.0
|
||||
# DNS resolution failures (gaierror) are transient — typically resolved by a
|
||||
# brief backoff and retry. Use a dedicated minimum attempt count + exponential
|
||||
# delays (1s, 2s, 4s) so callers that pass a small `retries` value still get a
|
||||
# meaningful chance to recover from a transient resolution failure.
|
||||
MIN_DNS_RETRIES = 3
|
||||
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
||||
|
||||
|
||||
def _is_dns_failure(err: urllib.error.URLError) -> bool:
|
||||
"""Return True if a URLError was caused by DNS resolution (gaierror)."""
|
||||
return isinstance(getattr(err, "reason", None), socket.gaierror)
|
||||
|
||||
|
||||
class HTTPError(Exception):
|
||||
"""HTTP request error with status code."""
|
||||
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
||||
@@ -96,13 +85,7 @@ def request(
|
||||
|
||||
last_error = None
|
||||
rate_limit_count = 0
|
||||
# DNS failures get a dedicated minimum attempt count + exponential backoff.
|
||||
# `effective_retries` is the actual loop bound; we expand it on the first
|
||||
# gaierror if the caller passed a smaller `retries` value than MIN_DNS_RETRIES.
|
||||
effective_retries = retries
|
||||
dns_attempts = 0
|
||||
attempt = 0
|
||||
while attempt < effective_retries:
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
body = response.read().decode('utf-8')
|
||||
@@ -132,8 +115,6 @@ def request(
|
||||
if rate_limit_count >= max_429_retries:
|
||||
raise last_error
|
||||
|
||||
# HTTP errors respect the caller's original `retries`; only DNS
|
||||
# failures get the widened `effective_retries` budget.
|
||||
if attempt < retries - 1:
|
||||
if e.code == 429:
|
||||
# Respect Retry-After header, fall back to exponential backoff
|
||||
@@ -149,43 +130,11 @@ def request(
|
||||
else:
|
||||
delay = RETRY_DELAY * (2 ** attempt)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure may have widened `effective_retries`, but that
|
||||
# widening is DNS-only — don't grant extra HTTP attempts.
|
||||
break
|
||||
except urllib.error.URLError as e:
|
||||
log(f"URL Error: {e.reason}")
|
||||
last_error = HTTPError(f"URL Error: {e.reason}")
|
||||
if _is_dns_failure(e):
|
||||
# DNS resolution failures are transient; expand the retry budget
|
||||
# to MIN_DNS_RETRIES if the caller passed fewer, and use
|
||||
# exponential backoff (1s, 2s, 4s, ...) instead of the linear
|
||||
# default. Counts DNS attempts separately so other URLError
|
||||
# causes don't bypass the regular retry budget.
|
||||
dns_attempts += 1
|
||||
if effective_retries < MIN_DNS_RETRIES:
|
||||
log(
|
||||
f"DNS resolution failed; expanding retry budget from "
|
||||
f"{effective_retries} to {MIN_DNS_RETRIES}"
|
||||
)
|
||||
effective_retries = MIN_DNS_RETRIES
|
||||
if attempt < effective_retries - 1:
|
||||
delay = 2 ** (dns_attempts - 1) # 1s, 2s, 4s, 8s, ...
|
||||
log(
|
||||
f"DNS resolution failure (attempt {dns_attempts}); "
|
||||
f"retrying in {delay:.1f}s"
|
||||
)
|
||||
time.sleep(delay)
|
||||
elif attempt < retries - 1:
|
||||
# Non-DNS URLError (e.g. ConnectionRefused) respects the
|
||||
# caller's original retry budget, not the DNS-widened bound.
|
||||
if attempt < retries - 1:
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure widening `effective_retries` does not carry over
|
||||
# to non-DNS error paths.
|
||||
break
|
||||
except json.JSONDecodeError as e:
|
||||
log(f"JSON decode error: {e}")
|
||||
last_error = HTTPError(f"Invalid JSON response: {e}")
|
||||
@@ -195,13 +144,7 @@ def request(
|
||||
log(f"Connection error: {type(e).__name__}: {e}")
|
||||
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
||||
if attempt < retries - 1:
|
||||
# Socket errors respect the caller's original retry budget.
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Original budget exhausted; DNS widening doesn't apply here.
|
||||
break
|
||||
|
||||
attempt += 1
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
|
||||
@@ -7,14 +7,12 @@ Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
|
||||
API docs: https://scrapecreators.com/docs
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from . import dates, http, log
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||
|
||||
@@ -28,42 +26,7 @@ DEPTH_CONFIG = {
|
||||
# Max words to keep from each caption
|
||||
CAPTION_MAX_WORDS = 500
|
||||
|
||||
# Default transcript fetch timeout (seconds). SC's
|
||||
# /v2/instagram/media/transcript regularly takes >15s on real workloads,
|
||||
# so the default is generous; override via LAST30DAYS_TRANSCRIPT_TIMEOUT.
|
||||
DEFAULT_TRANSCRIPT_TIMEOUT = 30
|
||||
|
||||
|
||||
def _resolve_transcript_timeout(
|
||||
timeout: Optional[float] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> float:
|
||||
"""Resolve the IG transcript-fetch timeout.
|
||||
|
||||
Priority (highest wins):
|
||||
1. Explicit ``timeout`` kwarg
|
||||
2. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in os.environ
|
||||
3. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in caller-supplied config dict
|
||||
4. ``DEFAULT_TRANSCRIPT_TIMEOUT`` (30s)
|
||||
|
||||
Mirrors the ``os.environ.get(X) or config.get(X)`` pattern used for
|
||||
LAST30DAYS_STORE in last30days.py so the env var works whether it's
|
||||
shell-exported or set in ~/.config/last30days/.env.
|
||||
"""
|
||||
if timeout is not None:
|
||||
try:
|
||||
return float(timeout)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
raw = os.environ.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||
if not raw and config:
|
||||
raw = config.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||
if raw:
|
||||
try:
|
||||
return float(raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return float(DEFAULT_TRANSCRIPT_TIMEOUT)
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
@@ -81,17 +44,6 @@ def _extract_core_subject(topic: str) -> str:
|
||||
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
||||
|
||||
|
||||
def _to_hashtag_form(query: str) -> str:
|
||||
"""Collapse a multi-word query to hashtag form (no spaces, lowercase).
|
||||
|
||||
SC's /v2/instagram/reels/search wraps Google Search and is documented
|
||||
to be flaky on multi-token queries. Single-token queries map to a
|
||||
hashtag page lookup which is the stable path. Used as a 500-retry
|
||||
fallback before the request bubbles up as a silent failure.
|
||||
"""
|
||||
return ''.join(query.split()).lower()
|
||||
|
||||
|
||||
def _infer_query_intent(topic: str) -> str:
|
||||
"""Tiny local intent classifier for Instagram query expansion."""
|
||||
text = topic.lower().strip()
|
||||
@@ -331,26 +283,6 @@ def search_instagram(
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except http.HTTPError as e:
|
||||
# SC's v2 reels search wraps Google Search and 500s frequently on
|
||||
# multi-token queries. Single tokens hit the stable hashtag-page
|
||||
# path. Retry once with hashtag form before bubbling up.
|
||||
if getattr(e, "status_code", None) == 500 and ' ' in core_topic:
|
||||
_log(f"IG search 500 on '{core_topic}', retrying with hashtag form")
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": _to_hashtag_form(core_topic)},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as retry_e:
|
||||
_log(f"IG search retry failed: {retry_e}")
|
||||
return {"items": [], "error": f"{type(retry_e).__name__}: {retry_e}"}
|
||||
else:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
@@ -385,8 +317,6 @@ def fetch_captions(
|
||||
video_items: List[Dict[str, Any]],
|
||||
token: str,
|
||||
depth: str = "default",
|
||||
timeout: Optional[float] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
||||
|
||||
@@ -398,19 +328,12 @@ def fetch_captions(
|
||||
video_items: Items from search_instagram()
|
||||
token: ScrapeCreators API key
|
||||
depth: Depth level for caption limit
|
||||
timeout: Optional per-request transcript timeout in seconds. When
|
||||
None, resolves from LAST30DAYS_TRANSCRIPT_TIMEOUT (env or
|
||||
config), defaulting to DEFAULT_TRANSCRIPT_TIMEOUT (30s).
|
||||
config: Optional config dict (from env.get_config()) used as a
|
||||
fallback source for LAST30DAYS_TRANSCRIPT_TIMEOUT when the
|
||||
value is not exported in os.environ.
|
||||
|
||||
Returns:
|
||||
Dict mapping video_id -> caption text (truncated to 500 words)
|
||||
"""
|
||||
depth_cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = depth_cfg["max_captions"]
|
||||
transcript_timeout = _resolve_transcript_timeout(timeout, config)
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = config["max_captions"]
|
||||
|
||||
if not video_items or not token:
|
||||
return {}
|
||||
@@ -441,7 +364,7 @@ def fetch_captions(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||
params={"url": url},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=transcript_timeout,
|
||||
timeout=15,
|
||||
retries=1,
|
||||
)
|
||||
transcripts = data.get("transcripts") or []
|
||||
|
||||
@@ -251,11 +251,6 @@ def _normalize_youtube(
|
||||
metadata: dict[str, Any] = {}
|
||||
if highlights:
|
||||
metadata["transcript_highlights"] = highlights
|
||||
if item.get("captions_disabled"):
|
||||
# Surfaced for quality_nudge: uploader disabled captions, so this
|
||||
# video should be subtracted from the degraded-transcript-ratio
|
||||
# denominator (it was never going to produce a transcript).
|
||||
metadata["captions_disabled"] = True
|
||||
metadata["top_comments"] = _remap_comments(
|
||||
item.get("top_comments") or [],
|
||||
score_keys=("score", "likes"),
|
||||
|
||||
@@ -79,8 +79,6 @@ MOCK_AVAILABLE_SOURCES = [
|
||||
"xiaohongshu",
|
||||
"github",
|
||||
"perplexity",
|
||||
"threads",
|
||||
"pinterest",
|
||||
"xquik",
|
||||
"digg",
|
||||
]
|
||||
@@ -120,9 +118,7 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("grounding")
|
||||
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
|
||||
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
|
||||
if config.get("OPENROUTER_API_KEY") and (
|
||||
"perplexity" in include_sources or (requested_sources and "perplexity" in requested_sources)
|
||||
):
|
||||
if config.get("OPENROUTER_API_KEY") and "perplexity" in include_sources:
|
||||
available.append("perplexity")
|
||||
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
|
||||
available.append("xiaohongshu")
|
||||
@@ -132,9 +128,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("pinterest")
|
||||
if env.is_xquik_available(config):
|
||||
available.append("xquik")
|
||||
exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()}
|
||||
if exclude:
|
||||
available = [s for s in available if s not in exclude]
|
||||
return available
|
||||
|
||||
|
||||
@@ -207,7 +200,7 @@ def run(
|
||||
available = [source for source in available if source in requested_sources]
|
||||
if web_backend == "none":
|
||||
available = [s for s in available if s != "grounding"]
|
||||
elif web_backend in ("brave", "exa", "serper", "parallel") and "grounding" not in available:
|
||||
elif web_backend in ("brave", "exa", "serper") and "grounding" not in available:
|
||||
available.append("grounding")
|
||||
if not available:
|
||||
raise RuntimeError("No sources are available for this run.")
|
||||
|
||||
@@ -19,14 +19,14 @@ ALLOWED_INTENTS = {
|
||||
}
|
||||
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
|
||||
QUICK_SOURCE_PRIORITY = {
|
||||
"factual": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||
"product": ["youtube", "reddit", "x", "xquik", "tiktok"],
|
||||
"concept": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||
"opinion": ["reddit", "x", "xquik", "youtube", "hackernews"],
|
||||
"how_to": ["youtube", "reddit", "x", "xquik", "hackernews"],
|
||||
"comparison": ["reddit", "x", "xquik", "hackernews", "youtube"],
|
||||
"breaking_news": ["x", "xquik", "reddit", "hackernews", "youtube", "polymarket"],
|
||||
"prediction": ["polymarket", "x", "xquik", "hackernews", "reddit", "youtube"],
|
||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||
"product": ["youtube", "reddit", "x", "tiktok"],
|
||||
"concept": ["hackernews", "reddit", "x", "youtube"],
|
||||
"opinion": ["reddit", "x", "youtube", "hackernews"],
|
||||
"how_to": ["youtube", "reddit", "x", "hackernews"],
|
||||
"comparison": ["reddit", "x", "hackernews", "youtube"],
|
||||
"breaking_news": ["x", "reddit", "hackernews", "youtube", "polymarket"],
|
||||
"prediction": ["polymarket", "x", "hackernews", "reddit", "youtube"],
|
||||
}
|
||||
SOURCE_PRIORITY = {
|
||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||
@@ -60,7 +60,6 @@ INTENT_SOURCE_EXCLUSIONS: dict[str, set[str]] = {
|
||||
SOURCE_CAPABILITIES = {
|
||||
"reddit": {"discussion", "social"},
|
||||
"x": {"discussion", "social"},
|
||||
"xquik": {"discussion", "social"},
|
||||
"youtube": {"video", "video_longform", "discussion"},
|
||||
"tiktok": {"video", "video_shortform", "social"},
|
||||
"instagram": {"video", "video_shortform", "social"},
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
from . import env, http, schema
|
||||
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite"
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_PRO = "gemini-3.1-pro-preview"
|
||||
OPENAI_DEFAULT = "gpt-5.4-nano"
|
||||
XAI_DEFAULT = "grok-4-1-fast"
|
||||
@@ -19,11 +19,7 @@ OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
||||
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
|
||||
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
|
||||
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||
# OpenRouter routes the Gemini Flash Lite tier as the -preview slug; that is the
|
||||
# stable form on that routing layer even though native Gemini's GEMINI_FLASH_LITE
|
||||
# constant is suffix-free. If GEMINI_FLASH_LITE moves to a non-preview stable ID,
|
||||
# double-check that OpenRouter's slug still maps to the same upstream model.
|
||||
OPENROUTER_DEFAULT = "google/gemini-3.1-flash-lite-preview"
|
||||
OPENROUTER_DEFAULT = "google/gemini-flash-2.0"
|
||||
|
||||
|
||||
class ReasoningClient:
|
||||
@@ -236,8 +232,8 @@ def _resolve_model_pins(config: dict[str, Any], depth: str, provider_name: str)
|
||||
rerank_model = config.get("LAST30DAYS_RERANK_MODEL") or default_rerank
|
||||
|
||||
if provider_name == "gemini":
|
||||
_require_gemini_31(planner_model, role="planner")
|
||||
_require_gemini_31(rerank_model, role="rerank")
|
||||
_require_gemini_31_preview(planner_model, role="planner")
|
||||
_require_gemini_31_preview(rerank_model, role="rerank")
|
||||
|
||||
return planner_model, rerank_model
|
||||
|
||||
@@ -348,11 +344,11 @@ def _resolve_x_backend(config: dict[str, Any]) -> str | None:
|
||||
return env.get_x_source(config)
|
||||
|
||||
|
||||
def _require_gemini_31(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-"):
|
||||
def _require_gemini_31_preview(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-") and model.endswith("-preview"):
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"{role} must use a Gemini 3.1 model. Got: {model}"
|
||||
f"{role} must use a Gemini 3.1 preview model. Got: {model}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -45,100 +45,26 @@ def _is_youtube_active(config: dict, research_results: dict) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
# Below this transcript-fetch ratio, YouTube is considered "degraded" rather
|
||||
# than active. Picked at 50% so a single legitimate caption-disabled video in a
|
||||
# multi-video result does not trip the nudge, but a stale-yt-dlp run that fails
|
||||
# every transcript does. Tunable via DEGRADED_TRANSCRIPT_THRESHOLD env var if
|
||||
# operators need to adjust without code changes.
|
||||
DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD = 0.5
|
||||
|
||||
|
||||
def _is_youtube_degraded(research_results: dict, threshold: float) -> bool:
|
||||
"""YouTube is degraded when videos were returned but the transcript-fetch
|
||||
ratio is below threshold. The canonical cause is a stale yt-dlp binary -
|
||||
YouTube's caption format changes frequently and old binaries silently fail
|
||||
every transcript while the search itself still succeeds.
|
||||
|
||||
Captions-disabled videos are subtracted from the denominator: an uploader
|
||||
who turned off captions can never produce a transcript, so counting that
|
||||
video toward "fetch failures" produces false positives. A single
|
||||
captions-disabled video in a small result set was tripping the nudge.
|
||||
"""
|
||||
videos = int(research_results.get("youtube_videos_count") or 0)
|
||||
transcripts = int(research_results.get("youtube_transcripts_count") or 0)
|
||||
captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0)
|
||||
if videos <= 0:
|
||||
return False
|
||||
eligible = videos - captions_disabled
|
||||
if eligible <= 0:
|
||||
# Every returned video had captions disabled - upstream content fact,
|
||||
# not a yt-dlp problem. Don't flag.
|
||||
return False
|
||||
return (transcripts / eligible) < threshold
|
||||
|
||||
|
||||
def _is_instagram_silent_failure(config: dict, research_results: dict) -> bool:
|
||||
"""Instagram is silently failing when SC is configured but the source
|
||||
returned zero items. The canonical cause is SC's v2 reels endpoint
|
||||
500'ing on multi-token queries (it wraps Google Search and is documented
|
||||
to be flaky there). Pre-fix the user got no signal at all - no Instagram
|
||||
section in the brief, no error in the footer, just unexplained absence.
|
||||
"""
|
||||
if not config.get("SCRAPECREATORS_API_KEY"):
|
||||
return False # not configured — not a silent failure
|
||||
# Honor EXCLUDE_SOURCES: a user who set EXCLUDE_SOURCES=instagram
|
||||
# intentionally turned the source off, so a zero-item count is
|
||||
# expected, not a silent failure. Mirror the canonical parsing
|
||||
# pattern from pipeline.available_sources().
|
||||
excluded = {
|
||||
s.strip().lower()
|
||||
for s in (config.get("EXCLUDE_SOURCES") or "").split(",")
|
||||
if s.strip()
|
||||
}
|
||||
# Symmetric case: INCLUDE_SOURCES is an opt-in allowlist. If it is
|
||||
# non-empty and does not name instagram, the source was intentionally
|
||||
# filtered out, so a zero-item count is expected — not a silent failure.
|
||||
included = {
|
||||
s.strip().lower()
|
||||
for s in (config.get("INCLUDE_SOURCES") or "").split(",")
|
||||
if s.strip()
|
||||
}
|
||||
if "instagram" in excluded or (included and "instagram" not in included):
|
||||
return False
|
||||
count = research_results.get("instagram_items_count")
|
||||
if count is None:
|
||||
return False # source not run this invocation
|
||||
return int(count) == 0
|
||||
|
||||
|
||||
def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||
"""Compute research quality score based on 5 core sources.
|
||||
|
||||
Args:
|
||||
config: Configuration dict from env.get_config()
|
||||
research_results: Dict with keys like x_error, youtube_error,
|
||||
reddit_error reflecting what happened this run. Optional keys
|
||||
``youtube_videos_count`` and ``youtube_transcripts_count`` enable
|
||||
degraded-YouTube detection (transcript-fetch ratio below threshold).
|
||||
Optional key ``instagram_items_count`` enables silent-failure
|
||||
detection for the bonus Instagram source.
|
||||
reddit_error reflecting what happened this run.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"score_pct": 40-100,
|
||||
"core_active": ["hn", "polymarket", ...],
|
||||
"core_missing": ["x", "youtube"],
|
||||
"core_errored": [], # configured but errored at top level
|
||||
"core_degraded": [], # configured and returned items but quality below threshold
|
||||
"bonus_errored": [], # bonus sources (Instagram, etc.) configured but silent
|
||||
"nudge_text": "..." or None if all sources healthy
|
||||
"core_errored": [], # configured but errored
|
||||
"nudge_text": "..." or None if 100%
|
||||
}
|
||||
"""
|
||||
core_active: List[str] = []
|
||||
core_missing: List[str] = []
|
||||
core_errored: List[str] = []
|
||||
core_degraded: List[str] = []
|
||||
bonus_errored: List[str] = []
|
||||
|
||||
# HN, Polymarket, and Reddit are always active
|
||||
core_active.append("hn")
|
||||
@@ -158,13 +84,6 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||
yt_active = _is_youtube_active(config, research_results)
|
||||
if yt_active:
|
||||
core_active.append("youtube")
|
||||
# Active means yt-dlp is installed and search did not error at the top
|
||||
# level. But search-success + transcript-failure is the canonical
|
||||
# stale-binary failure mode that the footer used to hide. Flag as
|
||||
# degraded so the user gets an actionable nudge to update the binary.
|
||||
threshold = float(config.get("DEGRADED_TRANSCRIPT_THRESHOLD") or DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD)
|
||||
if _is_youtube_degraded(research_results, threshold):
|
||||
core_degraded.append("youtube")
|
||||
else:
|
||||
core_missing.append("youtube")
|
||||
# Check if configured but errored (yt-dlp installed but failed this run)
|
||||
@@ -176,54 +95,28 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||
if has_ytdlp and research_results.get("youtube_error"):
|
||||
core_errored.append("youtube")
|
||||
|
||||
# Bonus sources (Instagram, etc.): SC-key holders expect content from
|
||||
# these but until now the pipeline fell silent on configured-but-zero.
|
||||
if _is_instagram_silent_failure(config, research_results):
|
||||
bonus_errored.append("instagram")
|
||||
|
||||
score_pct = int(len(core_active) / 5 * 100)
|
||||
|
||||
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||
active_sources = research_results.get("active_sources") or []
|
||||
nudge_text = _build_nudge_text(
|
||||
core_missing,
|
||||
core_errored,
|
||||
core_degraded,
|
||||
research_results,
|
||||
has_sc=has_sc,
|
||||
active_sources=active_sources,
|
||||
bonus_errored=bonus_errored,
|
||||
) if (core_missing or core_degraded or bonus_errored) else None
|
||||
nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None
|
||||
|
||||
return {
|
||||
"score_pct": score_pct,
|
||||
"core_active": core_active,
|
||||
"core_missing": core_missing,
|
||||
"core_errored": core_errored,
|
||||
"core_degraded": core_degraded,
|
||||
"bonus_errored": bonus_errored,
|
||||
"nudge_text": nudge_text,
|
||||
}
|
||||
|
||||
|
||||
def _build_nudge_text(
|
||||
core_missing: List[str],
|
||||
core_errored: List[str],
|
||||
core_degraded: List[str] = None,
|
||||
research_results: dict = None,
|
||||
has_sc: bool = False,
|
||||
active_sources: list = None,
|
||||
bonus_errored: List[str] = None,
|
||||
) -> str:
|
||||
"""Build human-readable nudge text describing what was missed or degraded.
|
||||
def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str:
|
||||
"""Build human-readable nudge text describing what was missed.
|
||||
|
||||
Prioritizes free suggestions. Optionally mentions bonus sources
|
||||
(TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured.
|
||||
"""
|
||||
lines: List[str] = []
|
||||
core_degraded = core_degraded or []
|
||||
bonus_errored = bonus_errored or []
|
||||
research_results = research_results or {}
|
||||
|
||||
# Describe what was missed
|
||||
missed_parts: List[str] = []
|
||||
@@ -236,14 +129,7 @@ def _build_nudge_text(
|
||||
|
||||
active_count = 5 - len(core_missing)
|
||||
lines.append(f"Research quality: {active_count}/5 core sources.")
|
||||
if missed_parts:
|
||||
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
||||
if core_degraded:
|
||||
degraded_labels = ", ".join(SOURCE_LABELS[s] for s in core_degraded)
|
||||
lines.append(f"Degraded: {degraded_labels}.")
|
||||
if bonus_errored:
|
||||
bonus_labels = ", ".join(s.capitalize() for s in bonus_errored)
|
||||
lines.append(f"Bonus source silent: {bonus_labels}.")
|
||||
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
||||
lines.append("")
|
||||
|
||||
# Free suggestions
|
||||
@@ -273,35 +159,6 @@ def _build_nudge_text(
|
||||
"explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
|
||||
)
|
||||
|
||||
if "youtube" in core_degraded:
|
||||
videos = int(research_results.get("youtube_videos_count") or 0)
|
||||
transcripts = int(research_results.get("youtube_transcripts_count") or 0)
|
||||
captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0)
|
||||
captions_note = ""
|
||||
if captions_disabled > 0:
|
||||
captions_note = (
|
||||
f" ({captions_disabled} of those had captions disabled by the "
|
||||
"uploader, which is a separate cause and not fixable on your end)"
|
||||
)
|
||||
free_suggestions.append(
|
||||
f"YouTube returned {videos} videos but only {transcripts} transcripts "
|
||||
f"captured{captions_note}. The most common remaining cause is a stale "
|
||||
"yt-dlp binary - YouTube's caption format changes frequently and old "
|
||||
"binaries silently fail every transcript. Update via your package "
|
||||
"manager: scoop update yt-dlp (Windows), brew upgrade yt-dlp (macOS), "
|
||||
"or pip install -U yt-dlp."
|
||||
)
|
||||
|
||||
if "instagram" in bonus_errored:
|
||||
free_suggestions.append(
|
||||
"Instagram returned 0 reels despite SC being configured. SC's "
|
||||
"v2 reels endpoint wraps Google Search and 500's frequently on "
|
||||
"multi-token queries. The skill now retries with hashtag-form "
|
||||
"automatically; if zero items still appear, the topic may have "
|
||||
"no reel coverage on Instagram. Try a single-word topic like "
|
||||
"the most distinctive noun in your query."
|
||||
)
|
||||
|
||||
# Mention bonus opt-in sources when SC key is present
|
||||
if has_sc:
|
||||
bonus_hints = []
|
||||
|
||||
@@ -334,7 +334,7 @@ def _global_search(
|
||||
)
|
||||
return data.get("posts", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
if e.status_code in (401, 403):
|
||||
raise
|
||||
_log(f"Global search error: {e}")
|
||||
return []
|
||||
@@ -376,11 +376,6 @@ def _subreddit_search(
|
||||
retries=2,
|
||||
)
|
||||
return data.get("posts", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
raise
|
||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||
return []
|
||||
@@ -408,11 +403,6 @@ def fetch_post_comments(
|
||||
retries=2,
|
||||
)
|
||||
return data.get("comments", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
raise
|
||||
_log(f"Comment fetch error: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
_log(f"Comment fetch error: {e}")
|
||||
return []
|
||||
|
||||
@@ -11,7 +11,6 @@ Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
|
||||
network timeouts, and missing subreddits.
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
@@ -22,11 +21,7 @@ from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeou
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
USER_AGENT = "last30days/3.0 (research tool)"
|
||||
|
||||
# Depth-aware limits for thread counts
|
||||
DEPTH_LIMITS = {
|
||||
@@ -65,9 +60,6 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
|
||||
headers = {
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
|
||||
@@ -79,10 +71,7 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
|
||||
_log(f"Anti-bot HTML response (Content-Type: {content_type})")
|
||||
return None
|
||||
|
||||
raw = resp.read()
|
||||
if resp.headers.get("Content-Encoding", "").lower() == "gzip":
|
||||
raw = gzip.decompress(raw)
|
||||
body = raw.decode("utf-8")
|
||||
body = resp.read().decode("utf-8")
|
||||
return json.loads(body)
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
@@ -209,7 +198,7 @@ def search(
|
||||
encoded_query = _url_encode(query)
|
||||
|
||||
if subreddit:
|
||||
sub = subreddit.removeprefix("r/").strip()
|
||||
sub = subreddit.lstrip("r/").strip()
|
||||
url = (
|
||||
f"https://www.reddit.com/r/{sub}/search.json"
|
||||
f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
|
||||
|
||||
@@ -4,11 +4,18 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from collections import Counter
|
||||
from datetime import date
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import dates, schema, skill_meta
|
||||
from . import dates, schema
|
||||
|
||||
|
||||
_VERSION_RE = re.compile(
|
||||
r'''^version:\s*(?:"([^"]+)"|'([^']+)'|(\S+))\s*$''',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
@@ -18,12 +25,11 @@ def _skill_version() -> str:
|
||||
Hermes, etc.) do not always carry `.claude-plugin/plugin.json` — that file ships with
|
||||
plugin-cache installs but not with per-harness skill installs. SKILL.md frontmatter is
|
||||
the fallback that keeps the badge from emitting v? on those installs. Returns "?" only
|
||||
if no usable version string is found from either source (missing files, corrupt JSON,
|
||||
or SKILL.md without a version line).
|
||||
if both sources are missing.
|
||||
|
||||
A corrupt manifest at one ancestor does not shadow a valid manifest at a deeper one
|
||||
(continue, not break). SKILL.md parsing accepts double-quoted, single-quoted, or
|
||||
unquoted YAML version scalars (delegated to skill_meta.read_skill_version).
|
||||
(continue, not break). YAML frontmatter accepts double-quoted, single-quoted, or
|
||||
unquoted version scalars.
|
||||
"""
|
||||
here = pathlib.Path(__file__).resolve()
|
||||
for parent in here.parents:
|
||||
@@ -37,11 +43,16 @@ def _skill_version() -> str:
|
||||
return version
|
||||
|
||||
# No usable manifest found at any ancestor — fall back to SKILL.md frontmatter.
|
||||
# First SKILL.md found in the walk is THIS skill's; never traverse past it.
|
||||
for parent in here.parents:
|
||||
skill_md = parent / "SKILL.md"
|
||||
if skill_md.is_file():
|
||||
return skill_meta.read_skill_version(skill_md) or "?"
|
||||
try:
|
||||
match = _VERSION_RE.search(skill_md.read_text())
|
||||
except (OSError, UnicodeDecodeError):
|
||||
break
|
||||
if match:
|
||||
return next(g for g in match.groups() if g is not None)
|
||||
break
|
||||
return "?"
|
||||
|
||||
|
||||
@@ -96,7 +107,7 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
f"# last30days v{_skill_version()}: {report.topic}",
|
||||
f"# last30days v3.0.0: {report.topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Date range: {report.range_from} to {report.range_to}",
|
||||
@@ -602,7 +613,7 @@ def render_comparison_multi(
|
||||
|
||||
lines: list[str] = [
|
||||
*_render_badge(),
|
||||
f"# last30days v{_skill_version()}: {synthesized_topic}",
|
||||
f"# last30days v3.0.0: {synthesized_topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Comparison mode: {len(entities)} entities ({', '.join(entities)})",
|
||||
@@ -790,7 +801,7 @@ def render_full(report: schema.Report) -> str:
|
||||
# Start with the same header as compact
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
lines = [
|
||||
f"# last30days v{_skill_version()}: {report.topic}",
|
||||
f"# last30days v3.0.0: {report.topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Date range: {report.range_from} to {report.range_to}",
|
||||
@@ -1285,16 +1296,15 @@ def _build_source_footer_lines(report: schema.Report) -> list[str]:
|
||||
if total > 0:
|
||||
total_str = f"{total:,}" if total >= 1000 else str(total)
|
||||
parts.append(f"{total_str} {word}")
|
||||
# YouTube: always append "M/N with transcripts" so a zero-transcript run
|
||||
# (typically caused by a stale yt-dlp binary) is visible at the conclusion
|
||||
# surface. Hiding zero converts a problem signal into an absence; the very
|
||||
# case that needs to be loud is the one previously omitted from the footer.
|
||||
# YouTube: append "N with transcripts" instead of a third likes-based column.
|
||||
# Transcripts are a more meaningful research-depth signal than likes.
|
||||
if source_key == "youtube":
|
||||
with_transcripts = sum(
|
||||
1 for it in items
|
||||
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
||||
)
|
||||
parts.append(f"{with_transcripts}/{len(items)} with transcripts")
|
||||
if with_transcripts > 0:
|
||||
parts.append(f"{with_transcripts} with transcripts")
|
||||
stats = " │ ".join(parts)
|
||||
out.append(_footer_line_for_source(emoji, label, len(items), item_word, stats))
|
||||
|
||||
|
||||
@@ -160,93 +160,6 @@ def _extract_github_repos(items: list[dict]) -> list[str]:
|
||||
return repos[:5] # cap at 5 repos
|
||||
|
||||
|
||||
_INTEGRATION_SUFFIX_KEYWORDS: dict[str, set[str]] = {
|
||||
"-action": {"action", "actions", "workflow", "workflows"},
|
||||
"-sdk": {"sdk", "client", "library"},
|
||||
"-plugin": {"plugin", "plugins", "extension", "extensions"},
|
||||
"-plugins": {"plugin", "plugins", "extension", "extensions"},
|
||||
"-docs": {"docs", "documentation"},
|
||||
"-examples": {"example", "examples", "sample", "samples"},
|
||||
"-template": {"template", "templates", "starter", "boilerplate"},
|
||||
}
|
||||
|
||||
|
||||
def _topic_tokens(topic: str) -> set[str]:
|
||||
return set(re.findall(r"[a-z0-9]+", (topic or "").lower()))
|
||||
|
||||
|
||||
def _topic_entity_slugs(topic: str) -> list[str]:
|
||||
entities = re.split(r"\b(?:vs|versus)\b", (topic or "").lower())
|
||||
slugs: list[str] = []
|
||||
for entity in entities:
|
||||
tokens = re.findall(r"[a-z0-9]+", entity)
|
||||
if tokens:
|
||||
slugs.append("-".join(tokens))
|
||||
return slugs
|
||||
|
||||
|
||||
def _repo_slug(repo: str) -> str:
|
||||
parts = repo.split("/", 1)
|
||||
if len(parts) != 2:
|
||||
return ""
|
||||
return parts[1].lower()
|
||||
|
||||
|
||||
def _canonicalize_integration_repo(topic: str, repo: str) -> str:
|
||||
"""Map integration repos back to canonical product repos when intent allows.
|
||||
|
||||
Example:
|
||||
anthropics/claude-code-action -> anthropics/claude-code
|
||||
unless topic explicitly asks for "action"/"workflow".
|
||||
"""
|
||||
parts = repo.split("/", 1)
|
||||
if len(parts) != 2:
|
||||
return repo
|
||||
owner, name = parts[0], parts[1]
|
||||
lower_name = name.lower()
|
||||
topic_words = _topic_tokens(topic)
|
||||
for suffix, intent_words in _INTEGRATION_SUFFIX_KEYWORDS.items():
|
||||
if not lower_name.endswith(suffix):
|
||||
continue
|
||||
if topic_words.intersection(intent_words):
|
||||
return repo
|
||||
base = name[: -len(suffix)]
|
||||
if base:
|
||||
return f"{owner}/{base}"
|
||||
return repo
|
||||
|
||||
|
||||
def canonicalize_github_repos(topic: str, repos: list[str], *, cap: int | None = 5) -> list[str]:
|
||||
"""Normalize/priority-sort GitHub repos for the current topic.
|
||||
|
||||
- Rewrites common integration suffixes to canonical product repos when
|
||||
topic intent does not mention those integrations.
|
||||
- Promotes exact topic slug matches (e.g., `claude-code`) over partials.
|
||||
"""
|
||||
canonicalized: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for repo in repos:
|
||||
candidate = _canonicalize_integration_repo(topic, repo.strip())
|
||||
if "/" not in candidate:
|
||||
continue
|
||||
key = candidate.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
canonicalized.append(candidate)
|
||||
|
||||
topic_slugs = set(_topic_entity_slugs(topic))
|
||||
if topic_slugs:
|
||||
exact = [r for r in canonicalized if _repo_slug(r) in topic_slugs]
|
||||
prefixed = [r for r in canonicalized if any(_repo_slug(r).startswith(f"{slug}-") for slug in topic_slugs) and r not in exact]
|
||||
rest = [r for r in canonicalized if r not in exact and r not in prefixed]
|
||||
canonicalized = exact + prefixed + rest
|
||||
|
||||
if cap is not None:
|
||||
return canonicalized[:cap]
|
||||
return canonicalized
|
||||
|
||||
|
||||
def _build_context_summary(items: list[dict]) -> str:
|
||||
"""Build a 1-2 sentence current events summary from news search results."""
|
||||
snippets: list[str] = []
|
||||
@@ -327,7 +240,7 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
||||
subreddits = _extract_subreddits(results.get("subreddit", []))
|
||||
x_handle = _extract_x_handle(results.get("x_handle", []))
|
||||
github_user = _extract_github_user(results.get("github", []))
|
||||
github_repos = canonicalize_github_repos(topic, _extract_github_repos(results.get("github", [])))
|
||||
github_repos = _extract_github_repos(results.get("github", []))
|
||||
context = _build_context_summary(results.get("news", []))
|
||||
|
||||
subreddits, category = _merge_category_peers(topic, subreddits)
|
||||
|
||||
@@ -107,18 +107,7 @@ def extract_safari_cookies_macos(
|
||||
if sys.platform != "darwin":
|
||||
return None
|
||||
|
||||
cookie_paths = [
|
||||
Path.home()
|
||||
/ "Library"
|
||||
/ "Containers"
|
||||
/ "com.apple.Safari"
|
||||
/ "Data"
|
||||
/ "Library"
|
||||
/ "Cookies"
|
||||
/ "Cookies.binarycookies",
|
||||
Path.home() / "Library" / "Cookies" / "Cookies.binarycookies",
|
||||
]
|
||||
cookie_path = next((path for path in cookie_paths if path.exists()), cookie_paths[0])
|
||||
cookie_path = Path.home() / "Library" / "Cookies" / "Cookies.binarycookies"
|
||||
|
||||
try:
|
||||
raw = cookie_path.read_bytes()
|
||||
|
||||
@@ -335,9 +335,8 @@ def poll_device_auth(
|
||||
"""
|
||||
import sys
|
||||
|
||||
started_at = time.time()
|
||||
deadline = started_at + timeout
|
||||
last_reminder = started_at
|
||||
deadline = time.time() + timeout
|
||||
last_reminder = time.time()
|
||||
reminder_count = 0
|
||||
max_reminders = 4
|
||||
reminder_interval = 30 # seconds between reminders
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
"""SKILL.md metadata helpers — single source of truth for parsing skill frontmatter.
|
||||
|
||||
Centralizes the version regex that previously lived in render.py and was
|
||||
duplicated in tests/test_plugin_contract.py and tests/test_version_consistency.py.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# Matches `version: "x.y.z"`, `version: 'x.y.z'`, or `version: x.y.z` in YAML
|
||||
# frontmatter. Multiline so the pattern can be applied to a full SKILL.md text.
|
||||
# Three alternation groups — exactly one captures per successful match.
|
||||
_VERSION_RE = re.compile(
|
||||
r'''^version:\s*(?:"([^"]+)"|'([^']+)'|(\S+))\s*$''',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def read_skill_version(skill_md_path: Path) -> str | None:
|
||||
"""Return the version string from a SKILL.md's frontmatter, or None.
|
||||
|
||||
Returns None if the file can't be read (missing, permission, decode error)
|
||||
or if no `version:` line is found. Accepts double-quoted, single-quoted,
|
||||
or unquoted YAML version scalars.
|
||||
"""
|
||||
try:
|
||||
text = skill_md_path.read_text()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return None
|
||||
match = _VERSION_RE.search(text)
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1) or match.group(2) or match.group(3)
|
||||
@@ -6,8 +6,6 @@ import threading
|
||||
import random
|
||||
from typing import Optional
|
||||
|
||||
from .render import _skill_version
|
||||
|
||||
# Check if we're in a real terminal (not captured by Claude Code)
|
||||
IS_TTY = sys.stderr.isatty()
|
||||
|
||||
@@ -200,7 +198,7 @@ Just start with "last30" and talk to me like normal.
|
||||
|
||||
# Shorter promo for single missing key
|
||||
PROMO_SINGLE_KEY = {
|
||||
"reddit": "\n💡 Unlock TikTok and Instagram with SCRAPECREATORS_API_KEY - 100 free credits, no CC - scrapecreators.com\n",
|
||||
"reddit": "\n💡 Unlock TikTok and Instagram with SCRAPECREATORS_API_KEY - 10,000 free calls, no CC - scrapecreators.com\n",
|
||||
"x": "\n💡 Unlock X: log into x.com in Firefox or Safari, then re-run. Or add AUTH_TOKEN/CT0 or XAI_API_KEY.\n",
|
||||
"web": "\n💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY.\n",
|
||||
}
|
||||
@@ -511,8 +509,7 @@ def show_diagnostic_banner(diag: dict):
|
||||
|
||||
if IS_TTY:
|
||||
lines.append(f"{Colors.DIM}┌─────────────────────────────────────────────────────┐{Colors.RESET}")
|
||||
_header = f"/last30days v{_skill_version()} - Source Status"
|
||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.BOLD}{_header}{Colors.RESET}{' ' * (52 - len(_header))}{Colors.DIM}│{Colors.RESET}")
|
||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.BOLD}/last30days v3.0.0 - Source Status{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
||||
|
||||
# Reddit
|
||||
@@ -559,8 +556,7 @@ def show_diagnostic_banner(diag: dict):
|
||||
else:
|
||||
# Plain text for non-TTY (Claude Code / Codex)
|
||||
lines.append("┌─────────────────────────────────────────────────────┐")
|
||||
_header_plain = f"/last30days v{_skill_version()} - Source Status"
|
||||
lines.append(f"│ {_header_plain}{' ' * (52 - len(_header_plain))}│")
|
||||
lines.append("│ /last30days v3.0.0 - Source Status │")
|
||||
lines.append("│ │")
|
||||
|
||||
if has_reddit and has_scrapecreators:
|
||||
|
||||
@@ -175,24 +175,16 @@ def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
break
|
||||
|
||||
if not output_text:
|
||||
response_preview = str(response)[:200] if response else "(empty)"
|
||||
raise http.HTTPError(
|
||||
f"xAI API returned empty response (no output text found; response preview: {response_preview})"
|
||||
)
|
||||
return items
|
||||
|
||||
# Extract JSON from the response
|
||||
json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
|
||||
if not json_match:
|
||||
raise http.HTTPError(
|
||||
f"xAI API returned output without valid JSON items structure (output: {output_text[:200]})"
|
||||
)
|
||||
try:
|
||||
data = json.loads(json_match.group())
|
||||
items = data.get("items", [])
|
||||
except json.JSONDecodeError:
|
||||
raise http.HTTPError(
|
||||
f"xAI API returned valid output but invalid JSON structure (output: {output_text[:200]})"
|
||||
)
|
||||
if json_match:
|
||||
try:
|
||||
data = json.loads(json_match.group())
|
||||
items = data.get("items", [])
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Failed to parse xAI response JSON: {output_text[:200]}")
|
||||
|
||||
# Validate and clean items
|
||||
clean_items = []
|
||||
|
||||
@@ -8,9 +8,7 @@ Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI).
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -98,76 +96,10 @@ def _log(msg: str):
|
||||
|
||||
|
||||
def is_ytdlp_installed() -> bool:
|
||||
"""Check if yt-dlp is available locally, or if SSH routing is configured.
|
||||
|
||||
When LAST30DAYS_YOUTUBE_SSH_HOST is set, returns True without a local check —
|
||||
yt-dlp lives on the remote host. Failures surface naturally on first use.
|
||||
"""
|
||||
if _ytdlp_ssh_host():
|
||||
return True
|
||||
"""Check if yt-dlp is available in PATH."""
|
||||
return shutil.which("yt-dlp") is not None
|
||||
|
||||
|
||||
# Host aliases must be plain hostnames / SSH config aliases — no flags, no
|
||||
# shell metacharacters. Rejects any value that could be reinterpreted by ssh
|
||||
# (or the surrounding shell) as something other than a destination.
|
||||
_SSH_HOST_ALIAS_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
|
||||
|
||||
|
||||
def _ytdlp_ssh_host() -> Optional[str]:
|
||||
"""Return SSH host alias if yt-dlp should be routed via SSH, else None.
|
||||
|
||||
Set LAST30DAYS_YOUTUBE_SSH_HOST=<ssh-alias> (e.g. 'macmini') in the environment
|
||||
to route yt-dlp through SSH for residential IP egress. This bypasses
|
||||
YouTube's bot-wall on datacenter IPs (Hetzner, DigitalOcean, AWS, etc.)
|
||||
where ytsearch returns 0 results regardless of cookies.
|
||||
|
||||
The remote host must have yt-dlp installed and reachable via the named
|
||||
SSH alias (configured in ~/.ssh/config). On macOS hosts with Homebrew,
|
||||
add brew shellenv to ~/.zshenv (not just ~/.zprofile) so non-login SSH
|
||||
shells find yt-dlp on PATH.
|
||||
|
||||
Validation: host value must match ``[A-Za-z0-9._-]+``. Anything starting
|
||||
with ``-`` or containing shell/SSH metacharacters is rejected with a
|
||||
stderr warning and treated as unset, so a misconfigured or attacker-
|
||||
controlled value can't slip through as an SSH option flag or proxy command.
|
||||
The ``--`` option terminator in ``_wrap_ytdlp_cmd`` is a second line of
|
||||
defense; this regex closes the door on the env var ever reaching ssh
|
||||
in the first place.
|
||||
|
||||
To use a value from ~/.config/last30days/.env, export it into the
|
||||
environment before invoking the engine, e.g. in a wrapper:
|
||||
set -a; source ~/.config/last30days/.env; set +a
|
||||
python3 last30days.py "..."
|
||||
"""
|
||||
host = os.environ.get("LAST30DAYS_YOUTUBE_SSH_HOST", "").strip()
|
||||
if not host:
|
||||
return None
|
||||
if not _SSH_HOST_ALIAS_RE.match(host):
|
||||
sys.stderr.write(
|
||||
f"[youtube_yt] WARNING: LAST30DAYS_YOUTUBE_SSH_HOST={host!r} "
|
||||
"does not look like a plain hostname/alias; ignoring. "
|
||||
"Expected pattern: letters, digits, dot, underscore, hyphen.\n"
|
||||
)
|
||||
return None
|
||||
return host
|
||||
|
||||
|
||||
def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]:
|
||||
"""Wrap a yt-dlp command list with `ssh <host>` when SSH routing is set.
|
||||
|
||||
Args are shell-quoted to survive the remote shell. Uses BatchMode=yes so
|
||||
a misconfigured key fails fast instead of hanging on a password prompt.
|
||||
The `--` option terminator prevents an SSH option-injection if
|
||||
LAST30DAYS_YOUTUBE_SSH_HOST were ever set to a value starting with `-`.
|
||||
"""
|
||||
host = _ytdlp_ssh_host()
|
||||
if not host:
|
||||
return cmd
|
||||
remote_cmd = " ".join(shlex.quote(a) for a in cmd)
|
||||
return ["ssh", "-o", "BatchMode=yes", "--", host, remote_cmd]
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for YouTube search.
|
||||
|
||||
@@ -291,7 +223,6 @@ def search_youtube(
|
||||
"--no-warnings",
|
||||
"--no-download",
|
||||
]
|
||||
cmd = _wrap_ytdlp_cmd(cmd)
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=120)
|
||||
@@ -385,11 +316,7 @@ def _clean_vtt(vtt_text: str) -> str:
|
||||
_YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
|
||||
|
||||
def _fetch_transcript_direct(
|
||||
video_id: str,
|
||||
timeout: int = 30,
|
||||
status: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[str]:
|
||||
def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]:
|
||||
"""Fetch YouTube transcript via direct HTTP without yt-dlp.
|
||||
|
||||
Scrapes the watch page HTML for the captions track URL in
|
||||
@@ -398,9 +325,6 @@ def _fetch_transcript_direct(
|
||||
Args:
|
||||
video_id: YouTube video ID
|
||||
timeout: HTTP request timeout in seconds
|
||||
status: Optional dict mutated to record per-video signals. Sets
|
||||
``status["no_caption_tracks"] = True`` when the player response
|
||||
confirms the uploader has no caption tracks (vs. fetch failure).
|
||||
|
||||
Returns:
|
||||
Raw VTT text, or None if captions are unavailable.
|
||||
@@ -449,8 +373,6 @@ def _fetch_transcript_direct(
|
||||
|
||||
if not caption_tracks:
|
||||
_log(f"Direct transcript: no caption tracks for {video_id}")
|
||||
if status is not None:
|
||||
status["no_caption_tracks"] = True
|
||||
return None
|
||||
|
||||
# Find English track (prefer exact 'en', then any en variant, then first track)
|
||||
@@ -536,11 +458,7 @@ def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def fetch_transcript(
|
||||
video_id: str,
|
||||
temp_dir: str,
|
||||
status: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[str]:
|
||||
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
||||
"""Fetch auto-generated transcript for a YouTube video.
|
||||
|
||||
Uses yt-dlp when available (preferred, more robust). Falls back to
|
||||
@@ -549,32 +467,19 @@ def fetch_transcript(
|
||||
Args:
|
||||
video_id: YouTube video ID
|
||||
temp_dir: Temporary directory for subtitle files
|
||||
status: Optional dict mutated by the direct-HTTP path to record
|
||||
per-video signals like ``no_caption_tracks``. Used to surface a
|
||||
captions-disabled count so the quality nudge avoids false-positive
|
||||
"stale yt-dlp" flags.
|
||||
|
||||
Returns:
|
||||
Plaintext transcript string, or None if no captions available.
|
||||
"""
|
||||
raw_vtt = None
|
||||
# When SSH-routing is on, the yt-dlp transcript path would write a VTT
|
||||
# file on the remote host that we can't easily read back. Skip it and
|
||||
# use the HTTP transcript fallback (different YouTube endpoint, less
|
||||
# bot-walled, works fine from datacenter IPs).
|
||||
ssh_host = _ytdlp_ssh_host()
|
||||
use_ytdlp = is_ytdlp_installed() and not ssh_host
|
||||
if use_ytdlp:
|
||||
if is_ytdlp_installed():
|
||||
raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
|
||||
if not raw_vtt:
|
||||
_log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback")
|
||||
raw_vtt = _fetch_transcript_direct(video_id, status=status)
|
||||
raw_vtt = _fetch_transcript_direct(video_id)
|
||||
else:
|
||||
if ssh_host:
|
||||
_log("SSH-routing active, using direct HTTP transcript fetch")
|
||||
else:
|
||||
_log("yt-dlp not installed, using direct HTTP transcript fetch")
|
||||
raw_vtt = _fetch_transcript_direct(video_id, status=status)
|
||||
_log("yt-dlp not installed, using direct HTTP transcript fetch")
|
||||
raw_vtt = _fetch_transcript_direct(video_id)
|
||||
|
||||
if not raw_vtt:
|
||||
_log(f"No transcript available for {video_id} (no captions found)")
|
||||
@@ -593,16 +498,12 @@ def fetch_transcript(
|
||||
def fetch_transcripts_parallel(
|
||||
video_ids: List[str],
|
||||
max_workers: int = 5,
|
||||
out_captions_disabled: Optional[Set[str]] = None,
|
||||
) -> Dict[str, Optional[str]]:
|
||||
"""Fetch transcripts for multiple videos in parallel.
|
||||
|
||||
Args:
|
||||
video_ids: List of YouTube video IDs
|
||||
max_workers: Max parallel fetches
|
||||
out_captions_disabled: Optional set mutated to record video_ids whose
|
||||
uploader confirmed no caption tracks (vs. transient fetch failures).
|
||||
Backward-compatible: callers that don't care can omit.
|
||||
|
||||
Returns:
|
||||
Dict mapping video_id to transcript text (or None).
|
||||
@@ -613,11 +514,10 @@ def fetch_transcripts_parallel(
|
||||
_log(f"Fetching transcripts for {len(video_ids)} videos")
|
||||
|
||||
results = {}
|
||||
statuses: Dict[str, Dict[str, Any]] = {vid: {} for vid in video_ids}
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {
|
||||
executor.submit(fetch_transcript, vid, temp_dir, statuses[vid]): vid
|
||||
executor.submit(fetch_transcript, vid, temp_dir): vid
|
||||
for vid in video_ids
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
@@ -631,11 +531,6 @@ def fetch_transcripts_parallel(
|
||||
_log(f"Unexpected transcript error for {vid}: {type(exc).__name__}: {exc}")
|
||||
results[vid] = None
|
||||
|
||||
if out_captions_disabled is not None:
|
||||
for vid, st in statuses.items():
|
||||
if st.get("no_caption_tracks"):
|
||||
out_captions_disabled.add(vid)
|
||||
|
||||
got = sum(1 for v in results.values() if v)
|
||||
errors = sum(1 for v in results.values() if v is None)
|
||||
_log(f"Got transcripts for {got}/{len(video_ids)} videos ({errors} failed)")
|
||||
@@ -686,21 +581,15 @@ def search_and_transcribe(
|
||||
# good chance of reaching the target number of successful transcripts.
|
||||
transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
|
||||
transcripts: Dict[str, Optional[str]] = {}
|
||||
captions_disabled_ids: Set[str] = set()
|
||||
if transcript_limit > 0:
|
||||
attempt_count = min(len(items), transcript_limit * 3)
|
||||
candidate_ids = [item["video_id"] for item in items[:attempt_count]]
|
||||
_log(f"Fetching transcripts for up to {attempt_count} videos (target: {transcript_limit}): {candidate_ids}")
|
||||
transcripts = fetch_transcripts_parallel(
|
||||
candidate_ids, out_captions_disabled=captions_disabled_ids,
|
||||
)
|
||||
transcripts = fetch_transcripts_parallel(candidate_ids)
|
||||
else:
|
||||
_log(f"Transcript limit is 0 for depth={depth}, skipping transcript fetch")
|
||||
|
||||
# Step 3: Attach transcripts and extract highlights. Mark captions_disabled
|
||||
# so quality_nudge can subtract those videos from the degraded-ratio
|
||||
# denominator (uploader-disabled captions can never produce a transcript;
|
||||
# counting them was producing false-positive stale-yt-dlp nudges).
|
||||
# Step 3: Attach transcripts and extract highlights
|
||||
core_topic = _extract_core_subject(topic)
|
||||
for item in items:
|
||||
vid = item["video_id"]
|
||||
@@ -709,7 +598,6 @@ def search_and_transcribe(
|
||||
item["transcript_highlights"] = extract_transcript_highlights(
|
||||
transcript or "", core_topic,
|
||||
)
|
||||
item["captions_disabled"] = vid in captions_disabled_ids
|
||||
|
||||
return {"items": items}
|
||||
|
||||
@@ -978,12 +866,9 @@ def _sc_youtube_search(keyword: str, token: str) -> List[Dict[str, Any]]:
|
||||
List of raw video dicts from the API.
|
||||
"""
|
||||
try:
|
||||
# SC's /v1/youtube/search rejects ?keyword= with HTTP 400; the canonical
|
||||
# parameter for that endpoint is `query`. Other SC endpoints use their
|
||||
# own per-endpoint param names so this was the lone outlier.
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_YT_BASE}/search",
|
||||
params={"query": keyword},
|
||||
params={"keyword": keyword},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Store last30days API keys in the macOS Keychain.
|
||||
#
|
||||
# Keys are stored as generic passwords with service name `last30days-<KEY>`
|
||||
# for the current user. The lib/env.py loader picks them up automatically as
|
||||
# the lowest-priority credential source on Darwin.
|
||||
#
|
||||
# Usage:
|
||||
# ./setup-keychain.sh # interactive: prompts for each key
|
||||
# ./setup-keychain.sh KEY [KEY..] # prompt only for the listed keys
|
||||
# ./setup-keychain.sh --list # list which last30days-* items exist
|
||||
# ./setup-keychain.sh --delete KEY # remove a stored key
|
||||
#
|
||||
# Existing values are shown as "(set)" and skipped unless --replace is passed.
|
||||
# Skip any prompt with empty input.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PREFIX="last30days-"
|
||||
# Mirrors lib/env.py::KEYCHAIN_KEYS — kept in sync via
|
||||
# tests/test_env_keychain.py::test_keychain_keys_match_setup_script.
|
||||
ALL_KEYS=(
|
||||
OPENAI_API_KEY
|
||||
XAI_API_KEY
|
||||
GOOGLE_API_KEY
|
||||
GEMINI_API_KEY
|
||||
GOOGLE_GENAI_API_KEY
|
||||
SCRAPECREATORS_API_KEY
|
||||
APIFY_API_TOKEN
|
||||
AUTH_TOKEN
|
||||
CT0
|
||||
BSKY_HANDLE
|
||||
BSKY_APP_PASSWORD
|
||||
TRUTHSOCIAL_TOKEN
|
||||
BRAVE_API_KEY
|
||||
EXA_API_KEY
|
||||
SERPER_API_KEY
|
||||
OPENROUTER_API_KEY
|
||||
PARALLEL_API_KEY
|
||||
XQUIK_API_KEY
|
||||
XIAOHONGSHU_API_BASE
|
||||
)
|
||||
|
||||
if [[ "${OSTYPE:-}" != darwin* ]]; then
|
||||
echo "setup-keychain.sh requires macOS (security command). Got: $OSTYPE" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v security >/dev/null 2>&1; then
|
||||
echo "security command not found on PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPLACE=0
|
||||
ACTION="prompt"
|
||||
TARGETS=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--list) ACTION="list"; shift ;;
|
||||
--delete) ACTION="delete"; shift ;;
|
||||
--replace) REPLACE=1; shift ;;
|
||||
--help|-h) sed -n '2,/^$/p' "$0" | sed 's/^# //; s/^#//'; exit 0 ;;
|
||||
-*) echo "unknown flag: $1" >&2; exit 2 ;;
|
||||
*) TARGETS+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "$ACTION" in
|
||||
list)
|
||||
echo "Stored last30days-* keychain items:"
|
||||
for key in "${ALL_KEYS[@]}"; do
|
||||
if security find-generic-password -a "$USER" -s "${PREFIX}${key}" -w >/dev/null 2>&1; then
|
||||
echo " $key"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
;;
|
||||
delete)
|
||||
if [[ ${#TARGETS[@]} -eq 0 ]]; then
|
||||
echo "--delete needs at least one KEY name" >&2; exit 2
|
||||
fi
|
||||
for key in "${TARGETS[@]}"; do
|
||||
if security delete-generic-password -a "$USER" -s "${PREFIX}${key}" >/dev/null 2>&1; then
|
||||
echo "deleted: $key"
|
||||
else
|
||||
echo "not found: $key"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ ${#TARGETS[@]} -eq 0 ]]; then
|
||||
TARGETS=("${ALL_KEYS[@]}")
|
||||
fi
|
||||
|
||||
added=0; skipped=0; replaced=0
|
||||
for key in "${TARGETS[@]}"; do
|
||||
existing="$(security find-generic-password -a "$USER" -s "${PREFIX}${key}" -w 2>/dev/null || true)"
|
||||
if [[ -n "$existing" && "$REPLACE" -eq 0 ]]; then
|
||||
printf " %-28s (set, skipping — use --replace to overwrite)\n" "$key"
|
||||
skipped=$((skipped + 1))
|
||||
continue
|
||||
fi
|
||||
printf " %-28s " "$key"
|
||||
IFS= read -rs value
|
||||
echo
|
||||
if [[ -z "$value" ]]; then
|
||||
skipped=$((skipped + 1))
|
||||
continue
|
||||
fi
|
||||
security add-generic-password -U -a "$USER" -s "${PREFIX}${key}" -w "$value"
|
||||
if [[ -n "$existing" ]]; then
|
||||
replaced=$((replaced + 1))
|
||||
else
|
||||
added=$((added + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Done. added=$added replaced=$replaced skipped=$skipped"
|
||||
echo "Verify with: $0 --list"
|
||||
@@ -14,7 +14,7 @@ import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -159,30 +159,7 @@ _UPDATABLE_FINDING_COLUMNS = frozenset({
|
||||
})
|
||||
|
||||
# Future migrations keyed by version number
|
||||
MIGRATIONS: Dict[int, str] = {
|
||||
2: """
|
||||
CREATE TABLE IF NOT EXISTS finding_sightings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
finding_id INTEGER NOT NULL REFERENCES findings(id) ON DELETE CASCADE,
|
||||
run_id INTEGER REFERENCES research_runs(id) ON DELETE CASCADE,
|
||||
topic_id INTEGER REFERENCES topics(id) ON DELETE CASCADE,
|
||||
source TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL,
|
||||
source_title TEXT,
|
||||
engagement_score REAL,
|
||||
relevance_score REAL,
|
||||
seen_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(run_id, finding_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_finding_sightings_run
|
||||
ON finding_sightings(run_id, topic_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_finding_sightings_topic_seen
|
||||
ON finding_sightings(topic_id, seen_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_finding_sightings_url
|
||||
ON finding_sightings(source_url);
|
||||
""",
|
||||
}
|
||||
MIGRATIONS: Dict[int, str] = {}
|
||||
|
||||
|
||||
def _connect(db_path: Optional[Path] = None) -> sqlite3.Connection:
|
||||
@@ -446,7 +423,6 @@ def store_findings(
|
||||
|
||||
new_count = len(insert_rows)
|
||||
updated_count = len(update_rows)
|
||||
_record_sightings(conn, run_id, topic_id, with_urls, existing_by_url)
|
||||
conn.execute(
|
||||
"UPDATE research_runs SET findings_new = ?, findings_updated = ? WHERE id = ?",
|
||||
(new_count, updated_count, run_id),
|
||||
@@ -458,84 +434,6 @@ def store_findings(
|
||||
return {"new": new_count, "updated": updated_count}
|
||||
|
||||
|
||||
def _record_sightings(
|
||||
conn: sqlite3.Connection,
|
||||
run_id: int,
|
||||
topic_id: int,
|
||||
findings_with_urls: List[tuple[str, Dict[str, Any]]],
|
||||
existing_by_url: Optional[Dict[str, sqlite3.Row]] = None,
|
||||
) -> None:
|
||||
"""Record the findings observed during this run.
|
||||
|
||||
The aggregate findings table keeps one row per URL and updates that row on
|
||||
re-sighting. This ledger preserves the run/topic membership needed for
|
||||
watchlist deltas and dossiers.
|
||||
"""
|
||||
if not findings_with_urls:
|
||||
return
|
||||
|
||||
by_url = {url: finding for url, finding in findings_with_urls}
|
||||
rows_by_url = dict(existing_by_url or {})
|
||||
|
||||
missing_urls = [url for url in by_url if url not in rows_by_url]
|
||||
if missing_urls:
|
||||
placeholders = ",".join("?" for _ in missing_urls)
|
||||
rows = conn.execute(
|
||||
f"SELECT id, source_url FROM findings WHERE source_url IN ({placeholders})",
|
||||
missing_urls,
|
||||
).fetchall()
|
||||
rows_by_url.update({row["source_url"]: row for row in rows})
|
||||
|
||||
sighting_rows = []
|
||||
for url, finding in by_url.items():
|
||||
row = rows_by_url.get(url)
|
||||
if row is None:
|
||||
continue
|
||||
sighting_rows.append((
|
||||
row["id"],
|
||||
run_id,
|
||||
topic_id,
|
||||
finding.get("source", "unknown"),
|
||||
url,
|
||||
finding.get("source_title") or finding.get("title", ""),
|
||||
finding.get("engagement_score", 0),
|
||||
finding.get("relevance_score", 0),
|
||||
))
|
||||
|
||||
if not sighting_rows:
|
||||
return
|
||||
|
||||
conn.executemany(
|
||||
"""INSERT INTO finding_sightings
|
||||
(finding_id, run_id, topic_id, source, source_url, source_title,
|
||||
engagement_score, relevance_score)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, finding_id) DO UPDATE SET
|
||||
topic_id = excluded.topic_id,
|
||||
source = excluded.source,
|
||||
source_url = excluded.source_url,
|
||||
source_title = excluded.source_title,
|
||||
engagement_score = excluded.engagement_score,
|
||||
relevance_score = excluded.relevance_score""",
|
||||
sighting_rows,
|
||||
)
|
||||
|
||||
|
||||
def get_sightings_for_run(topic_id: int, run_id: int) -> List[Dict[str, Any]]:
|
||||
"""Return findings observed for a topic during a specific run."""
|
||||
conn = _connect()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"""SELECT * FROM finding_sightings
|
||||
WHERE topic_id = ? AND run_id = ?
|
||||
ORDER BY id""",
|
||||
(topic_id, run_id),
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_new_findings(
|
||||
topic_id: int,
|
||||
since: Optional[str] = None,
|
||||
@@ -621,7 +519,7 @@ def get_daily_cost(date: Optional[str] = None) -> float:
|
||||
conn = _connect()
|
||||
try:
|
||||
if not date:
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
date = datetime.now().strftime("%Y-%m-%d")
|
||||
row = conn.execute(
|
||||
"""SELECT COALESCE(SUM(token_cost), 0) as total
|
||||
FROM research_runs
|
||||
@@ -677,7 +575,7 @@ def get_stats() -> Dict[str, Any]:
|
||||
topic_count = conn.execute("SELECT COUNT(*) FROM topics WHERE enabled = 1").fetchone()[0]
|
||||
finding_count = conn.execute("SELECT COUNT(*) FROM findings").fetchone()[0]
|
||||
|
||||
week_ago = (datetime.now(timezone.utc) - timedelta(days=7)).strftime("%Y-%m-%d")
|
||||
week_ago = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
|
||||
runs_7d = conn.execute(
|
||||
"SELECT COUNT(*) FROM research_runs WHERE run_date >= ?", (week_ago,)
|
||||
).fetchone()[0]
|
||||
@@ -723,7 +621,7 @@ def get_trending(days: int = 7) -> List[Dict[str, Any]]:
|
||||
"""Get topics ranked by recent finding activity."""
|
||||
conn = _connect()
|
||||
try:
|
||||
since = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
since = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
rows = conn.execute(
|
||||
"""SELECT t.name, t.id,
|
||||
COUNT(f.id) as new_findings,
|
||||
@@ -775,31 +673,27 @@ def findings_from_report(
|
||||
limit: Optional[int] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convert report into persisted findings.
|
||||
|
||||
|
||||
Uses ranked candidates (post-rerank) when available for quality scores and explanations.
|
||||
Supplements with raw items from items_by_source for HN/PM that didn't rank highly
|
||||
but are valuable for watchlist persistence. When ranked_candidates is empty
|
||||
(degraded path — rerank failed or was skipped), falls back to supplementing
|
||||
all sources from items_by_source so findings aren't silently dropped.
|
||||
but are valuable for watchlist persistence.
|
||||
"""
|
||||
findings = []
|
||||
seen_urls = set()
|
||||
|
||||
|
||||
# Phase 1: Process ranked candidates (high-quality data with explanations and corroboration)
|
||||
for candidate in report.ranked_candidates:
|
||||
findings.append(finding_from_candidate(candidate))
|
||||
finding = finding_from_candidate(candidate)
|
||||
findings.append(finding)
|
||||
seen_urls.add(candidate.url)
|
||||
|
||||
supplement_sources = (
|
||||
list(report.items_by_source)
|
||||
if not report.ranked_candidates
|
||||
else ["hackernews", "polymarket"]
|
||||
)
|
||||
for source_name in supplement_sources:
|
||||
|
||||
# Phase 2: Add HN/PM items not already captured in ranked candidates
|
||||
for source_name in ["hackernews", "polymarket"]:
|
||||
if source_name not in report.items_by_source:
|
||||
continue
|
||||
for item in report.items_by_source[source_name]:
|
||||
if item.url in seen_urls:
|
||||
continue
|
||||
continue # Already captured with rich data
|
||||
findings.append({
|
||||
"source": source_name,
|
||||
"source_url": item.url,
|
||||
@@ -811,7 +705,8 @@ def findings_from_report(
|
||||
"relevance_score": item.local_relevance or 0.5,
|
||||
})
|
||||
seen_urls.add(item.url)
|
||||
|
||||
|
||||
# Apply global limit after collecting all findings (fix: was per-source, now global)
|
||||
return findings[:limit] if limit is not None else findings
|
||||
|
||||
|
||||
@@ -827,10 +722,9 @@ def _cli_query(args):
|
||||
|
||||
since = None
|
||||
if args.since:
|
||||
# Parse duration like "7d", "30d". Use UTC to match SQLite's
|
||||
# datetime('now') which writes first_seen in UTC.
|
||||
# Parse duration like "7d", "30d"
|
||||
days = int(args.since.rstrip("d"))
|
||||
since = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
since = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
|
||||
findings = get_new_findings(topic["id"], since)
|
||||
print(json.dumps({"topic": topic["name"], "findings": findings, "count": len(findings)}, default=str))
|
||||
|
||||
@@ -6,8 +6,7 @@ set -euo pipefail
|
||||
# using `claude --print` to capture real end-to-end output.
|
||||
|
||||
SKILL_DIR="$HOME/.claude/skills/last30days"
|
||||
REPO_DIR="${REPO_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
|
||||
CLAUDE="${CLAUDE:-$(command -v claude || echo claude)}"
|
||||
REPO_DIR="/Users/mvanhorn/last30days-skill"
|
||||
|
||||
# Safety: always restore V2 SKILL.md on exit/crash
|
||||
cleanup() {
|
||||
@@ -102,7 +101,7 @@ run_version() {
|
||||
|
||||
# Run claude --print with the skill invocation
|
||||
# No timeout — claude --print exits on its own; kill manually if stuck
|
||||
if "$CLAUDE" --print \
|
||||
if /Users/mvanhorn/.local/bin/claude --print \
|
||||
"/last30days $query" \
|
||||
> "$outfile" 2>"$errfile"; then
|
||||
local end_time
|
||||
|
||||
@@ -232,79 +232,5 @@ class TestVendoredBirdRuntime(unittest.TestCase):
|
||||
self.assertEqual(5, items[0]["engagement"]["likes"])
|
||||
|
||||
|
||||
class TestRunBirdSearchJsonDecodeRetry(unittest.TestCase):
|
||||
"""When bird-search returns non-JSON stdout, retry the subprocess.
|
||||
|
||||
Twitter's edge sometimes serves an HTML anti-bot interstitial in place of
|
||||
JSON. Before this fix, that response made json.loads raise JSONDecodeError
|
||||
and the function returned {"items": []} with no diagnostic — silent-empty
|
||||
against an orchestrator that can't distinguish "Twitter blocked us" from
|
||||
"no tweets matched the query."
|
||||
"""
|
||||
|
||||
def _make_result(self, stdout: str, stderr: str = "", returncode: int = 0):
|
||||
from lib.subproc import SubprocResult
|
||||
return SubprocResult(returncode=returncode, stdout=stdout, stderr=stderr)
|
||||
|
||||
def test_retries_subprocess_on_html_interstitial_then_succeeds(self):
|
||||
"""First subprocess attempt returns HTML; second returns JSON → success."""
|
||||
from unittest import mock
|
||||
from lib import bird_x
|
||||
|
||||
html_interstitial = "<!DOCTYPE html><html><body>Rate limited</body></html>"
|
||||
json_success = '[{"id": "1", "text": "tweet"}]'
|
||||
|
||||
results = [
|
||||
(self._make_result(stdout=html_interstitial), None),
|
||||
(self._make_result(stdout=json_success), None),
|
||||
]
|
||||
|
||||
with mock.patch.object(bird_x, "_invoke_bird_subprocess", side_effect=results), \
|
||||
mock.patch.object(bird_x.time, "sleep") as mock_sleep:
|
||||
response = bird_x._run_bird_search("test", count=10, timeout=30)
|
||||
|
||||
self.assertNotIn("error", response)
|
||||
self.assertEqual(response["items"], [{"id": "1", "text": "tweet"}])
|
||||
# Should have slept between the failed first attempt and the retry.
|
||||
mock_sleep.assert_called_once_with(bird_x.JSON_DECODE_RETRY_DELAY)
|
||||
|
||||
def test_returns_error_after_all_retries_exhausted(self):
|
||||
"""All attempts return HTML → error dict with diagnostic + items=[]."""
|
||||
from unittest import mock
|
||||
from lib import bird_x
|
||||
|
||||
html_interstitial = "<!DOCTYPE html><html>blocked</html>"
|
||||
results = [
|
||||
(self._make_result(stdout=html_interstitial), None),
|
||||
(self._make_result(stdout=html_interstitial), None),
|
||||
]
|
||||
|
||||
with mock.patch.object(bird_x, "_invoke_bird_subprocess", side_effect=results), \
|
||||
mock.patch.object(bird_x.time, "sleep"):
|
||||
response = bird_x._run_bird_search("test", count=10, timeout=30)
|
||||
|
||||
self.assertIn("error", response)
|
||||
self.assertIn("Invalid JSON response", response["error"])
|
||||
# Diagnostic message names the anti-bot interstitial so it's
|
||||
# distinguishable from a genuine no-results case in logs.
|
||||
self.assertIn("anti-bot interstitial", response["error"].lower())
|
||||
self.assertEqual(response["items"], [])
|
||||
|
||||
def test_terminal_subprocess_error_is_not_retried(self):
|
||||
"""Subprocess timeout / spawn failure → terminal error, no retry."""
|
||||
from unittest import mock
|
||||
from lib import bird_x
|
||||
|
||||
timeout_error = {"error": "Search timed out after 30s", "items": []}
|
||||
results = [(None, timeout_error)]
|
||||
|
||||
with mock.patch.object(bird_x, "_invoke_bird_subprocess", side_effect=results), \
|
||||
mock.patch.object(bird_x.time, "sleep") as mock_sleep:
|
||||
response = bird_x._run_bird_search("test", count=10, timeout=30)
|
||||
|
||||
self.assertEqual(response, timeout_error)
|
||||
mock_sleep.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Tests for bluesky module."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -212,151 +211,5 @@ class TestSearchBlueskyAuth(unittest.TestCase):
|
||||
self.assertEqual(mock_request.call_args_list[3].kwargs.get("headers", {}), {"Authorization": "Bearer tok-new"})
|
||||
|
||||
|
||||
class TestSearchEndpointHostResolution(unittest.TestCase):
|
||||
"""The default search host moved from `public.api.bsky.app` (the
|
||||
unauthenticated public mirror, now BunnyCDN-blocked for searchPosts) to
|
||||
`api.bsky.app` (the canonical authenticated AppView). BSKY_SEARCH_HOST
|
||||
env var or config value can override the default if Bluesky migrates
|
||||
infrastructure again. Same os.environ-or-config hybrid pattern as
|
||||
LAST30DAYS_STORE.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
# Snapshot env so per-test overrides don't leak
|
||||
self._saved_env = os.environ.pop("BSKY_SEARCH_HOST", None)
|
||||
|
||||
def tearDown(self):
|
||||
if self._saved_env is not None:
|
||||
os.environ["BSKY_SEARCH_HOST"] = self._saved_env
|
||||
else:
|
||||
os.environ.pop("BSKY_SEARCH_HOST", None)
|
||||
|
||||
def test_resolver_default_uses_canonical_appview(self):
|
||||
# Regression guard against the public mirror reappearing as the default.
|
||||
# Anchored at the resolver because that is the code path search_bluesky
|
||||
# actually calls; a module-level constant would not catch a resolver
|
||||
# regression.
|
||||
self.assertIn("api.bsky.app", bluesky._resolve_search_url())
|
||||
|
||||
def test_resolver_default_does_not_use_public_mirror(self):
|
||||
# Hard regression guard — the exact host that BunnyCDN was blocking.
|
||||
# Asserted at the resolver level (the runtime path) so a default-host
|
||||
# regression in _resolve_search_url is actually caught.
|
||||
self.assertNotIn("public.api.bsky.app", bluesky._resolve_search_url())
|
||||
|
||||
def test_resolver_default_when_no_override(self):
|
||||
self.assertEqual(
|
||||
bluesky._resolve_search_url(),
|
||||
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||
)
|
||||
|
||||
def test_resolver_env_var_override(self):
|
||||
os.environ["BSKY_SEARCH_HOST"] = "staging.bsky.app"
|
||||
self.assertEqual(
|
||||
bluesky._resolve_search_url(),
|
||||
"https://staging.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||
)
|
||||
|
||||
def test_resolver_config_dict_override(self):
|
||||
# User has BSKY_SEARCH_HOST only in .env file (project loads .env into
|
||||
# config, not os.environ). Resolver must read both.
|
||||
url = bluesky._resolve_search_url({"BSKY_SEARCH_HOST": "pds.example.com"})
|
||||
self.assertEqual(url, "https://pds.example.com/xrpc/app.bsky.feed.searchPosts")
|
||||
|
||||
def test_resolver_env_var_wins_over_config(self):
|
||||
# When both are set, os.environ takes precedence (matches LAST30DAYS_STORE)
|
||||
os.environ["BSKY_SEARCH_HOST"] = "shell-host.example"
|
||||
url = bluesky._resolve_search_url({"BSKY_SEARCH_HOST": "config-host.example"})
|
||||
self.assertIn("shell-host.example", url)
|
||||
self.assertNotIn("config-host.example", url)
|
||||
|
||||
def test_resolver_output_does_not_use_public_mirror(self):
|
||||
# Regression guard at the resolver level (not just the constant) —
|
||||
# this is what runtime actually calls. The constant-level guard
|
||||
# above doesn't catch a regression where the resolver reverts.
|
||||
self.assertNotIn("public.api.bsky.app", bluesky._resolve_search_url())
|
||||
|
||||
def test_resolver_strips_surrounding_whitespace(self):
|
||||
# Pre-fix: " api.bsky.app " produced "https:// api.bsky.app /xrpc/..."
|
||||
# which urllib raises ValueError on with no hint the env var caused it.
|
||||
os.environ["BSKY_SEARCH_HOST"] = " api.bsky.app "
|
||||
self.assertEqual(
|
||||
bluesky._resolve_search_url(),
|
||||
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||
)
|
||||
|
||||
def test_resolver_rejects_embedded_path(self):
|
||||
# "my-proxy.com/xrpc/prefix" would have doubled the /xrpc/ segment.
|
||||
# We fall back to the default to avoid a guaranteed 404.
|
||||
os.environ["BSKY_SEARCH_HOST"] = "my-proxy.example.com/xrpc/prefix"
|
||||
self.assertEqual(
|
||||
bluesky._resolve_search_url(),
|
||||
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||
)
|
||||
|
||||
def test_resolver_strips_embedded_scheme(self):
|
||||
# Users who paste a full URL get a sane outcome, not a malformed URL.
|
||||
os.environ["BSKY_SEARCH_HOST"] = "https://api.bsky.app"
|
||||
self.assertEqual(
|
||||
bluesky._resolve_search_url(),
|
||||
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||
)
|
||||
|
||||
def test_resolver_empty_string_falls_back_to_default(self):
|
||||
os.environ["BSKY_SEARCH_HOST"] = ""
|
||||
self.assertEqual(
|
||||
bluesky._resolve_search_url(),
|
||||
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||
)
|
||||
|
||||
|
||||
class TestAppPasswordFormat(unittest.TestCase):
|
||||
"""Bluesky app passwords are 19-char xxxx-xxxx-xxxx-xxxx (lowercase
|
||||
alphanumeric, three hyphens at fixed positions). Main-account passwords
|
||||
are accepted by createSession but are bad hygiene. The validator detects
|
||||
the format mismatch without gating any caller.
|
||||
"""
|
||||
|
||||
def test_accepts_valid_app_password_form(self):
|
||||
# Use a fake example — never a real password
|
||||
self.assertTrue(bluesky._validate_app_password_format("wfwp-cq7o-5six-7wy5"))
|
||||
|
||||
def test_rejects_length_15_string(self):
|
||||
# The exact failure mode that triggered the 2026-05-04 investigation:
|
||||
# user stored their main login password (15 chars) in BSKY_APP_PASSWORD
|
||||
self.assertFalse(bluesky._validate_app_password_format("mainpassword123"))
|
||||
|
||||
def test_rejects_16_char_no_hyphen_string(self):
|
||||
# Hex-style API key shape — common confusion with other services
|
||||
self.assertFalse(bluesky._validate_app_password_format("abcdef0123456789"))
|
||||
|
||||
def test_rejects_uppercase_letters(self):
|
||||
# Bluesky app passwords are all-lowercase by spec
|
||||
self.assertFalse(bluesky._validate_app_password_format("WFWP-cq7o-5six-7wy5"))
|
||||
|
||||
def test_rejects_underscore_separator(self):
|
||||
# Wrong separator
|
||||
self.assertFalse(bluesky._validate_app_password_format("wfwp_cq7o_5six_7wy5"))
|
||||
|
||||
def test_rejects_special_chars_in_groups(self):
|
||||
# Special characters are not part of the alphanumeric class
|
||||
self.assertFalse(bluesky._validate_app_password_format("wfwp-cq7o-5six-7wy@"))
|
||||
|
||||
def test_rejects_empty_string(self):
|
||||
self.assertFalse(bluesky._validate_app_password_format(""))
|
||||
|
||||
def test_rejects_none(self):
|
||||
# Callers may pass config.get('BSKY_APP_PASSWORD') which is None when unset
|
||||
self.assertFalse(bluesky._validate_app_password_format(None))
|
||||
|
||||
def test_rejects_integer(self):
|
||||
# Defensive: don't crash if a numeric value sneaks in
|
||||
self.assertFalse(bluesky._validate_app_password_format(123456789012345))
|
||||
|
||||
def test_rejects_list(self):
|
||||
# Defensive: don't crash on iterables
|
||||
self.assertFalse(bluesky._validate_app_password_format(["wfwp", "cq7o", "5six", "7wy5"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -286,7 +286,7 @@ class TestFullExtraction:
|
||||
|
||||
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||
with mock.patch(
|
||||
"scripts.lib.chrome_cookies._get_chromium_encryption_key",
|
||||
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
|
||||
return_value=KNOWN_PASSPHRASE,
|
||||
):
|
||||
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
|
||||
@@ -319,7 +319,7 @@ class TestFullExtraction:
|
||||
|
||||
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||
with mock.patch(
|
||||
"scripts.lib.chrome_cookies._get_chromium_encryption_key",
|
||||
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
|
||||
return_value=KNOWN_PASSPHRASE,
|
||||
):
|
||||
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
||||
|
||||
+4
-94
@@ -1,7 +1,6 @@
|
||||
# ruff: noqa: E402
|
||||
import json
|
||||
import io
|
||||
import shutil
|
||||
import tempfile
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -28,8 +27,8 @@ class CliV3Tests(unittest.TestCase):
|
||||
generated_at="2026-03-16T00:00:00+00:00",
|
||||
provider_runtime=schema.ProviderRuntime(
|
||||
reasoning_provider="gemini",
|
||||
planner_model="gemini-3.1-flash-lite",
|
||||
rerank_model="gemini-3.1-flash-lite",
|
||||
planner_model="gemini-3.1-flash-lite-preview",
|
||||
rerank_model="gemini-3.1-flash-lite-preview",
|
||||
),
|
||||
query_plan=schema.QueryPlan(
|
||||
intent="comparison",
|
||||
@@ -72,26 +71,6 @@ class CliV3Tests(unittest.TestCase):
|
||||
cli.parse_search_flag("web, reddit, hn, web"),
|
||||
)
|
||||
|
||||
def test_parse_search_flag_accepts_optional_social_sources(self):
|
||||
self.assertEqual(
|
||||
["threads", "pinterest"],
|
||||
cli.parse_search_flag("threads, pinterest"),
|
||||
)
|
||||
|
||||
def test_explicit_threads_search_uses_scrapecreators_key_without_include_sources(self):
|
||||
available = cli.pipeline.available_sources(
|
||||
{"SCRAPECREATORS_API_KEY": "test-key", "INCLUDE_SOURCES": ""},
|
||||
requested_sources=["threads"],
|
||||
)
|
||||
self.assertIn("threads", available)
|
||||
|
||||
def test_explicit_perplexity_search_uses_openrouter_key_without_include_sources(self):
|
||||
available = cli.pipeline.available_sources(
|
||||
{"OPENROUTER_API_KEY": "test-key", "INCLUDE_SOURCES": ""},
|
||||
requested_sources=["perplexity"],
|
||||
)
|
||||
self.assertIn("perplexity", available)
|
||||
|
||||
def test_parse_search_flag_rejects_invalid_or_empty_inputs(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
cli.parse_search_flag("unknown")
|
||||
@@ -135,13 +114,13 @@ class CliV3Tests(unittest.TestCase):
|
||||
def test_slugify_and_emit_output_cover_supported_modes(self):
|
||||
report = self.make_report()
|
||||
self.assertEqual("openclaw-vs-nanoclaw", cli.slugify(report.topic))
|
||||
self.assertEqual("last30days CLI.", cli.__doc__)
|
||||
self.assertEqual("last30days v3.0.0 CLI.", cli.__doc__)
|
||||
|
||||
compact = cli.emit_output(report, "compact")
|
||||
json_output = cli.emit_output(report, "json")
|
||||
context = cli.emit_output(report, "context")
|
||||
|
||||
self.assertIn("# last30days v", compact)
|
||||
self.assertIn("# last30days v3.0.0", compact)
|
||||
self.assertIn('"topic": "OpenClaw vs NanoClaw"', json_output)
|
||||
self.assertIsInstance(context, str)
|
||||
|
||||
@@ -164,30 +143,6 @@ class CliV3Tests(unittest.TestCase):
|
||||
_, kwargs = write_text.call_args
|
||||
self.assertEqual("utf-8", kwargs.get("encoding"))
|
||||
|
||||
def test_compute_save_path_display_uses_posix_slashes_under_home(self):
|
||||
# Regression: f"~/{relative}" stringified pathlib.Path with the
|
||||
# OS-native separator, producing "~/Documents\\Last30Days\\..." on
|
||||
# Windows that no shell or File Explorer could open. The fix is
|
||||
# f"~/{relative.as_posix()}" which forces forward slashes regardless
|
||||
# of host OS. On POSIX hosts this asserts the contract for
|
||||
# cross-platform safety; on Windows hosts it would fail without the fix.
|
||||
real_home = Path.home()
|
||||
tmp_under_home = Path(tempfile.mkdtemp(prefix="l30d_save_path_", dir=str(real_home)))
|
||||
try:
|
||||
save_dir = tmp_under_home / "Documents" / "Last30Days"
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
display = cli.compute_save_path_display(
|
||||
str(save_dir), "british airways middle east", "v3", "compact"
|
||||
)
|
||||
self.assertTrue(display.startswith("~/"), f"Expected '~/' prefix, got: {display}")
|
||||
self.assertNotIn("\\", display, f"Backslash leaked into display: {display}")
|
||||
self.assertTrue(
|
||||
display.endswith("british-airways-middle-east-raw-v3.md"),
|
||||
f"Expected slug+suffix at end, got: {display}",
|
||||
)
|
||||
finally:
|
||||
shutil.rmtree(tmp_under_home, ignore_errors=True)
|
||||
|
||||
def test_persist_report_updates_run_status_on_success_and_failure(self):
|
||||
report = self.make_report()
|
||||
|
||||
@@ -260,51 +215,6 @@ class CliV3Tests(unittest.TestCase):
|
||||
fake_progress.show_promo.assert_called_once_with("both", diag=diag)
|
||||
self.assertIn("# rendered", stdout.getvalue())
|
||||
|
||||
def test_main_canonicalizes_explicit_github_repo_flags(self):
|
||||
report = self.make_report()
|
||||
diag = {
|
||||
"available_sources": ["grounding"],
|
||||
"providers": {"google": True, "openai": False, "xai": False},
|
||||
"x_backend": None,
|
||||
"bird_installed": True,
|
||||
"bird_authenticated": False,
|
||||
"bird_username": None,
|
||||
"native_web_backend": "brave",
|
||||
}
|
||||
with mock.patch.object(cli.env, "get_config", return_value={}), \
|
||||
mock.patch.object(cli.pipeline, "diagnose", return_value=diag), \
|
||||
mock.patch.object(cli.pipeline, "run", return_value=report) as run_mock, \
|
||||
mock.patch.object(cli, "emit_output", return_value="# rendered"), \
|
||||
mock.patch.object(sys, "argv", [
|
||||
"last30days.py",
|
||||
"claude",
|
||||
"code",
|
||||
"vs",
|
||||
"codex",
|
||||
"--github-repo",
|
||||
"openai/codex,anthropics/claude-code-action",
|
||||
]):
|
||||
stdout = io.StringIO()
|
||||
stderr = io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
rc = cli.main()
|
||||
self.assertEqual(0, rc)
|
||||
# In vs-mode main + competitors run in parallel via ThreadPoolExecutor,
|
||||
# so the order of pipeline.run invocations is non-deterministic. Find
|
||||
# the main runner's call by predicate on the canonicalized github_repos
|
||||
# rather than by index.
|
||||
expected_repos = ["openai/codex", "anthropics/claude-code"]
|
||||
main_call = next(
|
||||
(c for c in run_mock.call_args_list if c.kwargs.get("github_repos") == expected_repos),
|
||||
None,
|
||||
)
|
||||
self.assertIsNotNone(
|
||||
main_call,
|
||||
f"No pipeline.run call had github_repos={expected_repos}; "
|
||||
f"saw {[c.kwargs.get('github_repos') for c in run_mock.call_args_list]}",
|
||||
)
|
||||
self.assertIn("[GitHub] Canonicalized repos:", stderr.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -114,10 +114,9 @@ class TestGetConfigCookieIntegration:
|
||||
@patch("lib.cookie_extract.extract_cookies")
|
||||
@patch("lib.env._find_project_env", return_value=None)
|
||||
@patch("lib.env.load_env_file", return_value={})
|
||||
@patch("lib.env._load_keychain", return_value={})
|
||||
@patch("lib.env.get_openai_auth")
|
||||
def test_get_config_injects_cookies(
|
||||
self, mock_openai, mock_keychain, mock_load, mock_proj, mock_extract
|
||||
self, mock_openai, mock_load, mock_proj, mock_extract
|
||||
):
|
||||
from lib.env import get_config, OpenAIAuth
|
||||
mock_openai.return_value = OpenAIAuth(
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
"""Tests for macOS Keychain credential source in lib/env.py.
|
||||
|
||||
Covers:
|
||||
- non-Darwin returns {}
|
||||
- missing `security` binary returns {}
|
||||
- successful lookups return parsed key/value pairs
|
||||
- subprocess timeout / OSError are swallowed
|
||||
- get_config merges keychain at lowest priority and labels _CONFIG_SOURCE
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
||||
|
||||
from lib import env # noqa: E402
|
||||
|
||||
SETUP_KEYCHAIN_SH = Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts" / "setup-keychain.sh"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _load_keychain unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_load_keychain_returns_empty_on_non_darwin():
|
||||
with mock.patch("platform.system", return_value="Linux"):
|
||||
assert env._load_keychain(["XAI_API_KEY"]) == {}
|
||||
|
||||
|
||||
def test_load_keychain_returns_empty_when_security_missing():
|
||||
with mock.patch("platform.system", return_value="Darwin"), \
|
||||
mock.patch("shutil.which", return_value=None):
|
||||
assert env._load_keychain(["XAI_API_KEY"]) == {}
|
||||
|
||||
|
||||
def _run_result(returncode: int, stdout: str = "") -> subprocess.CompletedProcess:
|
||||
return subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr="")
|
||||
|
||||
|
||||
def test_load_keychain_loads_present_keys_skips_missing():
|
||||
def fake_run(cmd, **kwargs):
|
||||
service = cmd[cmd.index("-s") + 1]
|
||||
if service == "last30days-XAI_API_KEY":
|
||||
return _run_result(0, "xai-abc\n")
|
||||
if service == "last30days-BRAVE_API_KEY":
|
||||
return _run_result(0, "brv-xyz\n")
|
||||
return _run_result(44) # security's "not found" exit code
|
||||
|
||||
with mock.patch("platform.system", return_value="Darwin"), \
|
||||
mock.patch("shutil.which", return_value="/usr/bin/security"), \
|
||||
mock.patch("subprocess.run", side_effect=fake_run):
|
||||
result = env._load_keychain(["XAI_API_KEY", "BRAVE_API_KEY", "OPENAI_API_KEY"])
|
||||
|
||||
assert result == {"XAI_API_KEY": "xai-abc", "BRAVE_API_KEY": "brv-xyz"}
|
||||
|
||||
|
||||
def test_load_keychain_strips_whitespace_and_newlines():
|
||||
with mock.patch("platform.system", return_value="Darwin"), \
|
||||
mock.patch("shutil.which", return_value="/usr/bin/security"), \
|
||||
mock.patch("subprocess.run", return_value=_run_result(0, " hello-key \n")):
|
||||
result = env._load_keychain(["FOO"])
|
||||
assert result == {"FOO": "hello-key"}
|
||||
|
||||
|
||||
def test_load_keychain_swallows_subprocess_errors():
|
||||
def fake_run(cmd, **kwargs):
|
||||
raise subprocess.TimeoutExpired(cmd=cmd, timeout=5)
|
||||
|
||||
with mock.patch("platform.system", return_value="Darwin"), \
|
||||
mock.patch("shutil.which", return_value="/usr/bin/security"), \
|
||||
mock.patch("subprocess.run", side_effect=fake_run):
|
||||
assert env._load_keychain(["XAI_API_KEY"]) == {}
|
||||
|
||||
|
||||
def test_load_keychain_swallows_oserror():
|
||||
with mock.patch("platform.system", return_value="Darwin"), \
|
||||
mock.patch("shutil.which", return_value="/usr/bin/security"), \
|
||||
mock.patch("subprocess.run", side_effect=OSError("boom")):
|
||||
assert env._load_keychain(["XAI_API_KEY"]) == {}
|
||||
|
||||
|
||||
def test_load_keychain_skips_empty_stdout():
|
||||
with mock.patch("platform.system", return_value="Darwin"), \
|
||||
mock.patch("shutil.which", return_value="/usr/bin/security"), \
|
||||
mock.patch("subprocess.run", return_value=_run_result(0, "")):
|
||||
assert env._load_keychain(["XAI_API_KEY"]) == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_config integration tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_env(monkeypatch, tmp_path):
|
||||
"""Hide every key get_config might touch and point CONFIG_FILE at a
|
||||
non-existent path so no real user config bleeds in."""
|
||||
for var in [
|
||||
"OPENAI_API_KEY", "XAI_API_KEY", "BRAVE_API_KEY", "AUTH_TOKEN", "CT0",
|
||||
"SCRAPECREATORS_API_KEY", "APIFY_API_TOKEN", "BSKY_HANDLE",
|
||||
"BSKY_APP_PASSWORD", "TRUTHSOCIAL_TOKEN", "EXA_API_KEY",
|
||||
"SERPER_API_KEY", "OPENROUTER_API_KEY", "PARALLEL_API_KEY",
|
||||
"XQUIK_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY", "INCLUDE_SOURCES", "FROM_BROWSER",
|
||||
]:
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", tmp_path / "does-not-exist.env")
|
||||
monkeypatch.chdir(tmp_path) # no project .env in this tree either
|
||||
|
||||
|
||||
def test_get_config_reports_keychain_source(clean_env):
|
||||
with mock.patch.object(env, "_load_keychain", return_value={"XAI_API_KEY": "xai-from-kc"}):
|
||||
cfg = env.get_config()
|
||||
assert cfg["_CONFIG_SOURCE"] == "keychain"
|
||||
assert cfg["XAI_API_KEY"] == "xai-from-kc"
|
||||
|
||||
|
||||
def test_get_config_env_var_overrides_keychain(clean_env, monkeypatch):
|
||||
monkeypatch.setenv("XAI_API_KEY", "xai-from-env")
|
||||
with mock.patch.object(env, "_load_keychain", return_value={"XAI_API_KEY": "xai-from-kc"}):
|
||||
cfg = env.get_config()
|
||||
assert cfg["XAI_API_KEY"] == "xai-from-env"
|
||||
|
||||
|
||||
def test_get_config_reports_env_only_when_keychain_empty(clean_env):
|
||||
with mock.patch.object(env, "_load_keychain", return_value={}):
|
||||
cfg = env.get_config()
|
||||
assert cfg["_CONFIG_SOURCE"] == "env_only"
|
||||
|
||||
|
||||
def test_get_config_global_file_outranks_keychain(clean_env, tmp_path, monkeypatch):
|
||||
cfg_file = tmp_path / "global.env"
|
||||
cfg_file.write_text("XAI_API_KEY=xai-from-file\n")
|
||||
monkeypatch.setattr(env, "CONFIG_FILE", cfg_file)
|
||||
with mock.patch.object(env, "_load_keychain", return_value={"XAI_API_KEY": "xai-from-kc"}):
|
||||
cfg = env.get_config()
|
||||
assert cfg["XAI_API_KEY"] == "xai-from-file"
|
||||
assert cfg["_CONFIG_SOURCE"].startswith("global:")
|
||||
|
||||
|
||||
def test_get_config_openai_key_can_come_from_keychain(clean_env):
|
||||
"""OPENAI_API_KEY must be visible to get_openai_auth via the keychain
|
||||
merge — wiring regression test."""
|
||||
with mock.patch.object(env, "_load_keychain", return_value={"OPENAI_API_KEY": "sk-from-kc"}):
|
||||
cfg = env.get_config()
|
||||
assert cfg["OPENAI_API_KEY"] == "sk-from-kc"
|
||||
assert cfg["OPENAI_AUTH_SOURCE"] == "api_key"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Drift guard: lib/env.py KEYCHAIN_KEYS and setup-keychain.sh ALL_KEYS must
|
||||
# stay in lockstep. A mismatch means users storing a key via the helper script
|
||||
# wouldn't see it picked up by the loader, or vice versa.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_all_keys_from_shell(script: Path) -> list[str]:
|
||||
text = script.read_text(encoding="utf-8")
|
||||
match = re.search(r"ALL_KEYS=\(\s*(.*?)\s*\)", text, re.DOTALL)
|
||||
if not match:
|
||||
raise AssertionError(f"ALL_KEYS=( ... ) array not found in {script}")
|
||||
body = match.group(1)
|
||||
# Strip shell comments and split on whitespace
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
return [tok for tok in body.split() if tok]
|
||||
|
||||
|
||||
def test_keychain_keys_match_setup_script():
|
||||
shell_keys = _parse_all_keys_from_shell(SETUP_KEYCHAIN_SH)
|
||||
python_keys = list(env.KEYCHAIN_KEYS)
|
||||
assert shell_keys == python_keys, (
|
||||
"lib/env.py::KEYCHAIN_KEYS and scripts/setup-keychain.sh::ALL_KEYS "
|
||||
f"have drifted.\n python: {python_keys}\n shell: {shell_keys}"
|
||||
)
|
||||
@@ -41,34 +41,6 @@ class EnvV3Tests(unittest.TestCase):
|
||||
with mock.patch.dict(os.environ, {}, clear=False):
|
||||
self.assertIsNone(bird_x.is_bird_authenticated())
|
||||
|
||||
def test_file_permission_check_skips_windows_posix_mode_bits(self):
|
||||
path = mock.Mock(spec=Path)
|
||||
with mock.patch.object(env.os, "name", "nt"), mock.patch.object(env.sys.stderr, "write") as write:
|
||||
env._check_file_permissions(path)
|
||||
|
||||
path.stat.assert_not_called()
|
||||
write.assert_not_called()
|
||||
|
||||
|
||||
class ThreadsAvailabilityTests(unittest.TestCase):
|
||||
"""Threads is in the SC default-on family: same key, same per-call cost
|
||||
shape as TikTok / Instagram, so the same default-on rule applies.
|
||||
Suppression goes through EXCLUDE_SOURCES, not gated opt-in."""
|
||||
|
||||
def test_threads_available_with_sc_key_only(self):
|
||||
self.assertTrue(env.is_threads_available({"SCRAPECREATORS_API_KEY": "k"}))
|
||||
|
||||
def test_threads_unavailable_without_sc_key(self):
|
||||
self.assertFalse(env.is_threads_available({}))
|
||||
self.assertFalse(env.is_threads_available({"INCLUDE_SOURCES": "threads"}))
|
||||
|
||||
def test_threads_does_not_require_include_sources(self):
|
||||
"""Regression guard: INCLUDE_SOURCES should not be needed."""
|
||||
self.assertTrue(env.is_threads_available({
|
||||
"SCRAPECREATORS_API_KEY": "k",
|
||||
"INCLUDE_SOURCES": "",
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -125,7 +125,7 @@ class EvaluatorV3Tests(unittest.TestCase):
|
||||
topic="test topic",
|
||||
query_type="general",
|
||||
items=[{"key": "a"}],
|
||||
judge_model="gemini-3.1-flash-lite",
|
||||
judge_model="gemini-3.1-flash-lite-preview",
|
||||
gemini_api_key="key",
|
||||
)
|
||||
self.assertEqual({"a": 3}, cached)
|
||||
@@ -136,7 +136,7 @@ class EvaluatorV3Tests(unittest.TestCase):
|
||||
topic="test topic",
|
||||
query_type="general",
|
||||
items=[],
|
||||
judge_model="gemini-3.1-flash-lite",
|
||||
judge_model="gemini-3.1-flash-lite-preview",
|
||||
gemini_api_key=None,
|
||||
)
|
||||
self.assertEqual({}, skipped)
|
||||
|
||||
@@ -6,7 +6,6 @@ from __future__ import annotations
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
@@ -28,31 +27,13 @@ class FooterNudgeSuppressionTests(unittest.TestCase):
|
||||
"--emit=md",
|
||||
*argv,
|
||||
]
|
||||
env = {
|
||||
**os.environ,
|
||||
"LAST30DAYS_SKIP_PREFLIGHT": "1",
|
||||
# Skip ~/.config/last30days/.env so a contributor's saved
|
||||
# BRAVE/EXA/SERPER/PARALLEL key doesn't make grounding "available"
|
||||
# and suppress the promo we're checking for.
|
||||
"LAST30DAYS_CONFIG_DIR": "",
|
||||
# Pin X as available so _missing_sources_for_promo selects "web"
|
||||
# (otherwise the "x" promo wins and the BRAVE_API_KEY string never
|
||||
# appears).
|
||||
"XAI_API_KEY": "test-stub",
|
||||
}
|
||||
env = {**os.environ, "LAST30DAYS_SKIP_PREFLIGHT": "1"}
|
||||
# Strip any grounded-web keys the host might have so the promo path
|
||||
# triggers deterministically in mock + no-backend. Also strip X cookie
|
||||
# credentials so XAI_API_KEY is the unambiguous X backend.
|
||||
# triggers deterministically in mock + no-backend.
|
||||
for key in ("BRAVE_API_KEY", "EXA_API_KEY", "SERPER_API_KEY",
|
||||
"PARALLEL_API_KEY", "OPENROUTER_API_KEY",
|
||||
"AUTH_TOKEN", "CT0", "LAST30DAYS_X_BACKEND"):
|
||||
"PARALLEL_API_KEY", "OPENROUTER_API_KEY"):
|
||||
env.pop(key, None)
|
||||
# Run from a tmpdir so _find_project_env() can't walk up into any
|
||||
# .claude/last30days.env above the repo on the contributor's machine.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
return subprocess.run(
|
||||
cmd, capture_output=True, text=True, env=env, cwd=tmp,
|
||||
)
|
||||
return subprocess.run(cmd, capture_output=True, text=True, env=env)
|
||||
|
||||
def test_bare_run_emits_web_promo(self):
|
||||
result = self._run(topic="OpenAI")
|
||||
|
||||
@@ -122,54 +122,6 @@ class ExaSearchTests(unittest.TestCase):
|
||||
self.assertEqual(0, artifact["resultCount"])
|
||||
|
||||
|
||||
class ParallelSearchTests(unittest.TestCase):
|
||||
def test_parallel_search_filters_to_in_range_dated_items(self):
|
||||
mock_response = {
|
||||
"results": [
|
||||
{
|
||||
"title": "Parallel Result",
|
||||
"url": "https://example.com/parallel",
|
||||
"snippet": "A parallel snippet",
|
||||
"publish_date": "2026-03-15T00:00:00Z",
|
||||
},
|
||||
{
|
||||
"title": "Old Parallel Result",
|
||||
"url": "https://example.com/old-parallel",
|
||||
"snippet": "Should be filtered",
|
||||
"publish_date": "2025-12-01T00:00:00Z",
|
||||
},
|
||||
{
|
||||
"title": "Undated Parallel Result",
|
||||
"url": "https://example.com/undated-parallel",
|
||||
"snippet": "Should also be filtered",
|
||||
},
|
||||
]
|
||||
}
|
||||
with patch("lib.grounding.http.request", return_value=mock_response) as mock_req:
|
||||
items, artifact = grounding.parallel_search(
|
||||
"test", ("2026-02-25", "2026-03-27"), "fake-parallel-key"
|
||||
)
|
||||
self.assertEqual(1, len(items))
|
||||
self.assertEqual("Parallel Result", items[0]["title"])
|
||||
self.assertEqual("https://example.com/parallel", items[0]["url"])
|
||||
self.assertEqual("2026-03-15", items[0]["date"])
|
||||
self.assertTrue(items[0]["id"].startswith("WP"))
|
||||
self.assertEqual("parallel", artifact["label"])
|
||||
self.assertEqual(1, artifact["resultCount"])
|
||||
self.assertEqual("POST", mock_req.call_args.args[0])
|
||||
self.assertEqual("https://api.parallel.ai/v1/search", mock_req.call_args.args[1])
|
||||
self.assertEqual(
|
||||
"Bearer fake-parallel-key",
|
||||
mock_req.call_args.kwargs["headers"]["Authorization"],
|
||||
)
|
||||
|
||||
def test_parallel_search_returns_empty_for_no_results(self):
|
||||
with patch("lib.grounding.http.request", return_value={"results": []}):
|
||||
items, artifact = grounding.parallel_search("test", ("2026-02-25", "2026-03-27"), "key")
|
||||
self.assertEqual([], items)
|
||||
self.assertEqual(0, artifact["resultCount"])
|
||||
|
||||
|
||||
class WebSearchDispatchTests(unittest.TestCase):
|
||||
def test_auto_selects_brave_when_key_present(self):
|
||||
config = {"BRAVE_API_KEY": "test-key"}
|
||||
@@ -189,12 +141,6 @@ class WebSearchDispatchTests(unittest.TestCase):
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
|
||||
mock.assert_called_once()
|
||||
|
||||
def test_auto_selects_parallel_when_only_parallel_key(self):
|
||||
config = {"PARALLEL_API_KEY": "test-key"}
|
||||
with patch("lib.grounding.parallel_search", return_value=([], {})) as mock:
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
|
||||
mock.assert_called_once()
|
||||
|
||||
def test_auto_returns_empty_when_no_keys(self):
|
||||
items, artifact = grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="auto")
|
||||
self.assertEqual([], items)
|
||||
@@ -221,14 +167,6 @@ class WebSearchDispatchTests(unittest.TestCase):
|
||||
mock_exa.assert_called_once()
|
||||
mock_serper.assert_not_called()
|
||||
|
||||
def test_auto_prefers_serper_over_parallel(self):
|
||||
config = {"SERPER_API_KEY": "serper-key", "PARALLEL_API_KEY": "parallel-key"}
|
||||
with patch("lib.grounding.serper_search", return_value=([], {})) as mock_serper, \
|
||||
patch("lib.grounding.parallel_search", return_value=([], {})) as mock_parallel:
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
|
||||
mock_serper.assert_called_once()
|
||||
mock_parallel.assert_not_called()
|
||||
|
||||
def test_auto_prefers_brave_when_all_keys_present(self):
|
||||
config = {"BRAVE_API_KEY": "brave-key", "EXA_API_KEY": "exa-key", "SERPER_API_KEY": "serper-key"}
|
||||
with patch("lib.grounding.brave_search", return_value=([], {})) as mock_brave, \
|
||||
@@ -247,97 +185,10 @@ class WebSearchDispatchTests(unittest.TestCase):
|
||||
with self.assertRaises(RuntimeError):
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="brave")
|
||||
|
||||
def test_explicit_parallel_without_key_raises(self):
|
||||
with self.assertRaises(RuntimeError):
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="parallel")
|
||||
|
||||
def test_unsupported_backend_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="google")
|
||||
|
||||
|
||||
class RedditEnrichmentGateTests(unittest.TestCase):
|
||||
"""EXCLUDE_SOURCES=reddit must suppress the web-search Reddit enrichment.
|
||||
|
||||
Otherwise a user who explicitly excluded Reddit would still get Reddit
|
||||
content smuggled back in via web-search URLs that happen to point at
|
||||
reddit.com threads.
|
||||
"""
|
||||
|
||||
def test_reddit_excluded_via_exclude_sources_skips_enrichment(self):
|
||||
config = {"BRAVE_API_KEY": "k", "EXCLUDE_SOURCES": "reddit"}
|
||||
items = [{"url": "https://www.reddit.com/r/python/comments/abc/title/", "snippet": "original"}]
|
||||
with patch("lib.grounding.brave_search", return_value=(items, {})), \
|
||||
patch("lib.grounding._enrich_reddit_items") as enrich_mock:
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
|
||||
enrich_mock.assert_not_called()
|
||||
|
||||
def test_reddit_excluded_case_insensitive(self):
|
||||
for value in ("REDDIT", "Reddit", " reddit ", "x,reddit,y"):
|
||||
config = {"BRAVE_API_KEY": "k", "EXCLUDE_SOURCES": value}
|
||||
self.assertTrue(
|
||||
grounding._reddit_excluded(config),
|
||||
msg=f"_reddit_excluded should be True for EXCLUDE_SOURCES={value!r}",
|
||||
)
|
||||
|
||||
def test_reddit_not_excluded_when_other_sources_listed(self):
|
||||
config = {"EXCLUDE_SOURCES": "tiktok,instagram"}
|
||||
self.assertFalse(grounding._reddit_excluded(config))
|
||||
|
||||
def test_enrichment_runs_when_reddit_not_excluded(self):
|
||||
config = {"BRAVE_API_KEY": "k"}
|
||||
items = [{"url": "https://www.reddit.com/r/python/comments/abc/title/", "snippet": "original"}]
|
||||
with patch("lib.grounding.brave_search", return_value=(items, {})), \
|
||||
patch("lib.grounding._enrich_reddit_items", return_value=items) as enrich_mock:
|
||||
grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
|
||||
enrich_mock.assert_called_once()
|
||||
|
||||
|
||||
class RedditEnrichItemsTests(unittest.TestCase):
|
||||
"""Direct tests for `_enrich_reddit_items` covering the selftext key path
|
||||
and the RedditRateLimitError early-exit behavior.
|
||||
"""
|
||||
|
||||
def test_selftext_under_submission_populates_snippet(self):
|
||||
from lib import reddit_enrich
|
||||
|
||||
item = {
|
||||
"url": "https://www.reddit.com/r/python/comments/abc/title/",
|
||||
"snippet": "original",
|
||||
}
|
||||
parsed = {
|
||||
"submission": {"selftext": "thread body content"},
|
||||
"comments": [],
|
||||
}
|
||||
with patch.object(reddit_enrich, "fetch_thread_data", return_value={"raw": True}), \
|
||||
patch.object(reddit_enrich, "parse_thread_data", return_value=parsed):
|
||||
result = grounding._enrich_reddit_items([item])
|
||||
self.assertEqual("thread body content", result[0]["snippet"])
|
||||
self.assertEqual("reddit_json_api", result[0]["enriched_via"])
|
||||
|
||||
def test_rate_limit_error_halts_iteration(self):
|
||||
from lib import reddit_enrich
|
||||
|
||||
item1 = {"url": "https://www.reddit.com/r/python/comments/aaa/x/"}
|
||||
item2 = {"url": "https://www.reddit.com/r/python/comments/bbb/y/"}
|
||||
|
||||
def fake_fetch(url, *args, **kwargs):
|
||||
raise reddit_enrich.RedditRateLimitError(f"429 for {url}")
|
||||
|
||||
captured_stderr: list[str] = []
|
||||
|
||||
with patch.object(reddit_enrich, "fetch_thread_data", side_effect=fake_fetch) as fetch_mock, \
|
||||
patch("lib.grounding.sys.stderr.write", side_effect=lambda s: captured_stderr.append(s)):
|
||||
grounding._enrich_reddit_items([item1, item2])
|
||||
|
||||
# Only the first item should have triggered a fetch attempt
|
||||
self.assertEqual(1, fetch_mock.call_count)
|
||||
# A stderr message about the rate-limit halt should have been emitted
|
||||
self.assertTrue(
|
||||
any("rate-limited" in msg.lower() or "rate limited" in msg.lower() for msg in captured_stderr),
|
||||
msg=f"Expected a rate-limit stderr message, got: {captured_stderr!r}",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -160,44 +160,12 @@ def test_title_matches_query_empty_query():
|
||||
|
||||
|
||||
def test_title_matches_query_partial_match():
|
||||
"""Any-word matching: at least one query token in title is enough.
|
||||
|
||||
Previously required *all* tokens, which killed every hit on multi-keyword
|
||||
theme queries like 'claude, personal agents, agentic infra' since no real
|
||||
HN title contains all 5 tokens verbatim. Token-overlap relevance at parse
|
||||
time still demotes weak matches, so the loosened gate is safe.
|
||||
"""
|
||||
"""Test that all query words must match."""
|
||||
title = "New AI framework"
|
||||
query = "AI blockchain"
|
||||
|
||||
# "AI" matches as a whole word, even though "blockchain" doesn't appear
|
||||
assert hackernews._title_matches_query(title, query) is True
|
||||
|
||||
|
||||
def test_title_matches_query_no_token_in_title():
|
||||
"""If no query token appears in the title at all, reject."""
|
||||
assert hackernews._title_matches_query("New rust compiler", "AI blockchain") is False
|
||||
|
||||
|
||||
def test_title_matches_query_word_boundary_not_substring():
|
||||
"""Short tokens must match on word boundaries, not as substrings.
|
||||
|
||||
Without word-boundary matching, 'ai' would falsely match 'email',
|
||||
'rail', 'artists', etc.
|
||||
"""
|
||||
# 'ai' as a substring of 'email' must not match
|
||||
assert hackernews._title_matches_query("New email service", "ai blockchain") is False
|
||||
# 'ai' as a whole word does match
|
||||
assert hackernews._title_matches_query("Cool AI tool launched", "ai blockchain") is True
|
||||
|
||||
|
||||
def test_title_matches_query_flattens_hyphens_and_commas():
|
||||
"""Query tokens split on hyphens/commas the same way search_hackernews
|
||||
flattens them, so the post-filter stays aligned with what Algolia saw."""
|
||||
# query 'ts-bun-node' flattens to ['ts', 'bun', 'node']; title contains 'bun'
|
||||
assert hackernews._title_matches_query("Bun 1.2 released", "ts-bun-node") is True
|
||||
# query 'rust, go, zig' flattens; title contains 'go'
|
||||
assert hackernews._title_matches_query("Go 1.24 generics update", "rust, go, zig") is True
|
||||
|
||||
# "blockchain" is not in title, so should fail
|
||||
assert hackernews._title_matches_query(title, query) is False
|
||||
|
||||
|
||||
# === Tests for search_hackernews() ===
|
||||
|
||||
@@ -277,26 +277,6 @@ class HtmlCliIntegrationTests(unittest.TestCase):
|
||||
path = cli.compute_save_path_display("/tmp", report.topic, "v3", "html")
|
||||
self.assertTrue(path.endswith("/ai-agent-frameworks-raw-html-v3.html"))
|
||||
|
||||
def test_save_output_can_persist_comparison_html(self):
|
||||
reports = [
|
||||
("OpenClaw", _report("OpenClaw", ["Containers"])),
|
||||
("Hermes", _report("Hermes", ["Memory"])),
|
||||
]
|
||||
rendered = cli.emit_comparison_output(reports, "html")
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = cli.save_output(
|
||||
reports[0][1],
|
||||
"html",
|
||||
tmpdir,
|
||||
topic_override=cli.comparison_topic(reports),
|
||||
rendered_content=rendered,
|
||||
)
|
||||
self.assertEqual("openclaw-vs-hermes-raw-html.html", path.name)
|
||||
saved = path.read_text(encoding="utf-8")
|
||||
self.assertIn("last30days · OpenClaw vs Hermes", saved)
|
||||
self.assertIn("comparing 2: OpenClaw, Hermes", saved)
|
||||
self.assertNotIn("last30days · OpenClaw</title>", saved)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -104,117 +104,3 @@ class TestParamsEncoding(unittest.TestCase):
|
||||
sent_url = self._sent_url(mock_urlopen)
|
||||
self.assertIn("count=25", sent_url)
|
||||
self.assertIn("raw=True", sent_url)
|
||||
|
||||
|
||||
class TestDNSResolutionRetry(unittest.TestCase):
|
||||
"""DNS resolution failures (gaierror) must retry with exponential backoff.
|
||||
|
||||
Caller-passed `retries` values smaller than MIN_DNS_RETRIES are expanded
|
||||
on the first gaierror so a transient resolution failure doesn't wipe a
|
||||
request just because the caller passed retries=2.
|
||||
"""
|
||||
|
||||
@patch("lib.http.urllib.request.urlopen")
|
||||
@patch("lib.http.time.sleep")
|
||||
def test_gaierror_retries_up_to_min_dns_retries_even_when_caller_passes_fewer(
|
||||
self, mock_sleep, mock_urlopen
|
||||
):
|
||||
"""Caller passed retries=2; gaierror should still get MIN_DNS_RETRIES attempts."""
|
||||
import socket
|
||||
err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
|
||||
mock_urlopen.side_effect = err
|
||||
|
||||
with self.assertRaises(http.HTTPError):
|
||||
http.request("GET", "http://nonexistent.example", retries=2)
|
||||
|
||||
# Caller passed retries=2, but the budget expanded to MIN_DNS_RETRIES=3.
|
||||
self.assertEqual(mock_urlopen.call_count, http.MIN_DNS_RETRIES)
|
||||
|
||||
@patch("lib.http.urllib.request.urlopen")
|
||||
@patch("lib.http.time.sleep")
|
||||
def test_gaierror_succeeds_after_transient_failure(self, mock_sleep, mock_urlopen):
|
||||
"""gaierror on attempt 1, then success — should NOT raise."""
|
||||
import socket
|
||||
success_response = MagicMock()
|
||||
success_response.read.return_value = b'{"ok": true}'
|
||||
success_response.status = 200
|
||||
success_response.__enter__ = lambda self: self
|
||||
success_response.__exit__ = lambda *args: None
|
||||
|
||||
err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
|
||||
mock_urlopen.side_effect = [err, success_response]
|
||||
|
||||
result = http.request("GET", "http://flaky.example", retries=2)
|
||||
|
||||
self.assertEqual(result, {"ok": True})
|
||||
self.assertEqual(mock_urlopen.call_count, 2)
|
||||
|
||||
@patch("lib.http.urllib.request.urlopen")
|
||||
@patch("lib.http.time.sleep")
|
||||
def test_gaierror_uses_exponential_backoff(self, mock_sleep, mock_urlopen):
|
||||
"""Backoff delays for gaierror should be 1s, 2s, 4s — not the linear default."""
|
||||
import socket
|
||||
err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
|
||||
mock_urlopen.side_effect = err
|
||||
|
||||
with self.assertRaises(http.HTTPError):
|
||||
http.request("GET", "http://nonexistent.example", retries=3)
|
||||
|
||||
# Expected sleep calls: 1s (after attempt 1), 2s (after attempt 2).
|
||||
# No sleep after the final attempt (the loop exits to raise).
|
||||
sleep_delays = [call.args[0] for call in mock_sleep.call_args_list]
|
||||
self.assertEqual(sleep_delays, [1, 2])
|
||||
|
||||
@patch("lib.http.urllib.request.urlopen")
|
||||
@patch("lib.http.time.sleep")
|
||||
def test_non_dns_urlerror_uses_linear_backoff_not_dns_branch(
|
||||
self, mock_sleep, mock_urlopen
|
||||
):
|
||||
"""A URLError that's NOT a gaierror must NOT expand the retry budget."""
|
||||
# ConnectionRefusedError-style URLError reason (not gaierror)
|
||||
err = urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
|
||||
mock_urlopen.side_effect = err
|
||||
|
||||
with self.assertRaises(http.HTTPError):
|
||||
http.request("GET", "http://refused.example", retries=2)
|
||||
|
||||
# Caller passed retries=2, and non-DNS URLError doesn't expand it.
|
||||
self.assertEqual(mock_urlopen.call_count, 2)
|
||||
|
||||
@patch("lib.http.urllib.request.urlopen")
|
||||
@patch("lib.http.time.sleep")
|
||||
def test_dns_widening_does_not_leak_into_subsequent_non_dns_urlerror(
|
||||
self, mock_sleep, mock_urlopen
|
||||
):
|
||||
"""Mixed sequence: DNS-then-non-DNS must respect caller's original retries.
|
||||
|
||||
Without the fix, the first gaierror widens effective_retries from 2 to
|
||||
MIN_DNS_RETRIES=3, and a subsequent ConnectionRefused on attempt 1
|
||||
slips into a third overall attempt — exceeding what the caller asked
|
||||
for. Each non-DNS error path must gate on the original `retries`.
|
||||
"""
|
||||
import socket
|
||||
dns_err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
|
||||
conn_err = urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
|
||||
mock_urlopen.side_effect = [dns_err, conn_err, conn_err] # 3rd would only fire if budget leaked
|
||||
|
||||
with self.assertRaises(http.HTTPError):
|
||||
http.request("GET", "http://flaky.example", retries=2)
|
||||
|
||||
# Caller asked for at most 2 attempts. DNS widening must not give us a 3rd.
|
||||
self.assertEqual(mock_urlopen.call_count, 2)
|
||||
|
||||
@patch("lib.http.urllib.request.urlopen")
|
||||
@patch("lib.http.time.sleep")
|
||||
def test_dns_widening_does_not_leak_into_subsequent_oserror(
|
||||
self, mock_sleep, mock_urlopen
|
||||
):
|
||||
"""Mixed sequence: DNS-then-OSError must respect caller's original retries."""
|
||||
import socket
|
||||
dns_err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
|
||||
mock_urlopen.side_effect = [dns_err, TimeoutError("timed out"), TimeoutError("timed out")]
|
||||
|
||||
with self.assertRaises(http.HTTPError):
|
||||
http.request("GET", "http://flaky.example", retries=2)
|
||||
|
||||
self.assertEqual(mock_urlopen.call_count, 2)
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
"""Tests for instagram.py — ScrapeCreators Instagram search module."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Add lib to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "last30days" / "scripts"))
|
||||
@@ -87,172 +85,5 @@ class TestInstagramDepthConfig(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestHashtagFormCollapse(unittest.TestCase):
|
||||
"""Tests for _to_hashtag_form() — the multi-word retry workaround."""
|
||||
|
||||
def test_collapses_spaces(self):
|
||||
self.assertEqual(instagram._to_hashtag_form("toronto real estate"), "torontorealestate")
|
||||
|
||||
def test_lowercases(self):
|
||||
self.assertEqual(instagram._to_hashtag_form("Toronto REAL Estate"), "torontorealestate")
|
||||
|
||||
def test_idempotent_on_single_word(self):
|
||||
self.assertEqual(instagram._to_hashtag_form("ozempic"), "ozempic")
|
||||
|
||||
def test_handles_extra_whitespace(self):
|
||||
self.assertEqual(instagram._to_hashtag_form(" toronto real estate "), "torontorealestate")
|
||||
|
||||
|
||||
class TestSearchRetryOn500(unittest.TestCase):
|
||||
"""Tests for the multi-word -> hashtag retry on SC's flaky 500 path.
|
||||
|
||||
SC's /v2/instagram/reels/search wraps Google Search and is documented
|
||||
to be unreliable on multi-token queries. The retry collapses to a
|
||||
hashtag form which hits the stable hashtag-page lookup path.
|
||||
"""
|
||||
|
||||
def test_multiword_500_triggers_retry_with_hashtag_form(self):
|
||||
"""Multi-word query 500 -> retry with collapsed hashtag form."""
|
||||
from lib import http as http_module
|
||||
first_error = http_module.HTTPError("HTTP 500: Server Error", 500, "")
|
||||
second_payload = {"reels": []}
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.side_effect = [first_error, second_payload]
|
||||
instagram.search_instagram(
|
||||
"toronto real estate", "2026-04-01", "2026-05-04",
|
||||
depth="default", token="fake-token",
|
||||
)
|
||||
self.assertEqual(mock_http_get.call_count, 2)
|
||||
# First call: original multi-word query
|
||||
first_params = mock_http_get.call_args_list[0].kwargs["params"]
|
||||
self.assertEqual(first_params["query"], "toronto real estate")
|
||||
# Second call: collapsed hashtag form
|
||||
second_params = mock_http_get.call_args_list[1].kwargs["params"]
|
||||
self.assertEqual(second_params["query"], "torontorealestate")
|
||||
|
||||
def test_singleword_500_does_not_retry(self):
|
||||
"""Single-word query 500 has no spaces to collapse - no retry."""
|
||||
from lib import http as http_module
|
||||
only_error = http_module.HTTPError("HTTP 500: Server Error", 500, "")
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.side_effect = only_error
|
||||
result = instagram.search_instagram(
|
||||
"ozempic", "2026-04-01", "2026-05-04",
|
||||
depth="default", token="fake-token",
|
||||
)
|
||||
self.assertEqual(mock_http_get.call_count, 1)
|
||||
self.assertIn("error", result)
|
||||
self.assertEqual(result["items"], [])
|
||||
|
||||
def test_first_call_succeeds_no_retry(self):
|
||||
"""200 on first call -> retry path is never entered."""
|
||||
from lib import http as http_module
|
||||
ok_payload = {"reels": []}
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.return_value = ok_payload
|
||||
instagram.search_instagram(
|
||||
"toronto real estate", "2026-04-01", "2026-05-04",
|
||||
depth="default", token="fake-token",
|
||||
)
|
||||
self.assertEqual(mock_http_get.call_count, 1)
|
||||
|
||||
def test_no_token_short_circuits(self):
|
||||
"""No SCRAPECREATORS_API_KEY -> error returned without HTTP call."""
|
||||
from lib import http as http_module
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
result = instagram.search_instagram(
|
||||
"toronto real estate", "2026-04-01", "2026-05-04",
|
||||
depth="default", token=None,
|
||||
)
|
||||
mock_http_get.assert_not_called()
|
||||
self.assertIn("error", result)
|
||||
self.assertIn("SCRAPECREATORS_API_KEY", result["error"])
|
||||
|
||||
|
||||
class TestTranscriptTimeoutConfig(unittest.TestCase):
|
||||
"""Tests for LAST30DAYS_TRANSCRIPT_TIMEOUT configuration.
|
||||
|
||||
SC's /v2/instagram/media/transcript endpoint regularly takes >15s,
|
||||
so the timeout must be configurable. Default is DEFAULT_TRANSCRIPT_TIMEOUT
|
||||
(30s); the env var or per-call kwarg overrides it.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
# Snapshot any pre-existing env so we don't leak across tests
|
||||
self._saved_env = os.environ.pop("LAST30DAYS_TRANSCRIPT_TIMEOUT", None)
|
||||
|
||||
def tearDown(self):
|
||||
os.environ.pop("LAST30DAYS_TRANSCRIPT_TIMEOUT", None)
|
||||
if self._saved_env is not None:
|
||||
os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = self._saved_env
|
||||
|
||||
def _ok_payload(self):
|
||||
return {"transcripts": [{"text": "hello world"}]}
|
||||
|
||||
def _video_item(self, vid="abc123"):
|
||||
return {
|
||||
"video_id": vid,
|
||||
"url": f"https://www.instagram.com/reel/{vid}/",
|
||||
"text": "",
|
||||
}
|
||||
|
||||
def test_default_timeout_is_30s_when_nothing_set(self):
|
||||
"""No env var, no kwarg -> request uses 30s, not the legacy 15s."""
|
||||
from lib import http as http_module
|
||||
items = [self._video_item()]
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.return_value = self._ok_payload()
|
||||
instagram.fetch_captions(items, token="fake-token")
|
||||
kwargs = mock_http_get.call_args.kwargs
|
||||
self.assertEqual(kwargs["timeout"], 30.0)
|
||||
|
||||
def test_env_var_override(self):
|
||||
"""LAST30DAYS_TRANSCRIPT_TIMEOUT='60' -> request uses 60s."""
|
||||
from lib import http as http_module
|
||||
os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = "60"
|
||||
items = [self._video_item()]
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.return_value = self._ok_payload()
|
||||
instagram.fetch_captions(items, token="fake-token")
|
||||
kwargs = mock_http_get.call_args.kwargs
|
||||
self.assertEqual(kwargs["timeout"], 60.0)
|
||||
|
||||
def test_explicit_timeout_kwarg_wins_over_env(self):
|
||||
"""Explicit timeout= kwarg trumps the env var."""
|
||||
from lib import http as http_module
|
||||
os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = "60"
|
||||
items = [self._video_item()]
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.return_value = self._ok_payload()
|
||||
instagram.fetch_captions(items, token="fake-token", timeout=10)
|
||||
kwargs = mock_http_get.call_args.kwargs
|
||||
self.assertEqual(kwargs["timeout"], 10.0)
|
||||
|
||||
def test_config_dict_fallback_when_env_unset(self):
|
||||
"""config={'LAST30DAYS_TRANSCRIPT_TIMEOUT': '45'} -> request uses 45s."""
|
||||
from lib import http as http_module
|
||||
items = [self._video_item()]
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.return_value = self._ok_payload()
|
||||
instagram.fetch_captions(
|
||||
items,
|
||||
token="fake-token",
|
||||
config={"LAST30DAYS_TRANSCRIPT_TIMEOUT": "45"},
|
||||
)
|
||||
kwargs = mock_http_get.call_args.kwargs
|
||||
self.assertEqual(kwargs["timeout"], 45.0)
|
||||
|
||||
def test_invalid_env_value_falls_back_to_default(self):
|
||||
"""Garbage env var doesn't crash; falls back to 30s."""
|
||||
from lib import http as http_module
|
||||
os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = "not-a-number"
|
||||
items = [self._video_item()]
|
||||
with patch.object(http_module, "get") as mock_http_get:
|
||||
mock_http_get.return_value = self._ok_payload()
|
||||
instagram.fetch_captions(items, token="fake-token")
|
||||
kwargs = mock_http_get.call_args.kwargs
|
||||
self.assertEqual(kwargs["timeout"], 30.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
LAST30DAYS_SCRIPT = REPO_ROOT / "skills" / "last30days" / "scripts" / "last30days.py"
|
||||
|
||||
|
||||
def run_last30days(topic: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(LAST30DAYS_SCRIPT), topic, "--mock", "--emit=json"],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
class LastRunStateTests(unittest.TestCase):
|
||||
def test_empty_config_override_disables_last_run_write(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
home = Path(tmp) / "home"
|
||||
env = os.environ.copy()
|
||||
env["HOME"] = str(home)
|
||||
env["LAST30DAYS_CONFIG_DIR"] = ""
|
||||
|
||||
result = run_last30days("synthetic eval query", env)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse((home / ".config" / "last30days" / "last-run.json").exists())
|
||||
|
||||
def test_custom_config_override_writes_last_run_to_custom_dir(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
config_dir = Path(tmp) / "custom-config"
|
||||
env = os.environ.copy()
|
||||
env["HOME"] = str(Path(tmp) / "home")
|
||||
env["LAST30DAYS_CONFIG_DIR"] = str(config_dir)
|
||||
|
||||
result = run_last30days("custom config query", env)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
payload = json.loads((config_dir / "last-run.json").read_text())
|
||||
self.assertEqual(payload["topic"], "custom config query")
|
||||
self.assertGreaterEqual(payload["total"], 0)
|
||||
|
||||
def test_hook_reads_last_run_from_custom_config_dir(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
config_dir = Path(tmp) / "custom-config"
|
||||
config_dir.mkdir()
|
||||
(config_dir / "last-run.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"topic": "custom hook query",
|
||||
"timestamp": "2026-04-30T00:00:00+00:00",
|
||||
"sources": {"reddit": 2},
|
||||
"total": 2,
|
||||
}
|
||||
)
|
||||
)
|
||||
env = os.environ.copy()
|
||||
env["HOME"] = str(Path(tmp) / "home")
|
||||
env["LAST30DAYS_CONFIG_DIR"] = str(config_dir)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", "hooks/scripts/check-config.sh"],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn('Last run: "custom hook query"', result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -52,36 +52,6 @@ class PipelineV3Tests(unittest.TestCase):
|
||||
# At least one per-subquery line.
|
||||
self.assertIn("[Planner] sq1 label=", output)
|
||||
|
||||
def test_parallel_web_backend_enables_grounding_source(self):
|
||||
plan = {
|
||||
"intent": "news",
|
||||
"freshness_mode": "balanced_recent",
|
||||
"cluster_mode": "timeline",
|
||||
"subqueries": [
|
||||
{
|
||||
"label": "primary",
|
||||
"search_query": "test topic",
|
||||
"ranking_query": "What happened with test topic?",
|
||||
"sources": ["grounding"],
|
||||
}
|
||||
],
|
||||
"source_weights": {"grounding": 1.0},
|
||||
}
|
||||
report = pipeline.run(
|
||||
topic="test topic",
|
||||
config={"LAST30DAYS_REASONING_PROVIDER": "auto"},
|
||||
depth="quick",
|
||||
requested_sources=["grounding"],
|
||||
web_backend="parallel",
|
||||
external_plan=plan,
|
||||
)
|
||||
# Anchor on the stable source key, not the exact wording of the
|
||||
# grounding.py error message. Phrasing can shift (e.g., when the
|
||||
# missing-key check moves or the message is reworded) without
|
||||
# changing the contract that the grounding source registers an
|
||||
# error when its required backend key is unset.
|
||||
self.assertIn("grounding", report.errors_by_source)
|
||||
|
||||
|
||||
class TestSourceFetchCap(unittest.TestCase):
|
||||
"""X source fetch count must be capped by MAX_SOURCE_FETCHES."""
|
||||
@@ -934,81 +904,5 @@ class TestZeroKeyPipelineRun(unittest.TestCase):
|
||||
self.assertEqual("fallback-local-score", candidate.explanation)
|
||||
|
||||
|
||||
class TestExcludeSources(unittest.TestCase):
|
||||
"""EXCLUDE_SOURCES env var filters sources out of available_sources().
|
||||
|
||||
The existing INCLUDE_SOURCES allowlist (used by Perplexity opt-in) does
|
||||
not cover this case — tiktok and instagram are added unconditionally
|
||||
when SCRAPECREATORS_API_KEY is set, with no way to opt out short of
|
||||
unsetting the key. EXCLUDE_SOURCES gives runs a per-invocation denylist.
|
||||
"""
|
||||
|
||||
def test_excludes_tiktok_and_instagram(self):
|
||||
config = {
|
||||
"SCRAPECREATORS_API_KEY": "test-key",
|
||||
"EXCLUDE_SOURCES": "tiktok,instagram",
|
||||
}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertNotIn("tiktok", sources)
|
||||
self.assertNotIn("instagram", sources)
|
||||
self.assertIn("reddit", sources)
|
||||
self.assertIn("hackernews", sources)
|
||||
|
||||
def test_no_exclusion_when_unset(self):
|
||||
config = {"SCRAPECREATORS_API_KEY": "test-key"}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertIn("tiktok", sources)
|
||||
self.assertIn("instagram", sources)
|
||||
|
||||
def test_empty_exclude_sources_is_noop(self):
|
||||
config = {
|
||||
"SCRAPECREATORS_API_KEY": "test-key",
|
||||
"EXCLUDE_SOURCES": "",
|
||||
}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertIn("tiktok", sources)
|
||||
self.assertIn("instagram", sources)
|
||||
|
||||
def test_whitespace_and_case_insensitive(self):
|
||||
config = {
|
||||
"SCRAPECREATORS_API_KEY": "test-key",
|
||||
"EXCLUDE_SOURCES": " TikTok , INSTAGRAM ",
|
||||
}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertNotIn("tiktok", sources)
|
||||
self.assertNotIn("instagram", sources)
|
||||
|
||||
def test_excludes_non_scrapecreators_source(self):
|
||||
"""EXCLUDE_SOURCES applies to any source, not just SC-backed ones."""
|
||||
config = {"EXCLUDE_SOURCES": "hackernews"}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertNotIn("hackernews", sources)
|
||||
self.assertIn("reddit", sources)
|
||||
|
||||
|
||||
class TestExcludeSourcesEndToEnd(unittest.TestCase):
|
||||
"""Wiring regression: EXCLUDE_SOURCES from the process environment must
|
||||
reach available_sources() via env.get_config(). The unit tests above
|
||||
construct config dicts directly; this one exercises the env-to-config
|
||||
path so a missing entry in env.py's keys list is caught immediately."""
|
||||
|
||||
def test_exclude_sources_from_env_propagates_through_get_config(self):
|
||||
import os
|
||||
from unittest.mock import patch as _patch
|
||||
from lib import env as env_mod
|
||||
from importlib import reload
|
||||
with _patch.dict(os.environ, {
|
||||
"LAST30DAYS_CONFIG_DIR": "",
|
||||
"EXCLUDE_SOURCES": "tiktok,instagram",
|
||||
"SCRAPECREATORS_API_KEY": "fake",
|
||||
}, clear=False):
|
||||
reload(env_mod)
|
||||
cfg = env_mod.get_config()
|
||||
self.assertEqual(cfg.get("EXCLUDE_SOURCES"), "tiktok,instagram")
|
||||
sources = pipeline.available_sources(cfg)
|
||||
self.assertNotIn("tiktok", sources)
|
||||
self.assertNotIn("instagram", sources)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import json
|
||||
import sys
|
||||
import re
|
||||
import tomllib
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -8,19 +8,17 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_ROOT = ROOT / "skills" / "last30days"
|
||||
|
||||
sys.path.insert(0, str(SKILL_ROOT / "scripts"))
|
||||
from lib.skill_meta import read_skill_version # noqa: E402
|
||||
|
||||
|
||||
def _json(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
version = read_skill_version(SKILL_ROOT / "SKILL.md")
|
||||
if not version:
|
||||
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
|
||||
if not match:
|
||||
raise AssertionError("SKILL.md version frontmatter not found")
|
||||
return version
|
||||
return match.group(1)
|
||||
|
||||
|
||||
class TestPluginContract(unittest.TestCase):
|
||||
@@ -36,7 +34,6 @@ class TestPluginContract(unittest.TestCase):
|
||||
|
||||
self.assertEqual(version, _skill_version())
|
||||
self.assertEqual(version, _json(ROOT / ".claude-plugin" / "plugin.json")["version"])
|
||||
self.assertEqual(version, _json(ROOT / "gemini-extension.json")["version"])
|
||||
|
||||
marketplace = _json(ROOT / ".claude-plugin" / "marketplace.json")
|
||||
plugins = marketplace.get("plugins") or []
|
||||
@@ -52,11 +49,22 @@ class TestPluginContract(unittest.TestCase):
|
||||
self.assertIn("description", marketplace["metadata"])
|
||||
|
||||
def test_workflows_do_not_reference_removed_root_scripts_dir(self) -> None:
|
||||
# The root-level scripts/ directory was removed; workflows must not
|
||||
# reference it. Subdirectory scripts/ paths (skills/last30days/scripts/
|
||||
# for the Code-skill build, mcp/scripts/ for the .mcpb build) are
|
||||
# the legitimate replacements.
|
||||
allowed_prefixes = (
|
||||
"skills/last30days/scripts/",
|
||||
"mcp/scripts/",
|
||||
)
|
||||
offenders = []
|
||||
for path in sorted((ROOT / ".github" / "workflows").glob("*.yml")):
|
||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
if "scripts/" in line and "skills/last30days/scripts/" not in line:
|
||||
offenders.append(f"{path.relative_to(ROOT)}:{line_number}: {line.strip()}")
|
||||
if "scripts/" not in line:
|
||||
continue
|
||||
if any(prefix in line for prefix in allowed_prefixes):
|
||||
continue
|
||||
offenders.append(f"{path.relative_to(ROOT)}:{line_number}: {line.strip()}")
|
||||
|
||||
self.assertEqual([], offenders)
|
||||
|
||||
|
||||
+2
-380
@@ -5,11 +5,6 @@ HN, Polymarket, Reddit (always active), X, YouTube.
|
||||
ScrapeCreators adds TikTok + Instagram as bonus sources, not core.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -43,8 +38,8 @@ def _base_results(**overrides):
|
||||
|
||||
def _compute(config_overrides=None, result_overrides=None, ytdlp_installed=False):
|
||||
"""Helper to call compute_quality_score with mocked yt-dlp check."""
|
||||
from lib.quality_nudge import compute_quality_score
|
||||
from lib import youtube_yt
|
||||
from scripts.lib.quality_nudge import compute_quality_score
|
||||
from scripts.lib import youtube_yt
|
||||
|
||||
config = _base_config(**(config_overrides or {}))
|
||||
results = _base_results(**(result_overrides or {}))
|
||||
@@ -204,376 +199,3 @@ class TestRedditNeverInCoreErrored:
|
||||
# Reddit is always-active in core (public path), error doesn't demote it
|
||||
assert "reddit" in q["core_active"]
|
||||
assert q["score_pct"] == 100
|
||||
|
||||
|
||||
class TestYouTubeDegraded:
|
||||
"""YouTube is `degraded` when videos returned but transcripts below threshold.
|
||||
|
||||
Canonical failure mode: a stale yt-dlp binary still finds videos via search
|
||||
but silently fails every transcript fetch because YouTube's caption format
|
||||
has moved on. Pre-fix the user got no signal of this; the footer hid zero,
|
||||
and quality_nudge only checked top-level errors.
|
||||
"""
|
||||
|
||||
def test_zero_of_six_transcripts_flags_degraded(self):
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 6,
|
||||
"youtube_transcripts_count": 0,
|
||||
},
|
||||
)
|
||||
assert "youtube" in q["core_degraded"]
|
||||
assert q["nudge_text"] is not None
|
||||
# Counts surface in the message so the user sees the actual ratio
|
||||
assert "6 videos" in q["nudge_text"]
|
||||
assert "0 transcripts" in q["nudge_text"]
|
||||
assert "stale yt-dlp" in q["nudge_text"].lower()
|
||||
# Updates path mentions all three common package managers
|
||||
assert "scoop" in q["nudge_text"].lower()
|
||||
assert "brew" in q["nudge_text"].lower()
|
||||
assert "pip install" in q["nudge_text"].lower()
|
||||
|
||||
def test_five_of_six_transcripts_does_not_flag_degraded(self):
|
||||
# 83% transcript success - well above the 50% threshold
|
||||
# X is also enabled so all 5 cores are active and no nudge should fire
|
||||
q = _compute(
|
||||
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 6,
|
||||
"youtube_transcripts_count": 5,
|
||||
},
|
||||
)
|
||||
assert "youtube" not in q["core_degraded"]
|
||||
assert q["nudge_text"] is None # All 5 core sources active, no degradation
|
||||
|
||||
def test_zero_videos_does_not_flag_degraded(self):
|
||||
# No videos returned -> degraded check is meaningless and must not fire
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 0,
|
||||
"youtube_transcripts_count": 0,
|
||||
},
|
||||
)
|
||||
assert "youtube" not in q["core_degraded"]
|
||||
|
||||
def test_one_of_three_transcripts_flags_degraded(self):
|
||||
# 33% - below 50% threshold; the canonical "yt-dlp partially working" case
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 3,
|
||||
"youtube_transcripts_count": 1,
|
||||
},
|
||||
)
|
||||
assert "youtube" in q["core_degraded"]
|
||||
assert "Degraded: YouTube" in q["nudge_text"]
|
||||
|
||||
def test_threshold_tunable_via_config(self):
|
||||
# Operator overrides threshold via env-style config to be more permissive
|
||||
q = _compute(
|
||||
config_overrides={"DEGRADED_TRANSCRIPT_THRESHOLD": "0.1"},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 10,
|
||||
"youtube_transcripts_count": 2, # 20%, below default 50% but above override 10%
|
||||
},
|
||||
)
|
||||
assert "youtube" not in q["core_degraded"]
|
||||
|
||||
def test_degraded_does_not_affect_score(self):
|
||||
# Degradation is informational, not score-affecting; YouTube still counts as active
|
||||
q = _compute(
|
||||
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 6,
|
||||
"youtube_transcripts_count": 0,
|
||||
},
|
||||
)
|
||||
assert "youtube" in q["core_active"]
|
||||
assert q["score_pct"] == 100 # Full active count regardless of degradation
|
||||
# But nudge still fires
|
||||
assert q["nudge_text"] is not None
|
||||
assert "Degraded: YouTube" in q["nudge_text"]
|
||||
|
||||
|
||||
class TestYouTubeCaptionsDisabledDoesNotFalseFlag:
|
||||
"""Captions-disabled videos must not lower the transcript-fetch ratio.
|
||||
|
||||
A video where the uploader disabled captions can never produce a transcript,
|
||||
no matter how fresh yt-dlp is. Counting it in the denominator of the
|
||||
degraded-ratio check produces false positives - one captions-disabled video
|
||||
in a small result set was triggering a "stale yt-dlp binary" nudge that was
|
||||
wrong. Fix: subtract captions_disabled from the denominator.
|
||||
"""
|
||||
|
||||
def test_zero_captions_disabled_preserves_existing_behavior(self):
|
||||
# Pre-existing case: 0 of 6 transcripts is still degraded (no captions
|
||||
# disabled to discount). Behavior is unchanged from TestYouTubeDegraded.
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 6,
|
||||
"youtube_transcripts_count": 0,
|
||||
"youtube_captions_disabled_count": 0,
|
||||
},
|
||||
)
|
||||
assert "youtube" in q["core_degraded"]
|
||||
|
||||
def test_all_videos_captions_disabled_does_not_flag(self):
|
||||
# Every returned video had captions disabled by the uploader.
|
||||
# That's not a yt-dlp problem - it's an upstream content fact. Must not
|
||||
# flag degraded.
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 3,
|
||||
"youtube_transcripts_count": 0,
|
||||
"youtube_captions_disabled_count": 3,
|
||||
},
|
||||
)
|
||||
assert "youtube" not in q["core_degraded"]
|
||||
|
||||
def test_mixed_uses_corrected_denominator(self):
|
||||
# 6 videos, 3 captions_disabled, 2 transcripts.
|
||||
# Naive (buggy) ratio: 2/6 = 33% (would flag).
|
||||
# Corrected ratio: 2/(6-3) = 67% (does NOT flag).
|
||||
# This case demonstrates the fix changes the verdict.
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 6,
|
||||
"youtube_transcripts_count": 2,
|
||||
"youtube_captions_disabled_count": 3,
|
||||
},
|
||||
)
|
||||
assert "youtube" not in q["core_degraded"]
|
||||
|
||||
def test_mixed_still_flags_when_truly_degraded(self):
|
||||
# Even after discounting captions-disabled, the ratio is still bad.
|
||||
# 8 videos, 1 captions_disabled, 1 transcript -> 1/(8-1) = 14% (flags).
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 8,
|
||||
"youtube_transcripts_count": 1,
|
||||
"youtube_captions_disabled_count": 1,
|
||||
},
|
||||
)
|
||||
assert "youtube" in q["core_degraded"]
|
||||
# Nudge should still mention the stale yt-dlp possibility but also
|
||||
# acknowledge that captions-disabled is a separate cause.
|
||||
assert q["nudge_text"] is not None
|
||||
assert "captions disabled" in q["nudge_text"].lower()
|
||||
|
||||
def test_missing_count_defaults_to_zero(self):
|
||||
# Older callers that don't pass the new key still work (default 0).
|
||||
q = _compute(
|
||||
ytdlp_installed=True,
|
||||
result_overrides={
|
||||
"youtube_videos_count": 6,
|
||||
"youtube_transcripts_count": 0,
|
||||
# youtube_captions_disabled_count intentionally omitted
|
||||
},
|
||||
)
|
||||
assert "youtube" in q["core_degraded"]
|
||||
|
||||
|
||||
class TestInstagramSilentFailure:
|
||||
"""Instagram is a `bonus` source via SC. Silent-failure detection: if SC
|
||||
is configured but the source returned zero items, surface a nudge so the
|
||||
user understands why the brief lacks an Instagram section.
|
||||
|
||||
Pre-fix the user got no signal - SC's /v2/instagram/reels/search 500s
|
||||
frequently on multi-token queries and the pipeline silently returned
|
||||
empty without any indication.
|
||||
"""
|
||||
|
||||
def test_zero_items_with_sc_flags_bonus_errored(self):
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" in q["bonus_errored"]
|
||||
assert q["nudge_text"] is not None
|
||||
assert "Instagram" in q["nudge_text"]
|
||||
|
||||
def test_zero_items_without_sc_does_not_flag(self):
|
||||
q = _compute(
|
||||
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" not in q.get("bonus_errored", [])
|
||||
|
||||
def test_nonzero_items_does_not_flag(self):
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 5},
|
||||
)
|
||||
assert "instagram" not in q["bonus_errored"]
|
||||
assert q["nudge_text"] is None
|
||||
|
||||
def test_missing_key_means_source_did_not_run(self):
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
)
|
||||
assert "instagram" not in q["bonus_errored"]
|
||||
assert q["nudge_text"] is None
|
||||
|
||||
def test_nudge_text_explains_workaround(self):
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert q["nudge_text"] is not None
|
||||
text_lower = q["nudge_text"].lower()
|
||||
assert "instagram" in text_lower
|
||||
assert ("0 reels" in text_lower or "silent" in text_lower
|
||||
or "hashtag" in text_lower)
|
||||
|
||||
def test_bonus_errored_does_not_affect_core_score(self):
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert q["score_pct"] == 100
|
||||
assert "instagram" in q["bonus_errored"]
|
||||
assert q["nudge_text"] is not None
|
||||
assert "Bonus source silent" in q["nudge_text"]
|
||||
|
||||
def test_bonus_errored_field_always_present(self):
|
||||
q = _compute()
|
||||
assert q.get("bonus_errored") == []
|
||||
|
||||
def test_exclude_sources_instagram_suppresses_silent_failure(self):
|
||||
"""User set EXCLUDE_SOURCES=instagram - the source intentionally did
|
||||
not run, so the zero-count instagram_items_count written by
|
||||
last30days.py is a non-event, not a silent failure. Pre-fix: the
|
||||
nudge fired anyway because the gate only checked SC-key + count.
|
||||
"""
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
"EXCLUDE_SOURCES": "instagram",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" not in q["bonus_errored"]
|
||||
assert q["nudge_text"] is None
|
||||
|
||||
def test_exclude_sources_multi_value_with_instagram(self):
|
||||
"""Canonical parsing pattern is comma-separated; case-insensitive."""
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
"EXCLUDE_SOURCES": "threads, Instagram , pinterest",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" not in q["bonus_errored"]
|
||||
|
||||
def test_exclude_sources_other_value_still_flags(self):
|
||||
"""EXCLUDE_SOURCES that does not mention instagram must not suppress
|
||||
the silent-failure nudge for instagram.
|
||||
"""
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
"EXCLUDE_SOURCES": "threads",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" in q["bonus_errored"]
|
||||
|
||||
def test_include_sources_without_instagram_suppresses_silent_failure(self):
|
||||
"""User set INCLUDE_SOURCES to an opt-in allowlist that omits
|
||||
instagram — the pipeline skips the source by allowlist filter, so
|
||||
the zero-count instagram_items_count is intentional, not a silent
|
||||
failure. Symmetric to the EXCLUDE_SOURCES=instagram guard.
|
||||
"""
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
"INCLUDE_SOURCES": "reddit,hn,x,youtube",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" not in q["bonus_errored"]
|
||||
assert q["nudge_text"] is None
|
||||
|
||||
def test_include_sources_multi_value_without_instagram(self):
|
||||
"""Canonical parsing pattern is comma-separated; case-insensitive."""
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
"INCLUDE_SOURCES": " Reddit, HN , YouTube ",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" not in q["bonus_errored"]
|
||||
|
||||
def test_include_sources_with_instagram_still_flags(self):
|
||||
"""INCLUDE_SOURCES that explicitly names instagram must not suppress
|
||||
the silent-failure nudge — the source was opted in, so a zero count
|
||||
is a real silent failure.
|
||||
"""
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
"INCLUDE_SOURCES": "reddit,instagram",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" in q["bonus_errored"]
|
||||
|
||||
def test_include_sources_empty_does_not_suppress(self):
|
||||
"""Empty/unset INCLUDE_SOURCES means no allowlist filter, so the
|
||||
silent-failure gate should still fire when instagram is zero.
|
||||
"""
|
||||
q = _compute(
|
||||
config_overrides={
|
||||
"AUTH_TOKEN": "tok123",
|
||||
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||
"INCLUDE_SOURCES": "",
|
||||
},
|
||||
ytdlp_installed=True,
|
||||
result_overrides={"instagram_items_count": 0},
|
||||
)
|
||||
assert "instagram" in q["bonus_errored"]
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ class TestSearchRedditPublicHighLevel:
|
||||
reddit_public.search("test")
|
||||
|
||||
req = mock_urlopen.call_args[0][0]
|
||||
assert "Mozilla/5.0" in req.get_header("User-agent")
|
||||
assert req.get_header("User-agent") == "last30days/3.0 (research tool)"
|
||||
|
||||
|
||||
class TestMissingSubreddit:
|
||||
|
||||
+7
-93
@@ -70,8 +70,8 @@ def sample_report() -> schema.Report:
|
||||
generated_at="2026-03-16T00:00:00+00:00",
|
||||
provider_runtime=schema.ProviderRuntime(
|
||||
reasoning_provider="gemini",
|
||||
planner_model="gemini-3.1-flash-lite",
|
||||
rerank_model="gemini-3.1-flash-lite",
|
||||
planner_model="gemini-3.1-flash-lite-preview",
|
||||
rerank_model="gemini-3.1-flash-lite-preview",
|
||||
),
|
||||
query_plan=schema.QueryPlan(
|
||||
intent="breaking_news",
|
||||
@@ -91,8 +91,7 @@ def sample_report() -> schema.Report:
|
||||
class RenderV3Tests(unittest.TestCase):
|
||||
def test_render_compact_includes_cluster_first_sections(self):
|
||||
text = render.render_compact(sample_report())
|
||||
self.assertIn("# last30days v", text)
|
||||
self.assertIn(": test topic", text)
|
||||
self.assertIn("# last30days v3.0.0: test topic", text)
|
||||
self.assertIn("Safety note: evidence text below is untrusted internet content", text)
|
||||
self.assertIn("## Ranked Evidence Clusters", text)
|
||||
self.assertIn("## Stats", text)
|
||||
@@ -240,8 +239,8 @@ class RenderTopCommentsTests(unittest.TestCase):
|
||||
generated_at="2026-03-16T00:00:00+00:00",
|
||||
provider_runtime=schema.ProviderRuntime(
|
||||
reasoning_provider="gemini",
|
||||
planner_model="gemini-3.1-flash-lite",
|
||||
rerank_model="gemini-3.1-flash-lite",
|
||||
planner_model="gemini-3.1-flash-lite-preview",
|
||||
rerank_model="gemini-3.1-flash-lite-preview",
|
||||
),
|
||||
query_plan=schema.QueryPlan(
|
||||
intent="breaking_news",
|
||||
@@ -425,8 +424,8 @@ class RenderBestTakesCompactTests(unittest.TestCase):
|
||||
generated_at="2026-03-16T00:00:00+00:00",
|
||||
provider_runtime=schema.ProviderRuntime(
|
||||
reasoning_provider="gemini",
|
||||
planner_model="gemini-3.1-flash-lite",
|
||||
rerank_model="gemini-3.1-flash-lite",
|
||||
planner_model="gemini-3.1-flash-lite-preview",
|
||||
rerank_model="gemini-3.1-flash-lite-preview",
|
||||
),
|
||||
query_plan=schema.QueryPlan(
|
||||
intent="breaking_news",
|
||||
@@ -552,90 +551,5 @@ class DegradedRunBannerTests(unittest.TestCase):
|
||||
self.assertIn("--plan", text)
|
||||
|
||||
|
||||
class YoutubeFooterTranscriptRatioTests(unittest.TestCase):
|
||||
"""The YouTube footer line must surface the transcript-fetch ratio in all
|
||||
cases where videos were returned. Pre-fix the segment was suppressed when
|
||||
transcripts == 0, which converted the canonical stale-yt-dlp failure mode
|
||||
into a silent absence at the footer (the very surface users read for
|
||||
'did this work?'). Always-render the ratio so zero is loud.
|
||||
"""
|
||||
|
||||
def _build_youtube_report(self, transcript_flags: list[bool]) -> schema.Report:
|
||||
"""Build a Report with one YouTube item per entry in transcript_flags.
|
||||
True means the item has transcript data; False means it does not.
|
||||
"""
|
||||
items = []
|
||||
for idx, has_transcript in enumerate(transcript_flags):
|
||||
metadata = {"views": 1000}
|
||||
if has_transcript:
|
||||
metadata["transcript_highlights"] = ["Some pre-extracted quote."]
|
||||
items.append(schema.SourceItem(
|
||||
item_id=f"yt{idx}",
|
||||
source="youtube",
|
||||
title=f"Video {idx}",
|
||||
body=f"Description for video {idx}.",
|
||||
url=f"https://youtube.com/watch?v=v{idx}",
|
||||
container="some-channel",
|
||||
published_at="2026-04-15",
|
||||
date_confidence="high",
|
||||
engagement={"views": 1000, "likes": 100},
|
||||
metadata=metadata,
|
||||
))
|
||||
return schema.Report(
|
||||
topic="test topic",
|
||||
range_from="2026-04-01",
|
||||
range_to="2026-05-01",
|
||||
generated_at="2026-05-01T00:00:00+00:00",
|
||||
provider_runtime=schema.ProviderRuntime(
|
||||
reasoning_provider="gemini",
|
||||
planner_model="gemini",
|
||||
rerank_model="gemini",
|
||||
),
|
||||
query_plan=schema.QueryPlan(
|
||||
intent="general",
|
||||
freshness_mode="balanced_recent",
|
||||
cluster_mode="none",
|
||||
raw_topic="test topic",
|
||||
subqueries=[schema.SubQuery(
|
||||
label="primary", search_query="test topic",
|
||||
ranking_query="What about test topic?", sources=["youtube"],
|
||||
)],
|
||||
source_weights={"youtube": 1.0},
|
||||
),
|
||||
clusters=[],
|
||||
ranked_candidates=[],
|
||||
items_by_source={"youtube": items},
|
||||
errors_by_source={},
|
||||
)
|
||||
|
||||
def test_zero_transcripts_with_videos_present_renders_zero_over_total(self):
|
||||
# The canonical stale-yt-dlp case: 6 videos found, 0 transcripts captured.
|
||||
# Pre-fix the footer hid this entirely; post-fix it must say "0/6 with transcripts".
|
||||
report = self._build_youtube_report([False] * 6)
|
||||
text = render.render_compact(report)
|
||||
self.assertIn("0/6 with transcripts", text)
|
||||
|
||||
def test_partial_transcripts_renders_ratio(self):
|
||||
# 5 of 6 transcripts captured - shows ratio so user knows one was missed.
|
||||
report = self._build_youtube_report([True] * 5 + [False])
|
||||
text = render.render_compact(report)
|
||||
self.assertIn("5/6 with transcripts", text)
|
||||
|
||||
def test_full_transcripts_renders_ratio(self):
|
||||
# All 3 transcripts captured - still shows ratio for consistency.
|
||||
report = self._build_youtube_report([True] * 3)
|
||||
text = render.render_compact(report)
|
||||
self.assertIn("3/3 with transcripts", text)
|
||||
|
||||
def test_no_videos_no_transcript_segment(self):
|
||||
# When YouTube has no items at all, the YouTube footer line is
|
||||
# suppressed entirely (existing behavior) - the transcript segment
|
||||
# should not appear without a parent line.
|
||||
report = self._build_youtube_report([])
|
||||
text = render.render_compact(report)
|
||||
# No YouTube footer line at all - so no transcript segment either
|
||||
self.assertNotIn("with transcripts", text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -172,10 +172,10 @@ class RerankV3Tests(unittest.TestCase):
|
||||
plan=make_plan(),
|
||||
candidates=[first, second],
|
||||
provider=provider,
|
||||
model="gemini-3.1-flash-lite",
|
||||
model="gemini-3.1-flash-lite-preview",
|
||||
shortlist_size=1,
|
||||
)
|
||||
self.assertEqual("gemini-3.1-flash-lite", provider.model)
|
||||
self.assertEqual("gemini-3.1-flash-lite-preview", provider.model)
|
||||
self.assertEqual(95.0, first.rerank_score)
|
||||
self.assertEqual("high fit", first.explanation)
|
||||
# Tail is scored via the fallback (may or may not carry the entity-miss
|
||||
|
||||
@@ -109,24 +109,6 @@ class TestBuildContextSummary(unittest.TestCase):
|
||||
self.assertEqual(resolve._build_context_summary(items), "")
|
||||
|
||||
|
||||
class TestCanonicalizeGithubRepos(unittest.TestCase):
|
||||
def test_rewrites_integration_repo_to_canonical_product(self):
|
||||
repos = ["openai/codex", "anthropics/claude-code-action"]
|
||||
result = resolve.canonicalize_github_repos("claude code vs codex", repos, cap=None)
|
||||
self.assertEqual(result, ["openai/codex", "anthropics/claude-code"])
|
||||
|
||||
def test_preserves_action_repo_when_topic_intends_action(self):
|
||||
repos = ["anthropics/claude-code-action", "openai/codex"]
|
||||
result = resolve.canonicalize_github_repos("claude code action setup", repos, cap=None)
|
||||
self.assertIn("anthropics/claude-code-action", result)
|
||||
self.assertNotIn("anthropics/claude-code", result)
|
||||
|
||||
def test_dedupes_case_insensitive_after_canonicalization(self):
|
||||
repos = ["Anthropics/Claude-Code-Action", "anthropics/claude-code"]
|
||||
result = resolve.canonicalize_github_repos("claude code", repos, cap=None)
|
||||
self.assertEqual(result, ["Anthropics/Claude-Code"])
|
||||
|
||||
|
||||
class TestAutoResolve(unittest.TestCase):
|
||||
def test_no_backend_returns_empty(self):
|
||||
result = resolve.auto_resolve("test topic", {})
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user