Compare commits
110 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2ee0fc5c22 | |||
| c1ca1a4e9d | |||
| 57860aff1c | |||
| 4402dd317f | |||
| 6a92f63a56 | |||
| fd0e47d99f | |||
| 122158415a | |||
| 1bdc14878c | |||
| 26da1e157c | |||
| 4aae93ee5d | |||
| 8d3a9e4368 | |||
| 1e03af19e0 | |||
| f032e25e51 | |||
| 84a19cf44d | |||
| 861462689e | |||
| e74b0e1e93 | |||
| c5c0239dc9 | |||
| 269dda9f6c | |||
| a35677da77 | |||
| a78ab69ffe | |||
| 0bb01c2d6a | |||
| 444e07d141 | |||
| 850c7e0185 | |||
| d53121f035 | |||
| daca71f89e | |||
| d51e91ea26 | |||
| 170b570cbc | |||
| 4bae05e7fa | |||
| a4f1f94802 | |||
| 16ce073d0c | |||
| 5994b4f76a | |||
| bb5e6efbf9 | |||
| 2502a19d46 | |||
| 0f280245ac | |||
| 76b8df40d3 | |||
| eb2d8b55e0 | |||
| 1a8ffd4847 | |||
| 1814bb1967 | |||
| f236cff86a | |||
| 44971a6aae | |||
| a8e462c978 | |||
| d9a0ac31f2 | |||
| 3e60c0817d | |||
| d530c90239 | |||
| aef6f35460 | |||
| 77bd235f64 | |||
| e1017e95c9 | |||
| 71b1e8a411 | |||
| 618458eb7e | |||
| 8cccd3e982 | |||
| 17fb17222b | |||
| 8ccd778366 | |||
| a3f173dc8a | |||
| 5a3ac8ca37 | |||
| e8eb15102f | |||
| fd6e70c539 | |||
| d0b990e211 | |||
| 0f03a67166 | |||
| 5a625fda9f | |||
| b296a65515 | |||
| f2737fc035 | |||
| 9ce7264d43 | |||
| f458e0f5af | |||
| 8f565ee241 | |||
| dd7e6a1562 | |||
| 65313ce542 | |||
| 5ab8c3ba76 | |||
| 38bfb504e1 | |||
| e8f23b4205 | |||
| b78ce34922 | |||
| 0656b868e7 | |||
| 16a4fa9c39 | |||
| 3b75ff1537 | |||
| 321975e144 | |||
| 5f9c637bbe | |||
| 19132b0b5e | |||
| 1e4150ad78 | |||
| 6a15afd8e8 | |||
| 0ab7051bc5 | |||
| ec0b126af6 | |||
| d9e8a046ef | |||
| 87bf3debcc | |||
| 2f4b023db8 | |||
| 261ea5895c | |||
| 2692e0f4a2 | |||
| 10f35f82fe | |||
| 5b29b8f427 | |||
| 6a5a122195 | |||
| 7bda02169d | |||
| 6b40d2c46f | |||
| 4a99c4f557 | |||
| 5c802b0daa | |||
| 0e353ae03f | |||
| 0a5102e193 | |||
| cfde1dbbe4 | |||
| 608381a818 | |||
| af4cf7c03d | |||
| 5b0308b9e4 | |||
| 5817ef8387 | |||
| a87c1ba058 | |||
| 7214dd6051 | |||
| 6acf2fdbe2 | |||
| 07a3bdb3cf | |||
| 87577ff126 | |||
| 400fc4cc00 | |||
| e9ecce0b1c | |||
| c9cf3ef92f | |||
| 46cf2328aa | |||
| 7506cbd542 | |||
| a6bd481e61 |
@@ -11,7 +11,7 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||||
"version": "3.2.4",
|
"version": "3.3.2",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
"url": "https://github.com/mvanhorn"
|
"url": "https://github.com/mvanhorn"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"version": "3.2.4",
|
"version": "3.3.2",
|
||||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
|
|||||||
@@ -23,13 +23,11 @@ assets/ export-ignore
|
|||||||
# claude.ai-bundle-specific exclusions live in scripts/build-skill.sh.
|
# claude.ai-bundle-specific exclusions live in scripts/build-skill.sh.
|
||||||
|
|
||||||
# Historical + repo-only manifests
|
# Historical + repo-only manifests
|
||||||
SKILL-original.md export-ignore
|
|
||||||
SPEC.md export-ignore
|
SPEC.md export-ignore
|
||||||
TASKS.md export-ignore
|
TASKS.md export-ignore
|
||||||
test-run.log export-ignore
|
test-run.log export-ignore
|
||||||
CONTRIBUTORS.md export-ignore
|
CONTRIBUTORS.md export-ignore
|
||||||
HERMES_SETUP.md export-ignore
|
HERMES_SETUP.md export-ignore
|
||||||
release-notes.md export-ignore
|
|
||||||
CHANGELOG.md export-ignore
|
CHANGELOG.md export-ignore
|
||||||
uv.lock export-ignore
|
uv.lock export-ignore
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ body:
|
|||||||
label: Steps to Reproduce
|
label: Steps to Reproduce
|
||||||
description: How can we reproduce this?
|
description: How can we reproduce this?
|
||||||
placeholder: |
|
placeholder: |
|
||||||
1. Run `python3 scripts/last30days.py "topic" --emit compact`
|
1. Run `python3 skills/last30days/scripts/last30days.py "topic" --emit=compact`
|
||||||
2. ...
|
2. ...
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
|
|||||||
@@ -28,3 +28,7 @@ htmlcov/
|
|||||||
|
|
||||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||||
docs/plans/
|
docs/plans/
|
||||||
|
.context/
|
||||||
|
|
||||||
|
/work
|
||||||
|
/print
|
||||||
|
|||||||
@@ -3,12 +3,15 @@
|
|||||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||||
|
|
||||||
## Structure
|
## Structure
|
||||||
- `skills/last30days/SKILL.md` — canonical skill definition
|
- `skills/last30days/SKILL.md` — canonical skill definition / runtime spec the model reads when the slash command fires
|
||||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
||||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
||||||
|
- `CONFIGURATION.md` — user-facing knobs (env vars, flags, per-host install patterns); keep in sync per the rules below
|
||||||
|
- `CHANGELOG.md` — structured release history (launch copy lives in GitHub Releases)
|
||||||
|
- `HERMES_SETUP.md` — install instructions for the Hermes harness specifically
|
||||||
|
|
||||||
## Orientation
|
## Orientation
|
||||||
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
||||||
@@ -20,11 +23,20 @@ Agent Skills package for researching any topic across Reddit, X, YouTube, and we
|
|||||||
```bash
|
```bash
|
||||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
||||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||||
npx skills add . -g -y # one-time: symlink this repo into every detected harness's skill dir
|
npx skills add . -g -y # copies skill into ~/.agents/skills/<name>/ (frozen at install time); re-run to sync working-tree edits — see Rules below
|
||||||
|
|
||||||
|
# Tests (pytest, ~89 files under tests/, configured in pyproject.toml)
|
||||||
|
uv run pytest # full suite
|
||||||
|
uv run pytest tests/test_dedupe_v3.py # single file
|
||||||
|
uv run pytest tests/test_dedupe_v3.py -k some_case # single case
|
||||||
|
uv run pytest --cov # with coverage (skips lib/vendor/)
|
||||||
|
```
|
||||||
|
|
||||||
|
Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`.
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||||
- One-time setup: `npx skills add . -g -y` creates symlinks from each detected harness's skill dir to this repo. Edits in the working tree propagate live to every harness — no re-deploy step needed.
|
- One-time setup: `npx skills add . -g -y` copies the skill into `~/.agents/skills/<name>/` (real directory) and, for harnesses that support symlinked skill dirs, drops a per-host symlink pointing at that copy. **Working-tree edits do NOT propagate automatically** — the `~/.agents/skills/<name>/` copy is frozen at install time. To sync after edits, re-run `npx skills add . -g -y`. For live-edit on a dev machine, replace the install copy with a symlink to the working tree: `ln -sfn "$PWD/skills/last30days" ~/.agents/skills/last30days` (run from the repo root).
|
||||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||||
|
|
||||||
## Security hygiene
|
## Security hygiene
|
||||||
@@ -33,6 +45,22 @@ npx skills add . -g -y # one-time: symlink this repo into every detected harne
|
|||||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
||||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
||||||
|
|
||||||
|
## Maintaining CONFIGURATION.md
|
||||||
|
|
||||||
|
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
|
||||||
|
|
||||||
|
Update `CONFIGURATION.md` when:
|
||||||
|
|
||||||
|
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
|
||||||
|
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
|
||||||
|
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Hermes…)
|
||||||
|
- adding a new optional source that requires its own credential
|
||||||
|
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
|
||||||
|
|
||||||
|
Keep the existing structure organized by how often each layer is touched: per-run flags → env vars / `.env` → optional trend-monitoring stack → per-client patterns. Add new content into the right section rather than appending at the end.
|
||||||
|
|
||||||
|
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
|
||||||
|
|
||||||
## Beta channel
|
## Beta channel
|
||||||
|
|
||||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||||
|
|||||||
+159
-9
@@ -9,21 +9,171 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- `LAST30DAYS_YOUTUBE_SSH_HOST` env var: when set, yt-dlp YouTube search invocations are routed through `ssh <host>` for residential-IP egress. Bypasses YouTube's bot-wall on datacenter IPs (Hetzner/DigitalOcean/AWS) where `ytsearch:` returns 0 results regardless of cookies (the IP fingerprint is checked first). The named host must be configured in `~/.ssh/config` and have yt-dlp installed. Host value is validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection (e.g. a leading `-` masquerading as a flag). The transcript path is unchanged (uses the existing HTTP fallback when SSH-routing is on, since the timedtext API isn't bot-walled).
|
- **First-party positioning research + pitch-vs-pulse synthesis (company / product / service topics).** A new mandatory research step captures each entity's current stated positioning from first-party sources (homepage, docs, pricing) rather than from memory. The fetched pitch grounds `What it is` descriptions (entities described as they pitch themselves today), helps reject unrelated brand-name noise, and feeds an evidence-triggered prose beat: when the month's conversation directly supports a specific claim, cuts against one, or is squarely about the pitched ground, the synthesis says so anchored to the top thread — and stays silent when the pulse is orthogonal to the pitch, because a manufactured connection is worse than omission. Claims are tested at matched altitude (specific claims against specific threads; broad taglines are never graded against individual items), and statements stay windowed to the 30 days — no trend verdicts. Scoped to entities with an identifiable first party: people are always excluded (even founders whose companies qualify), as are events, abstract concepts, and ownerless topics like Bitcoin; the beat requires positioning fetched during the run, never from memory.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Entity-grounding rerank demotion now keys on the head token of the primary entity instead of requiring the full multi-word phrase as a contiguous substring. A high-engagement on-entity item (e.g. a 323-pt HN thread titled "Stripe is friendly to 'friendly fraud'") is no longer demoted to score 0 on a `Stripe payments` query just because it lacks the trailing search-hint word. The intended demotion still fires for items that never name the brand at all. The keyless Reddit comment-enrichment slot selection (`_slot_priority`), which mirrors this signal, was updated to the same head-token grounding so the two paths stay consistent.
|
||||||
|
|
||||||
|
## [3.3.2] - 2026-06-06
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Keyless Reddit comment enrichment now spends its limited slots on entity-matching posts first (mirroring rerank's entity-miss demotion signal) instead of raw upvote order, so off-topic high-upvote threads from broad subreddits no longer consume the comment budget only to be demoted afterward ([#484](https://github.com/mvanhorn/last30days-skill/pull/484))
|
||||||
|
|
||||||
|
## [3.3.1] - 2026-05-30
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Removed the redundant `commands/last30days.md` wrapper so the plugin exposes only the skill ([#461](https://github.com/mvanhorn/last30days-skill/issues/461)). Previously the plugin shipped both a command wrapper and the skill under the same name, so `/last30` surfaced two `last30days` entries with two different descriptions. The skill already carries its own `argument-hint`, so the `/last30days <topic>` picker UX is unchanged.
|
||||||
|
- Corrected the README install note that claimed Claude Code dedupes the slash command across install methods; it does not, so having both the marketplace plugin and the `npx skills` copy active shows two entries.
|
||||||
|
|
||||||
|
## [3.3.0] - 2026-05-17
|
||||||
|
|
||||||
|
A week-long shipping cycle: ~75 PRs merged plus 7 community fixes salvaged through PR triage. Big themes: install story modernized for the multi-harness world (Claude Code, Codex, Cursor, Gemini CLI, Copilot, Windsurf, and 50+ Agent Skills hosts), new emit and source modes, and a substantial reliability sweep across Reddit, X, Windows, YouTube, and the planner.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
**Emit modes and sources**
|
||||||
|
|
||||||
|
- `--emit=html` for shareable, print-friendly HTML research briefs ([#332](https://github.com/mvanhorn/last30days-skill/pull/332)).
|
||||||
|
- **Digg AI 1000 source**, auto-enabled when `digg-pp-cli` is on PATH ([#370](https://github.com/mvanhorn/last30days-skill/pull/370)). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief.
|
||||||
|
|
||||||
|
**Configuration knobs**
|
||||||
|
|
||||||
|
- `EXCLUDE_SOURCES` env var — the inverse of `INCLUDE_SOURCES`, honored in source count and pipeline filter ([#399](https://github.com/mvanhorn/last30days-skill/pull/399)).
|
||||||
|
- `LAST30DAYS_YOUTUBE_SSH_HOST` — opt-in SSH routing for `yt-dlp` through a residential-IP host, for users on datacenter VPS hit by YouTube's bot-wall ([#376](https://github.com/mvanhorn/last30days-skill/pull/376)). Host validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection. Transcript path unchanged (uses HTTP fallback).
|
||||||
|
- macOS Keychain as a credential source — reads from the system keychain when env vars and config files aren't set ([#407](https://github.com/mvanhorn/last30days-skill/pull/407)).
|
||||||
|
- Configuration enablement: env-var defaults and source-resilience patterns across the config layer ([#344](https://github.com/mvanhorn/last30days-skill/pull/344)).
|
||||||
|
|
||||||
|
**Pipeline and storage**
|
||||||
|
|
||||||
|
- Reddit URL auto-enrichment from web search via the public JSON API ([#366](https://github.com/mvanhorn/last30days-skill/pull/366)).
|
||||||
|
- Per-run finding sightings recorded in the SQLite store ([#373](https://github.com/mvanhorn/last30days-skill/pull/373)).
|
||||||
|
- Brave browser support for X/Twitter cookie extraction ([#320](https://github.com/mvanhorn/last30days-skill/pull/320)).
|
||||||
|
|
||||||
|
**Tests and CI**
|
||||||
|
|
||||||
|
- Full pytest suite restored to CI; 13 rotted tests repaired ([#416](https://github.com/mvanhorn/last30days-skill/pull/416)).
|
||||||
|
- `greptile.json` added with `triggerOnUpdates` + `statusCheck` ([#418](https://github.com/mvanhorn/last30days-skill/pull/418)).
|
||||||
|
- Advisory security workflow ([#368](https://github.com/mvanhorn/last30days-skill/pull/368)).
|
||||||
|
- Parallel grounding backend test coverage ([#355](https://github.com/mvanhorn/last30days-skill/pull/355)).
|
||||||
|
|
||||||
|
**Docs**
|
||||||
|
|
||||||
|
- New `CONFIGURATION.md` with README pointers ([#339](https://github.com/mvanhorn/last30days-skill/pull/339)).
|
||||||
|
- `docs/solutions/` learning capture for release-time consistency-test cascades ([#413](https://github.com/mvanhorn/last30days-skill/pull/413)) and the eval-not-in-CI design decision ([#417](https://github.com/mvanhorn/last30days-skill/pull/417)).
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- Replace the SKILL_ROOT resolver loops in Step 1 and comparison-mode with a single `SKILL_DIR` substitution pattern. The model templates the absolute path of the SKILL.md's own directory (which it always knows from the Read tool result); the bash block just validates that `scripts/last30days.py` lives there. Removes ~80 lines of bash across the two locations. Fixes a real bug: the previous resolver could pick a different install than the SKILL.md the model loaded from (spec-vs-engine divergence) and didn't enumerate harnesses like Hermes at all. The simplification works for any harness without enumeration because it just uses wherever SKILL.md was loaded from. STEP 0's marketplaces-stale-clone hop is unchanged.
|
**Install story modernized**
|
||||||
- Rename "Digg AI 1000" to just "Digg" in user-facing output (footer line, source label, inline-quote suffix, why_relevant, container attribution). Internal references to the upstream Digg AI 1000 product remain in code comments and docstrings.
|
|
||||||
- Bump `POSTS_PER_CLUSTER` from 3 to 5 and the render-side display limit from 2 to 3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous 3/2 caps routinely truncated cluster context (e.g. dropped a Jason Calacanis quote tweet on a `cli-printing-press` run).
|
|
||||||
- Rewrite SKILL.md path resolution. STEP 0 narrows from a global canonical-path enforcement to a Claude-Code-marketplaces-only stale-clone guard. Step 1 SKILL_ROOT resolver walks a single precedence list (Claude plugin cache, then `~/.codex/skills/`, `~/.agents/skills/`, repo checkout, `./.skills/last30days` for `npx skills add`, CWD, Gemini). Adds SKILL.md frontmatter fallback to `render.py::_skill_version` so the badge no longer prints `v?` on installs that don't include `.claude-plugin/plugin.json`.
|
|
||||||
|
|
||||||
- Switch SKILL.md's `--plan` and `--competitors-plan` invocation templates from inline single-quoted JSON to heredoc-written tmpfiles. Apostrophes in resolved context strings ("McDonald's", "people's choice", "developer's") previously closed the outer single-quote and broke shell parsing before the engine started — observed in a Codex run during PR #400 testing. The engine's `parse_plan()` / `parse_competitors_plan()` already supported file paths (via `os.path.isfile()` probe); only the template prose changed. Fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403).
|
- `npx skills add` is now the canonical install path for every harness ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). README and SKILL.md flipped to recommend `npx skills add . -g -y` over per-harness manual instructions. Surfaces Gemini CLI, Copilot, Windsurf, and 50+ other Agent Skills hosts that the install pattern reaches.
|
||||||
|
- README dropped the Gemini CLI native-extension install path (now covered by `npx skills add`).
|
||||||
|
- `hooks.json` made polyglot for Gemini CLI + Claude Code compatibility ([#318](https://github.com/mvanhorn/last30days-skill/pull/318)).
|
||||||
|
|
||||||
|
**Skill semantics and multi-harness reframe**
|
||||||
|
|
||||||
|
- `AGENTS.md` is now canonical; `CLAUDE.md` points at it ([#410](https://github.com/mvanhorn/last30days-skill/pull/410)). Reframes the project as a multi-harness Agent Skills package rather than a Claude-Code-specific tool.
|
||||||
|
- SKILL.md path resolution rewritten: STEP 0 narrows to a Claude-Code-marketplaces-only stale-clone guard; Step 1 walks a single `SKILL_DIR` substitution pattern ([#400](https://github.com/mvanhorn/last30days-skill/pull/400), [#409](https://github.com/mvanhorn/last30days-skill/pull/409)). Removes ~80 lines of bash and fixes a real spec-vs-engine divergence where the previous resolver could pick a different install than the SKILL.md the model loaded from.
|
||||||
|
- SKILL.md version regex consolidated into `lib/skill_meta.py` ([#412](https://github.com/mvanhorn/last30days-skill/pull/412)).
|
||||||
|
- `--plan` / `--competitors-plan` invocation templates switched from inline single-quoted JSON to heredoc-written tmpfiles ([#404](https://github.com/mvanhorn/last30days-skill/pull/404), fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403)). Apostrophes in resolved context strings ("McDonald's", "people's choice") no longer break shell parsing.
|
||||||
|
- `POSTS_PER_CLUSTER` raised 3→5 and render-side display limit 2→3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous caps routinely truncated cluster context.
|
||||||
|
- Digg AI 1000 renamed to "Digg" in user-facing output ([#372](https://github.com/mvanhorn/last30days-skill/pull/372)) — footer line, source label, inline-quote suffix, why_relevant, container attribution. Internal references retain the upstream product name.
|
||||||
|
- GitHub repo resolution canonicalized for ambiguous product comparisons ([#302](https://github.com/mvanhorn/last30days-skill/pull/302)).
|
||||||
|
|
||||||
|
**Dependencies and tooling**
|
||||||
|
|
||||||
|
- Dropped `requests` runtime dependency. All providers route through stdlib `urllib` via the `lib/http` wrapper ([#393](https://github.com/mvanhorn/last30days-skill/pull/393)).
|
||||||
|
- Migrated to `gemini-3.1-flash-lite` GA model ([#378](https://github.com/mvanhorn/last30days-skill/pull/378)).
|
||||||
|
- Aligned Codex/Claude plugin manifests + added Codex `AGENTS.md` ([#321](https://github.com/mvanhorn/last30days-skill/pull/321)).
|
||||||
|
- pytest dev dep bumped 9.0.2 → 9.0.3 ([#414](https://github.com/mvanhorn/last30days-skill/pull/414)).
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
|
|
||||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1. Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1 ([#400](https://github.com/mvanhorn/last30days-skill/pull/400)). Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||||
- **`skills/last30days/scripts/sync.sh`.** The maintainer dev-deploy script is gone. Every job it did has a better replacement: `npx skills add . -g -y` symlinks the working tree into every detected harness's skill dir (better than sync.sh's copy model — edits propagate live), `hermes skills install mvanhorn/last30days-skill --force` handles Hermes, `clawhub install last30days-official` handles OpenClaw, and the Claude marketplace cache target was a "test against the official install path" hack we shouldn't have been recommending in the first place. The `test_sync_cache_path_uses_skill_version` test was dropped along with it. CLAUDE.md, HERMES_SETUP.md, the PR template, and a render.py docstring were updated to drop references; CHANGELOG and historical docs (release notes, plan files) keep their existing mentions as accurate history.
|
- **`skills/last30days/scripts/sync.sh`** — maintainer dev-deploy script ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). Replaced by `npx skills add . -g -y` (live-symlink into every detected harness's skill dir — better than sync.sh's copy model since edits propagate live). Hermes uses `hermes skills install mvanhorn/last30days-skill --force`; OpenClaw uses `clawhub install last30days-official`.
|
||||||
|
- Orphaned `SPEC.md` and `TASKS.md` ([#419](https://github.com/mvanhorn/last30days-skill/pull/419)).
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
**Reddit**
|
||||||
|
|
||||||
|
- `lstrip("r/")` mangled subreddits starting with `r` (`r/robotics` → `obotics`, `r/ruby` → `uby`); replaced with `removeprefix("r/")` at 4 sites (Alex Key, salvaged from #288).
|
||||||
|
- Browser-like User-Agent + `Accept-Language`/`Accept-Encoding`/`Connection` headers + gzip decompression to fix `urllib` 403s on Reddit's public JSON endpoint (Franco Carballar, salvaged from #199).
|
||||||
|
- HTTP 402 re-raised across all three ScrapeCreators paths (`_global_search`, `_subreddit_search`, `fetch_post_comments`) so the OpenAI/public-JSON fallback chain triggers when credits are exhausted (Jonathan Oppenheim, salvaged from #170).
|
||||||
|
|
||||||
|
**Authentication and credentials**
|
||||||
|
|
||||||
|
- Restored multi-key rotation for `SCRAPECREATORS_API_KEY` accidentally dropped in v3.0.6 (Eric Oberhofer, salvaged from #287). Comma-separated keys round-robin via `random.choice` per run.
|
||||||
|
|
||||||
|
**Windows compatibility**
|
||||||
|
|
||||||
|
- `os.killpg` in `_cleanup_children()` guarded with `hasattr(os, "killpg")`, falls back to `os.kill(SIGTERM)` (gujishh, salvaged from #226).
|
||||||
|
- POSIX-style secret-permission warning skipped on Windows ([#357](https://github.com/mvanhorn/last30days-skill/pull/357)).
|
||||||
|
- Render uses forward slashes in save-path footer for Windows ([#338](https://github.com/mvanhorn/last30days-skill/pull/338)).
|
||||||
|
|
||||||
|
**xAI / X / xurl**
|
||||||
|
|
||||||
|
- `parse_x_response` now raises `http.HTTPError` on empty output, missing JSON, or decode failure — surfaces in `errors_by_source` instead of silently returning an empty result list (Kaustav Mishra, salvaged from #155).
|
||||||
|
- `xurl` treats `PermissionError` from PATH lookup as unavailable ([#322](https://github.com/mvanhorn/last30days-skill/pull/322)).
|
||||||
|
|
||||||
|
**YouTube**
|
||||||
|
|
||||||
|
- SC YouTube + multi-token HN searches unblocked ([#388](https://github.com/mvanhorn/last30days-skill/pull/388)).
|
||||||
|
- Transcript-fetch ratio surfaced + degraded-run nudge for stale `yt-dlp` ([#340](https://github.com/mvanhorn/last30days-skill/pull/340)).
|
||||||
|
|
||||||
|
**bird_x / HTTP**
|
||||||
|
|
||||||
|
- Subprocess retry on non-JSON stdout to handle X anti-bot HTML interstitials ([#383](https://github.com/mvanhorn/last30days-skill/pull/383)).
|
||||||
|
- HTTP retry budget expanded + exponential backoff on DNS resolution failure ([#382](https://github.com/mvanhorn/last30days-skill/pull/382)).
|
||||||
|
- Parallel AI search aligned with current API schema ([#341](https://github.com/mvanhorn/last30days-skill/pull/341)).
|
||||||
|
- Parallel web backend routed through grounding ([#354](https://github.com/mvanhorn/last30days-skill/pull/354)).
|
||||||
|
|
||||||
|
**Planner and sources**
|
||||||
|
|
||||||
|
- `xquik` registered in `SOURCE_CAPABILITIES` ([#336](https://github.com/mvanhorn/last30days-skill/pull/336), fixes [#319](https://github.com/mvanhorn/last30days-skill/issues/319)).
|
||||||
|
- Honor explicit optional source requests ([#356](https://github.com/mvanhorn/last30days-skill/pull/356)).
|
||||||
|
- ScrapeCreators source-gating aligned between code and docs ([#415](https://github.com/mvanhorn/last30days-skill/pull/415)).
|
||||||
|
- OpenClaw works without ScrapeCreators key ([#392](https://github.com/mvanhorn/last30days-skill/pull/392), by @thinkun).
|
||||||
|
|
||||||
|
**Render, version display, hosting paths**
|
||||||
|
|
||||||
|
- Hardcoded `v3.0.0` in render replaced with dynamic `_skill_version()` ([#365](https://github.com/mvanhorn/last30days-skill/pull/365)).
|
||||||
|
- Comparison HTML artifacts saved correctly ([#389](https://github.com/mvanhorn/last30days-skill/pull/389)).
|
||||||
|
- `OPENROUTER_DEFAULT` model ID corrected ([#323](https://github.com/mvanhorn/last30days-skill/pull/323)).
|
||||||
|
- OpenClaw poll-timing initialized once ([#358](https://github.com/mvanhorn/last30days-skill/pull/358)).
|
||||||
|
- Prefer sandboxed Safari cookie path ([#343](https://github.com/mvanhorn/last30days-skill/pull/343)).
|
||||||
|
- Preserve clean mode for last-run state ([#334](https://github.com/mvanhorn/last30days-skill/pull/334)).
|
||||||
|
- Replaced hardcoded `/Users/mvanhorn/...` paths in `test-v1-vs-v2.sh` with portable env-var overrides (Dave Morin, salvaged from #297).
|
||||||
|
|
||||||
|
**Hooks**
|
||||||
|
|
||||||
|
- `check-config.sh` path-quoting fix for paths with spaces ([#337](https://github.com/mvanhorn/last30days-skill/pull/337)).
|
||||||
|
- Replaced unsafe `eval` with `declare` in `check-config.sh` ([#364](https://github.com/mvanhorn/last30days-skill/pull/364)).
|
||||||
|
|
||||||
|
**Sync and version metadata**
|
||||||
|
|
||||||
|
- `sync.sh` pointed at this repo's plugin cache, not the private repo's ([#402](https://github.com/mvanhorn/last30days-skill/pull/402)).
|
||||||
|
- Sync cache target bumped to 3.2.1 to match SKILL.md ([#397](https://github.com/mvanhorn/last30days-skill/pull/397)).
|
||||||
|
- ScrapeCreators free-tier credit count corrected to 100 in docs ([#369](https://github.com/mvanhorn/last30days-skill/pull/369), fixes [#367](https://github.com/mvanhorn/last30days-skill/issues/367)).
|
||||||
|
- Gemini extension version synced ([#349](https://github.com/mvanhorn/last30days-skill/pull/349)).
|
||||||
|
- Various stale path/link fixes ([#345](https://github.com/mvanhorn/last30days-skill/pull/345), [#346](https://github.com/mvanhorn/last30days-skill/pull/346), [#347](https://github.com/mvanhorn/last30days-skill/pull/347), [#348](https://github.com/mvanhorn/last30days-skill/pull/348), [#351](https://github.com/mvanhorn/last30days-skill/pull/351)).
|
||||||
|
|
||||||
|
### Contributors
|
||||||
|
|
||||||
|
First-time contributors whose fixes shipped in this release (most via PR triage salvage — fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
||||||
|
|
||||||
|
- Dave Morin — portable test-harness paths
|
||||||
|
- Alex Key — `removeprefix("r/")` for subreddit names
|
||||||
|
- Eric Oberhofer — multi-key rotation restored
|
||||||
|
- gujishh — Windows process cleanup
|
||||||
|
- Franco Carballar — Reddit browser-like headers
|
||||||
|
- Jonathan Oppenheim — Reddit 402 fallback chain
|
||||||
|
- Kaustav Mishra — xAI error surfacing
|
||||||
|
- [@thinkun](https://github.com/thinkun) ([#363](https://github.com/mvanhorn/last30days-skill/pull/363)) — OpenClaw ScrapeCreators-key-optional fix
|
||||||
|
|
||||||
|
Full PR list at [github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0](https://github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0).
|
||||||
|
|
||||||
## [3.2.0] - 2026-05-09
|
## [3.2.0] - 2026-05-09
|
||||||
|
|
||||||
@@ -50,7 +200,7 @@ Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`,
|
|||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||||
- **Broken README link.** The README's "source of truth" link pointed at `skills/last30days/SKILL.md`, a path that does not exist. Fixed to point at root `SKILL.md`.
|
- **Broken README link.** The README's "source of truth" link pointed at root `SKILL.md`, which is no longer maintained after the plugin-layout restructure. Fixed to point at `skills/last30days/SKILL.md`.
|
||||||
|
|
||||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||||
|
|
||||||
|
|||||||
+24
@@ -16,6 +16,30 @@ The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do t
|
|||||||
|
|
||||||
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
||||||
|
|
||||||
|
## Research pipeline
|
||||||
|
|
||||||
|
### Primary entity
|
||||||
|
|
||||||
|
The brand or proper-noun core of a research topic — the topic with its Intent modifier stripped. It is what the research is *about*, as distinct from how the user phrased the search.
|
||||||
|
|
||||||
|
### Intent modifier
|
||||||
|
|
||||||
|
A trailing word or phrase in a topic that expresses what the user wants to know rather than what the topic is ("review", "use cases", "pricing"). Stripped when deriving the Primary entity.
|
||||||
|
|
||||||
|
### Entity grounding
|
||||||
|
|
||||||
|
The check that a candidate item plausibly mentions the Primary entity before final ranking. Grounding keys on the head token (first word) of the Primary entity rather than the full phrase — trailing words are usually search descriptors, so requiring them falsely demotes on-entity items.
|
||||||
|
|
||||||
|
An item that fails grounding receives a decisive entity-miss demotion, designed so engagement cannot rescue off-entity content. Because the demotion is decisive, the grounding bar is deliberately conservative: its failure modes degrade toward "no penalty," never toward burying on-entity signal.
|
||||||
|
|
||||||
|
### Keyless path
|
||||||
|
|
||||||
|
The research flow available with no API keys: source data is gathered by scraping and RSS rather than authenticated APIs, and ranking falls back to local scoring instead of LLM-based reranking. This is the free tier of the Skill; lexical quality safeguards like Entity grounding matter most here, because no LLM is available to judge relevance semantically.
|
||||||
|
|
||||||
|
### Comment-enrichment slots
|
||||||
|
|
||||||
|
The small, depth-dependent budget of Reddit posts whose comments get fetched in the Keyless path. Slot selection is relevance-aware: posts that pass Entity grounding claim slots first, so the budget is not spent on high-engagement posts that final ranking will demote anyway.
|
||||||
|
|
||||||
## Distribution
|
## Distribution
|
||||||
|
|
||||||
### Beta channel
|
### Beta channel
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
# Configuration
|
||||||
|
|
||||||
|
Everything you can tune in `/last30days` without editing the engine source.
|
||||||
|
Three layers, in order of how often you'll touch them:
|
||||||
|
|
||||||
|
1. **Per-run flags** - what you pass on the command line.
|
||||||
|
2. **Environment variables and `.env`** - what's enabled across all runs.
|
||||||
|
3. **Optional trend-monitoring stack** - SQLite store, watchlist, briefings.
|
||||||
|
|
||||||
|
Per-client patterns and the experimental beta channel are at the bottom.
|
||||||
|
|
||||||
|
> Skip ahead: [Where output is saved](#where-output-is-saved) - [API keys](#api-keys-env) - [Reasoning provider](#reasoning-provider-priority) - [Web search backend](#web-search-backend-priority) - [Trend monitoring](#trend-monitoring-store--watchlist--briefings) - [Per-client patterns](#per-client-patterns) - [Beta channel](#beta-channel)
|
||||||
|
|
||||||
|
## Why this document exists
|
||||||
|
|
||||||
|
This is a focused **configuration reference** maintained alongside the engine. The runtime contract (the voice rules, the planner protocol, the LAWs the synthesizing model follows) lives in [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md) - that file is authoritative when the two ever differ. This file's job is narrower: surface every knob a user or operator can turn, in one place, kept current with the code so client-facing setups stay reliable. New configuration knobs added to the engine should be reflected here in the same PR.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Where output is saved
|
||||||
|
|
||||||
|
| Platform | Default path | Override |
|
||||||
|
|---|---|---|
|
||||||
|
| Linux / macOS | `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` | set `LAST30DAYS_MEMORY_DIR=/path` |
|
||||||
|
| Windows | `LAST30DAYS_MEMORY_DIR` defaults to `C:\Users\<you>\Documents\Last30Days\` | set `LAST30DAYS_MEMORY_DIR=C:\path` |
|
||||||
|
|
||||||
|
Each run produces one file per topic, slug-named:
|
||||||
|
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
|
||||||
|
|
||||||
|
**Per-run overrides:**
|
||||||
|
- `--save-dir <path>` - one-off output location.
|
||||||
|
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
||||||
|
|
||||||
|
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## API keys (`.env`)
|
||||||
|
|
||||||
|
The skill reads keys from a `.env` file. Two locations are supported, in priority order:
|
||||||
|
|
||||||
|
1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present.
|
||||||
|
2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback.
|
||||||
|
|
||||||
|
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
||||||
|
|
||||||
|
The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case.
|
||||||
|
|
||||||
|
**Source-by-source** - what each key unlocks:
|
||||||
|
|
||||||
|
| Source | Key(s) | Required for | Free tier |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Reddit (public) | none | always on | yes |
|
||||||
|
| Hacker News | none | always on | yes |
|
||||||
|
| Polymarket | none | always on | yes |
|
||||||
|
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
|
||||||
|
| YouTube | `yt-dlp` CLI installed | always on if `yt-dlp` present | yes |
|
||||||
|
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; xAI / ScrapeCreators = paid |
|
||||||
|
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
|
||||||
|
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
|
||||||
|
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
|
||||||
|
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
|
||||||
|
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
|
||||||
|
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
|
||||||
|
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
|
||||||
|
| Perplexity Deep Research | `OPENROUTER_API_KEY` | `--deep-research` flag (~$0.90/query) | no |
|
||||||
|
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
|
||||||
|
|
||||||
|
**Example `.env` skeleton** (placeholders only - replace with your own values):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Reasoning + planning (one provider; see priority below)
|
||||||
|
GOOGLE_API_KEY=<your-gemini-key>
|
||||||
|
|
||||||
|
# Web search backend (one is enough; Brave is the cheapest)
|
||||||
|
BRAVE_API_KEY=<your-brave-key>
|
||||||
|
|
||||||
|
# Optional sources
|
||||||
|
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
|
||||||
|
INCLUDE_SOURCES=tiktok,instagram
|
||||||
|
|
||||||
|
# X authentication (one option only)
|
||||||
|
XAI_API_KEY=<your-xai-key>
|
||||||
|
# OR cookie-jar (no key needed; logs in via your browser session)
|
||||||
|
# FROM_BROWSER=firefox
|
||||||
|
|
||||||
|
# Bluesky
|
||||||
|
BSKY_HANDLE=<your-handle>.bsky.social
|
||||||
|
BSKY_APP_PASSWORD=<your-app-password>
|
||||||
|
```
|
||||||
|
|
||||||
|
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
||||||
|
|
||||||
|
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search.
|
||||||
|
|
||||||
|
### Bluesky app-password format and search host
|
||||||
|
|
||||||
|
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
|
||||||
|
|
||||||
|
The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reasoning provider priority
|
||||||
|
|
||||||
|
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
||||||
|
|
||||||
|
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
||||||
|
2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`)
|
||||||
|
3. **xAI** - `XAI_API_KEY`
|
||||||
|
4. **OpenRouter** - `OPENROUTER_API_KEY` (also unlocks `--deep-research`)
|
||||||
|
5. **Local / deterministic** - always available, lowest quality
|
||||||
|
|
||||||
|
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Web search backend priority
|
||||||
|
|
||||||
|
Used by `--auto-resolve` (when WebSearch isn't available from the host) and Step 2 supplements. Auto-detect priority (override per-run with `--web-backend=<name>`):
|
||||||
|
|
||||||
|
1. **Brave** - `BRAVE_API_KEY`
|
||||||
|
2. **Exa** - `EXA_API_KEY`
|
||||||
|
3. **Serper** - `SERPER_API_KEY`
|
||||||
|
4. **Parallel** - `PARALLEL_API_KEY`
|
||||||
|
5. **Host's native WebSearch** - Claude Code, Codex, Gemini all have one built in
|
||||||
|
|
||||||
|
Visible quality difference between hosts with vs without a configured backend. If your client setup produces thinner results than yours, this is usually why.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Trend monitoring (`--store` + watchlist + briefings)
|
||||||
|
|
||||||
|
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
|
||||||
|
|
||||||
|
### `--store` flag
|
||||||
|
|
||||||
|
Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate.
|
||||||
|
|
||||||
|
**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`.
|
||||||
|
|
||||||
|
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
|
||||||
|
|
||||||
|
### `watchlist.py` - recurring topics
|
||||||
|
|
||||||
|
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
|
||||||
|
|
||||||
|
Two-step flow (the watchlist holds the topic; an external scheduler invokes the run):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Add the topic to the watchlist
|
||||||
|
# Default schedule daily 8am; --weekly switches to Mondays 8am
|
||||||
|
python3 scripts/watchlist.py add "british airways middle east" --weekly
|
||||||
|
|
||||||
|
# 2. Configure delivery and budget (optional)
|
||||||
|
python3 scripts/watchlist.py config delivery "https://hooks.slack.com/services/..."
|
||||||
|
python3 scripts/watchlist.py config budget 5.00
|
||||||
|
|
||||||
|
# 3. Trigger via cron / Task Scheduler / GitHub Actions
|
||||||
|
python3 scripts/watchlist.py run-one "british airways middle east"
|
||||||
|
# or run every enabled topic, gated by daily_budget
|
||||||
|
python3 scripts/watchlist.py run-all
|
||||||
|
```
|
||||||
|
|
||||||
|
The schedule field stored on each topic is metadata - the actual cron / Task Scheduler invocation is your responsibility. Watchlist runs hardcode `--quick` and `--lookback-days 90` when spawning the underlying engine.
|
||||||
|
|
||||||
|
### `briefing.py` - daily / weekly digests
|
||||||
|
|
||||||
|
[`scripts/briefing.py`](skills/last30days/scripts/briefing.py) reads the SQLite store and emits structured data the agent then synthesizes into prose. Modes: `generate` (daily), `generate --weekly`, `show [--date DATE]` (display a saved briefing). Briefs save to `~/.local/share/last30days/briefs/`.
|
||||||
|
|
||||||
|
### Recommended cadence pattern
|
||||||
|
|
||||||
|
| Step | Cadence | Command |
|
||||||
|
|---|---|---|
|
||||||
|
| Baseline | one-time per topic | `/last30days "<topic>" --days=30 --store` |
|
||||||
|
| Add to watchlist | one-time per topic | `python3 scripts/watchlist.py add "<topic>" --weekly` |
|
||||||
|
| Recurring run | daily or weekly (external scheduler) | `python3 scripts/watchlist.py run-all` |
|
||||||
|
| Digest | weekly | `python3 scripts/briefing.py generate --weekly` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Per-client patterns
|
||||||
|
|
||||||
|
The skill is built to flex around different client environments. Four patterns that compose well:
|
||||||
|
|
||||||
|
### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders)
|
||||||
|
|
||||||
|
The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
||||||
|
SCRAPECREATORS_API_KEY=<acme-scoped-key-or-shared>
|
||||||
|
INCLUDE_SOURCES=tiktok,instagram
|
||||||
|
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
||||||
|
```
|
||||||
|
|
||||||
|
`cd` into the client folder, run `/last30days <topic>` as normal, no flags or wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||||
|
|
||||||
|
### 2. Per-client save dir + suffix wrapper
|
||||||
|
|
||||||
|
For workflows where you don't `cd` into a client folder (running from anywhere, scripted batches), a tiny shell function isolates each client's research without engine changes.
|
||||||
|
|
||||||
|
PowerShell example:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
function Run-L30D-Client {
|
||||||
|
param([string]$ClientSlug, [Parameter(ValueFromRemainingArguments=$true)]$Args)
|
||||||
|
$env:LAST30DAYS_MEMORY_DIR = "C:\Users\$env:USERNAME\Clients\$ClientSlug\Research\Last30Days"
|
||||||
|
/last30days @Args --save-suffix=$ClientSlug
|
||||||
|
}
|
||||||
|
# Usage: Run-L30D-Client acme "british airways middle east"
|
||||||
|
```
|
||||||
|
|
||||||
|
Bash example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
l30d-client() {
|
||||||
|
local client=$1; shift
|
||||||
|
LAST30DAYS_MEMORY_DIR="$HOME/Clients/$client/Research/Last30Days" \
|
||||||
|
/last30days "$@" --save-suffix="$client"
|
||||||
|
}
|
||||||
|
# Usage: l30d-client acme "british airways middle east"
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Custom category-peer subreddits
|
||||||
|
|
||||||
|
[`scripts/lib/categories.py`](skills/last30days/scripts/lib/categories.py) holds a table of `(category_id, trigger_keywords, peer_subreddits)`. If a client lives in a vertical that isn't covered (legal-tech, real-estate-tech, B2B HR SaaS), add a row. Pure data, no logic.
|
||||||
|
|
||||||
|
Section 2a of `SKILL.md` documents the merging rule the skill applies when your topic matches a category.
|
||||||
|
|
||||||
|
### 4. Pre-built `--competitors-plan` JSON
|
||||||
|
|
||||||
|
For competitor-vs-comparisons that recur, a pre-written JSON skeleton per client industry saves real time:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"Competitor B": {
|
||||||
|
"x_handle": "competitor_b_handle",
|
||||||
|
"subreddits": ["sub1", "sub2"],
|
||||||
|
"github_user": "competitor-b-org",
|
||||||
|
"context": "Founded 2019, focused on ..."
|
||||||
|
},
|
||||||
|
"Competitor C": { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Pass as `--competitors-plan @client/competitors-plan.json` (or as a string). See `SKILL.md` section "If QUERY_TYPE = COMPARISON" for the full schema.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Beta channel
|
||||||
|
|
||||||
|
Experimental customizations live on a private companion repo (`mvanhorn/last30days-skill-private`) installed as `/last30days-beta`. Never ship beta-only changes to the public marketplace without a review PR against the public repo. Workflow guide: `BETA.md` in the private repo.
|
||||||
|
|
||||||
|
This is the right home for client-specific changes you don't intend to upstream - custom category rows, internal subreddit lists, per-vertical plan templates.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Cross-references
|
||||||
|
|
||||||
|
- The CLI flag surface: `python3 scripts/last30days.py --help`
|
||||||
|
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
|
||||||
|
- Engine spec (some sections stale; SKILL.md wins on conflicts): [`SPEC.md`](SPEC.md)
|
||||||
|
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
|
||||||
@@ -12,11 +12,12 @@
|
|||||||
|
|
||||||
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
**An AI agent-led search engine scored by upvotes, likes, and real money - not editors.**
|
||||||
|
|
||||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [SKILL.md](SKILL.md), which is the source of truth for the latest command and setup behavior.
|
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||||
|
|
||||||
**Claude Code (recommended — auto-updates via marketplace):**
|
**Claude Code (recommended — auto-updates via marketplace):**
|
||||||
```
|
```
|
||||||
/plugin marketplace add mvanhorn/last30days-skill
|
/plugin marketplace add mvanhorn/last30days-skill
|
||||||
|
/plugin install last30days
|
||||||
```
|
```
|
||||||
|
|
||||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
||||||
@@ -188,7 +189,7 @@ If you'd rather use the agent-skills install path on Claude Code, that's also su
|
|||||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||||
```
|
```
|
||||||
|
|
||||||
The native plugin and the `npx skills` install can coexist; Claude Code dedupes the slash command.
|
The native plugin and the `npx skills` install can coexist. Note that Claude Code does not dedupe across install methods: if you have both the marketplace plugin and the `npx skills` copy active, `/last30days` will show two entries. Use one install method per machine.
|
||||||
|
|
||||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
||||||
|
|
||||||
@@ -280,6 +281,18 @@ skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
|||||||
|
|
||||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
||||||
|
|
||||||
|
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Two things you'll likely want to know on day one:
|
||||||
|
|
||||||
|
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each run produces `<slug>-raw[-suffix].md`.
|
||||||
|
|
||||||
|
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||||
|
|
||||||
|
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||||
|
|||||||
@@ -1,391 +0,0 @@
|
|||||||
---
|
|
||||||
name: last30days
|
|
||||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
|
||||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
|
||||||
context: fork
|
|
||||||
agent: Explore
|
|
||||||
disable-model-invocation: true
|
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
|
||||||
---
|
|
||||||
|
|
||||||
# last30days: Research Any Topic from the Last 30 Days
|
|
||||||
|
|
||||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
|
||||||
|
|
||||||
Use cases:
|
|
||||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
|
||||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
|
||||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
|
||||||
- **General**: any topic you're curious about → understand what the community is saying
|
|
||||||
|
|
||||||
## CRITICAL: Parse User Intent
|
|
||||||
|
|
||||||
Before doing anything, parse the user's input for:
|
|
||||||
|
|
||||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
|
||||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
|
||||||
3. **QUERY TYPE**: What kind of research they want:
|
|
||||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
|
||||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
|
||||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
|
||||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
|
||||||
|
|
||||||
Common patterns:
|
|
||||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
|
||||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
|
||||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
|
||||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
|
||||||
- If tool is specified in the query, use it
|
|
||||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
|
||||||
|
|
||||||
**Store these variables:**
|
|
||||||
- `TOPIC = [extracted topic]`
|
|
||||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
|
||||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Setup Check
|
|
||||||
|
|
||||||
The skill works in three modes based on available API keys:
|
|
||||||
|
|
||||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
|
||||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
|
||||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
|
||||||
|
|
||||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
|
||||||
|
|
||||||
### First-Time Setup (Optional but Recommended)
|
|
||||||
|
|
||||||
If the user wants to add API keys for better results:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mkdir -p ~/.config/last30days
|
|
||||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
|
||||||
# last30days API Configuration
|
|
||||||
# Both keys are optional - skill works with WebSearch fallback
|
|
||||||
|
|
||||||
# For Reddit research (uses OpenAI's web_search tool)
|
|
||||||
OPENAI_API_KEY=
|
|
||||||
|
|
||||||
# For X/Twitter research (uses xAI's x_search tool)
|
|
||||||
XAI_API_KEY=
|
|
||||||
ENVEOF
|
|
||||||
|
|
||||||
chmod 600 ~/.config/last30days/.env
|
|
||||||
echo "Config created at ~/.config/last30days/.env"
|
|
||||||
echo "Edit to add your API keys for enhanced research."
|
|
||||||
```
|
|
||||||
|
|
||||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Research Execution
|
|
||||||
|
|
||||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
|
||||||
|
|
||||||
**Step 1: Run the research script**
|
|
||||||
```bash
|
|
||||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
|
||||||
```
|
|
||||||
|
|
||||||
The script will automatically:
|
|
||||||
- Detect available API keys
|
|
||||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
|
||||||
- Run Reddit/X searches if keys exist
|
|
||||||
- Signal if WebSearch is needed
|
|
||||||
|
|
||||||
**Step 2: Check the output mode**
|
|
||||||
|
|
||||||
The script output will indicate the mode:
|
|
||||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
|
||||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
|
||||||
|
|
||||||
**Step 3: Do WebSearch**
|
|
||||||
|
|
||||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
|
||||||
|
|
||||||
Choose search queries based on QUERY_TYPE:
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
|
||||||
- Search for: `best {TOPIC} recommendations`
|
|
||||||
- Search for: `{TOPIC} list examples`
|
|
||||||
- Search for: `most popular {TOPIC}`
|
|
||||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
|
||||||
|
|
||||||
**If NEWS** ("what's happening with X", "X news"):
|
|
||||||
- Search for: `{TOPIC} news 2026`
|
|
||||||
- Search for: `{TOPIC} announcement update`
|
|
||||||
- Goal: Find current events and recent developments
|
|
||||||
|
|
||||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
|
||||||
- Search for: `{TOPIC} prompts examples 2026`
|
|
||||||
- Search for: `{TOPIC} techniques tips`
|
|
||||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
|
||||||
|
|
||||||
**If GENERAL** (default):
|
|
||||||
- Search for: `{TOPIC} 2026`
|
|
||||||
- Search for: `{TOPIC} discussion`
|
|
||||||
- Goal: Find what people are actually saying
|
|
||||||
|
|
||||||
For ALL query types:
|
|
||||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
|
||||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
|
||||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
|
||||||
- Your knowledge may be outdated - trust the user's terminology
|
|
||||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
|
||||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
|
||||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
|
||||||
|
|
||||||
**Step 3: Wait for background script to complete**
|
|
||||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
|
||||||
|
|
||||||
**Depth options** (passed through from user's command):
|
|
||||||
- `--quick` → Faster, fewer sources (8-12 each)
|
|
||||||
- (default) → Balanced (20-30 each)
|
|
||||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Judge Agent: Synthesize All Sources
|
|
||||||
|
|
||||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
|
||||||
|
|
||||||
The Judge Agent must:
|
|
||||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
|
||||||
2. Weight WebSearch sources LOWER (no engagement data)
|
|
||||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
|
||||||
4. Note any contradictions between sources
|
|
||||||
5. Extract the top 3-5 actionable insights
|
|
||||||
|
|
||||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FIRST: Internalize the Research
|
|
||||||
|
|
||||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
|
||||||
|
|
||||||
Read the research output carefully. Pay attention to:
|
|
||||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
|
||||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
|
||||||
- **What the sources actually say**, not what you assume the topic is about
|
|
||||||
|
|
||||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
|
||||||
|
|
||||||
### If QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
|
||||||
|
|
||||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
|
||||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
|
||||||
- Count how many times each is mentioned
|
|
||||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
|
||||||
- List them by popularity/mention count
|
|
||||||
|
|
||||||
**BAD synthesis for "best Claude Code skills":**
|
|
||||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
|
||||||
|
|
||||||
**GOOD synthesis for "best Claude Code skills":**
|
|
||||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
|
||||||
|
|
||||||
### For all QUERY_TYPEs
|
|
||||||
|
|
||||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
|
||||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
|
||||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
|
||||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
|
||||||
- Common pitfalls mentioned BY THE SOURCES
|
|
||||||
|
|
||||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## THEN: Show Summary + Invite Vision
|
|
||||||
|
|
||||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
|
||||||
|
|
||||||
**Display in this EXACT sequence:**
|
|
||||||
|
|
||||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
|
||||||
```
|
|
||||||
🏆 Most mentioned:
|
|
||||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
|
||||||
2. [Specific name] - mentioned {n}x (sources)
|
|
||||||
3. [Specific name] - mentioned {n}x (sources)
|
|
||||||
4. [Specific name] - mentioned {n}x (sources)
|
|
||||||
5. [Specific name] - mentioned {n}x (sources)
|
|
||||||
|
|
||||||
Notable mentions: [other specific things with 1-2 mentions]
|
|
||||||
```
|
|
||||||
|
|
||||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
|
||||||
```
|
|
||||||
What I learned:
|
|
||||||
|
|
||||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
|
||||||
|
|
||||||
KEY PATTERNS I'll use:
|
|
||||||
1. [Pattern from research]
|
|
||||||
2. [Pattern from research]
|
|
||||||
3. [Pattern from research]
|
|
||||||
```
|
|
||||||
|
|
||||||
**THEN - Stats (right before invitation):**
|
|
||||||
|
|
||||||
For **full/partial mode** (has API keys):
|
|
||||||
```
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
|
||||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
|
||||||
├─ 🌐 Web: {n} pages │ {domains}
|
|
||||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
|
||||||
```
|
|
||||||
|
|
||||||
For **web-only mode** (no API keys):
|
|
||||||
```
|
|
||||||
---
|
|
||||||
✅ Research complete!
|
|
||||||
├─ 🌐 Web: {n} pages │ {domains}
|
|
||||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
|
||||||
|
|
||||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
|
||||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
|
||||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
|
||||||
```
|
|
||||||
|
|
||||||
**LAST - Invitation:**
|
|
||||||
```
|
|
||||||
---
|
|
||||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
|
||||||
```
|
|
||||||
|
|
||||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
|
||||||
|
|
||||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
|
||||||
|
|
||||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
|
||||||
```
|
|
||||||
What tool will you use these prompts with?
|
|
||||||
|
|
||||||
Options:
|
|
||||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
|
||||||
2. Nano Banana Pro (image generation)
|
|
||||||
3. ChatGPT / Claude (text/code)
|
|
||||||
4. Other (tell me)
|
|
||||||
```
|
|
||||||
|
|
||||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WAIT FOR USER'S VISION
|
|
||||||
|
|
||||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
|
||||||
|
|
||||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
|
||||||
|
|
||||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
|
||||||
|
|
||||||
### CRITICAL: Match the FORMAT the research recommends
|
|
||||||
|
|
||||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
|
||||||
|
|
||||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
|
||||||
- Research says "structured parameters" → Use structured key: value format
|
|
||||||
- Research says "natural language" → Use conversational prose
|
|
||||||
- Research says "keyword lists" → Use comma-separated keywords
|
|
||||||
|
|
||||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
|
||||||
|
|
||||||
### Output Format:
|
|
||||||
|
|
||||||
```
|
|
||||||
Here's your prompt for {TARGET_TOOL}:
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
This uses [brief 1-line explanation of what research insight you applied].
|
|
||||||
```
|
|
||||||
|
|
||||||
### Quality Checklist:
|
|
||||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
|
||||||
- [ ] Directly addresses what the user said they want to create
|
|
||||||
- [ ] Uses specific patterns/keywords discovered in research
|
|
||||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
|
||||||
- [ ] Appropriate length and style for TARGET_TOOL
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## IF USER ASKS FOR MORE OPTIONS
|
|
||||||
|
|
||||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
|
||||||
|
|
||||||
After delivering a prompt, offer to write more:
|
|
||||||
|
|
||||||
> Want another prompt? Just tell me what you're creating next.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## CONTEXT MEMORY
|
|
||||||
|
|
||||||
For the rest of this conversation, remember:
|
|
||||||
- **TOPIC**: {topic}
|
|
||||||
- **TARGET_TOOL**: {tool}
|
|
||||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
|
||||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
|
||||||
|
|
||||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
|
||||||
|
|
||||||
When the user asks follow-up questions:
|
|
||||||
- **DO NOT run new WebSearches** - you already have the research
|
|
||||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
|
||||||
- **If they ask for a prompt** - write one using your expertise
|
|
||||||
- **If they ask a question** - answer it from your research findings
|
|
||||||
|
|
||||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Output Summary Footer (After Each Prompt)
|
|
||||||
|
|
||||||
After delivering a prompt, end with:
|
|
||||||
|
|
||||||
For **full/partial mode**:
|
|
||||||
```
|
|
||||||
---
|
|
||||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
|
||||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
|
||||||
|
|
||||||
Want another prompt? Just tell me what you're creating next.
|
|
||||||
```
|
|
||||||
|
|
||||||
For **web-only mode**:
|
|
||||||
```
|
|
||||||
---
|
|
||||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
|
||||||
📊 Based on: {n} web pages from {domains}
|
|
||||||
|
|
||||||
Want another prompt? Just tell me what you're creating next.
|
|
||||||
|
|
||||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
|
||||||
```
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
---
|
|
||||||
description: Research what people actually say about any topic in the last 30 days across Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.
|
|
||||||
argument-hint: <topic> — e.g. "nvidia earnings reaction" or "best noise cancelling headphones"
|
|
||||||
allowed-tools: [Bash, Read, Write, AskUserQuestion, WebSearch]
|
|
||||||
---
|
|
||||||
|
|
||||||
Invoke the `last30days` skill with the user's arguments: $ARGUMENTS
|
|
||||||
|
|
||||||
Use the skill's canonical pipeline (plan → retrieve → normalize → fuse → rerank → cluster → render). If the user provided no arguments, ask them for a topic before proceeding.
|
|
||||||
+12
-11
@@ -142,7 +142,7 @@ The repo vendors a search-only subset of Bird's Twitter GraphQL client and shell
|
|||||||
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
||||||
| Replies/quotes | Real | Real |
|
| Replies/quotes | Real | Real |
|
||||||
| Author handle | Real | Real |
|
| Author handle | Real | Real |
|
||||||
| Relevance score | Default 0.7 (re-ranked by score.py) | AI-assessed 0.0-1.0 |
|
| Relevance score | Default 0.7 (re-ranked by relevance.py) | AI-assessed 0.0-1.0 |
|
||||||
|
|
||||||
### Depth settings
|
### Depth settings
|
||||||
|
|
||||||
@@ -183,13 +183,14 @@ After both searches complete:
|
|||||||
|
|
||||||
| File | Purpose |
|
| File | Purpose |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `scripts/last30days.py` | Main orchestrator, concurrent execution |
|
| `skills/last30days/scripts/last30days.py` | Main CLI entry point |
|
||||||
| `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API |
|
| `skills/last30days/scripts/lib/pipeline.py` | Multi-source retrieval orchestration |
|
||||||
| `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
| `skills/last30days/scripts/lib/reddit_public.py` | Reddit public JSON search |
|
||||||
| `scripts/lib/xai_x.py` | X search via xAI API |
|
| `skills/last30days/scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||||
| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
| `skills/last30days/scripts/lib/xai_x.py` | X search via xAI API |
|
||||||
| `scripts/lib/models.py` | Auto-select best available model |
|
| `skills/last30days/scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||||
| `scripts/lib/env.py` | API key loading, source detection |
|
| `skills/last30days/scripts/lib/providers.py` | Reasoning provider and model selection |
|
||||||
| `scripts/lib/http.py` | HTTP transport with retries |
|
| `skills/last30days/scripts/lib/env.py` | API key loading, source detection |
|
||||||
| `scripts/lib/score.py` | Relevance scoring |
|
| `skills/last30days/scripts/lib/http.py` | HTTP transport with retries |
|
||||||
| `scripts/lib/dedupe.py` | URL-based deduplication |
|
| `skills/last30days/scripts/lib/relevance.py` | Query matching and relevance scoring |
|
||||||
|
| `skills/last30days/scripts/lib/dedupe.py` | URL-based deduplication |
|
||||||
|
|||||||
@@ -1,306 +0,0 @@
|
|||||||
---
|
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "feat: --competitors flag for auto-discovered comparison fan-out"
|
|
||||||
type: feat
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
---
|
|
||||||
|
|
||||||
# feat: --competitors flag for auto-discovered comparison fan-out
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Add a `--competitors` flag to the last30days engine that auto-discovers 2-4 peer entities for the topic, runs the full retrieval pipeline on each in parallel, and renders a multi-entity comparison. Invoking `last30days Kanye West --competitors` should resolve to "Kanye vs Drake vs Kendrick Lamar" and emit a comparison report covering all three. Invoking `last30days OpenAI --competitors` should resolve to "OpenAI vs Anthropic vs xAI vs Gemini" and emit a four-way comparison.
|
|
||||||
|
|
||||||
Discovery mirrors the existing `resolve.auto_resolve()` pattern used for X handles and subreddits at pipeline start — web search (Brave / Exa / Serper) plus deterministic extraction. Not an internal LLM call.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
Users who want a comparison today must type "OpenAI vs Anthropic vs xAI" themselves. The `planner._comparison_entities()` path already handles explicit multi-entity topics and `render._render_comparison_scaffold()` already emits a 9-axis comparison table. What is missing is the discovery half — a user who types a single entity with `--competitors` should get the comparison for free.
|
|
||||||
|
|
||||||
This is also the natural next step after the Step 0.55 category-peer subreddit work (PR #305, merged 2026-04-22). That feature widens the subreddit set within a single topic; this feature widens the entity set into peer entities.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. New `--competitors` boolean flag that triggers competitor discovery and multi-entity fan-out.
|
|
||||||
- R2. New `--competitors-list="A,B,C"` to explicitly skip discovery (mirrors `--plan`, `--subreddits`, `--x-handle` overrides).
|
|
||||||
- R3. New `--competitors=N` short form to set competitor count inline (N in 1..6).
|
|
||||||
- R4. Default count is 3 competitors (original + 3 = 4-way comparison).
|
|
||||||
- R5. Competitor retrieval depth inherits the main run's depth (`--quick` / `--deep`); all entities run in parallel so wall clock stays close to a single run.
|
|
||||||
- R6. Discovery mirrors `resolve.auto_resolve()`: web search for peers, deterministic text extraction. No internal LLM dependency.
|
|
||||||
- R7. If no web search backend is configured and no `--competitors-list` was passed, engine emits a LAW 7-style stderr telling the host agent to pass `--competitors-list` and exits non-zero.
|
|
||||||
- R8. Output rendering is a single comparison report covering all entities, reusing the existing 9-axis scaffold from `render._render_comparison_scaffold()` where applicable.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- Synthesis prompt changes beyond wiring N reports into the existing comparison scaffold are out of scope.
|
|
||||||
- `--competitors` does not replace the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`; both paths coexist.
|
|
||||||
- No caching layer for discovery results in v1.
|
|
||||||
- No UI/SKILL.md rewrite of the entire comparison section; only the new flag is documented.
|
|
||||||
- No new web search backend.
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Caching of competitor lookups: separate follow-up once hit rate justifies it.
|
|
||||||
- Disambiguation UX for topics with multiple common entities ("Amazon" the company vs the river): separate brainstorm.
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py:168-249` — `build_parser()` argparse definitions. Existing depth flags (`--quick`, `--deep`) and override flags (`--plan`, `--subreddits`, `--x-handle`, `--auto-resolve`) set the convention to mirror.
|
|
||||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the reference pattern: web search fan-out via `ThreadPoolExecutor`, per-query extraction functions, graceful empty-dict return when no backend is available.
|
|
||||||
- `scripts/lib/resolve.py:98-140` — `_extract_x_handle()` and sibling extractors show the deterministic text-mining style competitor extraction should mirror.
|
|
||||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature is the fan-out target. One call per entity, each returning a `schema.Report`.
|
|
||||||
- `scripts/lib/planner.py:430-564` — Existing comparison-intent handling and `_comparison_entities()` entity extraction. The new flag feeds the same mental model but populates entities from discovery instead of from the topic string.
|
|
||||||
- `scripts/lib/render.py:333-392` — `_render_comparison_scaffold()` already emits a 9-axis markdown comparison table. The new multi-report renderer should reuse this helper by assembling a synthetic "A vs B vs C" topic header for it.
|
|
||||||
- `scripts/lib/grounding.py` + `scripts/lib/providers.py` — Web search backend resolution (Brave / Exa / Serper). Reused as-is.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- No existing `docs/solutions/` entries for competitor discovery or multi-entity fan-out.
|
|
||||||
- Recent plan `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md` established the precedent of deterministic peer expansion; this plan extends that idea from subreddits to entities.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None gathered — local patterns are strong. `resolve.auto_resolve()` is a direct template.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Discovery mirrors auto_resolve, not plan_query.** Web search + regex extraction, not an LLM call. Matches the user's explicit direction ("use the python brain the same way it searches for X handles"). Cheaper, no provider credential requirement, deterministic.
|
|
||||||
- **Orchestration lives in `last30days.py` main, not inside `pipeline.run()`.** The fan-out is a top-level concern — one pipeline run per entity, each independent. Keeps `pipeline.run()` single-entity and unchanged except for sharing a `ThreadPoolExecutor` factory.
|
|
||||||
- **Sub-runs inherit main depth and run in parallel.** Wall clock ≈ single run; token cost scales linearly with N. User-controlled via the existing `--quick`/`--deep` flags.
|
|
||||||
- **New module `scripts/lib/competitors.py` instead of adding to `resolve.py`.** Keeps resolve focused on single-entity entity-bundle discovery (handles/subreddits/github); competitors.py owns peer-entity discovery. Similar shape, different responsibility.
|
|
||||||
- **Multi-report render is additive in `render.py`.** New `render_comparison_multi(reports: list[Report]) -> str` composes a synthetic "A vs B vs C" topic and delegates to the existing scaffold + synthesis path where possible. No rewrite of the single-entity render path.
|
|
||||||
- **Default count = 3 competitors (4-way comparison).** Hard cap at 6.
|
|
||||||
- **LAW 7-style stderr when no backend and no list.** Matches how `planner.plan_query()` already tells the hosting agent to pass `--plan`.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **Discovery mechanism:** Web search via `grounding.web_search()`, not an internal LLM. User confirmed the auto_resolve pattern is the target.
|
|
||||||
- **Default competitor count:** 3 (original + 3 = 4-way).
|
|
||||||
- **Sub-run depth:** Inherit main depth, parallel execution.
|
|
||||||
- **Flag naming:** `--competitors` (standard argparse double-dash). `--competitors=N` for inline count. `--competitors-list="A,B,C"` to skip discovery.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Exact extraction heuristics for competitor names across Brave / Exa / Serper result shapes. The SERP text varies (listicles, comparison pages, "vs" pages); the initial implementation will start with listicle parsing plus a "X vs Y" pattern match, and harden against real results in the test phase.
|
|
||||||
- Handling of topic ambiguity ("Amazon", "Apple"). Initial behavior: trust whatever web search returns for the topic verbatim; disambiguation is a separate concern.
|
|
||||||
- Merge strategy when two entities return overlapping URLs (e.g., an "OpenAI vs Anthropic" article shows up in both runs). Likely dedupe at the clustering step, but defer the exact policy until we see how often it happens.
|
|
||||||
- Whether to expose competitor discovery artifacts (the raw web search results) as a debug emit. Follow the existing `--debug` conventions.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: CLI flag parsing and validation**
|
|
||||||
|
|
||||||
**Goal:** Add `--competitors`, `--competitors=N`, and `--competitors-list` to the argparse surface, validate values, and thread them into the main orchestration.
|
|
||||||
|
|
||||||
**Requirements:** R1, R2, R3, R4
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py`
|
|
||||||
- Test: `tests/test_cli_competitors.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add three mutually cooperative flags near line 205 in `build_parser()`:
|
|
||||||
- `--competitors` with `nargs="?"` and `const=3` so bare `--competitors` defaults to 3, `--competitors=4` is honored, and `--competitors=0` is rejected
|
|
||||||
- `--competitors-list` free-text CSV
|
|
||||||
- Normalize in `main()`: if `--competitors-list` is present, skip discovery and use the list. If `--competitors` is set and no list, trigger discovery with count = the flag value. Clamp count to 1..6 with a stderr warning at boundary.
|
|
||||||
- Thread the resulting entity list into the orchestrator added in Unit 3.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `--plan` argument at `scripts/last30days.py:187` — same skip-discovery-when-explicit shape.
|
|
||||||
- `--subreddits` / `--x-handle` at `scripts/last30days.py:180,189` — same override semantics.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: bare `--competitors` parses to count=3, empty list.
|
|
||||||
- Happy path: `--competitors=4` parses to count=4.
|
|
||||||
- Happy path: `--competitors-list="A,B,C"` parses to count=3, list=["A","B","C"], and is preferred over any discovery signal.
|
|
||||||
- Edge case: `--competitors=0` and `--competitors=-1` are rejected with a clear error.
|
|
||||||
- Edge case: `--competitors=99` clamps to 6 with a stderr warning.
|
|
||||||
- Edge case: `--competitors` combined with `--competitors-list` uses the list and logs that discovery was skipped.
|
|
||||||
- Edge case: `--competitors-list` value with whitespace ("A, B , C") normalizes correctly.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Running the binary with each flag variation produces the expected post-parse state without calling out to the network.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: `scripts/lib/competitors.py` discovery module**
|
|
||||||
|
|
||||||
**Goal:** Discover peer entities for a topic using web search + deterministic extraction, mirroring `resolve.auto_resolve()`.
|
|
||||||
|
|
||||||
**Requirements:** R6, R7
|
|
||||||
|
|
||||||
**Dependencies:** None (pure module; wired by Unit 3)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `scripts/lib/competitors.py`
|
|
||||||
- Test: `tests/test_competitors.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Public entry point `discover_competitors(topic: str, count: int, config: dict) -> list[str]`.
|
|
||||||
- Early return `[]` when `_has_backend(config)` is false (reuse the helper from `resolve.py`; factor if needed).
|
|
||||||
- Fan out 2-3 web searches in a `ThreadPoolExecutor`:
|
|
||||||
- `"{topic} competitors"`
|
|
||||||
- `"{topic} alternatives"`
|
|
||||||
- `"{topic} vs"` (captures "X vs Y" articles)
|
|
||||||
- Feed results into a deterministic `_extract_peer_entities(results, topic)` that:
|
|
||||||
- Mines titles and snippets for capitalized noun phrases other than the topic itself
|
|
||||||
- Scores by frequency across results
|
|
||||||
- Filters stopwords and the topic's own tokens
|
|
||||||
- Returns top `count` unique entities ordered by score
|
|
||||||
- Emit a single-line stderr log mirroring the `resolve._log` format.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/lib/resolve.py:179-258` for the function shape, executor usage, and empty-result fallback.
|
|
||||||
- `scripts/lib/resolve.py:98-140` for extractor style (small, deterministic, no external state).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: canned SERP fixtures for "OpenAI" return ["Anthropic", "xAI", "Google"] or close peers in the top 3.
|
|
||||||
- Happy path: canned SERP fixtures for "Kanye West" return rap peers (Drake, Kendrick) in the top 3.
|
|
||||||
- Edge case: empty SERP results return `[]` without raising.
|
|
||||||
- Edge case: extractor filters out the topic itself (case- and punctuation-insensitive).
|
|
||||||
- Edge case: near-duplicate entities ("OpenAI" vs "Open AI") dedupe to one slot.
|
|
||||||
- Error path: web search backend raises — the failure is logged and the function returns `[]`.
|
|
||||||
- Edge case: count=1 returns a single-element list; count=6 returns up to six entities.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Unit tests pass with fixtures committed under `tests/fixtures/competitors-*.json`.
|
|
||||||
- Manual run against a live backend for one topic confirms sensible output (recorded as a notes file, not a test assertion).
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Parallel fan-out orchestrator**
|
|
||||||
|
|
||||||
**Goal:** Run `pipeline.run()` once per entity (topic + discovered competitors) in parallel, collect `schema.Report` per entity, and hand them to the comparison renderer.
|
|
||||||
|
|
||||||
**Requirements:** R5, R7
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1, Unit 2
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py`
|
|
||||||
- Possibly create: `scripts/lib/fanout.py` if the orchestrator grows past ~60 lines
|
|
||||||
- Test: `tests/test_competitor_fanout.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- After arg parsing and before the existing `pipeline.run()` call, branch on `args.competitors`:
|
|
||||||
- If a list was provided or discovery returned entities, build `entities = [topic, *competitors]`.
|
|
||||||
- Spawn one `pipeline.run()` per entity via `ThreadPoolExecutor(max_workers=len(entities))`, passing the same `config`, `depth`, and all sub-run-relevant args (mock, plan, etc.). Respect `--plan` — if a plan is passed it applies to the main topic only; competitors use the internal planner fallback for v1.
|
|
||||||
- Collect `{entity: Report}` mapping. A per-entity failure logs a stderr warning and drops that entity from the comparison; the run continues as long as 2 entities succeed.
|
|
||||||
- If fewer than 2 entities survive, exit with a clear error.
|
|
||||||
- LAW 7-style stderr:
|
|
||||||
- If `args.competitors` is set, no list was passed, no web search backend is configured, emit a LAW 7 stderr message pointing to the `--competitors-list` override and exit non-zero. Reuse the tone from `planner.plan_query()` fallback (`scripts/lib/planner.py:125-135`).
|
|
||||||
|
|
||||||
**Execution note:** Start with a failing integration test that exercises the full main → orchestrator → mocked pipeline.run path; the orchestrator is where bugs hide.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/lib/resolve.py:225-239` for ThreadPoolExecutor + as_completed + per-future error handling.
|
|
||||||
- `scripts/lib/pipeline.py:310+` for how ThreadPoolExecutor is already used inside a single run (same idiom, outer layer).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: main + 2 competitors, all three `pipeline.run()` calls succeed (mocked), orchestrator returns 3 Reports.
|
|
||||||
- Happy path: discovery returns the competitor list; orchestrator fans out accordingly.
|
|
||||||
- Edge case: one of three competitor pipelines raises — the run continues with the surviving 2 and emits a warning.
|
|
||||||
- Edge case: all competitors fail but the main topic succeeds — orchestrator exits non-zero with a clear error rather than silently degrading to a single-entity render.
|
|
||||||
- Edge case: `--competitors` set, no backend, no list — orchestrator emits the LAW 7 stderr and exits non-zero before any pipeline call.
|
|
||||||
- Integration: wall-clock time for 3 mocked pipelines in parallel is close to the slowest single run, not the sum (timing assertion with generous margin).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- End-to-end test with mocked `pipeline.run()` and mocked competitors discovery produces 3 Reports and hands them to a stubbed renderer.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: Multi-report comparison renderer**
|
|
||||||
|
|
||||||
**Goal:** Compose N `schema.Report`s into a single comparison-mode output, reusing the existing 9-axis scaffold.
|
|
||||||
|
|
||||||
**Requirements:** R8
|
|
||||||
|
|
||||||
**Dependencies:** Unit 3
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/render.py`
|
|
||||||
- Test: `tests/test_render_comparison_multi.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `render_comparison_multi(reports: list[schema.Report], *, emit: str) -> str`.
|
|
||||||
- Build a synthetic comparison topic: `f"{entity_a} vs {entity_b} vs {entity_c}"`.
|
|
||||||
- Reuse `_render_comparison_scaffold()` for the table skeleton. Each entity column is populated from its own Report's top clusters and citations.
|
|
||||||
- For the narrative synthesis block, concatenate per-entity highlights, clearly labeled by entity, under a shared "Comparison" header.
|
|
||||||
- Preserve existing emit modes (`compact`, `md`, `json`, `context`). In `json` emit, return a `{"entities": [...], "reports": [...]}` shape; single-Report consumers remain unaffected because the single-report render path is untouched.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/lib/render.py:333-392` (`_parse_comparison_entities`, `_render_comparison_scaffold`) — the scaffold is the contract.
|
|
||||||
- `scripts/lib/render.py` single-report rendering — for per-entity narrative blocks.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: 3 Reports with distinct clusters render into a 3-column table and a "Comparison" section that mentions each entity at least once.
|
|
||||||
- Happy path: 2 Reports render as a 2-column table without breaking the scaffold.
|
|
||||||
- Edge case: a Report with an empty cluster list renders as "(no significant discussion this month)" in its column rather than crashing.
|
|
||||||
- Edge case: Reports with overlapping URLs (same article cited by two entities) dedupe citations at the footer but keep both column entries.
|
|
||||||
- Emit variants: `--emit=compact`, `--emit=md`, `--emit=json`, `--emit=context` each produce valid output with all entities represented.
|
|
||||||
- Integration: end-to-end snapshot test using fixture Reports, checked against a stored expected output (with a clear update path when the scaffold intentionally evolves).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Snapshot tests pass. Manual review of one real 3-way comparison confirms readability.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: Docs, SKILL.md mention, and sync**
|
|
||||||
|
|
||||||
**Goal:** Document the new flag so the hosting agent and human users both know it exists, and run the sync script.
|
|
||||||
|
|
||||||
**Requirements:** R1-R8 (surfaces them to users)
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-4
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md`
|
|
||||||
- Modify: `README.md` (brief flag reference)
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add a compact "Competitor mode" subsection under the existing comparison docs in `SKILL.md`. Document the flag, the default count, the override flag, and the LAW 7 fallback stderr.
|
|
||||||
- Keep `README.md` addition to a single example line.
|
|
||||||
- CHANGELOG entry mirrors the voice of recent entries (imperative, outcome-first).
|
|
||||||
- Sync via `scripts/sync.sh` per CLAUDE.md rules so `~/.claude/`, `~/.agents/`, `~/.codex/` pick up the new SKILL.md.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — documentation and sync only. Verification is by inspection and by running `sync.sh` and confirming target directories updated.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `sync.sh` completes without errors.
|
|
||||||
- `SKILL.md` rendered preview mentions `--competitors` in the comparison section.
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** `last30days.py main()` now orchestrates multiple `pipeline.run()` calls instead of one. No other callers of `pipeline.run()` are affected (it remains single-entity).
|
|
||||||
- **Error propagation:** Per-entity failures degrade gracefully as long as ≥2 entities survive; fewer survivors exits non-zero. Discovery failure with `--competitors` and no list is fatal.
|
|
||||||
- **State lifecycle risks:** Each sub-run uses its own `pipeline.run()` state; no shared mutable config. The `config` dict is read-only in `pipeline.run()` today — verify before committing to shared-reference passing, else deep-copy per sub-run.
|
|
||||||
- **API surface parity:** `--competitors` coexists with the existing explicit "A vs B vs C" topic parsing in `planner._comparison_entities()`. Both produce comparable output formats; the only difference is where the entity list came from.
|
|
||||||
- **Integration coverage:** The fan-out orchestrator crosses CLI → discovery → N pipelines → render; integration tests in Unit 3 and Unit 4 must exercise the full path end to end, not just unit-level.
|
|
||||||
- **Unchanged invariants:** `pipeline.run()` signature and single-entity semantics are unchanged. The single-entity render path in `render.py` is unchanged. No changes to `planner.plan_query()`. No changes to existing flags.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Competitor discovery returns garbage entities for niche topics. | `--competitors-list` override lets the user (or hosting agent) correct it. Unit tests with edge-case fixtures. Log discovery output to stderr under `--debug`. |
|
|
||||||
| Token cost scales linearly with N sub-runs. | Default count capped at 3, hard max 6, inherit `--quick` to let users throttle. Wall clock stays parallel. Emit a cost hint to stderr when N ≥ 4. |
|
|
||||||
| Merge conflicts against the single-entity render path during refactoring. | Keep the multi-report renderer strictly additive; do not modify the single-Report code path. |
|
|
||||||
| Config dict mutation inside sub-runs could leak state between entities. | Verify read-only usage before sharing references. If any sub-component mutates, deep-copy per sub-run before spawning threads. |
|
|
||||||
| A SERP extractor that works on Brave fixtures breaks on Exa/Serper result shapes. | Test fixtures for all three backends. Extractor operates on a normalized shape from `grounding.web_search()` (already the case), not raw provider output. |
|
|
||||||
| Hosting agent (Claude Code, Codex) unaware of the new flag when it could usefully pass `--competitors-list`. | SKILL.md updated in Unit 5 documents the flag in the same style as `--plan` and `--auto-resolve`. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first: per `CLAUDE.md`, experimental changes go to `mvanhorn/last30days-skill-private` on the `/last30days-beta` command. Land this on the private repo first, shake out on real topics for a day or two, then cherry-pick to public.
|
|
||||||
- After land-merge: run `scripts/sync.sh` to deploy SKILL.md + scripts to `~/.claude/`, `~/.agents/`, `~/.codex/`.
|
|
||||||
- Release notes entry in CHANGELOG.md follows the v3.0.9 voice — outcome-first, one paragraph.
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Related code: `scripts/lib/resolve.py:179` (`auto_resolve`), `scripts/lib/pipeline.py:162` (`pipeline.run`), `scripts/lib/planner.py:80` (`plan_query` LAW 7 fallback), `scripts/lib/render.py:333` (comparison scaffold)
|
|
||||||
- Related PRs: #305 (Step 0.55 category-peer subreddit expansion — the precedent for deterministic peer expansion, merged 2026-04-22)
|
|
||||||
- Related plan: `docs/plans/2026-04-22-001-fix-category-peer-subreddit-resolution-plan.md`
|
|
||||||
@@ -1,352 +0,0 @@
|
|||||||
---
|
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "fix: per-entity resolution, default-2, and stale-path guard for --competitors"
|
|
||||||
type: fix
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
origin: docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md
|
|
||||||
---
|
|
||||||
|
|
||||||
# fix: per-entity resolution, default-2, and stale-path guard for --competitors
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Three test runs of v3.0.11 `--competitors` surfaced four real bugs plus one product tweak. This plan fixes all of them in a single follow-up:
|
|
||||||
|
|
||||||
1. Competitor sub-runs get no Step 0.55 resolution (no X handle, no subreddits, no GitHub repo). Drake / Kendrick / Travis ran with deterministic-fallback single-word queries while Kanye had the full targeting package. User called it "lazy" and was right.
|
|
||||||
2. Two of three test windows (Linear, Coinbase) never invoked the new flag at all. They loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (a Claude-Code-managed git clone pinned to origin/main, which predates PR #308) instead of `plugins/cache/last30days-skill/last30days/3.0.11/`, so `--help` showed no `--competitors` flag and the model fell back to the manual comparison path.
|
|
||||||
3. Each competitor sub-run emits a scary `[Planner] No --plan passed... deterministic fallback` stderr line because LAW 7 targets the hosting-model path, not internal fan-out sub-runs.
|
|
||||||
4. Default competitor count is 3 (→ 4-way comparison). User wants default 2 (→ 3-way: original + 2 peers). Flag keeps `--competitors=N` to customize.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
The 3 test runs (Kanye, Linear, Coinbase) showed a pattern:
|
|
||||||
|
|
||||||
| Window | Loaded SKILL.md from | Invoked --competitors? | Per-entity resolution? | Outcome |
|
|
||||||
|--------|----------------------|-----------------------|------------------------|---------|
|
|
||||||
| Kanye | cache/3.0.11/ (correct) | Yes | Only for main topic (Kanye) | Drake/Kendrick/Travis thin; Reddit 403 fallbacks |
|
|
||||||
| Linear | marketplaces/ (stale) | No — fell back to manual comparison | No | Thin run with noisy subreddits |
|
|
||||||
| Coinbase | marketplaces/ (stale) | No — fell back to manual comparison | Main only; keyword-search poisoned pool | Top subs: r/survivor, r/Airpodsmax (noise) |
|
|
||||||
|
|
||||||
Root causes:
|
|
||||||
- **Per-entity resolution gap:** `scripts/lib/fanout.py` calls `pipeline.run()` with topic + depth + web_backend + lookback_days only. It does not call `resolve.auto_resolve()` per entity, so sub-runs have no X handle, subreddit, or GitHub targeting. The original plan (`2026-04-22-002`) acknowledged this as a deliberate v1 simplification ("competitor sub-runs use planner defaults"). In practice this produces visibly asymmetric output and triggers downstream retrieval issues (403 fallbacks, keyword-search noise).
|
|
||||||
- **Stale-path loading:** Claude Code's skill loader alphabetizes `find` results with `marketplaces/` before `cache/`, and the model reads the first plausible SKILL.md it sees. SKILL.md line 823's `SKILL_ROOT` resolver is the correct path but only fires in engine-invocation blocks, not in the skill-load step.
|
|
||||||
- **LAW 7 in sub-runs:** LAW 7 exists because the *hosting reasoning model* is supposed to pass `--plan`. For competitor sub-runs, there is no hosting-model planning — it's an engine-internal fan-out. The warning is a false positive there.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. Default `--competitors` count is 2 peers (3-way comparison: original + 2).
|
|
||||||
- R2. Each competitor sub-run performs Step 0.55 resolution (X handle, subreddits, GitHub user/repos, news context) before its pipeline runs — not just the main topic.
|
|
||||||
- R3. Sub-runs do not emit the LAW 7 `No --plan passed` warning; they are internal fan-out, not hosting-model calls.
|
|
||||||
- R4. The rendered comparison output includes a visible "Resolved entities" block showing per-entity handles/subs/github for debug transparency (answers "did it resolve everyone?" without the user having to read stderr).
|
|
||||||
- R5. SKILL.md has a canonical-path self-check at the top: if the reader loaded it from anywhere other than `plugins/cache/last30days-skill/last30days/{VERSION}/`, re-read from the versioned path before proceeding.
|
|
||||||
- R6. Version bumps to 3.0.12; CHANGELOG entry; `scripts/sync.sh` deploys.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No new discovery strategy. The web-search + regex extraction in `scripts/lib/competitors.py` stays as-is.
|
|
||||||
- No new CLI flags beyond the behavior changes above. Specifically: no per-entity override flags like `--competitor-handles`. The hosting-model escape hatch remains `--competitors-list`.
|
|
||||||
- No changes to the explicit `A vs B` comparison path (topic-string parsing in `planner._comparison_entities`).
|
|
||||||
- No marketplace-clone auto-restore fix — that's Claude Code harness behavior. This plan only guards against the symptom on the skill side.
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Caching of per-entity resolution results: separate follow-up once hit rate justifies it.
|
|
||||||
- Fan-out rate-limiting tuning (currently `max_workers=len(entities)+1`, capped at 6): defer until we see real-world quota exhaustion.
|
|
||||||
- Pre-flight cost hint when N ≥ 4 (noted in `2026-04-22-002` risks): defer.
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py:205-219` — `--competitors` / `--competitors-list` argparse definition (const=3 today; changing to 2).
|
|
||||||
- `scripts/last30days.py:220-290` — `resolve_competitors_args()` validator; update `COMPETITORS_DEFAULT`.
|
|
||||||
- `scripts/last30days.py:438-520` — main() fan-out orchestration; currently passes only topic/depth to each `_competitor_runner`.
|
|
||||||
- `scripts/lib/fanout.py:40-95` — `run_competitor_fanout()` signature. The `competitor_runner` callable is where per-entity resolution needs to happen.
|
|
||||||
- `scripts/lib/resolve.py:179-258` — `auto_resolve()` is the exact per-entity resolver to reuse. Already does X handle + subreddits + GitHub user/repos + news context in parallel via ThreadPoolExecutor.
|
|
||||||
- `scripts/lib/planner.py:80-135` — `plan_query()` emits the LAW 7 stderr. A `quiet: bool` keyword or `internal_subrun: bool` flag will suppress it.
|
|
||||||
- `scripts/lib/pipeline.py:162-220` — `pipeline.run()` signature. Needs a new keyword to propagate quiet-mode down to the planner.
|
|
||||||
- `scripts/lib/render.py:render_comparison_multi` — where the "Resolved entities" block is inserted.
|
|
||||||
- `SKILL.md` line 823 — canonical `SKILL_ROOT` resolver already exists but fires in engine bash, not at skill-load time.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md` acknowledged the per-entity-resolution gap as a v1 tradeoff. This plan closes that gap.
|
|
||||||
- Kanye run stderr: `[Planner] No --plan passed... deterministic fallback` × 3 (once per competitor sub-run). That's the LAW 7 noise R3 targets.
|
|
||||||
- Linear / Coinbase runs loaded `plugins/marketplaces/last30days-skill/CLAUDE.md` as the first hit. That's the stale-path issue R5 targets.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None. All patterns are in-repo.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Per-entity resolve happens inside fanout, not in SKILL.md.** The user-facing promise of `--competitors` is "one flag, engine does the work." Pushing resolution onto the hosting model creates another path-of-least-resistance trap (model skips it, output looks lazy). Auto-resolve inside each sub-run when a web backend is available makes the feature self-contained.
|
|
||||||
- **Stale-path guard is a SKILL.md self-check, not a code change.** We cannot stop Claude Code from auto-restoring the marketplace clone. But we can put a 3-line banner at the top of SKILL.md that forces any path-mismatched read to re-read from the versioned cache. Both the marketplace copy (once main catches up) and the cache copy carry the guard.
|
|
||||||
- **LAW 7 suppression is opt-in via `internal_subrun=True` keyword.** Do not remove the warning from the default path — it's load-bearing for the hosting-model contract. Add an explicit bypass for engine-internal fan-out only.
|
|
||||||
- **Default 2, hard max 6 unchanged.** "Original + 2" matches the Kanye/Drake/Kendrick mental model from the feature description. Still allow `--competitors=N` from 1 to 6.
|
|
||||||
- **Resolved block is inside the EVIDENCE envelope, not above it.** Keeps the rendered output structure stable for the synthesis contract (LAW 1–8). The block is context, not output.
|
|
||||||
- **Skip auto-resolve when `--mock` or no web backend.** Mirrors the existing `resolve.auto_resolve()` fast-fail and keeps the mock test path deterministic.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **Where does per-entity resolve live?** Inside `fanout.run_competitor_fanout`, not in `main()`. Each sub-run calls `auto_resolve()` just before `pipeline.run()`.
|
|
||||||
- **Should the hosting model still be able to override?** Yes — `--competitors-list` remains the escape hatch. When an explicit list is passed, the engine still does auto-resolve per entity; the user's list just skips discovery.
|
|
||||||
- **Should sub-runs run auto-resolve in parallel with each other?** Yes. The existing `ThreadPoolExecutor` in fanout already parallelizes sub-runs; auto-resolve happens inside each sub-run's thread, so resolve calls for different entities run concurrently.
|
|
||||||
- **Default count:** 2 peers (3-way). Confirmed.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Whether to expose a `--no-auto-resolve-competitors` flag for power users who want the fast, shallow behavior. Probably not needed v2; ship auto-resolve always-on and revisit if someone complains about cost.
|
|
||||||
- Whether to surface the per-entity resolution context back into the main topic's planner (cross-entity context sharing). Stays deferred.
|
|
||||||
- Whether the Resolved block should be collapsible or always inline. Start inline; revisit based on output length feedback.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: Default `--competitors` to 2 peers**
|
|
||||||
|
|
||||||
**Goal:** Change the bare `--competitors` default from 3 to 2 per user feedback. `--competitors=N` still overrides; range 1..6 unchanged.
|
|
||||||
|
|
||||||
**Requirements:** R1
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (`COMPETITORS_DEFAULT`, `--competitors` const, stderr messages if any reference 3)
|
|
||||||
- Modify: `SKILL.md` Competitor mode section ("discovered 2-6" wording, bare-flag default line)
|
|
||||||
- Modify: `README.md` auto-discovered example line (if it references count)
|
|
||||||
- Test: `tests/test_cli_competitors.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Change `COMPETITORS_DEFAULT = 3` → `2` in `scripts/last30days.py`.
|
|
||||||
- Change argparse `--competitors` `const=3` → `const=2`.
|
|
||||||
- Update any SKILL.md / README copy referencing "3 peers" to "2 peers" (default) or "2-6 peers" (range).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing default constants in `scripts/last30days.py` argparse block.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: bare `--competitors` yields count=2, enabled=True, empty explicit_list.
|
|
||||||
- Edge case: `--competitors=3` still works (explicit override).
|
|
||||||
- Edge case: existing `test_bare_flag_defaults_to_three` test is updated to `test_bare_flag_defaults_to_two` and asserts count=2.
|
|
||||||
- Edge case: `--competitors=5` with a `--competitors-list` of length 2 still logs the mismatch warning and uses the list.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `pytest tests/test_cli_competitors.py -v` passes with the updated default.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: Per-entity Step 0.55 resolution inside fanout**
|
|
||||||
|
|
||||||
**Goal:** Each competitor sub-run auto-resolves its own X handle, subreddits, GitHub user/repos, and news context via `resolve.auto_resolve()` before its `pipeline.run()` call — just like the main topic.
|
|
||||||
|
|
||||||
**Requirements:** R2
|
|
||||||
|
|
||||||
**Dependencies:** None (but Unit 3 should land together so sub-runs don't emit LAW 7 stderr while the resolution context is being passed)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/fanout.py`
|
|
||||||
- Modify: `scripts/last30days.py` (`_competitor_runner` closure builds the resolved args)
|
|
||||||
- Test: `tests/test_competitor_fanout.py`
|
|
||||||
- Test: `tests/test_competitors_resolve_integration.py` (new; covers the auto-resolve path)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- `_competitor_runner(entity)` in main() does:
|
|
||||||
1. Call `resolve.auto_resolve(entity, config)` when `not args.mock` and a web backend is configured (reuse `_has_backend`).
|
|
||||||
2. Extract resolved x_handle, subreddits, github_user, github_repos, context.
|
|
||||||
3. Pass them to `pipeline.run()` for that sub-run.
|
|
||||||
4. Inject resolved context into a per-entity config copy (so `_auto_resolve_context` does not leak across sub-runs — deep-copy the config or use a local dict).
|
|
||||||
5. Store the resolved block on the Report's `artifacts` so the renderer can surface it (Unit 4).
|
|
||||||
- When `args.mock` is True or no backend is available, skip auto-resolve (fall through to planner defaults, matching the existing `auto_resolve()` early-return contract).
|
|
||||||
- Update `fanout.run_competitor_fanout` docstring to note that auto-resolve happens inside the caller-provided runner.
|
|
||||||
|
|
||||||
**Execution note:** Start with a failing integration test that exercises two-entity fanout + auto-resolve via a mocked `resolve.auto_resolve` and asserts that `pipeline.run` receives the resolved x_handle/subreddits for each entity.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `scripts/last30days.py` main topic branch (`if args.auto_resolve and not external_plan`) already calls `resolve.auto_resolve` and propagates results — mirror the shape for competitors.
|
|
||||||
- Config isolation: `scripts/lib/pipeline.py:162-220` reads config as-is; use `dict(config)` to avoid cross-sub-run mutation of `_auto_resolve_context`.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: 3 entities, mocked `auto_resolve` returns distinct handles per entity; `pipeline.run` receives `x_handle=@drake` for Drake, `x_handle=@kendricklamar` for Kendrick, etc.
|
|
||||||
- Happy path: the main topic still uses the user-supplied `--x-handle` / `--subreddits` overrides (not overwritten by auto-resolve for the main). Competitors use their own auto-resolved values.
|
|
||||||
- Edge case: `--mock` skips auto-resolve entirely for all sub-runs (no `resolve.auto_resolve` calls).
|
|
||||||
- Edge case: `resolve.auto_resolve` returns empty dicts for one entity (low-signal topic) — the sub-run still executes with planner defaults; doesn't crash.
|
|
||||||
- Edge case: no web backend configured — auto-resolve returns empty for every entity, sub-runs fall through to planner defaults, no stack trace.
|
|
||||||
- Error path: `resolve.auto_resolve` raises — the sub-run logs a warning and continues with planner defaults (does not fail the whole comparison).
|
|
||||||
- Integration: config `_auto_resolve_context` from entity A does not leak into entity B's `pipeline.run`. Assert each sub-run gets its own context string.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- New integration test passes.
|
|
||||||
- End-to-end smoke (mock mode + explicit list): each sub-run's stderr shows `[AutoResolve]` lines per entity with distinct values.
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Suppress LAW 7 warning for engine-internal sub-runs**
|
|
||||||
|
|
||||||
**Goal:** The `[Planner] No --plan passed... deterministic fallback` warning does not fire during competitor sub-runs. LAW 7 is load-bearing for hosting-model contracts and must stay on the default path; this is an opt-in bypass for internal fan-out only.
|
|
||||||
|
|
||||||
**Requirements:** R3
|
|
||||||
|
|
||||||
**Dependencies:** Unit 2 (so the sub-run call site is already being modified)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/planner.py` (`plan_query` signature + conditional stderr)
|
|
||||||
- Modify: `scripts/lib/pipeline.py` (`run` signature + propagation)
|
|
||||||
- Modify: `scripts/last30days.py` or `scripts/lib/fanout.py` (pass `internal_subrun=True` for competitor runners)
|
|
||||||
- Test: `tests/test_planner_v3.py` (or new `tests/test_planner_quiet_mode.py`)
|
|
||||||
- Test: `tests/test_competitor_fanout.py` (assert sub-runs don't emit LAW 7 stderr)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add a keyword `internal_subrun: bool = False` to `planner.plan_query`. When True, skip the two `print(..., file=sys.stderr)` blocks that emit the LAW 7 banner and the `[Planner] No --plan passed` capability message.
|
|
||||||
- Add the same keyword to `pipeline.run()`; pass through to `plan_query`.
|
|
||||||
- In main()/fanout, set `internal_subrun=True` for every competitor sub-run's pipeline.run call. The main topic's pipeline.run keeps the default (LAW 7 stays on for the hosting-model path).
|
|
||||||
- Also suppress the LAW 7-triggered degraded-run warning block in the render layer for sub-reports when the envelope is going to be merged into a comparison output (or accept that the block is per-entity and surfaces once per entity).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing keyword-only parameters on `pipeline.run` (`mock`, `x_handle`, etc.).
|
|
||||||
- `planner.plan_query` signature is already keyword-only.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `plan_query(..., internal_subrun=True, provider=None, model=None)` returns the deterministic fallback plan WITHOUT writing the LAW 7 stderr block.
|
|
||||||
- Happy path: `plan_query(...)` with default `internal_subrun=False` still writes the LAW 7 warning (unchanged behavior).
|
|
||||||
- Integration: end-to-end competitor fanout; assert captured stderr contains zero occurrences of `No --plan passed` and zero of `YOU ARE the planner`.
|
|
||||||
- Integration: main topic is not part of competitor mode; if the user invokes bare `/last30days OpenAI` without `--plan`, LAW 7 stderr fires exactly once (regression test).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Running the Kanye-style smoke test shows zero `[Planner] No --plan passed` lines for Drake / Kendrick / Travis sub-runs.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: "Resolved entities" block in comparison output**
|
|
||||||
|
|
||||||
**Goal:** The rendered comparison output includes a visible block listing per-entity handles, subreddits, GitHub user, and resolved context. Answers "did it resolve everyone?" at a glance without reading stderr.
|
|
||||||
|
|
||||||
**Requirements:** R4
|
|
||||||
|
|
||||||
**Dependencies:** Unit 2 (needs resolved data on report artifacts)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/render.py` (`render_comparison_multi` and `render_comparison_multi_context`)
|
|
||||||
- Test: `tests/test_render_comparison_multi.py`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- When each entity's `Report.artifacts` contains a `resolved` dict (populated by Unit 2), `render_comparison_multi` emits a `## Resolved Entities` block early in the EVIDENCE envelope:
|
|
||||||
```
|
|
||||||
## Resolved Entities
|
|
||||||
- **Kanye West**: X @kanyewest | Subs r/Kanye, r/hiphopheads | GitHub: — | Context: BULLY released, UK ban…
|
|
||||||
- **Drake**: X @Drake | Subs r/DrakeTheType, r/hiphopheads | GitHub: — | Context: ICEMAN rollout…
|
|
||||||
- **Kendrick Lamar**: X @kendricklamar | Subs r/KendrickLamar | GitHub: — | Context: Grammy wins, dormant…
|
|
||||||
```
|
|
||||||
- Missing fields render as `—` not empty.
|
|
||||||
- When no entity has a `resolved` payload (mock mode, no web backend), omit the block entirely rather than emit an empty section.
|
|
||||||
- Context strings are truncated at 120 chars to keep the block scannable.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing `render_comparison_multi` envelope structure (lines ~395-480 in render.py).
|
|
||||||
- Existing per-entity evidence block format (`## {label}`) for consistency.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: 3 entities each with a `resolved` artifact → block lists all 3 with their fields.
|
|
||||||
- Happy path: 2 entities, one with full resolution, one with partial (x_handle only) → missing fields render as `—`.
|
|
||||||
- Edge case: no entity has a resolved artifact → block is omitted entirely.
|
|
||||||
- Edge case: context string > 120 chars → truncated with ellipsis.
|
|
||||||
- Integration: rendered output passes through the same EVIDENCE envelope comments and synthesis contract (LAW 1–8 unchanged).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Snapshot tests confirm the block appears in the right spot with the right formatting.
|
|
||||||
- End-to-end smoke shows a realistic 3-entity Resolved block in the rendered output.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: SKILL.md canonical-path self-check**
|
|
||||||
|
|
||||||
**Goal:** A top-of-file SKILL.md directive forces any reader (Claude Code, Codex, Hermes, Gemini) to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` before proceeding. If loaded from `marketplaces/` or any other path, re-read from the pinned versioned cache.
|
|
||||||
|
|
||||||
**Requirements:** R5
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md` (prepend a STEP 0 block before the existing STEP 0 / LAW list)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add a numbered first step at the top (before or bundled with existing "STEP 0: ToolSearch preload"):
|
|
||||||
```
|
|
||||||
## STEP 0: Canonical Path Self-Check (must run first)
|
|
||||||
|
|
||||||
Before reading anything else below, verify you loaded this SKILL.md from
|
|
||||||
the versioned cache, not the marketplace clone:
|
|
||||||
|
|
||||||
CANONICAL=$HOME/.claude/plugins/cache/last30days-skill/last30days/
|
|
||||||
CANONICAL_LATEST=$(ls -d "$CANONICAL"*/ 2>/dev/null | sort -V | tail -1)
|
|
||||||
|
|
||||||
If the SKILL.md you just read is not under $CANONICAL_LATEST, STOP. Re-read
|
|
||||||
$CANONICAL_LATEST/SKILL.md and restart from here. Marketplace clones
|
|
||||||
(`plugins/marketplaces/last30days-skill/`) are pinned to origin/main and
|
|
||||||
can be stale; the versioned cache is the ground truth.
|
|
||||||
```
|
|
||||||
- Reinforce in the existing LAW 7 block that `--help` output must be read from the same pinned `SKILL_ROOT` to avoid flag-list skew.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing STEP 0 ToolSearch preload (top of SKILL.md) for tone / imperative voice.
|
|
||||||
- Existing `SKILL_ROOT` resolver snippet (line ~823).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — SKILL.md is documentation; no unit test, verified by follow-up user invocation.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- In a fresh Claude Code window, `/last30days Test --competitors` loads SKILL.md, the model executes the STEP 0 self-check, and (if it had loaded from marketplaces/) switches to the cache path before running `--help` or the engine. Observable via the model's announced reasoning / task list.
|
|
||||||
|
|
||||||
- [ ] **Unit 6: Version bump, CHANGELOG, sync**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.12 and deploy to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** R6
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-5
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json` (version 3.0.11 → 3.0.12)
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG entry under `## [3.0.12]` dated 2026-04-22 covering the four fixes (Fixed: per-entity resolution; Fixed: LAW 7 sub-run noise; Changed: default count 3→2; Added: Resolved entities block; Added: canonical-path self-check in SKILL.md).
|
|
||||||
- `sync.sh` deploys to `~/.claude/plugins/cache/last30days-skill-private/...`, `~/.agents/`, `~/.codex/`, Hermes.
|
|
||||||
- Manual hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.12/` so the public `/last30days` slash command picks up the new version before PR merge (matches the 3.0.11 testing pattern).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging only. Verification is by inspection.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` returns `3.0.12`.
|
|
||||||
- `sync.sh` exits 0 with "Import check: OK" for each target.
|
|
||||||
- Hot-copied 3.0.12 directory contains the new files and `/last30days` picks up the new version (highest-version resolver).
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** Fanout sub-runs now call `resolve.auto_resolve` per entity. Each sub-run is independent; no shared mutable state with other sub-runs or with the main topic.
|
|
||||||
- **Error propagation:** `auto_resolve` failures inside a sub-run log a warning and degrade to planner defaults; do not propagate up to abort the comparison. Same contract as today for the main topic.
|
|
||||||
- **State lifecycle risks:** Config dict is mutated by `auto_resolve` (via `config["_auto_resolve_context"]`). Must deep-copy per sub-run or scope context to a local mapping — otherwise two sub-runs' context strings race.
|
|
||||||
- **API surface parity:** `pipeline.run` gains a keyword (`internal_subrun`); callers that don't pass it get the existing behavior. `planner.plan_query` gains the same. Backward compatible.
|
|
||||||
- **Integration coverage:** New integration test for the fanout + auto-resolve + render chain. Existing snapshot tests update to include the Resolved block.
|
|
||||||
- **Unchanged invariants:** Single-entity `/last30days` invocations (no `--competitors`) behave identically. Explicit `A vs B` comparison topics behave identically. LAW 7 still fires on the default hosting-model path. `render_compact` path is untouched.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Auto-resolving per competitor triples the WebSearch call volume (4 queries × 3 competitors = 12 extra web searches). | Fast-fail when no backend; user can pass `--competitors-list` to skip discovery but still get auto-resolve. Cost note in CHANGELOG. |
|
|
||||||
| Config mutation across sub-runs via `_auto_resolve_context`. | Unit 2 deep-copies config per sub-run before each `auto_resolve` + `pipeline.run` call. Integration test asserts no cross-entity leak. |
|
|
||||||
| LAW 7 suppression leaks onto the hosting-model path via a wrong default. | Default `internal_subrun=False`. Only fanout's competitor sub-runs set True. Unit test asserts bare-topic invocation still emits LAW 7. |
|
|
||||||
| SKILL.md STEP 0 banner gets ignored by the model (same failure mode as line 823 today). | Put it in the guaranteed-read top band (before LAW 1, above all other content), imperative voice, concrete `STOP` verb. Still not bulletproof but strictly better than current. |
|
|
||||||
| Default count change breaks assumptions in downstream tools or existing user muscle memory. | Changelog calls it out as Changed; `--competitors=3` still works for users who want the old default. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first: merge behind `/last30days-beta` via the private repo before cherry-picking to public. Follows the same process as 3.0.11.
|
|
||||||
- Version 3.0.12 is a fix release; no marketing post required.
|
|
||||||
- After merge, add a line to the PR description pointing at this plan.
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Origin plan: `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
|
||||||
- Related PR: #308 (v3.0.11 shipping --competitors)
|
|
||||||
- Test windows that surfaced the bugs: Kanye, Linear, Coinbase (2026-04-22 session)
|
|
||||||
- Related code: `scripts/lib/fanout.py`, `scripts/lib/resolve.py` (`auto_resolve`), `scripts/lib/planner.py` (`plan_query`), `scripts/lib/render.py` (`render_comparison_multi`)
|
|
||||||
-394
@@ -1,394 +0,0 @@
|
|||||||
---
|
|
||||||
title: "fix: --competitors runs a full last30days per entity with hosting-model pre-resolve"
|
|
||||||
type: fix
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
origin: docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md
|
|
||||||
---
|
|
||||||
|
|
||||||
# fix: --competitors runs a full last30days per entity with hosting-model pre-resolve
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
User intent confirmed 2026-04-22: `--competitors` should run a full single-entity `last30days` pipeline for the main topic AND for each discovered peer — three independent full-depth passes, each with its own Step 0.55 resolution, own X handle primary weight, own subreddit targeting, own GitHub repo scoping. Then merge them into the comparison output.
|
|
||||||
|
|
||||||
3.0.12 already built the N-parallel-pipelines orchestration (`scripts/lib/fanout.py`). What it got wrong: it tried to do per-entity Step 0.55 engine-side via `resolve.auto_resolve()`, which requires a web search backend key (BRAVE/EXA/SERPER/PARALLEL/OPENROUTER). Matt runs from Claude Code, which has its own WebSearch tool. The engine has none of those keys, so per-entity auto_resolve silently no-ops and all peer sub-runs fall through to deterministic single-word planner queries.
|
|
||||||
|
|
||||||
Four 2026-04-22 test runs (Warriors, Seattle, Arizona Wildcats, Kanye West) confirmed this via engine receipts:
|
|
||||||
|
|
||||||
- Compact Resolved Entities block shows peers as `X - | Subs - | GitHub - | Context: -`.
|
|
||||||
- Sub-run planner lines show `source=deterministic, subqueries=1` — the "I gave up and keyword-searched" shape.
|
|
||||||
- Engine footer keeps nudging `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY`, which is wrong advice for a Claude Code user who already has WebSearch.
|
|
||||||
- Kanye run leaked main topic's `--subreddits` into Drake's and Kendrick's sub-runs (regression bug).
|
|
||||||
|
|
||||||
The fix is to flip the resolution responsibility: the hosting model (Claude Code, Codex, Hermes, Gemini) does Step 0.55 via its own WebSearch tool for every entity, then passes the resolved targeting to the engine via a new `--competitors-plan` JSON flag. Engine fan-out remains — each peer still runs a full `pipeline.run()`. The difference is the peers now arrive with full targeting, equivalent to the main topic, so retrieval is apples-to-apples.
|
|
||||||
|
|
||||||
Why not just reuse vs-mode? vs-mode is a SINGLE `pipeline.run()` with a comparison-optimized plan. It pre-resolves Step 0.55 per entity but merges everything into one retrieval pool with lower-weight `--x-related` for peers, merged subreddits, and cross-entity keyword noise. That is not "three full passes." The user explicitly wants three full passes.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
3.0.12's architecture was correct; its data dependency was wrong.
|
|
||||||
|
|
||||||
| Capability | 3.0.12 path | Target path (this plan) |
|
|
||||||
|---|---|---|
|
|
||||||
| Fan out to N parallel pipelines | Yes (`fanout.run_competitor_fanout`) | Same — keep |
|
|
||||||
| Per-entity Step 0.55 resolution | Engine-internal `resolve.auto_resolve()` — needs BRAVE/EXA/SERPER/PARALLEL key | Hosting model does it via its own WebSearch, passes to engine |
|
|
||||||
| Per-entity targeting threaded into `pipeline.run()` | Main topic only via outer flags; peers via auto_resolve (failing) or nothing | Main topic via outer flags; peers via `--competitors-plan` JSON |
|
|
||||||
| Footer nudge | Unconditional BRAVE/SERPER | Suppressed when `--plan` or `--competitors-plan` present |
|
|
||||||
| Resolved Entities block in raw save file | Stdout only | Also in `--save-dir` raw file |
|
|
||||||
| Override-leak from main into peers | Present (Kanye receipt) | Fixed via explicit per-entity kwargs scrub |
|
|
||||||
| Polymarket noise on ambiguous topics | Present (Warriors, Arizona receipts) | `--polymarket-keywords` + auto-skip for single-token-ambiguous |
|
|
||||||
|
|
||||||
The key architectural change is who owns per-entity resolution. The engine stops trying to do it itself; the hosting model does it upstream (it already has WebSearch) and passes results in.
|
|
||||||
|
|
||||||
This is the same pattern `--plan` already uses for the main topic: hosting model generates the plan via its own reasoning, passes it in, engine accepts. We apply the pattern to peers.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. New `--competitors-plan` JSON flag accepting per-entity targeting: `x_handle`, `x_related`, `subreddits`, `github_user`, `github_repos`, `context`. Implies `--competitors`. Per-entity values thread into that entity's `pipeline.run()`. Bypasses engine-internal `auto_resolve` for covered entities.
|
|
||||||
- R2. SKILL.md "Competitor mode" rewritten to make the hosting-model path canonical: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers) via WebSearch, (c) assemble `--competitors-plan` JSON, (d) invoke engine. Engine-internal auto_resolve remains as headless fallback.
|
|
||||||
- R3. The LAW 7-style stderr emitted when `--competitors` has no list, no plan, no backend is reframed: leads with "hosting reasoning model, use your WebSearch to run Step 0.55 per entity and pass `--competitors-plan`." Does not lead with BRAVE_API_KEY.
|
|
||||||
- R4. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` OR `--competitors-plan` was passed. Signal: hosting model is driving and already has WebSearch.
|
|
||||||
- R5. Override-leak fix: competitor sub-runs do not inherit main topic's `--subreddits`, `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`. Sub-runs use only their own per-entity targeting (from `--competitors-plan` if provided, else engine-internal auto_resolve if backend, else planner defaults).
|
|
||||||
- R6. The `## Resolved Entities` block is also appended to the saved raw file when `--save-dir` is in use. Each entity's effective targeting (whatever was actually passed to its `pipeline.run()`) is visible on audit.
|
|
||||||
- R6b. When `--save-dir` is in use with a comparison run, each entity's sub-run ALSO saves its own standalone raw file — same format as a single-entity run. `/last30days Kanye West --competitors` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md` (one per entity) plus the merged comparison file. Matches the historical vs-mode behavior when it ran as N passes.
|
|
||||||
- R7. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
|
||||||
- R8. Default `--competitors` count remains 2 (3-way: main + 2 peers). Unchanged from 3.0.12.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No changes to `scripts/lib/fanout.py` architecture. N parallel pipelines stays. Only the data each sub-run receives changes.
|
|
||||||
- No changes to the vs-mode (topic contains "vs" / "versus") behavior. That path is independent.
|
|
||||||
- No new emit modes. Comparison output format unchanged.
|
|
||||||
- No deprecation of `--competitors-list`. Stays as the minimum escape hatch for hosting models that skip per-entity Step 0.55 (names-only).
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Cache layer for hosting-model competitor resolution: separate plan once cost evidence exists.
|
|
||||||
- Cross-source disambiguation beyond Polymarket: separate plan.
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py` — `--competitors` / `--competitors-list` argparse block, `resolve_competitors_args` validator, `_main_runner` closure, `_competitor_runner` closure, the `[Competitors] --competitors requires...` stderr block. Primary file for this plan.
|
|
||||||
- `scripts/lib/fanout.py` — `run_competitor_fanout` orchestrator. Signature unchanged; `_competitor_runner` closure now builds kwargs from `--competitors-plan`.
|
|
||||||
- `scripts/lib/pipeline.py` — `pipeline.run()` signature; no changes required (all per-entity flags already exist as kwargs).
|
|
||||||
- `scripts/lib/planner.py` — existing `--plan` parsing and validation, pattern to mirror for `--competitors-plan`.
|
|
||||||
- `scripts/lib/render.py` `_render_resolved_entities_block` (added in 3.0.12) — already reads `report.artifacts["resolved"]`; no change needed.
|
|
||||||
- `scripts/last30days.py` `save_output` / `render.render_full` — the save path. Needs to include the Resolved Entities block for comparison runs.
|
|
||||||
- `scripts/lib/quality_nudge.py` — where the BRAVE/SERPER footer nudge is emitted. Needs a context-aware suppression check.
|
|
||||||
- `scripts/lib/polymarket.py` — source adapter. Entry point for `--polymarket-keywords` filter and single-token-ambiguous auto-skip.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- 3.0.11 plan (`2026-04-22-002`): built the initial fanout, deferred per-entity resolve as "v1 simplification."
|
|
||||||
- 3.0.12 plan (`2026-04-22-003`): tried to close the gap via engine-internal `auto_resolve`. Works only with backend keys. Fails silently without.
|
|
||||||
- 2026-04-22 test session receipts: confirmed all four fixes in this plan are real, reproducible bugs.
|
|
||||||
- User's architectural steer 2026-04-22: "runs a full last30days on all 3 topics" — this plan encodes that explicitly as N full `pipeline.run()` calls with pre-resolved targeting per entity.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None. All patterns in-repo.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **`--competitors-plan` is a single JSON flag, not a fan of separate flags.** Mirrors `--plan`. Stable schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Accept inline JSON or a file path (matches `--plan`).
|
|
||||||
- **Hosting-model-driven resolution is the documented default.** Engine-internal `auto_resolve` is the headless / cron fallback. SKILL.md routes hosting models to the JSON-flag path; engine keeps auto_resolve alive for BRAVE/EXA/SERPER users running CI.
|
|
||||||
- **Override-leak fix is call-site scrubbing, not a signature change.** `_competitor_runner` builds an explicit kwargs dict per entity from `_subrun_kwargs(entity, plan_entry)`. No closure-default fallthrough from main scope. The 3.0.12 `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag.
|
|
||||||
- **Footer nudge becomes context-aware.** Suppressed when `--plan` or `--competitors-plan` present. Not suppressed for bare `--competitors-list` or bare invocations. Headless cron without keys still sees the nudge.
|
|
||||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known list of single-token-ambiguous names (states, common nouns). Stderr notes the skip so it is observable and overridable.
|
|
||||||
- **Per-entity sub-runs get the full `pipeline.run()` pass.** Same depth, same sources, same API cost per entity as a single-topic run. This is the explicit user intent — three full passes, not one merged pass.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **JSON or multi-flag?** JSON. Matches `--plan`.
|
|
||||||
- **Default count?** 2 peers (3-way comparison). Unchanged from 3.0.12.
|
|
||||||
- **Does engine-internal auto_resolve stay alive?** Yes, for entities not covered by `--competitors-plan` when a backend is configured. Headless/cron users with keys keep the current 3.0.12 behavior.
|
|
||||||
- **vs-mode or fanout?** Fanout. User's explicit ask: three full passes, not one merged pass. vs-mode merges into one pipeline with lower peer weighting, which is not what the user wants.
|
|
||||||
- **Does the save file need per-entity clusters?** Start with the Resolved block appended. Per-entity cluster sections can follow in a separate task; they are nice-to-have, not blocking.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Exact trace of override-leak source. Candidates: closure capture of `subreddits` in `_competitor_runner`, shared `_auto_resolve_context` leak, Reddit adapter inheriting global config. Test-first; trace at implementation time.
|
|
||||||
- Heuristic for "single-token-ambiguous topic" auto-skip. Start with a short hard-coded list (US state names, US city names, common nouns like "Warriors", "Suns", "Jets"); revisit after dogfood.
|
|
||||||
- Whether per-entity coverage warnings fire when `--competitors-plan` under-resolves an entity (e.g., only `x_handle`, no subreddits). Start with stderr logging; revisit UX.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: `--competitors-plan` JSON flag + per-entity kwargs threading**
|
|
||||||
|
|
||||||
**Goal:** New CLI flag accepting per-entity targeting JSON. Each covered entity's `pipeline.run()` receives its own `x_handle` / `x_related` / `subreddits` / `github_user` / `github_repos` / `context`. Skips engine-internal `auto_resolve` for covered entities.
|
|
||||||
|
|
||||||
**Requirements:** R1, R5 (primary leak fix site)
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner`)
|
|
||||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; verify)
|
|
||||||
- Test: `tests/test_cli_competitors.py` (extend)
|
|
||||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON OR a file path (mirror `--plan`).
|
|
||||||
- Validation: parse JSON; must be a dict; each value must be a dict; unknown fields log warnings; malformed input exits 2.
|
|
||||||
- Schema per entity: optional fields `x_handle` (str), `x_related` (list), `subreddits` (list), `github_user` (str), `github_repos` (list), `context` (str).
|
|
||||||
- Case-insensitive matching against `--competitors-list` / discovered entities.
|
|
||||||
- Build `_subrun_kwargs(entity, plan_entry)` helper. Returns a complete, explicit kwargs dict for `pipeline.run()` with no closure-default fallthrough from main scope. This helper is the single source of truth for per-entity call args. It also fixes the override-leak (R5) by scrubbing all per-entity flags to None unless the plan (or auto_resolve) sets them.
|
|
||||||
- `_competitor_runner(entity)`:
|
|
||||||
1. Look up `plan_entry` from `--competitors-plan` (if any).
|
|
||||||
2. If plan covers entity fully, build kwargs from it; skip `auto_resolve`.
|
|
||||||
3. If plan partially covers or is absent, fall back to `auto_resolve` (3.0.12 behavior) when a backend is configured. Plan values win over auto_resolve values on conflict.
|
|
||||||
4. If neither plan nor backend, fall through to `pipeline.run()` with per-entity kwargs all None — engine uses planner defaults for that entity only (no leak).
|
|
||||||
- Deep-copy config per sub-run (already done in 3.0.12); merge per-entity `context` into `entity_config["_auto_resolve_context"]` only.
|
|
||||||
|
|
||||||
**Execution note:** Test-first for the override-leak regression (pass `--subreddits=A,B` on main + a peer, assert peer's `pipeline.run(subreddits=...)` is None or peer-specific).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `--plan` parsing at `scripts/last30days.py` (inline JSON or file path).
|
|
||||||
- 3.0.12's `_competitor_runner` closure for scope; extract the kwargs-build into `_subrun_kwargs` helper.
|
|
||||||
- `entity_config = dict(config)` deep-copy pattern from 3.0.12.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors-plan '{"Drake": {"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's `pipeline.run` receives `x_handle="Drake"` and `subreddits=["Drizzy"]`; no `auto_resolve` call for Drake.
|
|
||||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered entities skip auto_resolve; third falls back to auto_resolve.
|
|
||||||
- Happy path: plan file path accepted like `--plan` file path.
|
|
||||||
- Happy path: case-insensitive entity match (`Drake` in plan, `drake` in list).
|
|
||||||
- Edge case: unknown fields in plan entry → logged, ignored, run continues.
|
|
||||||
- Edge case: plan entry for entity not in list → ignored with warning.
|
|
||||||
- Error path: malformed JSON → exit 2.
|
|
||||||
- Error path: top-level JSON is list not dict → exit 2.
|
|
||||||
- Regression (leak fix): main `--subreddits=A,B` + `--competitors-list "Drake"` + no plan → Drake's `pipeline.run` receives `subreddits=None` (no leak).
|
|
||||||
- Regression (leak fix): same for `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`.
|
|
||||||
- Regression (leak fix): main `--x-handle=kanyewest` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanyewest"`.
|
|
||||||
- Integration: full main + 2 peers run via `--competitors-plan`; assert each sub-run's effective kwargs match expected per-entity values.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- All new and regression tests pass.
|
|
||||||
- Smoke run (mock mode + `--competitors-plan`): stderr shows `[Competitors] Drake: x=@Drake subs=Drizzy` line per entity; no `[AutoResolve]` calls for plan-covered entities; no leak of main topic's flags.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: Reframe LAW 7-style stderr for hosting-model context**
|
|
||||||
|
|
||||||
**Goal:** When `--competitors` has no `--competitors-list`, no `--competitors-plan`, and no backend, stderr tells the hosting reasoning model to use its WebSearch tool for Step 0.55 per entity and pass `--competitors-plan`. Stops leading with BRAVE_API_KEY.
|
|
||||||
|
|
||||||
**Requirements:** R3
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1 (flag must exist)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (the existing `[Competitors] --competitors requires...` block)
|
|
||||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Rewrite stderr in this order:
|
|
||||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent runtime with a WebSearch tool), YOU should: (a) discover N peers via WebSearch, (b) run Step 0.55 per entity (main + peers), (c) assemble a `--competitors-plan` JSON, (d) re-invoke. Skip this step and quality degrades — peer entities will run with planner defaults."
|
|
||||||
2. "If you are running headless (cron, CI, no hosting model), set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
|
||||||
3. "Minimum escape hatch: `--competitors-list "A,B,C"` skips discovery but does not pre-resolve peers. Use only for quick tests."
|
|
||||||
- Exits non-zero as today.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing LAW 7 stderr in `planner.plan_query` for tone.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: stderr leads with "If you are the hosting reasoning model" and names `--competitors-plan` before any backend key.
|
|
||||||
- Happy path: stderr explicitly names `--competitors-plan` as the preferred override.
|
|
||||||
- Happy path: stderr does NOT say "requires either a configured web search backend OR an explicit --competitors-list" (the current 3.0.12 wording).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Test asserts ordering and required phrases.
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Suppress BRAVE/SERPER footer nudge when hosting-model-driven**
|
|
||||||
|
|
||||||
**Goal:** The `💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY` footer is suppressed when `--plan` or `--competitors-plan` was passed (signal: hosting model is driving and already has WebSearch).
|
|
||||||
|
|
||||||
**Requirements:** R4
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever nudge is emitted; verify during implementation)
|
|
||||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Locate the nudge emission point.
|
|
||||||
- Add a suppression check: if `--plan` OR `--competitors-plan` was passed, skip the nudge. Otherwise, current behavior.
|
|
||||||
- Don't suppress the nudge for bare `--competitors-list` alone — that path isn't necessarily hosting-model-driven.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--plan` passed, no backend → nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-plan` passed, no backend → nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-list` only, no backend → nudge fires (current behavior).
|
|
||||||
- Happy path: no `--competitors`, no `--plan`, no backend → nudge fires (current behavior unchanged).
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- All four scenarios produce expected nudge presence/absence.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: Per-entity save files + Resolved block in each**
|
|
||||||
|
|
||||||
**Goal:** When `--save-dir` is in use with a comparison run, each entity's sub-run saves its own standalone raw file (same format as a single-entity run), and each file includes the `## Resolved Entities` block so audits can see what targeting that entity received. Matches the historical vs-mode behavior when it was N passes.
|
|
||||||
|
|
||||||
**Requirements:** R6, R6b
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (`save_output`, the save loop after fanout completes)
|
|
||||||
- Possibly modify: `scripts/lib/render.py` (`render_full` branch to include Resolved block when artifact is present)
|
|
||||||
- Test: `tests/test_save_raw_competitor_files.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]`. For each `(entity, entity_report)` tuple, call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)` — same path a single-entity run takes.
|
|
||||||
- Each saved file uses its entity's slug as the filename (`drake-raw.md`, `kendrick-lamar-raw.md`). Main topic keeps the existing `kanye-west-raw.md` filename.
|
|
||||||
- Each file includes its own `## Resolved Entities` block (single-entity variant: one row for that entity only). This makes each sub-run's file self-describing — you can see what targeting was used without opening the comparison file.
|
|
||||||
- The merged comparison output (stdout) still includes the 3-row Resolved Entities block.
|
|
||||||
- Optional: also save a comparison summary file (e.g., `kanye-west-comparison-raw.md`) holding the merged multi-entity render. Start with per-entity files only; comparison summary is a follow-up if stdout-plus-individual-files is insufficient.
|
|
||||||
- Single-entity runs unchanged (no additional files, no block change).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing `save_output` invocation for single-entity runs (line 501 of current `scripts/last30days.py`).
|
|
||||||
- Existing slug generation (`slugify(topic)`) for filename consistency.
|
|
||||||
- `_render_resolved_entities_block` from 3.0.12 for the single-entity variant.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors-list "Drake,Kendrick Lamar"` + `--save-dir=/tmp/x` → `/tmp/x/kanye-west-raw.md`, `/tmp/x/drake-raw.md`, `/tmp/x/kendrick-lamar-raw.md` all exist.
|
|
||||||
- Happy path: each peer file's first sections include that entity's Resolved Entities block with its own row only.
|
|
||||||
- Happy path: single-entity run with `--save-dir` → one file, unchanged from today's behavior.
|
|
||||||
- Edge case: entity slug collides with existing file → overwrite (matches single-entity behavior).
|
|
||||||
- Edge case: `--save-suffix=v3` → all 3 files get the suffix (`kanye-west-raw-v3.md`, `drake-raw-v3.md`, `kendrick-lamar-raw-v3.md`).
|
|
||||||
- Edge case: comparison run with one peer whose sub-run failed → that entity's file is NOT saved; others are.
|
|
||||||
- Integration: stderr after save shows three `[last30days] Saved output to <path>` lines, one per entity.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- After `/last30days Kanye West --competitors-list "Drake,Kendrick Lamar" --save-dir=/tmp/x`: `ls /tmp/x/*-raw.md` shows 3 files. Each contains its entity's Resolved block.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: SKILL.md "Competitor mode" rewrite — hosting-model Step 0.55 canonical**
|
|
||||||
|
|
||||||
**Goal:** SKILL.md documents the hosting-model-driven path as canonical: discover N peers via WebSearch, run Step 0.55 per entity, assemble `--competitors-plan`, invoke engine. Engine-internal `auto_resolve` is labeled the headless fallback.
|
|
||||||
|
|
||||||
**Requirements:** R2
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1 (flag must exist before documented)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md` (Competitor mode subsection)
|
|
||||||
- Modify: `README.md` (one-line example update)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Replace the 3.0.12 Competitor mode subsection with a clear flow:
|
|
||||||
1. User invokes with `--competitors` or `--competitors=N`.
|
|
||||||
2. Hosting model runs WebSearch for "[topic] competitors" / "[topic] alternatives" → picks top N peers.
|
|
||||||
3. Hosting model runs Step 0.55 for main + each peer (x_handle, subreddits, github_user, github_repos, context) — same protocol as vs-mode per SKILL.md §679.
|
|
||||||
4. Hosting model assembles a `--competitors-plan` JSON object.
|
|
||||||
5. Hosting model invokes the engine with `--competitors-list "A,B,C" --competitors-plan '{...}'`.
|
|
||||||
6. Engine fans out N full pipelines (main + peers), each with its own full Step 0.55-grade targeting. Each entity also saves its own `*-raw.md` file when `--save-dir` is set (three full passes → three save files, matching the historical vs-mode behavior). Comparison output merges them for display.
|
|
||||||
- Concrete JSON example in SKILL.md showing the schema.
|
|
||||||
- Failure-mode warning: a `## Resolved Entities` block with dashes for any entity means hosting model skipped Step 0.55 for that one. Re-run with corrected plan.
|
|
||||||
- "Headless fallback" sub-subsection: when BRAVE/EXA/SERPER/PARALLEL/OPENROUTER is set, engine's internal `auto_resolve` handles peers and `--competitors-plan` is optional.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- SKILL.md "Step 0.55" section for per-entity resolve protocol.
|
|
||||||
- SKILL.md "If QUERY_TYPE = COMPARISON" section for the same-protocol-as-vs-mode reference.
|
|
||||||
- Tone of existing 3.0.12 Competitor mode prose.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — documentation. Verification is a fresh Claude Code window dogfood run.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `/last30days Kanye West --competitors` in a new window: hosting model does Step 0.55 for Kanye + 2 discovered peers; passes `--competitors-plan`; rendered Resolved block shows non-empty fields for all 3; top voices include at least one peer-specific handle.
|
|
||||||
|
|
||||||
- [ ] **Unit 6: Polymarket disambiguation guard**
|
|
||||||
|
|
||||||
**Goal:** Support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
|
||||||
|
|
||||||
**Requirements:** R7
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` argparse (`--polymarket-keywords`)
|
|
||||||
- Modify: `scripts/lib/polymarket.py`
|
|
||||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--polymarket-keywords "kw1,kw2"` flag. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
|
||||||
- Auto-skip rule: if topic is one token AND token matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords` provided, skip Polymarket with a stderr note.
|
|
||||||
- SKILL.md Step 0.55 protocol gets a small addition: for ambiguous topics, hosting model passes `--polymarket-keywords` with topic-specific qualifiers.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing Polymarket adapter match logic.
|
|
||||||
- Single-token detection heuristic.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr notes the skip.
|
|
||||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs; matches filtered.
|
|
||||||
- Happy path: topic "OpenAI" (no ambiguity) → Polymarket runs as before.
|
|
||||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
|
||||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Warriors smoke run → Polymarket footer absent OR filtered to nba/gsw markets.
|
|
||||||
|
|
||||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.13 to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** Closes R1-R7
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-6
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json`
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG entry groups the fixes: Added `--competitors-plan` JSON flag for per-entity hosting-model pre-resolve. Fixed override-leak from main into peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when `--plan` / `--competitors-plan` is present. Added: Resolved Entities block persists to saved raw file. Added: `--polymarket-keywords` + auto-skip for ambiguous single-token topics.
|
|
||||||
- Beta channel first per CLAUDE.md.
|
|
||||||
- Hot-copy so public `/last30days` picks up 3.0.13 immediately.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` returns 3.0.13.
|
|
||||||
- `sync.sh` exits 0.
|
|
||||||
- Hot-copy contains the new files with competitors.py, fanout.py, the updated SKILL.md, and plugin.json 3.0.13.
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** `_competitor_runner` becomes the single source of truth for sub-run kwargs via `_subrun_kwargs(entity, plan_entry)`. Every per-entity flag flows through one helper. No closure-default leaks.
|
|
||||||
- **Error propagation:** `--competitors-plan` JSON parse errors exit 2 with stderr (same as `--plan`). Per-entity plan entries with malformed values log warnings and fall back; don't abort the whole run.
|
|
||||||
- **State lifecycle risks:** `entity_config = dict(config)` already deep-copies for `_auto_resolve_context`; extend the isolation discipline to every per-entity flag. Verified in Unit 1 regression tests.
|
|
||||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors` and `--competitors-list` unchanged. `--plan` unchanged. `--polymarket-keywords` additive.
|
|
||||||
- **Integration coverage:** New regression tests for override-leak. New integration test for plan-driven sub-run threading. New nudge-suppression test. New Polymarket disambiguation test.
|
|
||||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. `planner.plan_query` LAW 7 behavior for the default path unchanged. Single-entity render path unchanged. vs-mode behavior unchanged.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Hosting model takes the lazy path and uses `--competitors-list` names-only. | Unit 2 stderr explicitly steers to `--competitors-plan` with Step 0.55 protocol named. Unit 5 SKILL.md docs. Resolved Entities dashes in output make the gap visible. |
|
|
||||||
| JSON gets verbose for the hosting model to construct repeatedly. | Schema is small (≤6 fields per entity). Hosting model already runs Step 0.55 for main topic in every comparison run; peers use the same protocol. One JSON block replaces N CLI flags. |
|
|
||||||
| Override-leak source is deeper than `_competitor_runner` closure. | Test-first per Unit 1. Receipts from 2026-04-22 Kanye run are reproducible. Trace methodically from call site. |
|
|
||||||
| Plan-covered entity bypasses auto_resolve but plan data is incomplete (e.g., no subreddits). | Hosting model's own SKILL.md contract says Step 0.55 must cover all fields. Stderr logs per-entity coverage so under-resolved entities are visible. Next-run correction, not engine-side rescue. |
|
|
||||||
| Polymarket auto-skip false-positives on legitimate ambiguous topics with real markets. | Conservative match (single-token + known list). `--polymarket-keywords` override is explicit and unambiguous. Stderr notes the skip. |
|
|
||||||
| Footer nudge suppression hides the message from headless users who genuinely need it. | Suppression only fires when `--plan` or `--competitors-plan` is present. Cron / CI runs that pass neither still see the nudge. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first per CLAUDE.md (private repo `/last30days-beta`).
|
|
||||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
|
||||||
- CHANGELOG voice should call this out as the feedback-driven follow-up to 3.0.12. Reader should see "we tried engine-internal resolve in 3.0.12; it needs backend keys we don't have; we moved resolution to the hosting model in 3.0.13."
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Origin plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
|
||||||
- Earlier plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
|
||||||
- 2026-04-22 test session receipts: Warriors, Seattle, Arizona Wildcats, Kanye West
|
|
||||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol
|
|
||||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
|
||||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
|
||||||
@@ -1,454 +0,0 @@
|
|||||||
---
|
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "feat: vs mode runs N full passes and --competitors is vs with auto-discovery"
|
|
||||||
type: feat
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
origin: docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded
|
|
||||||
---
|
|
||||||
|
|
||||||
# feat: vs mode runs N full passes and --competitors is vs with auto-discovery
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Architectural unification driven by user correction 2026-04-22: vs mode and `--competitors` are the same thing. A user typing `/last30days OpenAI vs Anthropic vs xAI` should get a full single-entity last30days pass for each of the three entities — three full pipelines, three saved `*-raw.md` files, merged into one comparison output. A user typing `/last30days OpenAI --competitors` should get the same output after the hosting model auto-picks 2 peers; i.e., `--competitors` is a thin shortcut that expands "topic + `--competitors`" into "topic vs peer1 vs peer2" and then runs the unified vs pipeline.
|
|
||||||
|
|
||||||
Current state diverges from this:
|
|
||||||
|
|
||||||
- **vs mode today**: one `pipeline.run()` with a comparison-optimized plan that merges all entities' targeting into a single retrieval pool. Lower-weight `--x-related` for peers, merged subreddits, cross-entity keyword noise. One saved file.
|
|
||||||
- **`--competitors` today (3.0.12)**: N parallel `pipeline.run()` calls via `scripts/lib/fanout.py`, but per-entity Step 0.55 depends on an engine-side web backend key Matt doesn't have. Silently degrades to planner defaults for peers. One saved file (main topic only). Override-leak from main into peers.
|
|
||||||
|
|
||||||
After this plan:
|
|
||||||
|
|
||||||
- **vs mode**: N parallel `pipeline.run()` calls, one per entity, each with its own full Step 0.55-grade targeting, each saving its own `*-raw.md`. Merged into one comparison output.
|
|
||||||
- **`--competitors`**: SKILL.md shortcut. Hosting model discovers N peers, builds `"topic vs peer1 vs peer2"`, and invokes the same vs pipeline. No separate orchestration path.
|
|
||||||
- **Same fanout machinery (`scripts/lib/fanout.py`)** serves both. One fix, both behaviors improve.
|
|
||||||
|
|
||||||
## Problem Frame
|
|
||||||
|
|
||||||
The product insight from 2026-04-22 test runs is simple: the user wants three full last30days reports plus a comparison merge. Not one comparison pass with N-way targeting merged into a single retrieval pool. Not one save file. Not "main gets Step 0.55, peers get planner defaults." Three full passes. Three save files. Merged output.
|
|
||||||
|
|
||||||
The historical vs mode did that (it ran as 3 passes, saving 3 files). SKILL.md §551 currently says:
|
|
||||||
|
|
||||||
> "When the user asks 'X vs Y', run ONE research pass with a comparison-optimized plan that covers both entities AND their rivalry. This replaces the old 3-pass approach (which took 13+ minutes and produced tangential content)."
|
|
||||||
|
|
||||||
That change was a latency optimization that removed the user-visible behavior the user wants. The fix is to revert the architectural direction: N passes per entity, in parallel rather than serial (parallelism lowers wall-clock to ~1× a single pass, not N×), with per-entity save files.
|
|
||||||
|
|
||||||
The 3.0.11 `--competitors` flag already introduced parallel N-pass machinery (`fanout.run_competitor_fanout`). The 3.0.12 follow-up tried to wire per-entity Step 0.55 into it but failed when no web backend was configured. The elegant move: stop maintaining two architectures. vs-mode and `--competitors` both use `fanout.py`. `--competitors` becomes a SKILL.md-level shortcut that discovers 2 peers and hands off to vs-mode.
|
|
||||||
|
|
||||||
Four 2026-04-22 test receipts (Warriors, Seattle, Arizona Wildcats, Kanye West) all confirmed the user's pain points:
|
|
||||||
|
|
||||||
- Peers thin because they ran without per-entity handle/sub targeting.
|
|
||||||
- Only one `*-raw.md` per run — no per-entity audit.
|
|
||||||
- Kanye peers leaked main topic's `--subreddits`.
|
|
||||||
- Engine footer nudging `BRAVE_API_KEY` to Claude Code users who already have WebSearch.
|
|
||||||
- Polymarket noise on ambiguous topics (Warriors → Glasgow rugby; Arizona → Diamondbacks).
|
|
||||||
|
|
||||||
This plan closes all of them by unifying the architecture and making hosting-model-driven Step 0.55 per entity the canonical path.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. vs mode (any topic containing ` vs ` / ` versus `) runs N full `pipeline.run()` calls in parallel, one per entity. Each sub-run uses its entity's own Step 0.55 targeting (from the hosting model's pre-resolution, passed via a new `--competitors-plan` JSON).
|
|
||||||
- R2. `--competitors` (and `--competitors=N`) becomes a SKILL.md-level shortcut: the hosting model (a) discovers N peers via WebSearch, (b) runs Step 0.55 per entity (main + peers), (c) rewrites the topic to `"main vs peer1 vs peer2"`, (d) invokes the engine with `--competitors-plan` containing each entity's targeting.
|
|
||||||
- R3. New `--competitors-plan` JSON flag. Schema: `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Implies vs mode when present with a single-entity topic. Applies per-entity targeting to each sub-run. Accepts inline JSON or a file path (matches `--plan`).
|
|
||||||
- R4. Each entity's sub-run saves its own `*-raw.md` file when `--save-dir` is in use. Example: `/last30days "Kanye West vs Drake vs Kendrick Lamar" --save-dir=~/Documents/Last30Days` produces `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`. Same filenames a single-entity run of each topic would produce. Matches historical vs-mode behavior.
|
|
||||||
- R5. Each per-entity saved file includes its own single-row `## Resolved Entities` block so the audit survives. The merged comparison stdout still shows the full 3-row block.
|
|
||||||
- R6. Override-leak fix: no main-topic flags (`--subreddits`, `--x-handle`, `--x-related`, `--tiktok-*`, `--ig-creators`, `--github-*`) leak into peer sub-runs. Every per-entity kwarg is scrubbed at the sub-run call site.
|
|
||||||
- R7. LAW 7-style stderr for `--competitors` invocations with no list, no plan, no backend is reframed for hosting-model context: leads with "use your WebSearch to discover peers, resolve Step 0.55 per entity, re-invoke with `topic vs peer1 vs peer2 --competitors-plan '...'`." Does not lead with BRAVE_API_KEY.
|
|
||||||
- R8. Footer nudge `💡 You can unlock native grounded web search with BRAVE_API_KEY...` is suppressed when `--plan` or `--competitors-plan` was passed.
|
|
||||||
- R9. Polymarket disambiguation: support `--polymarket-keywords "kw1,kw2"` to filter market matches; auto-skip Polymarket when topic is single-token-ambiguous and no override is provided.
|
|
||||||
- R10. Default `--competitors` count stays 2 peers (3-way comparison). Unchanged from 3.0.12.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No changes to single-entity `pipeline.run()` semantics. Each sub-run in vs mode behaves identically to a bare `/last30days {entity}` invocation.
|
|
||||||
- No changes to the planner's comparison-intent logic for single-entity-containing topics. The `_should_force_deterministic_plan` shortcut for vs-topics routes to fanout, not to its current single-pipeline path.
|
|
||||||
- No new emit modes. Comparison output format unchanged.
|
|
||||||
- No removal of `--competitors-list`. Stays as a minimum escape hatch (names-only, no per-entity targeting) for scripted headless use.
|
|
||||||
- No removal of engine-internal `resolve.auto_resolve()` in fanout. Remains as headless / cron fallback for users with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys. The dominant Claude Code path bypasses it via `--competitors-plan`.
|
|
||||||
|
|
||||||
### Deferred to Separate Tasks
|
|
||||||
|
|
||||||
- Explicit "head-to-head" rivalry pass in vs-mode (a supplemental subquery like `"A vs B"` that catches rivalry articles missing from pure entity-scoped passes). Start with N independent passes; add a head-to-head supplemental pass if the rivalry-content gap shows up in dogfood.
|
|
||||||
- Cache layer for hosting-model pre-resolution.
|
|
||||||
- Cross-source disambiguation (not just Polymarket).
|
|
||||||
- Latency knob for users who want the old one-pass vs behavior (probably not needed; parallel N-pass is ~1× wall clock).
|
|
||||||
|
|
||||||
## Context & Research
|
|
||||||
|
|
||||||
### Relevant Code and Patterns
|
|
||||||
|
|
||||||
- `scripts/last30days.py` — main(), `_main_runner`, `_competitor_runner`, the competitor enable/discovery branch. Primary file.
|
|
||||||
- `scripts/lib/fanout.py` — existing orchestrator (3.0.11). Reused as-is; `competitor_runner` closure is where per-entity kwargs apply.
|
|
||||||
- `scripts/lib/planner.py` — `_should_force_deterministic_plan` detects vs-topics via regex. Current path synthesizes ONE comparison plan; new path routes to fanout.
|
|
||||||
- `scripts/lib/render.py` — `render_comparison_multi` (3.0.12) + `_render_resolved_entities_block`. Both reused. `render_full` needs a per-entity variant when saving sub-run files.
|
|
||||||
- `scripts/last30days.py` `save_output` — where raw files are written. Needs to iterate per entity when competitor_reports artifact present.
|
|
||||||
- `scripts/lib/quality_nudge.py` — BRAVE/SERPER nudge emission.
|
|
||||||
- `scripts/lib/polymarket.py` — source adapter for `--polymarket-keywords` and ambiguous-topic auto-skip.
|
|
||||||
- SKILL.md §551 "If QUERY_TYPE = COMPARISON" and §679 per-entity Step 0.55 protocol — the hosting-model contract that drives per-entity pre-resolution for both vs mode and `--competitors`.
|
|
||||||
|
|
||||||
### Institutional Learnings
|
|
||||||
|
|
||||||
- 3.0.11 plan (`2026-04-22-002`): built fanout.
|
|
||||||
- 3.0.12 plan (`2026-04-22-003`): tried engine-internal per-entity auto_resolve; failed without backend keys.
|
|
||||||
- 3.0.13 plan draft (`2026-04-22-004-...superseded`): proposed `--competitors-plan` JSON + vs-mode-shortcut path but kept them separate. User's 2026-04-22 correction unifies them.
|
|
||||||
- 2026-04-22 test receipts: Warriors, Seattle, Arizona Wildcats, Kanye West runs all reproduced the per-entity resolve gap.
|
|
||||||
- User's architectural steer: "vs mode should work that way too" + "--competitors is just vs mode with auto-discovery." This plan encodes that.
|
|
||||||
|
|
||||||
### External References
|
|
||||||
|
|
||||||
- None. All patterns in-repo.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Unify vs-mode and --competitors on one orchestrator.** `fanout.run_competitor_fanout` serves both. vs-mode is "topic contains ' vs '" detection → fanout. `--competitors` is "SKILL.md shortcut → hosting model rewrites topic to vs form → fanout." One code path.
|
|
||||||
- **Per-entity targeting via `--competitors-plan` JSON.** Schema `{entity_name: {x_handle, x_related, subreddits, github_user, github_repos, context}}`. Mirrors `--plan`. Applies to both vs-mode and `--competitors` paths. Hosting model passes it after running Step 0.55 per entity.
|
|
||||||
- **N save files, one per entity.** Each sub-run writes a `{entity-slug}-raw.md` file when `--save-dir` is set. Matches historical vs-mode behavior. Single-entity runs unchanged.
|
|
||||||
- **Revert the "one pass for latency" optimization that removed per-entity passes.** Parallel execution via `ThreadPoolExecutor` means wall-clock is ~max(per-entity-latency), not sum. The old latency concern (13+ minutes for 3 serial passes) does not apply to a parallel fan-out.
|
|
||||||
- **Override-leak fix at the call site.** `_subrun_kwargs(entity, plan_entry)` helper returns fully explicit per-entity kwargs; no closure-default fallthrough from main scope.
|
|
||||||
- **LAW 7 stderr reframed, not just updated.** Current message treats BRAVE_API_KEY as the solution. New message treats hosting-model Step 0.55 as the solution, with backend keys listed only as the headless fallback.
|
|
||||||
- **Polymarket disambiguation is additive and conservative.** `--polymarket-keywords` is explicit; auto-skip only fires for a known-ambiguous single-token list.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Resolved During Planning
|
|
||||||
|
|
||||||
- **vs mode N passes or single-pass?** N passes. User's architectural correction.
|
|
||||||
- **Should --competitors still be an engine flag at all?** Yes, kept for headless / cron contexts with backend keys. Dominant Claude Code path is SKILL.md shortcut → vs-mode fanout. Engine flag stays as compatibility surface.
|
|
||||||
- **`--competitors-plan` JSON or multi-flag?** JSON. Matches `--plan`.
|
|
||||||
- **Default count?** 2 peers → 3-way comparison. Unchanged.
|
|
||||||
- **Saved-file naming?** `{entity-slug}-raw.md` per entity, same as single-entity runs would produce.
|
|
||||||
|
|
||||||
### Deferred to Implementation
|
|
||||||
|
|
||||||
- Exact trace of override-leak path (closure capture vs shared config vs Reddit adapter fallback). Test-first per Unit 2; patch at the right layer.
|
|
||||||
- Heuristic for single-token-ambiguous Polymarket auto-skip. Start with a short hard-coded list; iterate.
|
|
||||||
- Whether to include a head-to-head rivalry supplemental pass in vs-mode. Ship N-independent passes first; revisit after dogfood if rivalry content is missing.
|
|
||||||
- Exact filename convention when the comparison merged output is saved (if saved at all). Not blocking — per-entity files are the primary save artifact.
|
|
||||||
|
|
||||||
## High-Level Technical Design
|
|
||||||
|
|
||||||
> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.*
|
|
||||||
|
|
||||||
```
|
|
||||||
User invokes:
|
|
||||||
/last30days "OpenAI vs Anthropic vs xAI"
|
|
||||||
OR
|
|
||||||
/last30days OpenAI --competitors (hosting model rewrites to vs form)
|
|
||||||
OR
|
|
||||||
/last30days OpenAI --competitors-list "Anthropic,xAI"
|
|
||||||
OR
|
|
||||||
/last30days "OpenAI vs Anthropic vs xAI" --competitors-plan '{...per-entity...}'
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
scripts/last30days.py main():
|
|
||||||
- Detect: topic has " vs " OR --competitors enabled
|
|
||||||
- If --competitors and no list/plan: emit LAW 7-style stderr with hosting-model instruction
|
|
||||||
- If --competitors with list or discovery: rewrite topic to vs form, continue
|
|
||||||
- Parse --competitors-plan JSON, map to entities
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
fanout.run_competitor_fanout (shared path):
|
|
||||||
- For each entity (main + peers):
|
|
||||||
- entity_config = dict(config) [deep copy to prevent leak]
|
|
||||||
- kwargs = _subrun_kwargs(entity, plan_entry) [explicit; no main-topic leak]
|
|
||||||
- If plan_entry missing a field AND backend available: auto_resolve() fill
|
|
||||||
- pipeline.run(topic=entity, **kwargs, internal_subrun=True)
|
|
||||||
- Parallel ThreadPoolExecutor
|
|
||||||
- Collect per-entity Reports
|
|
||||||
- Attach resolved targeting to each Report.artifacts["resolved"]
|
|
||||||
|
|
||||||
↓
|
|
||||||
|
|
||||||
scripts/last30days.py after fanout:
|
|
||||||
- If --save-dir: save each entity's Report as {entity-slug}-raw.md
|
|
||||||
Each file includes its own single-row Resolved Entities block
|
|
||||||
- emit_comparison_output → render_comparison_multi (merged stdout)
|
|
||||||
Includes full N-row Resolved Entities block
|
|
||||||
```
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: vs-topic detection routes to fanout (not single-pipeline)**
|
|
||||||
|
|
||||||
**Goal:** A topic containing ` vs ` / ` versus ` triggers `fanout.run_competitor_fanout` with the parsed entities. Each entity runs a full `pipeline.run()`. Replace the current single-pipeline-with-comparison-plan behavior.
|
|
||||||
|
|
||||||
**Requirements:** R1
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (main() — detect vs-topic, route to fanout)
|
|
||||||
- Modify: `scripts/lib/planner.py` (remove / bypass the `_should_force_deterministic_plan` special case for vs topics; vs topics no longer go through `plan_query` as a single comparison plan)
|
|
||||||
- Test: `tests/test_vs_mode_fanout.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Parse the incoming topic: if it contains ` vs ` or ` versus ` (case-insensitive), split into entities (reuse `planner._comparison_entities`-style logic or move that utility into main()).
|
|
||||||
- When vs-entities are detected, route to the same fanout branch `--competitors` uses today. The entity list comes from the topic string; no discovery step needed.
|
|
||||||
- Each entity runs `pipeline.run()` with its own plan (either from `--competitors-plan[entity]` or from the engine's per-entity fallback path).
|
|
||||||
- For back-compat, if the user passes both a vs-topic AND `--plan`, honor `--plan` for the main (first) entity and use per-entity defaults for peers unless `--competitors-plan` is also provided.
|
|
||||||
|
|
||||||
**Execution note:** Start with an integration test that runs `"A vs B"` via mock mode and asserts fanout was called with two entities + two pipeline.run calls.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- 3.0.11 fanout wiring in `scripts/last30days.py`'s `--competitors` branch.
|
|
||||||
- `planner._comparison_entities` for the split logic.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: topic `"A vs B"` → two pipeline.run calls, two Reports returned, merged render.
|
|
||||||
- Happy path: topic `"A vs B vs C"` → three pipeline.run calls.
|
|
||||||
- Happy path: topic `"A versus B"` → matches the same regex, two pipelines.
|
|
||||||
- Edge case: topic `"OpenAI vs"` (trailing empty entity) → treated as single-entity `"OpenAI"`, not vs mode.
|
|
||||||
- Edge case: topic contains "vs." (dot, no trailing space) → existing regex tolerates it; verify.
|
|
||||||
- Edge case: topic `"A vs B"` plus `--plan` → plan applies to first entity only, peers use per-entity defaults.
|
|
||||||
- Integration: full vs-mode run end-to-end in mock mode; verify rendered output, stderr has one `[Competitors] Comparing: A vs B vs ...` line.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Test assertions pass.
|
|
||||||
- Mock-mode smoke of `/last30days "OpenAI vs Anthropic"` shows fanout invocation, per-entity Reports, merged comparison output.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: `--competitors-plan` JSON flag + `_subrun_kwargs` helper + override-leak fix**
|
|
||||||
|
|
||||||
**Goal:** New JSON flag threads per-entity targeting into each sub-run's `pipeline.run()`. A `_subrun_kwargs(entity, plan_entry)` helper is the single source of truth for per-entity kwargs, eliminating override-leak.
|
|
||||||
|
|
||||||
**Requirements:** R3, R6
|
|
||||||
|
|
||||||
**Dependencies:** None (can land alongside or before Unit 1)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (argparse + parse + `_competitor_runner` + `_subrun_kwargs` helper)
|
|
||||||
- Possibly modify: `scripts/lib/fanout.py` (no signature change expected; the competitor_runner contract is unchanged)
|
|
||||||
- Test: `tests/test_cli_competitors.py` (extend)
|
|
||||||
- Test: `tests/test_competitors_plan_threading.py` (new)
|
|
||||||
- Test: `tests/test_competitor_subrun_isolation.py` (new, regression)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--competitors-plan` argparse flag. Accepts inline JSON or file path (mirror `--plan`).
|
|
||||||
- Validation: top-level dict; each value is a dict; unknown fields log warnings; malformed input exits 2. Case-insensitive entity matching.
|
|
||||||
- Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`.
|
|
||||||
- Build `_subrun_kwargs(entity, plan_entry)` — returns an explicit dict with every per-entity flag. No closure-default fallthrough. This is the leak fix.
|
|
||||||
- `_competitor_runner(entity)`:
|
|
||||||
1. Get `plan_entry` from `--competitors-plan` if present.
|
|
||||||
2. Build base kwargs with `_subrun_kwargs(entity, plan_entry)`.
|
|
||||||
3. Fill missing fields via `resolve.auto_resolve(entity, entity_config)` only if backend is configured (3.0.12 fallback path).
|
|
||||||
4. Call `pipeline.run(topic=entity, internal_subrun=True, **kwargs)`.
|
|
||||||
5. Attach `resolved` dict to `report.artifacts`.
|
|
||||||
- Verify no per-entity flag from main() leaks via closure. The helper is the only source of per-entity values.
|
|
||||||
|
|
||||||
**Execution note:** Test-first for the override-leak regression. Use the Kanye 2026-04-22 receipt as the failing test input (main `--subreddits=Kanye,hiphopheads` + `--competitors-list "Drake"` → assert Drake's pipeline.run receives `subreddits=None`).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- `--plan` parsing block in `scripts/last30days.py`.
|
|
||||||
- 3.0.12's `entity_config = dict(config)` deep-copy pattern.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors-plan '{"Drake":{"x_handle":"Drake","subreddits":["Drizzy"]}}'` → Drake's pipeline.run receives `x_handle="Drake"`, `subreddits=["Drizzy"]`. No auto_resolve call for Drake.
|
|
||||||
- Happy path: plan covers 2 of 3 entities, backend configured → covered skip auto_resolve; third falls back.
|
|
||||||
- Happy path: plan file path accepted like `--plan`.
|
|
||||||
- Happy path: case-insensitive entity match.
|
|
||||||
- Edge case: unknown fields → warn, ignore.
|
|
||||||
- Edge case: plan entry for entity not in list → warn, ignore.
|
|
||||||
- Error path: malformed JSON → exit 2.
|
|
||||||
- Error path: top-level JSON is list → exit 2.
|
|
||||||
- Regression (leak): main `--subreddits=A,B` + `--competitors-list "X"` + no plan → X's pipeline.run gets `subreddits=None`.
|
|
||||||
- Regression (leak): same for `--x-handle`, `--x-related`, `--tiktok-hashtags`, `--tiktok-creators`, `--ig-creators`, `--github-user`, `--github-repo`.
|
|
||||||
- Regression (leak): main `--x-handle=kanye` + plan `{"Drake":{"x_handle":"Drake"}}` → Drake's sub-run gets `x_handle="Drake"`, NOT `"kanye"`.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- All regression tests pass.
|
|
||||||
- Smoke run (mock mode + plan): stderr shows per-entity `[Competitors] {entity}: x=... subs=...` line; no leak from main topic's flags.
|
|
||||||
|
|
||||||
- [ ] **Unit 3: Per-entity save files**
|
|
||||||
|
|
||||||
**Goal:** When `--save-dir` is set in a vs-mode or `--competitors` run, each entity's sub-run saves its own `{entity-slug}-raw.md` file — same format as a single-entity run would produce.
|
|
||||||
|
|
||||||
**Requirements:** R4, R5
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1, Unit 2
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (`save_output` iteration after fanout)
|
|
||||||
- Modify: `scripts/lib/render.py` (`render_full` includes single-row Resolved Entities block when that entity's `artifacts["resolved"]` is present)
|
|
||||||
- Test: `tests/test_save_raw_per_entity.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- After fanout completes, iterate `report.artifacts["competitor_reports"]` (or equivalent). For each `(entity, entity_report)`:
|
|
||||||
- Call `save_output(entity_report, emit="md", save_dir=args.save_dir, suffix=args.save_suffix)`.
|
|
||||||
- Uses entity's `slugify(entity)` for the filename. Same pattern a single-entity run uses.
|
|
||||||
- Each saved file invokes `render_full` (or the save-variant). `render_full` now checks for `report.artifacts["resolved"]` and prepends a single-row Resolved Entities block.
|
|
||||||
- Stderr logs one `[last30days] Saved output to <path>` line per entity.
|
|
||||||
- Single-entity runs unchanged (no extra files, render_full unchanged for them).
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing `save_output` invocation in main() for single-entity runs.
|
|
||||||
- `slugify(topic)` for filename.
|
|
||||||
- 3.0.12's `_render_resolved_entities_block` (reused, single-row mode).
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `/last30days "A vs B vs C" --save-dir=/tmp/x` → `/tmp/x/a-raw.md`, `/tmp/x/b-raw.md`, `/tmp/x/c-raw.md` exist.
|
|
||||||
- Happy path: `--competitors-list "Drake,Kendrick" --save-dir=/tmp/x` on topic Kanye → three files: `kanye-west-raw.md`, `drake-raw.md`, `kendrick-lamar-raw.md`.
|
|
||||||
- Happy path: each file includes a single-row Resolved Entities block for its entity.
|
|
||||||
- Happy path: single-entity run with `--save-dir` → one file, no Resolved block (unchanged).
|
|
||||||
- Edge case: `--save-suffix=v3` → all N files get the suffix.
|
|
||||||
- Edge case: one entity sub-run failed → its file is NOT saved; the others are.
|
|
||||||
- Integration: `ls {save-dir}/*-raw.md` returns N files after a vs-mode run.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Test assertions pass.
|
|
||||||
- Manual vs-mode smoke saves N files.
|
|
||||||
|
|
||||||
- [ ] **Unit 4: LAW 7-style stderr reframe + footer-nudge suppression**
|
|
||||||
|
|
||||||
**Goal:** The `--competitors`-with-no-backend stderr tells the hosting model to do Step 0.55 per entity and pass `--competitors-plan`. The BRAVE/SERPER footer nudge is suppressed when `--plan` or `--competitors-plan` is present.
|
|
||||||
|
|
||||||
**Requirements:** R7, R8
|
|
||||||
|
|
||||||
**Dependencies:** Unit 2 (flag must exist)
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (the `[Competitors] --competitors requires...` stderr block)
|
|
||||||
- Modify: `scripts/lib/quality_nudge.py` (or wherever footer nudge emits; verify during implementation)
|
|
||||||
- Test: `tests/test_competitors_no_backend_message.py` (new)
|
|
||||||
- Test: `tests/test_footer_nudge_suppression.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Rewrite stderr in this order:
|
|
||||||
1. "If you are the hosting reasoning model (Claude Code, Codex, Hermes, Gemini, or any agent with WebSearch), the recommended path: (a) discover N peers via WebSearch, (b) run Step 0.55 for main + each peer, (c) re-invoke as `/last30days 'topic vs peer1 vs peer2' --competitors-plan '{...}'`. See SKILL.md 'Competitor mode'."
|
|
||||||
2. "Headless / cron path: set BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY and re-run."
|
|
||||||
3. "Minimum escape hatch: `--competitors-list 'A,B,C'` skips discovery but does not pre-resolve peers."
|
|
||||||
- Suppress footer nudge when `external_plan` OR `competitors_plan` was passed.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: `--competitors` with no backend, no list, no plan → stderr leads with "If you are the hosting reasoning model" and references `--competitors-plan` before naming API keys.
|
|
||||||
- Happy path: `--plan` passed → footer nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-plan` passed → footer nudge does NOT fire.
|
|
||||||
- Happy path: `--competitors-list` only (no plan, no backend) → footer nudge still fires (hosting model didn't fully engage).
|
|
||||||
- Happy path: no `--competitors`, no `--plan` → footer nudge unchanged.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Tests pass.
|
|
||||||
|
|
||||||
- [ ] **Unit 5: Polymarket disambiguation guard**
|
|
||||||
|
|
||||||
**Goal:** `--polymarket-keywords "kw1,kw2"` filters market matches; auto-skip Polymarket on single-token-ambiguous topics without override.
|
|
||||||
|
|
||||||
**Requirements:** R9
|
|
||||||
|
|
||||||
**Dependencies:** None
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `scripts/last30days.py` (argparse)
|
|
||||||
- Modify: `scripts/lib/polymarket.py`
|
|
||||||
- Test: `tests/test_polymarket_disambiguation.py` (new)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Add `--polymarket-keywords "kw1,kw2"`. When provided, Polymarket adapter filters market titles to those whose normalized text contains at least one keyword.
|
|
||||||
- Auto-skip: if topic is one token AND matches a known-ambiguous list (US state names, US city names, common sports/color/animal words) AND no `--polymarket-keywords`, skip Polymarket with stderr note.
|
|
||||||
- SKILL.md update (small): mention `--polymarket-keywords` in Step 0.55 instructions for ambiguous topics.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Happy path: topic "Warriors", no override → Polymarket skipped; stderr note.
|
|
||||||
- Happy path: topic "Warriors", `--polymarket-keywords "nba,gsw"` → Polymarket runs, filtered.
|
|
||||||
- Happy path: topic "OpenAI" → Polymarket runs as before.
|
|
||||||
- Happy path: topic "Arizona Wildcats" (multi-token) → Polymarket runs as before.
|
|
||||||
- Edge case: `--polymarket-keywords ""` → treated as empty, no filter.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- Warriors smoke → Polymarket footer absent or filtered.
|
|
||||||
|
|
||||||
- [ ] **Unit 6: SKILL.md rewrite — vs mode is the canonical path, `--competitors` is a shortcut**
|
|
||||||
|
|
||||||
**Goal:** SKILL.md documents the unified architecture. vs mode runs N full passes. `--competitors` is a SKILL.md-level shortcut that discovers 2 peers and invokes vs mode with `--competitors-plan`.
|
|
||||||
|
|
||||||
**Requirements:** R1, R2, R10 (surfaces them)
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-4
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md` (§551 "If QUERY_TYPE = COMPARISON" rewrite; Competitor mode subsection rewrite)
|
|
||||||
- Modify: `README.md` (one-line example)
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- Rewrite §551 to describe the N-pass architecture: "When the user asks 'X vs Y' (or 'X vs Y vs Z'), run Step 0.55 per entity, then invoke the engine. The engine fans out N full pipelines in parallel. Each entity gets its own single-entity-grade coverage. Wall clock is close to a single run."
|
|
||||||
- Remove the "ONE research pass with a comparison-optimized plan that replaces the old 3-pass approach" language.
|
|
||||||
- Add a `--competitors-plan` JSON example.
|
|
||||||
- Rewrite the Competitor mode subsection: "`--competitors` is a shortcut. The hosting model: (1) runs WebSearch to discover N=2 peers, (2) runs Step 0.55 for main + each peer, (3) rewrites topic to `'main vs peer1 vs peer2'`, (4) invokes engine with `--competitors-plan '{...}'`. Engine flag `--competitors` and `--competitors-list` remain for headless fallback."
|
|
||||||
- Cross-reference §679 (per-entity Step 0.55 protocol).
|
|
||||||
- Warning: a thin `## Resolved Entities` block (dashes for any entity) means the hosting model skipped Step 0.55 for that one.
|
|
||||||
|
|
||||||
**Patterns to follow:**
|
|
||||||
- Existing §679 per-entity Step 0.55 protocol for tone.
|
|
||||||
- 3.0.12 Competitor mode prose for terseness.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — documentation. Verification is dogfood.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `/last30days "OpenAI vs Anthropic vs xAI"` in a fresh Claude Code window produces 3 save files with populated Resolved blocks and non-dash per-entity targeting.
|
|
||||||
- `/last30days OpenAI --competitors` produces same after discovery step.
|
|
||||||
|
|
||||||
- [ ] **Unit 7: Version 3.0.13, CHANGELOG, sync, hot-copy**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.13 to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** Closes R1-R10
|
|
||||||
|
|
||||||
**Dependencies:** Units 1-6
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json`
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG: group the changes. "Changed: vs mode now runs N full passes in parallel, one per entity — reverting the one-pass optimization to restore per-entity depth. Added: --competitors-plan JSON for per-entity Step 0.55 targeting (applies to vs mode and --competitors). Changed: --competitors is now a SKILL.md shortcut for vs-with-discovery. Added: per-entity *-raw.md save files. Fixed: override-leak from main to peer sub-runs. Changed: LAW 7 stderr framing for hosting-model context. Changed: BRAVE/SERPER footer nudge suppressed when --plan / --competitors-plan present. Added: --polymarket-keywords + auto-skip for ambiguous topics."
|
|
||||||
- Beta channel first per CLAUDE.md.
|
|
||||||
- Hot-copy so public `/last30days` picks up 3.0.13.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` → 3.0.13.
|
|
||||||
- `sync.sh` exits 0.
|
|
||||||
- Hot-copy contains the new files.
|
|
||||||
|
|
||||||
## System-Wide Impact
|
|
||||||
|
|
||||||
- **Interaction graph:** vs-mode and `--competitors` share one orchestrator (`fanout.run_competitor_fanout`). `_subrun_kwargs` is the single source of per-entity kwargs. Save loop iterates per entity.
|
|
||||||
- **Error propagation:** Per-entity sub-run failure → logged, dropped, continue (3.0.11 behavior unchanged). `--competitors-plan` JSON parse errors exit 2 (same shape as `--plan`).
|
|
||||||
- **State lifecycle risks:** `entity_config = dict(config)` deep-copy pattern extends to every per-entity flag (Unit 2 fix). No cross-entity context leak.
|
|
||||||
- **API surface parity:** `--competitors-plan` is additive. `--competitors`, `--competitors-list`, `--plan` unchanged. `--polymarket-keywords` additive. vs-mode keeps its topic-string surface.
|
|
||||||
- **Integration coverage:** New vs-mode-fanout integration test. New override-leak regression test. New plan-threading test. New nudge-suppression test. New per-entity-save test. New Polymarket disambiguation test.
|
|
||||||
- **Unchanged invariants:** `pipeline.run()` signature unchanged. Single-entity render path unchanged. LAW 7 on the default path unchanged (still fires when a single-entity run lacks `--plan`).
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| vs-mode N-pass latency feels slower for users who remember the one-pass shortcut. | Parallel execution keeps wall-clock ~= max(per-entity-latency), not sum. `--quick` on a vs-topic still applies to each sub-run. CHANGELOG calls out the revert + parallelism. |
|
|
||||||
| API cost scales linearly with N (per source). | Default count 2 caps it. Hard max 6 on `--competitors`. vs-mode users opted into N entities explicitly. |
|
|
||||||
| Rivalry content ("A vs B" articles) missed in N-independent passes. | Deferred to separate task (head-to-head supplemental pass). Start shipping and observe whether this is actually a gap. |
|
|
||||||
| Hosting model skips `--competitors-plan` and uses `--competitors-list` only. | Unit 4 stderr reframe steers explicitly. SKILL.md Unit 6 makes the plan-path canonical. Thin Resolved block in output makes skipped-Step-0.55 visible. |
|
|
||||||
| Override-leak fix misses a subtle closure path. | Unit 2 is test-first with the Kanye receipt as the failing input. Regression test asserts every per-entity flag is None unless plan provides it. |
|
|
||||||
|
|
||||||
## Documentation / Operational Notes
|
|
||||||
|
|
||||||
- Beta channel first per CLAUDE.md.
|
|
||||||
- After merge: hot-copy to `~/.claude/plugins/cache/last30days-skill/last30days/3.0.13/`.
|
|
||||||
- CHANGELOG explicitly frames the vs-mode change as an architectural revert-with-parallelism, not a regression to the old serial N-pass.
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- Superseded plan: `docs/plans/2026-04-22-004-fix-competitors-hosting-model-resolve-and-leak-plan.md.superseded`
|
|
||||||
- Previous plan (3.0.12): `docs/plans/2026-04-22-003-fix-competitors-per-entity-resolution-plan.md`
|
|
||||||
- Initial plan (3.0.11): `docs/plans/2026-04-22-002-feat-competitors-flag-comparison-fanout-plan.md`
|
|
||||||
- 2026-04-22 test session receipts (Warriors, Seattle, Arizona Wildcats, Kanye West)
|
|
||||||
- SKILL.md §551 + §679 — the per-entity Step 0.55 protocol the hosting model uses for both paths
|
|
||||||
- Related code: `scripts/lib/fanout.py`, `scripts/last30days.py` `_competitor_runner`, `scripts/lib/planner.py` vs-topic special-case, `scripts/lib/render.py` `_render_resolved_entities_block`, `scripts/lib/polymarket.py`, `scripts/lib/quality_nudge.py`
|
|
||||||
- Related PRs: #308 (3.0.11), #309 (3.0.12)
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
---
|
|
||||||
|
|
||||||
> **NOTE (added 2026-05-16):** This plan references `bash scripts/sync.sh`. That script was deleted in [PR #405](https://github.com/mvanhorn/last30days-skill/pull/405); the install workflow is now `npx skills add . -g -y` (symlinks the working tree across every detected harness). For context on why sync.sh went away, see [docs/solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md](../solutions/workflow-issues/release-consistency-test-cascade-2026-05-16.md). The decisions captured in this plan remain accurate; only the deploy mechanism changed.
|
|
||||||
|
|
||||||
title: "fix: comparison title says (/Last30Days) instead of (Last 30 Days)"
|
|
||||||
type: fix
|
|
||||||
status: active
|
|
||||||
date: 2026-04-22
|
|
||||||
---
|
|
||||||
|
|
||||||
# fix: comparison title says (/Last30Days) instead of (Last 30 Days)
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
User feedback 2026-04-22 on the 3.0.13 release runs (Kanye vs Drake, Mercer Island, Figma): the comparison title currently reads `# Kanye West vs Drake: What the Community Says (Last 30 Days)`. It should read `# Kanye West vs Drake: What the Community Says (/Last30Days)` — attributing the output to the slash command rather than describing the date range generically.
|
|
||||||
|
|
||||||
Single-line change in SKILL.md, three occurrences. No code change.
|
|
||||||
|
|
||||||
## Requirements Trace
|
|
||||||
|
|
||||||
- R1. Comparison title pattern in SKILL.md changes from `(Last 30 Days)` to `(/Last30Days)` so synthesis outputs read `... What the Community Says (/Last30Days)`.
|
|
||||||
- R2. Both the rule statement (line 113) and the COMPARISON-exception statement (line 131) and the synthesis template example (line 1208) all use the new suffix.
|
|
||||||
- R3. Version bumps to 3.0.14, CHANGELOG entry, sync, hot-copy. Public cache picks up the new title pattern.
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- No changes to the single-entity output title (no `(/Last30Days)` suffix there — only comparison topics carry it).
|
|
||||||
- No changes to engine code. Pure SKILL.md content.
|
|
||||||
- No changes to anything else surfaced in the test runs.
|
|
||||||
|
|
||||||
## Key Technical Decisions
|
|
||||||
|
|
||||||
- **Replace all three occurrences of the suffix string in one pass.** They are identical strings; changing one without the others would cause synthesis-time confusion when the model reaches a different reference.
|
|
||||||
- **Ship as 3.0.14, not 3.0.13.x.** Patch-level bump matches the small scope and keeps the release log clean.
|
|
||||||
|
|
||||||
## Implementation Units
|
|
||||||
|
|
||||||
- [ ] **Unit 1: Replace `(Last 30 Days)` → `(/Last30Days)` in SKILL.md**
|
|
||||||
|
|
||||||
**Goal:** All three SKILL.md references to the comparison title use the new suffix.
|
|
||||||
|
|
||||||
**Requirements:** R1, R2
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `SKILL.md`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- `replace_all` swap of `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Three occurrences, no other strings overlap.
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — pure documentation. Verification by inspection + dogfood run.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep -c "What the Community Says (/Last30Days)" SKILL.md` returns 3.
|
|
||||||
- `grep -c "What the Community Says (Last 30 Days)" SKILL.md` returns 0.
|
|
||||||
|
|
||||||
- [ ] **Unit 2: Version 3.0.14 + CHANGELOG + sync + hot-copy**
|
|
||||||
|
|
||||||
**Goal:** Ship 3.0.14 to all local targets.
|
|
||||||
|
|
||||||
**Requirements:** R3
|
|
||||||
|
|
||||||
**Dependencies:** Unit 1
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.claude-plugin/plugin.json`
|
|
||||||
- Modify: `CHANGELOG.md`
|
|
||||||
- Run: `bash scripts/sync.sh`
|
|
||||||
- Hot-copy: `~/.claude/plugins/cache/last30days-skill/last30days/3.0.14/`
|
|
||||||
|
|
||||||
**Approach:**
|
|
||||||
- CHANGELOG: "Changed: comparison-mode title attribution — `What the Community Says (Last 30 Days)` → `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range."
|
|
||||||
|
|
||||||
**Test scenarios:**
|
|
||||||
- Test expectation: none — packaging.
|
|
||||||
|
|
||||||
**Verification:**
|
|
||||||
- `grep version .claude-plugin/plugin.json` → 3.0.14.
|
|
||||||
- Hot-copy contains the updated SKILL.md.
|
|
||||||
|
|
||||||
## Risks & Dependencies
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|------------|
|
|
||||||
| Hosting model has the old title pattern memorized from a prior run and re-emits `(Last 30 Days)`. | SKILL.md is read top-to-bottom each invocation. STEP 0 canonical-path self-check (3.0.12) ensures the model loads the new SKILL.md, not the marketplace stale copy. |
|
|
||||||
|
|
||||||
## Sources & References
|
|
||||||
|
|
||||||
- 2026-04-22 dogfood runs (Kanye West vs Drake, Mercer Island --competitors, Figma --competitors)
|
|
||||||
- Related code: `SKILL.md` lines 113, 131, 1208
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Search Quality Eval
|
# Search Quality Eval
|
||||||
|
|
||||||
`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
`skills/last30days/scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||||
|
|
||||||
What it does:
|
What it does:
|
||||||
|
|
||||||
@@ -18,13 +18,13 @@ What it does:
|
|||||||
Recommended usage:
|
Recommended usage:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv run python scripts/evaluate_search_quality.py
|
uv run python skills/last30days/scripts/evaluate_search_quality.py
|
||||||
```
|
```
|
||||||
|
|
||||||
Useful flags:
|
Useful flags:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv run python scripts/evaluate_search_quality.py \
|
uv run python skills/last30days/scripts/evaluate_search_quality.py \
|
||||||
--baseline-rev origin/main \
|
--baseline-rev origin/main \
|
||||||
--candidate-rev HEAD \
|
--candidate-rev HEAD \
|
||||||
--no-default-topics \
|
--no-default-topics \
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
---
|
||||||
|
title: Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
|
||||||
|
date: 2026-06-09
|
||||||
|
category: docs/solutions/logic-errors
|
||||||
|
module: lib/rerank
|
||||||
|
problem_type: logic_error
|
||||||
|
component: search_ranking
|
||||||
|
severity: high
|
||||||
|
symptoms:
|
||||||
|
- on-entity, high-engagement items that name the brand but omit the trailing descriptor of a multi-word query are demoted in keyless/fallback rerank results
|
||||||
|
- observed case is a 323-point HN thread about Stripe scoring 0 on a "Stripe payments" query
|
||||||
|
- the entity-miss demotion lands twice (ENTITY_MISS_PENALTY on rerank_score plus a secondary final_score penalty), so a false miss guarantees burial regardless of engagement
|
||||||
|
- reddit keyless comment-enrichment slot selection skips the same on-entity threads via an independently duplicated full-phrase check in _slot_priority
|
||||||
|
root_cause: logic_error
|
||||||
|
resolution_type: code_fix
|
||||||
|
related_components:
|
||||||
|
- reddit_keyless
|
||||||
|
- comment_enrichment
|
||||||
|
tags:
|
||||||
|
- entity-grounding
|
||||||
|
- rerank
|
||||||
|
- keyless-fallback
|
||||||
|
- multi-word-entity
|
||||||
|
- substring-match
|
||||||
|
- false-demotion
|
||||||
|
- reddit-keyless
|
||||||
|
- duplicated-logic
|
||||||
|
---
|
||||||
|
|
||||||
|
# Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The keyless/fallback rerank path's entity-grounding demotion required the FULL multi-word primary-entity phrase as a contiguous substring of the candidate's text (`primary_entity.lower() not in haystack`), so on-entity items that omitted a trailing search descriptor were falsely flagged as entity misses and buried by a deliberately decisive double penalty.
|
||||||
|
|
||||||
|
## Symptoms
|
||||||
|
|
||||||
|
- On a "Stripe payments" query, a 323-point HN thread titled "Stripe is friendly to 'friendly fraud'" was demoted to score 0 — purely because its text never contained the literal phrase "stripe payments" (the trailing word "payments" was missing).
|
||||||
|
- The burial is guaranteed by design, not incidental: a flagged entity miss takes −25 `ENTITY_MISS_PENALTY` on `rerank_score` in `_fallback_tuple`, PLUS `ENTITY_MISS_FINAL_PENALTY` applied directly in `_final_score` (added 2026-04-19 after engagement + freshness drowned the diluted penalty). A false positive on the check means confirmed-good signal cannot recover.
|
||||||
|
- The same over-strict check had been independently re-implemented in `reddit_keyless._slot_priority` (keyless Reddit comment-enrichment slot selection), so scarce comment slots were also steered away from head-token-only posts.
|
||||||
|
|
||||||
|
## What Didn't Work
|
||||||
|
|
||||||
|
- **Naively relaxing the check** — the full-phrase check existed for a real reason: on 2026-04-19 an off-topic video with zero brand mentions ranked #2 on a Hermes query (documented in the `ENTITY_MISS_FINAL_PENALTY` comment in `skills/last30days/scripts/lib/rerank.py`). Any fix had to keep that demotion firing.
|
||||||
|
- **Word-boundary matching** — rejected; it re-introduces over-demotion on plurals/possessives/compounds ("stripes", "Stripe's").
|
||||||
|
- **Graded penalty** (full-phrase = 0, head-only = half, none = full) — rejected; it half-punishes items that are 100% about the entity. Lexical coverage is not topical degree.
|
||||||
|
- **Any-token grounding** — rejected; "payments" alone would ground completely generic posts.
|
||||||
|
- **Distinctiveness gate for generic heads** — rejected as complexity to patch a failure mode that is already a safe no-op (see Why This Works).
|
||||||
|
- **Trusting the docstring** — `reddit_keyless._slot_priority`'s docstring claimed to "mirror rerank's demotion signal," but its inline reimplementation (`entity in _post_text(post).lower()`) had silently drifted from being a mirror into being a second copy of the bug. It was found only by a code-reuse review, not by tests.
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
Ground on the **head token** of the primary entity instead of the full phrase, via one shared helper used by both paths.
|
||||||
|
|
||||||
|
**Site 1 — new helper in `skills/last30days/scripts/lib/rerank.py`:**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def _entity_grounded(haystack: str, primary_entity: str) -> bool:
|
||||||
|
tokens = primary_entity.lower().split()
|
||||||
|
if not tokens:
|
||||||
|
return True
|
||||||
|
return tokens[0] in haystack
|
||||||
|
```
|
||||||
|
|
||||||
|
`_fallback_tuple` switches from the inline phrase check to the helper:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# before
|
||||||
|
if haystack.strip() and primary_entity.lower() not in haystack:
|
||||||
|
# after
|
||||||
|
if haystack.strip() and not _entity_grounded(haystack, primary_entity):
|
||||||
|
```
|
||||||
|
|
||||||
|
**Site 2 — secondary penalty in `_final_score`: no code change needed.** It keys off the explanation string set by site 1, so it inherits the fix automatically:
|
||||||
|
|
||||||
|
```python
|
||||||
|
if candidate.explanation and "entity-miss" in candidate.explanation:
|
||||||
|
base = max(0.0, base - ENTITY_MISS_FINAL_PENALTY)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Site 3 — `skills/last30days/scripts/lib/reddit_keyless.py` `_slot_priority`:** replace the drifted reimplementation with a call to the shared helper:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# before
|
||||||
|
return entity in _post_text(post).lower()
|
||||||
|
# after
|
||||||
|
return rerank._entity_grounded(_post_text(post).lower(), entity)
|
||||||
|
```
|
||||||
|
|
||||||
|
Tests: `tests/test_rerank_v3.py` gained `test_fallback_grounds_on_head_token_not_full_phrase` (the Stripe regression) and `test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` (guards the 2026-04-19 behavior). `tests/test_reddit_keyless.py`'s two old-contract tests were rewritten as `test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match`.
|
||||||
|
|
||||||
|
## Why This Works
|
||||||
|
|
||||||
|
- **Root cause:** trailing tokens of a multi-word query ("payments" in "Stripe payments") are usually category descriptors the user/planner appended for search, not part of the entity name. Requiring the whole phrase conflates "doesn't repeat my search phrasing" with "isn't about my entity." The brand head token alone is sufficient grounding; items that never name the brand at all still miss the head token and stay demoted — so the original 2026-04-19 fix keeps firing.
|
||||||
|
- **Asymmetry argument:** the demotion is engineered to be decisive (double penalty across `rerank_score` and `final_score`), so a false entity-miss is fatal-by-design, while a false grounding merely defers the item to normal relevance/freshness/quality ranking. When the punishment is capital, the conviction standard should be conservative.
|
||||||
|
- **Substring (not word-boundary) is deliberate:** it catches plurals/possessives/compounds ("stripes", "Stripe's"). Degenerate short heads ("X", "Go", "C") make the check vacuously true, which merely **disables** the penalty — reverting to the pre-grounding baseline — rather than burying good items. Every failure mode of this rule degrades toward "no penalty," never toward "bury good signal."
|
||||||
|
- **Accepted, bounded limitation:** head-collision with a different famous entity ("Hermes Agent" → a "Hermes Birkin" thread now escapes demotion). This is lexically unfixable — any token rule strong enough to kill the collision re-kills the Stripe case; the discriminator is semantic. The LLM rerank path (which receives the full phrase as prompt guidance and judges semantically) covers this when API keys exist; the keyless path accepts the bounded risk.
|
||||||
|
|
||||||
|
## Prevention
|
||||||
|
|
||||||
|
- **Shared helper as single source of truth:** when one module's behavior must "mirror" another's signal, it must *call* the same function, not re-implement the check. The `reddit_keyless._slot_priority` drift happened precisely because the mirror was a copy. The fix wires it to `rerank._entity_grounded`, and the docstring now states this explicitly: "keying on the same head token keeps the two paths from diverging."
|
||||||
|
- **Docstrings record deliberate trade-offs:** `_entity_grounded`'s docstring documents WHY head-token (not phrase), why substring (not word-boundary), and the safe-failure direction. Future readers see the rejected alternatives were considered, not overlooked — and won't "tighten" the check into a regression.
|
||||||
|
- **Both directions pinned by named tests:**
|
||||||
|
- `tests/test_rerank_v3.py::test_fallback_grounds_on_head_token_not_full_phrase` — false-demotion regression (the Stripe HN thread must not be flagged).
|
||||||
|
- `tests/test_rerank_v3.py::test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` — the fix must not neuter the demotion (guards the 2026-04-19 off-topic-video incident).
|
||||||
|
- `tests/test_reddit_keyless.py::test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match` — the mirrored path asserts the same contract.
|
||||||
|
- **Audit tests when changing a contract:** tests that encode the old behavior as correct must be rewritten to the new contract, not worked around — the two old `test_reddit_keyless.py` tests would have silently re-blessed the bug.
|
||||||
|
- **For decisive penalties, route through one flag:** the `_final_score` backstop keys off `"entity-miss" in candidate.explanation` rather than re-running the check — so there was exactly one site to fix and the second penalty inherited it for free. Prefer this signal-propagation pattern over duplicating predicate logic at each penalty site.
|
||||||
|
|
||||||
|
## Related Issues
|
||||||
|
|
||||||
|
- [PR #484](https://github.com/mvanhorn/last30days-skill/pull/484) — "fix(reddit): relevance-aware comment-enrichment slot selection in keyless path" — introduced the `_slot_priority` mirror this fix reroutes through the shared helper.
|
||||||
|
- [PR #457](https://github.com/mvanhorn/last30days-skill/pull/457) — "fix(reddit): restore free path via keyless RSS + shreddit scrape" — established the keyless Reddit path.
|
||||||
|
- [PR #488](https://github.com/mvanhorn/last30days-skill/pull/488) (open) — "fix(reddit): relevance floor + relevance-first ranking" — external PR touching the same ranking surface; coordinate before merging both.
|
||||||
|
- [Issue #468](https://github.com/mvanhorn/last30days-skill/issues/468) (open) — relevance scoring over-pruning on-topic YouTube items; same symptom family in a different source.
|
||||||
|
- [../architecture/search-quality-eval-manual-by-default-2026-05-10.md](../architecture/search-quality-eval-manual-by-default-2026-05-10.md) — how to validate ranking/grounding changes like this one (manual eval, not CI-gated).
|
||||||
|
- [../workflow-issues/release-consistency-test-cascade-2026-05-16.md](../workflow-issues/release-consistency-test-cascade-2026-05-16.md) — sibling prevention pattern: lockstep artifacts drift unless mechanically unified.
|
||||||
@@ -1,388 +0,0 @@
|
|||||||
---
|
|
||||||
name: last30days
|
|
||||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
|
||||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
|
||||||
---
|
|
||||||
|
|
||||||
# last30days: Research Any Topic from the Last 30 Days
|
|
||||||
|
|
||||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
|
||||||
|
|
||||||
Use cases:
|
|
||||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
|
||||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
|
||||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
|
||||||
- **General**: any topic you're curious about → understand what the community is saying
|
|
||||||
|
|
||||||
## CRITICAL: Parse User Intent
|
|
||||||
|
|
||||||
Before doing anything, parse the user's input for:
|
|
||||||
|
|
||||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
|
||||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
|
||||||
3. **QUERY TYPE**: What kind of research they want:
|
|
||||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
|
||||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
|
||||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
|
||||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
|
||||||
|
|
||||||
Common patterns:
|
|
||||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
|
||||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
|
||||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
|
||||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
|
||||||
- If tool is specified in the query, use it
|
|
||||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
|
||||||
|
|
||||||
**Store these variables:**
|
|
||||||
- `TOPIC = [extracted topic]`
|
|
||||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
|
||||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Setup Check
|
|
||||||
|
|
||||||
The skill works in three modes based on available API keys:
|
|
||||||
|
|
||||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
|
||||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
|
||||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
|
||||||
|
|
||||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
|
||||||
|
|
||||||
### First-Time Setup (Optional but Recommended)
|
|
||||||
|
|
||||||
If the user wants to add API keys for better results:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mkdir -p ~/.config/last30days
|
|
||||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
|
||||||
# last30days API Configuration
|
|
||||||
# Both keys are optional - skill works with WebSearch fallback
|
|
||||||
|
|
||||||
# For Reddit research (uses OpenAI's web_search tool)
|
|
||||||
OPENAI_API_KEY=
|
|
||||||
|
|
||||||
# For X/Twitter research (uses xAI's x_search tool)
|
|
||||||
XAI_API_KEY=
|
|
||||||
ENVEOF
|
|
||||||
|
|
||||||
chmod 600 ~/.config/last30days/.env
|
|
||||||
echo "Config created at ~/.config/last30days/.env"
|
|
||||||
echo "Edit to add your API keys for enhanced research."
|
|
||||||
```
|
|
||||||
|
|
||||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Research Execution
|
|
||||||
|
|
||||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
|
||||||
|
|
||||||
**Step 1: Run the research script**
|
|
||||||
```bash
|
|
||||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
|
||||||
```
|
|
||||||
|
|
||||||
The script will automatically:
|
|
||||||
- Detect available API keys
|
|
||||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
|
||||||
- Run Reddit/X searches if keys exist
|
|
||||||
- Signal if WebSearch is needed
|
|
||||||
|
|
||||||
**Step 2: Check the output mode**
|
|
||||||
|
|
||||||
The script output will indicate the mode:
|
|
||||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
|
||||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
|
||||||
|
|
||||||
**Step 3: Do WebSearch**
|
|
||||||
|
|
||||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
|
||||||
|
|
||||||
Choose search queries based on QUERY_TYPE:
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
|
||||||
- Search for: `best {TOPIC} recommendations`
|
|
||||||
- Search for: `{TOPIC} list examples`
|
|
||||||
- Search for: `most popular {TOPIC}`
|
|
||||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
|
||||||
|
|
||||||
**If NEWS** ("what's happening with X", "X news"):
|
|
||||||
- Search for: `{TOPIC} news 2026`
|
|
||||||
- Search for: `{TOPIC} announcement update`
|
|
||||||
- Goal: Find current events and recent developments
|
|
||||||
|
|
||||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
|
||||||
- Search for: `{TOPIC} prompts examples 2026`
|
|
||||||
- Search for: `{TOPIC} techniques tips`
|
|
||||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
|
||||||
|
|
||||||
**If GENERAL** (default):
|
|
||||||
- Search for: `{TOPIC} 2026`
|
|
||||||
- Search for: `{TOPIC} discussion`
|
|
||||||
- Goal: Find what people are actually saying
|
|
||||||
|
|
||||||
For ALL query types:
|
|
||||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
|
||||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
|
||||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
|
||||||
- Your knowledge may be outdated - trust the user's terminology
|
|
||||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
|
||||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
|
||||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
|
||||||
|
|
||||||
**Step 3: Wait for background script to complete**
|
|
||||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
|
||||||
|
|
||||||
**Depth options** (passed through from user's command):
|
|
||||||
- `--quick` → Faster, fewer sources (8-12 each)
|
|
||||||
- (default) → Balanced (20-30 each)
|
|
||||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Judge Agent: Synthesize All Sources
|
|
||||||
|
|
||||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
|
||||||
|
|
||||||
The Judge Agent must:
|
|
||||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
|
||||||
2. Weight WebSearch sources LOWER (no engagement data)
|
|
||||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
|
||||||
4. Note any contradictions between sources
|
|
||||||
5. Extract the top 3-5 actionable insights
|
|
||||||
|
|
||||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FIRST: Internalize the Research
|
|
||||||
|
|
||||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
|
||||||
|
|
||||||
Read the research output carefully. Pay attention to:
|
|
||||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
|
||||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
|
||||||
- **What the sources actually say**, not what you assume the topic is about
|
|
||||||
|
|
||||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
|
||||||
|
|
||||||
### If QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
|
||||||
|
|
||||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
|
||||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
|
||||||
- Count how many times each is mentioned
|
|
||||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
|
||||||
- List them by popularity/mention count
|
|
||||||
|
|
||||||
**BAD synthesis for "best Claude Code skills":**
|
|
||||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
|
||||||
|
|
||||||
**GOOD synthesis for "best Claude Code skills":**
|
|
||||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
|
||||||
|
|
||||||
### For all QUERY_TYPEs
|
|
||||||
|
|
||||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
|
||||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
|
||||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
|
||||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
|
||||||
- Common pitfalls mentioned BY THE SOURCES
|
|
||||||
|
|
||||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## THEN: Show Summary + Invite Vision
|
|
||||||
|
|
||||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
|
||||||
|
|
||||||
**Display in this EXACT sequence:**
|
|
||||||
|
|
||||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
|
||||||
```
|
|
||||||
🏆 Most mentioned:
|
|
||||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
|
||||||
2. [Specific name] - mentioned {n}x (sources)
|
|
||||||
3. [Specific name] - mentioned {n}x (sources)
|
|
||||||
4. [Specific name] - mentioned {n}x (sources)
|
|
||||||
5. [Specific name] - mentioned {n}x (sources)
|
|
||||||
|
|
||||||
Notable mentions: [other specific things with 1-2 mentions]
|
|
||||||
```
|
|
||||||
|
|
||||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
|
||||||
```
|
|
||||||
What I learned:
|
|
||||||
|
|
||||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
|
||||||
|
|
||||||
KEY PATTERNS I'll use:
|
|
||||||
1. [Pattern from research]
|
|
||||||
2. [Pattern from research]
|
|
||||||
3. [Pattern from research]
|
|
||||||
```
|
|
||||||
|
|
||||||
**THEN - Stats (right before invitation):**
|
|
||||||
|
|
||||||
For **full/partial mode** (has API keys):
|
|
||||||
```
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
|
||||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
|
||||||
├─ 🌐 Web: {n} pages │ {domains}
|
|
||||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
|
||||||
```
|
|
||||||
|
|
||||||
For **web-only mode** (no API keys):
|
|
||||||
```
|
|
||||||
---
|
|
||||||
✅ Research complete!
|
|
||||||
├─ 🌐 Web: {n} pages │ {domains}
|
|
||||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
|
||||||
|
|
||||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
|
||||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
|
||||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
|
||||||
```
|
|
||||||
|
|
||||||
**LAST - Invitation:**
|
|
||||||
```
|
|
||||||
---
|
|
||||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
|
||||||
```
|
|
||||||
|
|
||||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
|
||||||
|
|
||||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
|
||||||
|
|
||||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
|
||||||
```
|
|
||||||
What tool will you use these prompts with?
|
|
||||||
|
|
||||||
Options:
|
|
||||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
|
||||||
2. Nano Banana Pro (image generation)
|
|
||||||
3. ChatGPT / Claude (text/code)
|
|
||||||
4. Other (tell me)
|
|
||||||
```
|
|
||||||
|
|
||||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WAIT FOR USER'S VISION
|
|
||||||
|
|
||||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
|
||||||
|
|
||||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
|
||||||
|
|
||||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
|
||||||
|
|
||||||
### CRITICAL: Match the FORMAT the research recommends
|
|
||||||
|
|
||||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
|
||||||
|
|
||||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
|
||||||
- Research says "structured parameters" → Use structured key: value format
|
|
||||||
- Research says "natural language" → Use conversational prose
|
|
||||||
- Research says "keyword lists" → Use comma-separated keywords
|
|
||||||
|
|
||||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
|
||||||
|
|
||||||
### Output Format:
|
|
||||||
|
|
||||||
```
|
|
||||||
Here's your prompt for {TARGET_TOOL}:
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
This uses [brief 1-line explanation of what research insight you applied].
|
|
||||||
```
|
|
||||||
|
|
||||||
### Quality Checklist:
|
|
||||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
|
||||||
- [ ] Directly addresses what the user said they want to create
|
|
||||||
- [ ] Uses specific patterns/keywords discovered in research
|
|
||||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
|
||||||
- [ ] Appropriate length and style for TARGET_TOOL
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## IF USER ASKS FOR MORE OPTIONS
|
|
||||||
|
|
||||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
|
||||||
|
|
||||||
After delivering a prompt, offer to write more:
|
|
||||||
|
|
||||||
> Want another prompt? Just tell me what you're creating next.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## CONTEXT MEMORY
|
|
||||||
|
|
||||||
For the rest of this conversation, remember:
|
|
||||||
- **TOPIC**: {topic}
|
|
||||||
- **TARGET_TOOL**: {tool}
|
|
||||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
|
||||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
|
||||||
|
|
||||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
|
||||||
|
|
||||||
When the user asks follow-up questions:
|
|
||||||
- **DO NOT run new WebSearches** - you already have the research
|
|
||||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
|
||||||
- **If they ask for a prompt** - write one using your expertise
|
|
||||||
- **If they ask a question** - answer it from your research findings
|
|
||||||
|
|
||||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Output Summary Footer (After Each Prompt)
|
|
||||||
|
|
||||||
After delivering a prompt, end with:
|
|
||||||
|
|
||||||
For **full/partial mode**:
|
|
||||||
```
|
|
||||||
---
|
|
||||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
|
||||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
|
||||||
|
|
||||||
Want another prompt? Just tell me what you're creating next.
|
|
||||||
```
|
|
||||||
|
|
||||||
For **web-only mode**:
|
|
||||||
```
|
|
||||||
---
|
|
||||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
|
||||||
📊 Based on: {n} web pages from {domains}
|
|
||||||
|
|
||||||
Want another prompt? Just tell me what you're creating next.
|
|
||||||
|
|
||||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
|
||||||
```
|
|
||||||
@@ -1,310 +0,0 @@
|
|||||||
# V1 vs V2 Comparison Analysis
|
|
||||||
|
|
||||||
**Date:** 2026-02-06
|
|
||||||
**Queries tested:** 4 (1 head-to-head, 3 V1-only)
|
|
||||||
**Scope:** Quick smoke test, not full 17-query matrix
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Part 1: Head-to-Head -- "kanye west" (NEWS Query)
|
|
||||||
|
|
||||||
### Dimension-by-Dimension Scoring
|
|
||||||
|
|
||||||
#### 1. Query Parsing Display
|
|
||||||
|
|
||||||
Does it show the `🔍 **{TOPIC}** · {QUERY_TYPE}` line before running tools?
|
|
||||||
|
|
||||||
| Version | Score | Evidence |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| V1 | 1 | No parsing display at all. Output starts with "## What I learned:" -- jumps straight into synthesis. No acknowledgment of topic or query type before research. |
|
|
||||||
| V2 | 1 | No parsing display either. Output starts with "Here's what I found:" then "## What I learned:" -- same problem as V1. |
|
|
||||||
|
|
||||||
**Analysis:** Neither version actually rendered the query parsing display. V2 SKILL.md explicitly requires `🔍 **kanye west** · News` before any tools run, but the agent did not produce it. This is a V2 instruction that failed to land. Both score 1/5.
|
|
||||||
|
|
||||||
Possible cause: The parsing display is supposed to appear *before* tools are called -- it may have been shown during execution but not captured in the final output text. If so, both outputs represent only the post-research synthesis, not the full session. Regardless, based on what is in the output files, neither shows it.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
#### 2. Source Coverage (Reddit/X/Web counts)
|
|
||||||
|
|
||||||
| Version | Score | Evidence |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| V1 | 3 | `Reddit: 0 relevant threads` / `X: 30 posts │ ~10 likes` / `Web: 20+ pages`. Two of three sources returned results. Reddit was zero. |
|
|
||||||
| V2 | 3 | `Reddit: 0 threads (no results this cycle)` / `X: 29 posts │ 33 likes │ 14 reposts` / `Web: 30+ pages`. Same pattern: two of three returned results. |
|
|
||||||
|
|
||||||
**Analysis:** Nearly identical coverage. Both got zero Reddit results (likely a script/API issue for this topic, not a SKILL.md problem). V2 has slightly more precise X metrics (33 likes, 14 reposts vs. V1's vague "~10 likes"). V2 has more web pages (30+ vs 20+). Both miss the 10+ Reddit threshold for a score of 4+.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
#### 3. Citation Quality (sparse vs every-sentence)
|
|
||||||
|
|
||||||
| Version | Score | Evidence |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| V1 | 2 | No inline citations at all. The body text makes claims ("full-page Wall Street Journal apology," "Hellwatt Festival in Italy") but never attributes them to a specific source. The stats box lists "Washington Post, Billboard, AllHipHop" but the body has zero `per @handle` or `per Rolling Stone` attributions. |
|
|
||||||
| V2 | 5 | Every bold section ends with a sparse, clean citation. Examples: `"per Rolling Stone"`, `"per The Washington Post"`, `"per Billboard"`, `"per AllHipHop"`, `"per The News International"`. One citation per topic, never chained. Exactly what V2 SKILL.md specifies. |
|
|
||||||
|
|
||||||
**Analysis:** This is the single biggest quality gap between V1 and V2. V1's output reads like a Wikipedia summary -- informative but ungrounded. V2 reads like a researched briefing where every claim has a named source. V2 nails the "sparse citation" rule from its SKILL.md: `"cite 1 source per pattern, short format: 'per @handle' or 'per r/sub'"`.
|
|
||||||
|
|
||||||
V1 quote (no citation): `"He'll headline the new Hellwatt Festival in Italy (July 4-18, 2026)."`
|
|
||||||
V2 quote (cited): `"Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard."`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
#### 4. Summary Structure (bold topic headers, organized sections)
|
|
||||||
|
|
||||||
| Version | Score | Evidence |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| V1 | 3 | Has a coherent narrative structure with a paragraph of synthesis, then a `**KEY THEMES:**` numbered list. But the opening is a single dense paragraph, not broken into scannable sections with bold headers. |
|
|
||||||
| V2 | 5 | Each storyline gets its own bold header: `**BULLY Album — March 20, 2026 via Gamma**`, `**Public Apology for Antisemitism**`, `**Hellwatt Festival in Italy**`, `**Health Concerns**`, `**Grammys Ban**`, `**Kim & Lewis Hamilton Buzz**`. Each is a standalone scannable unit with 1-3 sentences. |
|
|
||||||
|
|
||||||
**Analysis:** V2 follows the SKILL.md template exactly: `**{Topic 1}** — [1-2 sentences, per source]`. V1 uses a blob + list approach which is readable but less scannable. V2 is notably better for a user who wants to skim and find the story they care about.
|
|
||||||
|
|
||||||
V1 structure: 1 dense paragraph -> 5-item `KEY THEMES` list
|
|
||||||
V2 structure: 6 bold topic cards, each self-contained -> no KEY THEMES list (but doesn't need one because the structure itself is the organization)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
#### 5. Stats Box Format (emoji tree vs plain text)
|
|
||||||
|
|
||||||
| Version | Score | Evidence |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| V1 | 4 | Uses `├─` tree format with emoji: `├─ 🟠 Reddit: 0 relevant threads` / `├─ 🔵 X: 30 posts` / `├─ 🌐 Web: 20+ pages` / `└─ Top voices:`. Minor deviation: says "0 relevant threads (filtered out noise)" instead of the V1 SKILL.md template "0 threads (no results this cycle)". Also omits the `🗣️` emoji on the Top voices line. |
|
|
||||||
| V2 | 5 | Perfect match to V2 SKILL.md template: `├─ 🟠 Reddit: 0 threads (no results this cycle)` / `├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)` / `├─ 🌐 Web: 30+ pages │ rollingstone.com, ...` / `└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex`. Includes `(via xAI)` notation, `🗣️` emoji, @handles with engagement counts. |
|
|
||||||
|
|
||||||
**Analysis:** V2 is tighter and matches its template exactly. V1 is close but has minor deviations (custom "filtered out noise" text, missing `🗣️` emoji, no @handles or engagement counts on Top voices). V2's inclusion of actual @handles with like counts (`@honest30bgfan_ (33 likes)`) adds credibility.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
#### 6. Research Grounding (actual research vs generic knowledge)
|
|
||||||
|
|
||||||
| Version | Score | Evidence |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| V1 | 4 | Clearly grounded: mentions specific details like "Wall Street Journal apology (Jan 26, 2026)," "four-month-long manic episode," "frontal-lobe brain injury," "North West collaborated on 'Piercings on My Hand,'" "Monumental Plaza de Toros." These are specific enough to be from research, not pre-training. Minor generic leakage: the "KEY THEMES" list uses editorial framing ("Accountability arc," "Mental health transparency") that feels more like analysis than research extraction. |
|
|
||||||
| V2 | 5 | Every fact is specific and attributed: "12th studio album," "13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign," "earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded," "103,000-capacity RCF Arena." The AI-deepfaked vocals detail is a standout -- it is clearly from research, not something a model would know from pre-training. The Kim/Lewis Hamilton item (`"X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton"`) is explicitly sourced from X data, not general knowledge. |
|
|
||||||
|
|
||||||
**Analysis:** Both are well-grounded, but V2 has more "could only come from research" details. The deepfaked vocals story, the exact venue capacity, and the explicit X chatter observation are details that prove the synthesis is from the research output, not hallucinated.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
#### 7. Prompt Quality (invitation to share vision, not dumping prompts)
|
|
||||||
|
|
||||||
| Version | Score | Evidence |
|
|
||||||
|---------|-------|----------|
|
|
||||||
| V1 | 3 | Ends with: `"Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in."` This is a follow-up invitation, but it is NOT the SKILL.md-specified invitation. It is topic-specific and conversational, which is nice, but it does not ask the user to "share your vision for what you want to create." It misses the prompt-generation angle entirely. |
|
|
||||||
| V2 | 5 | Ends with exactly: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice."` This matches the V2 SKILL.md template verbatim. It positions the skill correctly: not a news summarizer but a research-to-prompt pipeline. |
|
|
||||||
|
|
||||||
**Analysis:** V1's closing is friendly but off-brand. It treats the skill as a research tool, not a research-to-prompt tool. V2 correctly frames the next step as "tell me what to create and I'll write the prompt." This is a meaningful difference -- V1 would leave a user thinking they just got a summary, while V2 primes them to get a usable output.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Head-to-Head Scorecard
|
|
||||||
|
|
||||||
| Dimension | V1 | V2 | Winner |
|
|
||||||
|-----------|----|----|--------|
|
|
||||||
| 1. Query Parsing Display | 1 | 1 | Tie (both failed) |
|
|
||||||
| 2. Source Coverage | 3 | 3 | Tie |
|
|
||||||
| 3. Citation Quality | 2 | 5 | **V2 (+3)** |
|
|
||||||
| 4. Summary Structure | 3 | 5 | **V2 (+2)** |
|
|
||||||
| 5. Stats Box Format | 4 | 5 | **V2 (+1)** |
|
|
||||||
| 6. Research Grounding | 4 | 5 | **V2 (+1)** |
|
|
||||||
| 7. Prompt Quality (invitation) | 3 | 5 | **V2 (+2)** |
|
|
||||||
| **TOTAL** | **20/35** | **29/35** | **V2 wins by 9 points** |
|
|
||||||
|
|
||||||
**V2 is clearly better.** The biggest gaps are citation quality (+3) and summary structure (+2). V2's output reads like a professional research briefing; V1's reads like a decent but unstructured summary.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Part 2: V1-Only Outputs Analysis
|
|
||||||
|
|
||||||
### Output 1: "open claw" (GENERAL query)
|
|
||||||
|
|
||||||
**What V1 does well:**
|
|
||||||
- Strong research grounding. Mentions exact numbers: "145,000+ GitHub stars," "20,000+ forks," "700+ skills," "341 malicious skills." These are clearly from research.
|
|
||||||
- The KEY PATTERNS section is excellent: 5 well-organized patterns with community quotes (`"I give it sudo and let it configure everything"` vs `"prompt injection is terrifying when you give the bot access to your actual bank account"`).
|
|
||||||
- Good synthesis of the security vs. enthusiasm tension -- captures the community split accurately.
|
|
||||||
- Stats box uses the emoji tree format correctly with `├──` (though note: uses double-dash `──` instead of single `─`, minor inconsistency).
|
|
||||||
|
|
||||||
**What V1 is missing (per V2 SKILL.md features):**
|
|
||||||
- No query parsing display (`🔍 **open claw** · General`).
|
|
||||||
- No inline citations in the body text. The 5 KEY PATTERNS have no `per @handle` or `per r/sub` attribution. Which Reddit thread said "I give it sudo"? Which X post raised the security concern? We do not know.
|
|
||||||
- The stats box says `├── 🟠 Reddit: 25 threads │ ~750+ upvotes` -- the tilde and plus are imprecise. V2 SKILL.md wants exact parsed numbers.
|
|
||||||
- Top voices line lists subreddits and handles but no engagement counts: `@grok, @Starlink` -- are these the highest-engagement handles? No like counts shown.
|
|
||||||
- No bold topic headers in the body -- it is a single paragraph followed by a numbered list, not the `**{Topic}** — sentence, per source` format V2 requires.
|
|
||||||
|
|
||||||
**V1 Score (estimated):** 22/35
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Output 2: "nano banana pro prompting" (PROMPTING query)
|
|
||||||
|
|
||||||
**What V1 does well:**
|
|
||||||
- Correctly identifies two prompting styles (JSON structured vs. natural language "Creative Director") and explains when each works best. This is excellent PROMPTING-type synthesis.
|
|
||||||
- KEY PATTERNS are specific and actionable: "85mm lens at f/1.8," "three-point lighting with key at 45 degrees," "text rendering works -- keep text under 3 words for best results (75% success rate)." These are concrete tips a user can apply immediately.
|
|
||||||
- Research grounding is strong: cites specific upvote counts ("149-259 upvotes"), subreddit names (`r/nanobanana2pro`), and the Google AI blog.
|
|
||||||
- The invitation correctly targets Nano Banana Pro: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro."`
|
|
||||||
|
|
||||||
**What V1 is missing (per V2 SKILL.md features):**
|
|
||||||
- No query parsing display.
|
|
||||||
- Stats box uses plain text dashes: `- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments` instead of the tree format `├─ 🟠 Reddit:`. Uses `|` pipe instead of `│` box-drawing character. V2 SKILL.md explicitly says: "NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji."
|
|
||||||
- No inline body citations. KEY PATTERNS mention Reddit upvote ranges but no specific `per @handle` attributions.
|
|
||||||
- Missing `✅ All agents reported back!` header -- just says "All agents reported back!" without the checkmark.
|
|
||||||
- Body structure is paragraph + numbered list, not bold topic headers.
|
|
||||||
|
|
||||||
**V1 Score (estimated):** 23/35 (slightly higher than open claw due to better actionability)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Output 3: "how to best setup clawdbot" (HOW-TO query)
|
|
||||||
|
|
||||||
**What V1 does well:**
|
|
||||||
- This is the best V1 output of the batch. It goes beyond synthesis and actually delivers a **Quick-Start guide** with numbered steps, a **Security Hardening** checklist, and a **Budget Option** -- all grounded in research.
|
|
||||||
- Excellent research grounding: `"per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0"` -- this is an actual citation with an @handle!
|
|
||||||
- Specific, actionable recommendations: exact commands (`curl -fsSL https://clawd.bot/install.sh | bash`), specific model recommendations (Claude Opus 4.5 for best results, GLM 4.7 Flash for local), specific channel advice (Telegram first, WhatsApp QR code fails).
|
|
||||||
- Stats box is correct emoji tree format with engagement counts: `@aashatwt (452 likes), @recap_david (329 likes)`.
|
|
||||||
- Captures the naming confusion accurately: "Clawdbot -> Moltbot -> OpenClaw."
|
|
||||||
|
|
||||||
**What V1 is missing (per V2 SKILL.md features):**
|
|
||||||
- No query parsing display.
|
|
||||||
- Body text has no inline citations except the Budget Option section. The 5 KEY PATTERNS have no `per @handle` attribution.
|
|
||||||
- Bold topic headers are used only in the Quick-Start and Security sections, not in the KEY PATTERNS or intro.
|
|
||||||
- The output delivers the "answer" directly (setup guide) rather than waiting for the user's vision and offering to write a prompt. For a HOW-TO query this might be the right call, but it skips the SKILL.md flow of "show research -> invite vision -> write prompt."
|
|
||||||
|
|
||||||
**V1 Score (estimated):** 26/35 (best of the V1 outputs)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Patterns Across All V1 Outputs
|
|
||||||
|
|
||||||
**Consistent strengths:**
|
|
||||||
1. Research grounding is solid across all three. V1 does not hallucinate -- the facts are clearly from the research output, not pre-training.
|
|
||||||
2. KEY PATTERNS lists are consistently useful and actionable.
|
|
||||||
3. Stats boxes are present in all outputs (though formatting varies).
|
|
||||||
4. The invitation/closing line is present in all outputs.
|
|
||||||
|
|
||||||
**Consistent weaknesses:**
|
|
||||||
1. **No query parsing display** in any output (0 for 4, including Kanye West).
|
|
||||||
2. **No inline citations** in the body text (except one @handle in the clawdbot output). The research feels real but is unattributed.
|
|
||||||
3. **Stats box formatting is inconsistent.** Open claw uses `├──` (double dash), nano banana pro uses `- 🟠` (plain dash + pipe), clawdbot uses `├─` (correct). Three different formats in three outputs.
|
|
||||||
4. **Body structure defaults to paragraph + numbered list** instead of bold topic headers. Only clawdbot partially uses bold headers (in the guide section, not the research section).
|
|
||||||
5. **No `(via Bird/xAI)` notation** on X stats in any output.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Part 3: SKILL.md Feature Diff
|
|
||||||
|
|
||||||
### Features in V2 but NOT V1
|
|
||||||
|
|
||||||
| Feature | V2 Lines | Impact |
|
|
||||||
|---------|----------|--------|
|
|
||||||
| **Query parsing display** (`🔍 **{TOPIC}** · {QUERY_TYPE}`) | 40-53 | HIGH -- confirms to user the skill understood their request before spending time on research. |
|
|
||||||
| **Sparse citation rules** with BAD/GOOD examples | 186-193 | HIGH -- this is the #1 quality differentiator in the Kanye head-to-head. `"per @handle"` format, never chain multiple citations. |
|
|
||||||
| **Bold topic headers** template (`**{Topic 1}** — [1-2 sentences, per source]`) | 195-208 | HIGH -- makes output scannable. |
|
|
||||||
| **Strict stats template** with "NEVER use plain text dashes" instruction | 217-230 | MEDIUM -- prevents the formatting inconsistency seen across V1 outputs. |
|
|
||||||
| **RECOMMENDATIONS source attribution** (each item MUST have Sources: line with @handles) | 178-182 | MEDIUM -- only affects RECOMMENDATIONS queries. |
|
|
||||||
| **Reddit 0 results handling** (explicit instruction for what to write) | 229 | LOW -- edge case, but prevents ad-hoc text like V1's "filtered out noise." |
|
|
||||||
| **Bird CLI / xAI notation** in stats | 223 | LOW -- cosmetic transparency about data source. |
|
|
||||||
| **Step 2 phrasing: "DO WEBSEARCH WHILE SCRIPT RUNS"** | 71-73 | LOW -- execution optimization, no output impact. |
|
|
||||||
|
|
||||||
### Features in V1 but NOT V2
|
|
||||||
|
|
||||||
| Feature | V1 Lines | Impact | Should Restore? |
|
|
||||||
|---------|----------|--------|-----------------|
|
|
||||||
| **Use cases block** (4 examples in intro) | 12-17 | LOW | No |
|
|
||||||
| **Setup Check section** (3 modes, bash script, "keys are OPTIONAL") | 50-78 | MEDIUM for new users | Yes, for public release |
|
|
||||||
| **BAD/GOOD synthesis anti-pattern examples** | 172-191 | MEDIUM-HIGH | YES |
|
|
||||||
| **Self-check instruction** ("Re-read your 'What I learned' section...") | 269 | MEDIUM | YES |
|
|
||||||
| **Quality Checklist** (5-point checklist before delivering prompt) | 306-324 | HIGH | YES |
|
|
||||||
| **Prompt format anti-pattern** ("Research says JSON but you write prose") | 302 | MEDIUM | YES |
|
|
||||||
| **"IF USER ASKS FOR MORE OPTIONS"** section | 327-329 | LOW-MEDIUM | YES |
|
|
||||||
| **Web-only mode stats template + promo** | 248-259 | MEDIUM for no-key users | For public release |
|
|
||||||
| **TARGET_TOOL question template** (4 options) | 272-280 | LOW | No |
|
|
||||||
| **Context Memory: explicit "don't re-search" instructions** | 342-358 | MEDIUM | YES |
|
|
||||||
| **Output footer emoji + engagement counts** | 366-380 | LOW | YES |
|
|
||||||
|
|
||||||
### Features in BOTH (Shared)
|
|
||||||
|
|
||||||
| Feature | Notes |
|
|
||||||
|---------|-------|
|
|
||||||
| Parse User Intent (TOPIC, TARGET_TOOL, QUERY_TYPE) | Same 4 query types, same detection logic |
|
|
||||||
| "Don't ask about tool before research" rule | Identical |
|
|
||||||
| Research script execution command | Same `python3` command |
|
|
||||||
| WebSearch queries by QUERY_TYPE | Same search strategies |
|
|
||||||
| "Use user's exact terminology" instruction | V2 shorter but same intent |
|
|
||||||
| Judge Agent synthesis logic | Same 5-step weighting process |
|
|
||||||
| "Ground in actual research" instruction | Same core instruction, V1 has more examples |
|
|
||||||
| RECOMMENDATIONS: extract specific names | Same logic |
|
|
||||||
| Prompt format matching | Same instruction |
|
|
||||||
| Wait for user's vision | Same |
|
|
||||||
| Write ONE perfect prompt | Same structure |
|
|
||||||
| Context Memory | V2 shorter version |
|
|
||||||
| Output summary footer | Both have it, V1 has emoji |
|
|
||||||
| Depth options (quick/default/deep) | Same |
|
|
||||||
| "After each prompt: Stay in Expert Mode" | Same |
|
|
||||||
|
|
||||||
### Overall Assessment
|
|
||||||
|
|
||||||
**V2 is a clear upgrade in output formatting and citation quality.** The three features V2 adds (query parsing display, sparse citation rules, bold topic headers) directly address the three biggest weaknesses seen across all V1 outputs. The Kanye West head-to-head proves it: V2 scores 29/35 vs V1's 20/35.
|
|
||||||
|
|
||||||
**However, V2 dropped several quality guardrails from V1** that do not affect formatting but affect *correctness*: the self-check instruction, the anti-pattern examples, the quality checklist for prompts, and the "don't re-search" context memory rule. These are cheap to restore (under 25 lines total) and protect against subtle failure modes that may not show up in a 1-query test but will appear over dozens of uses.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Part 4: Verdict
|
|
||||||
|
|
||||||
### Ship V2 or Not?
|
|
||||||
|
|
||||||
**Ship V2 -- but restore the guardrails first.**
|
|
||||||
|
|
||||||
V2 is unambiguously better on every formatting dimension. The citation quality improvement alone (V1: 2/5 -> V2: 5/5) makes it worth shipping. The bold topic headers and strict stats template fix the inconsistency problems visible across all V1 outputs.
|
|
||||||
|
|
||||||
But V2 dropped 6 guardrail features from V1 that cost almost nothing to include and protect against real failure modes. These should be restored before V2 goes public.
|
|
||||||
|
|
||||||
### Remaining Gaps
|
|
||||||
|
|
||||||
**Must fix before shipping (affects correctness):**
|
|
||||||
|
|
||||||
1. **Restore the quality checklist for prompts.** This is the test plan's #1 priority item. V1 had a 5-point checklist; V2 reduced it to one line. The checklist is what makes prompts feel polished -- it is the "that's a great prompt" mechanism. Add 8 lines.
|
|
||||||
|
|
||||||
2. **Restore BAD/GOOD anti-pattern examples.** V2 says "ground in actual research" but does not show what *bad* grounding looks like. V1's ClawdBot/Claude Code conflation example is exactly the kind of concrete negative example that prevents real failures. Add 5 lines.
|
|
||||||
|
|
||||||
3. **Restore self-check instruction.** One sentence: "Re-read your 'What I learned' section -- does it match what the research ACTUALLY says?" Zero cost, catches hallucination. Add 2 lines.
|
|
||||||
|
|
||||||
4. **Restore "don't re-search" context memory rule.** V2 only says "only do new research if user asks about a DIFFERENT topic." V1 explicitly bans re-searching and tells the agent to answer from existing research. Add 3 lines.
|
|
||||||
|
|
||||||
**Should fix (polish):**
|
|
||||||
|
|
||||||
5. Restore prompt format anti-pattern ("Research says JSON but you write prose"). Add 2 lines.
|
|
||||||
6. Restore "IF USER ASKS FOR MORE OPTIONS" section. Add 2 lines.
|
|
||||||
7. Add emoji + engagement counts back to the output summary footer. Edit 3 lines.
|
|
||||||
|
|
||||||
**Skip for now:**
|
|
||||||
|
|
||||||
8. Setup Check section -- add back for public release, not needed for execution.
|
|
||||||
9. Web-only mode stats template -- lower priority, most testers have API keys.
|
|
||||||
10. TARGET_TOOL question template -- agent handles this naturally.
|
|
||||||
|
|
||||||
### Query Parsing Display: Investigate
|
|
||||||
|
|
||||||
Both V1 and V2 scored 1/5 on query parsing display. V2 has the feature in its SKILL.md but the agent did not render it in the captured output. This could mean:
|
|
||||||
- The display was shown during execution but not captured (likely -- it appears before tools run, and the output files may only contain post-research content).
|
|
||||||
- The instruction is not strong enough and the agent skips it.
|
|
||||||
|
|
||||||
**Recommendation:** Verify in a live session whether the parsing display actually appears. If it does not, strengthen the instruction (e.g., "This line MUST be the first thing you output, before any tool calls").
|
|
||||||
|
|
||||||
### Total Effort
|
|
||||||
|
|
||||||
Restoring all 7 priority items: approximately 25 lines added to V2 SKILL.md. Under 15 minutes of work. The V2 formatting wins are substantial and proven; the V1 guardrails are small and proven. Combining both produces the best version.
|
|
||||||
|
|
||||||
### Final Score Summary
|
|
||||||
|
|
||||||
| | V1 (Kanye) | V2 (Kanye) | Delta |
|
|
||||||
|--|-----------|-----------|-------|
|
|
||||||
| Total | 20/35 | 29/35 | **V2 +9** |
|
|
||||||
|
|
||||||
| | V1 (Open Claw) | V1 (Nano Banana) | V1 (Clawdbot) | V1 Average |
|
|
||||||
|--|---------------|-----------------|--------------|------------|
|
|
||||||
| Estimated Total | 22/35 | 23/35 | 26/35 | **23.7/35** |
|
|
||||||
|
|
||||||
V2 at 29/35 beats every V1 output, including V1's best (clawdbot at 26/35).
|
|
||||||
|
|
||||||
**Decision: Ship V2 with guardrails restored.**
|
|
||||||
@@ -1,388 +0,0 @@
|
|||||||
---
|
|
||||||
name: last30days
|
|
||||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
|
||||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
|
||||||
---
|
|
||||||
|
|
||||||
# last30days: Research Any Topic from the Last 30 Days
|
|
||||||
|
|
||||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
|
||||||
|
|
||||||
Use cases:
|
|
||||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
|
||||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
|
||||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
|
||||||
- **General**: any topic you're curious about → understand what the community is saying
|
|
||||||
|
|
||||||
## CRITICAL: Parse User Intent
|
|
||||||
|
|
||||||
Before doing anything, parse the user's input for:
|
|
||||||
|
|
||||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
|
||||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
|
||||||
3. **QUERY TYPE**: What kind of research they want:
|
|
||||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
|
||||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
|
||||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
|
||||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
|
||||||
|
|
||||||
Common patterns:
|
|
||||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
|
||||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
|
||||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
|
||||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
|
||||||
- If tool is specified in the query, use it
|
|
||||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
|
||||||
|
|
||||||
**Store these variables:**
|
|
||||||
- `TOPIC = [extracted topic]`
|
|
||||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
|
||||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Setup Check
|
|
||||||
|
|
||||||
The skill works in three modes based on available API keys:
|
|
||||||
|
|
||||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
|
||||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
|
||||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
|
||||||
|
|
||||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
|
||||||
|
|
||||||
### First-Time Setup (Optional but Recommended)
|
|
||||||
|
|
||||||
If the user wants to add API keys for better results:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mkdir -p ~/.config/last30days
|
|
||||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
|
||||||
# last30days API Configuration
|
|
||||||
# Both keys are optional - skill works with WebSearch fallback
|
|
||||||
|
|
||||||
# For Reddit research (uses OpenAI's web_search tool)
|
|
||||||
OPENAI_API_KEY=
|
|
||||||
|
|
||||||
# For X/Twitter research (uses xAI's x_search tool)
|
|
||||||
XAI_API_KEY=
|
|
||||||
ENVEOF
|
|
||||||
|
|
||||||
chmod 600 ~/.config/last30days/.env
|
|
||||||
echo "Config created at ~/.config/last30days/.env"
|
|
||||||
echo "Edit to add your API keys for enhanced research."
|
|
||||||
```
|
|
||||||
|
|
||||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Research Execution
|
|
||||||
|
|
||||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
|
||||||
|
|
||||||
**Step 1: Run the research script**
|
|
||||||
```bash
|
|
||||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
|
||||||
```
|
|
||||||
|
|
||||||
The script will automatically:
|
|
||||||
- Detect available API keys
|
|
||||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
|
||||||
- Run Reddit/X searches if keys exist
|
|
||||||
- Signal if WebSearch is needed
|
|
||||||
|
|
||||||
**Step 2: Check the output mode**
|
|
||||||
|
|
||||||
The script output will indicate the mode:
|
|
||||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
|
||||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
|
||||||
|
|
||||||
**Step 3: Do WebSearch**
|
|
||||||
|
|
||||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
|
||||||
|
|
||||||
Choose search queries based on QUERY_TYPE:
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
|
||||||
- Search for: `best {TOPIC} recommendations`
|
|
||||||
- Search for: `{TOPIC} list examples`
|
|
||||||
- Search for: `most popular {TOPIC}`
|
|
||||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
|
||||||
|
|
||||||
**If NEWS** ("what's happening with X", "X news"):
|
|
||||||
- Search for: `{TOPIC} news 2026`
|
|
||||||
- Search for: `{TOPIC} announcement update`
|
|
||||||
- Goal: Find current events and recent developments
|
|
||||||
|
|
||||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
|
||||||
- Search for: `{TOPIC} prompts examples 2026`
|
|
||||||
- Search for: `{TOPIC} techniques tips`
|
|
||||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
|
||||||
|
|
||||||
**If GENERAL** (default):
|
|
||||||
- Search for: `{TOPIC} 2026`
|
|
||||||
- Search for: `{TOPIC} discussion`
|
|
||||||
- Goal: Find what people are actually saying
|
|
||||||
|
|
||||||
For ALL query types:
|
|
||||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
|
||||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
|
||||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
|
||||||
- Your knowledge may be outdated - trust the user's terminology
|
|
||||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
|
||||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
|
||||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
|
||||||
|
|
||||||
**Step 3: Wait for background script to complete**
|
|
||||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
|
||||||
|
|
||||||
**Depth options** (passed through from user's command):
|
|
||||||
- `--quick` → Faster, fewer sources (8-12 each)
|
|
||||||
- (default) → Balanced (20-30 each)
|
|
||||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Judge Agent: Synthesize All Sources
|
|
||||||
|
|
||||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
|
||||||
|
|
||||||
The Judge Agent must:
|
|
||||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
|
||||||
2. Weight WebSearch sources LOWER (no engagement data)
|
|
||||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
|
||||||
4. Note any contradictions between sources
|
|
||||||
5. Extract the top 3-5 actionable insights
|
|
||||||
|
|
||||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FIRST: Internalize the Research
|
|
||||||
|
|
||||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
|
||||||
|
|
||||||
Read the research output carefully. Pay attention to:
|
|
||||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
|
||||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
|
||||||
- **What the sources actually say**, not what you assume the topic is about
|
|
||||||
|
|
||||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
|
||||||
|
|
||||||
### If QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
|
||||||
|
|
||||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
|
||||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
|
||||||
- Count how many times each is mentioned
|
|
||||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
|
||||||
- List them by popularity/mention count
|
|
||||||
|
|
||||||
**BAD synthesis for "best Claude Code skills":**
|
|
||||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
|
||||||
|
|
||||||
**GOOD synthesis for "best Claude Code skills":**
|
|
||||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
|
||||||
|
|
||||||
### For all QUERY_TYPEs
|
|
||||||
|
|
||||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
|
||||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
|
||||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
|
||||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
|
||||||
- Common pitfalls mentioned BY THE SOURCES
|
|
||||||
|
|
||||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## THEN: Show Summary + Invite Vision
|
|
||||||
|
|
||||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
|
||||||
|
|
||||||
**Display in this EXACT sequence:**
|
|
||||||
|
|
||||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
|
||||||
```
|
|
||||||
🏆 Most mentioned:
|
|
||||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
|
||||||
2. [Specific name] - mentioned {n}x (sources)
|
|
||||||
3. [Specific name] - mentioned {n}x (sources)
|
|
||||||
4. [Specific name] - mentioned {n}x (sources)
|
|
||||||
5. [Specific name] - mentioned {n}x (sources)
|
|
||||||
|
|
||||||
Notable mentions: [other specific things with 1-2 mentions]
|
|
||||||
```
|
|
||||||
|
|
||||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
|
||||||
```
|
|
||||||
What I learned:
|
|
||||||
|
|
||||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
|
||||||
|
|
||||||
KEY PATTERNS I'll use:
|
|
||||||
1. [Pattern from research]
|
|
||||||
2. [Pattern from research]
|
|
||||||
3. [Pattern from research]
|
|
||||||
```
|
|
||||||
|
|
||||||
**THEN - Stats (right before invitation):**
|
|
||||||
|
|
||||||
For **full/partial mode** (has API keys):
|
|
||||||
```
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
|
||||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
|
||||||
├─ 🌐 Web: {n} pages │ {domains}
|
|
||||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
|
||||||
```
|
|
||||||
|
|
||||||
For **web-only mode** (no API keys):
|
|
||||||
```
|
|
||||||
---
|
|
||||||
✅ Research complete!
|
|
||||||
├─ 🌐 Web: {n} pages │ {domains}
|
|
||||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
|
||||||
|
|
||||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
|
||||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
|
||||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
|
||||||
```
|
|
||||||
|
|
||||||
**LAST - Invitation:**
|
|
||||||
```
|
|
||||||
---
|
|
||||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
|
||||||
```
|
|
||||||
|
|
||||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
|
||||||
|
|
||||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
|
||||||
|
|
||||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
|
||||||
```
|
|
||||||
What tool will you use these prompts with?
|
|
||||||
|
|
||||||
Options:
|
|
||||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
|
||||||
2. Nano Banana Pro (image generation)
|
|
||||||
3. ChatGPT / Claude (text/code)
|
|
||||||
4. Other (tell me)
|
|
||||||
```
|
|
||||||
|
|
||||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WAIT FOR USER'S VISION
|
|
||||||
|
|
||||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
|
||||||
|
|
||||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
|
||||||
|
|
||||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
|
||||||
|
|
||||||
### CRITICAL: Match the FORMAT the research recommends
|
|
||||||
|
|
||||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
|
||||||
|
|
||||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
|
||||||
- Research says "structured parameters" → Use structured key: value format
|
|
||||||
- Research says "natural language" → Use conversational prose
|
|
||||||
- Research says "keyword lists" → Use comma-separated keywords
|
|
||||||
|
|
||||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
|
||||||
|
|
||||||
### Output Format:
|
|
||||||
|
|
||||||
```
|
|
||||||
Here's your prompt for {TARGET_TOOL}:
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
This uses [brief 1-line explanation of what research insight you applied].
|
|
||||||
```
|
|
||||||
|
|
||||||
### Quality Checklist:
|
|
||||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
|
||||||
- [ ] Directly addresses what the user said they want to create
|
|
||||||
- [ ] Uses specific patterns/keywords discovered in research
|
|
||||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
|
||||||
- [ ] Appropriate length and style for TARGET_TOOL
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## IF USER ASKS FOR MORE OPTIONS
|
|
||||||
|
|
||||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
|
||||||
|
|
||||||
After delivering a prompt, offer to write more:
|
|
||||||
|
|
||||||
> Want another prompt? Just tell me what you're creating next.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## CONTEXT MEMORY
|
|
||||||
|
|
||||||
For the rest of this conversation, remember:
|
|
||||||
- **TOPIC**: {topic}
|
|
||||||
- **TARGET_TOOL**: {tool}
|
|
||||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
|
||||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
|
||||||
|
|
||||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
|
||||||
|
|
||||||
When the user asks follow-up questions:
|
|
||||||
- **DO NOT run new WebSearches** - you already have the research
|
|
||||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
|
||||||
- **If they ask for a prompt** - write one using your expertise
|
|
||||||
- **If they ask a question** - answer it from your research findings
|
|
||||||
|
|
||||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Output Summary Footer (After Each Prompt)
|
|
||||||
|
|
||||||
After delivering a prompt, end with:
|
|
||||||
|
|
||||||
For **full/partial mode**:
|
|
||||||
```
|
|
||||||
---
|
|
||||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
|
||||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
|
||||||
|
|
||||||
Want another prompt? Just tell me what you're creating next.
|
|
||||||
```
|
|
||||||
|
|
||||||
For **web-only mode**:
|
|
||||||
```
|
|
||||||
---
|
|
||||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
|
||||||
📊 Based on: {n} web pages from {domains}
|
|
||||||
|
|
||||||
Want another prompt? Just tell me what you're creating next.
|
|
||||||
|
|
||||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
|
||||||
```
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
## What I learned:
|
|
||||||
|
|
||||||
The last 30 days have been one of Kanye West's most significant periods in years. The biggest story is his **full-page Wall Street Journal apology** (Jan 26, 2026) titled "To Those I've Hurt" — a comprehensive open letter addressing his antisemitic remarks and erratic behavior. He attributed his actions to a "four-month-long manic episode" and revealed that a car accident 25 years ago caused a frontal-lobe brain injury that wasn't properly diagnosed until 2023, leading to his bipolar type-1 diagnosis. Multiple sources describe wife **Bianca Censori** as instrumental in getting him to seek treatment during a manic depressive episode.
|
|
||||||
|
|
||||||
On the music side, he's announced **two upcoming albums**: *BULLY* (now scheduled for March 20, 2026) and *IN A PERFECT WORLD*. Daughter **North West** collaborated with him on a track called "Piercings on My Hand." He performed a massive show in Mexico City (Jan 30) at the Monumental Plaza de Toros, and he'll headline the new **Hellwatt Festival** in Italy (July 4-18, 2026).
|
|
||||||
|
|
||||||
The **2026 Grammys** remain contentious — he's reportedly not invited after last year's red carpet drama. Health concerns also surfaced after a January LA outing where he appeared "noticeably bloated and moving with sluggish energy."
|
|
||||||
|
|
||||||
**KEY THEMES:**
|
|
||||||
1. **Accountability arc** — The WSJ apology is being widely discussed as either genuine growth or calculated PR
|
|
||||||
2. **Mental health transparency** — Brain injury + bipolar diagnosis framing his past behavior
|
|
||||||
3. **Bianca Censori's role** — Increasingly seen as a stabilizing force (despite continued provocative public appearances)
|
|
||||||
4. **Music comeback** — Two albums announced, active touring in Latin America and Europe
|
|
||||||
5. **Grammy exile** — Still persona non grata at major award shows
|
|
||||||
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├─ 🟠 Reddit: 0 relevant threads (filtered out noise)
|
|
||||||
├─ 🔵 X: 30 posts │ ~10 likes │ casual fan chatter + Bianca Censori discourse
|
|
||||||
├─ 🌐 Web: 20+ pages │ washingtonpost.com, billboard.com, allhiphop.com, euronews.com, rap-up.com
|
|
||||||
└─ Top voices: Washington Post, Billboard, AllHipHop, Euronews, The Mirror
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in.
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
## What I learned:
|
|
||||||
|
|
||||||
OpenClaw (formerly ClawdBot, formerly Moltbot) is an open-source autonomous AI agent created by Peter Steinberger that went massively viral in late January 2026 — hitting 145,000+ GitHub stars and 20,000+ forks. It runs locally on your machine, connects to messaging platforms (Telegram, WhatsApp, Discord, Signal, etc.), and uses LLMs like Claude, GPT, or Gemini to actually perform tasks on your computer — file management, scheduling, web browsing, purchases, and more. Think "JARVIS for everyone."
|
|
||||||
|
|
||||||
The community is simultaneously excited and alarmed. The "ClawHub" skill marketplace has 700+ skills but a VirusTotal/Koi Security audit found **341 malicious skills** doing data exfiltration and crypto theft. A high-severity RCE vulnerability was also disclosed. The #1 most downloaded skill on ClawHub was confirmed malware.
|
|
||||||
|
|
||||||
**KEY PATTERNS from the research:**
|
|
||||||
|
|
||||||
1. **Cost is the top complaint** — API usage burns money fast; many users recommend Gemini CLI (free 1,000 req/day) over Claude/GPT for OpenClaw to control spend
|
|
||||||
2. **Security is a dumpster fire** — Supply chain attacks via ClawHub skills, prompt injection risks when browsing the web, RCE vulnerabilities, and orgs are banning it at the EDR level
|
|
||||||
3. **Setup is nontrivial** — Even after install, most users get stuck on skills/tools/automation config; the `openclaw onboard` wizard is the recommended path
|
|
||||||
4. **The skill ecosystem is powerful but dangerous** — 700+ skills available, but no code signing, no real moderation, and active malware campaigns
|
|
||||||
5. **Community is split** — Enthusiasts love the autonomy ("I give it sudo and let it configure everything"), while security folks are terrified ("prompt injection is terrifying when you give the bot access to your actual bank account")
|
|
||||||
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├── 🟠 Reddit: 25 threads │ ~750+ upvotes │ ~600+ comments
|
|
||||||
├── 🔵 X: 27 posts │ 1,750+ likes │ 325+ reposts
|
|
||||||
├── 🌐 Web: 20+ pages │ Wikipedia, Bloomberg, The Register, Hacker News, Nature, VirusTotal, DigitalOcean, Codecademy, Tom's Hardware
|
|
||||||
└── Top voices: r/openclaw, r/cybersecurity, r/clawdbot, r/ArtificialInteligence │ @grok, @Starlink │ Nature, Bloomberg, ACM
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly.
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
Now I have a comprehensive picture. Let me synthesize everything.
|
|
||||||
|
|
||||||
**What I learned:**
|
|
||||||
|
|
||||||
Nano Banana Pro is fundamentally different from older image models — it's a "thinking" model that understands intent, physics, and composition. The community has converged on two distinct prompting styles, and **both work well** depending on the use case:
|
|
||||||
|
|
||||||
1. **JSON structured prompts** — Dominant on Reddit (r/nanobanana2pro) and X. Power users share elaborate JSON objects with nested fields for subject, appearance, lighting, composition, and style. These produce the most consistent, controlled results for portraits, editorials, and product shots.
|
|
||||||
|
|
||||||
2. **Natural language "Creative Director" prompts** — Recommended by Google's official guide. Full sentences describing the scene as if briefing a photographer. Better for cinematic shots, complex edits, and iterative refinement.
|
|
||||||
|
|
||||||
**KEY PATTERNS I'll use:**
|
|
||||||
1. **JSON format for maximum control** — The top Reddit posts (149-259 upvotes) all use structured JSON with nested objects: `subject > appearance > hair/skin/expression`, `lighting > key/fill/rim`, `camera > lens/angle/depth_of_field`
|
|
||||||
2. **Be a Creative Director, not a tag spammer** — Replace "4k, realistic, beautiful" with specific technical directions: "85mm lens at f/1.8", "three-point lighting with key at 45°", "shallow depth of field with bokeh"
|
|
||||||
3. **Provide context for WHY** — Adding purpose ("for a luxury fashion editorial", "for a high-end gourmet cookbook") helps the model infer appropriate styling, plating, lighting
|
|
||||||
4. **Edit, don't regenerate** — When 80% is right, ask for the specific change conversationally
|
|
||||||
5. **Text rendering works** — Keep text under 3 words for best results (75% success rate), specify font style and placement
|
|
||||||
|
|
||||||
---
|
|
||||||
All agents reported back!
|
|
||||||
- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments
|
|
||||||
- 🔵 X: 29 posts | 312 likes | 117 reposts
|
|
||||||
- 🌐 Web: 10+ pages | blog.google, dev.to, eweek.com, imagine.art, github.com, leonardo.ai, medium.com
|
|
||||||
- Top voices: r/nanobanana2pro | @Xmira_belle, @doctorwasif, @chatgptpaglu | Google AI blog, DEV Community
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro.
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
**What I learned:**
|
|
||||||
|
|
||||||
Clawdbot (now officially renamed **OpenClaw**) is a self-hosted, open-source personal AI assistant that connects to messaging apps (Telegram, WhatsApp, Discord, Slack, Signal, iMessage) and uses LLMs to act as an autonomous agent on your behalf. It's roughly one week old as a public project and the community is moving fast — the name has already changed twice (Clawdbot → Moltbot → OpenClaw).
|
|
||||||
|
|
||||||
**KEY PATTERNS from the research:**
|
|
||||||
|
|
||||||
1. **Security is the #1 concern** — Multiple Reddit threads and guides warn that many setups are wide open to the internet with zero auth. Run it on dedicated/isolated hardware (VM, Mac Mini, NAS), never your main machine. Block dangerous commands via allowlist. Never add it to group chats.
|
|
||||||
|
|
||||||
2. **Use the onboarding wizard, not Docker** — Docker was reported as unreliable (shims, bridges). The single-line install script (`curl -fsSL https://clawd.bot/install.sh | bash`) followed by `openclaw onboard` is the smoothest path. Docker is a "no-go" per multiple users.
|
|
||||||
|
|
||||||
3. **Model choice matters a lot** — Claude Opus 4.5 is recommended for best results (trained to resist prompt injection, ~99% resistance). For budget setups, Claude Haiku works at $0. Local models (Ollama/LM Studio) work but cause hallucination and token issues — GLM 4.7 Flash (30B) was recommended as the best local option.
|
|
||||||
|
|
||||||
4. **Telegram is the easiest channel to start with** — Most guides and users start with a Telegram bot token. WhatsApp QR code pairing fails frequently. Get Telegram working first, then expand.
|
|
||||||
|
|
||||||
5. **Sandbox and isolate** — Set `agents.defaults.sandbox.mode` to `"non-main"` for Docker sandboxing. Use a Linux VM or UTM on Mac to segregate traffic. Tailscale for secure remote access.
|
|
||||||
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├─ 🟠 Reddit: 19 threads │ 415+ upvotes │ 290+ comments
|
|
||||||
├─ 🔵 X: 29 posts │ 815+ likes │ 65+ reposts
|
|
||||||
├─ 🌐 Web: 10+ pages │ Medium, DataCamp, DEV, GitHub, QNAP, docs.openclaw.ai
|
|
||||||
└─ Top voices: r/clawdbot, r/LocalLLM, r/AskVibecoders │ @aashatwt (452 likes), @recap_david (329 likes) │ Lukas Niessen on Medium
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Here's the consensus best setup path based on what the community actually recommends:**
|
|
||||||
|
|
||||||
### Quick-Start (30 min)
|
|
||||||
1. **Get dedicated hardware** — Mac Mini, old laptop, NAS (QNAP has an official guide), or a VPS (DigitalOcean gist available). Don't use your daily driver.
|
|
||||||
2. **Install via script**: `curl -fsSL https://clawd.bot/install.sh | bash`
|
|
||||||
3. **Run the wizard**: `openclaw onboard` — it walks through gateway, workspace, channels, and skills
|
|
||||||
4. **Connect Telegram first** — create a bot via @BotFather, paste the token
|
|
||||||
5. **Set your API key** — Anthropic API key for Claude, or connect Ollama for local models
|
|
||||||
6. **Customize SOUL.md** — this controls your bot's personality and behavior
|
|
||||||
|
|
||||||
### Security Hardening (do this immediately)
|
|
||||||
- Bind gateway to localhost only, use Tailscale for remote access
|
|
||||||
- Set sandbox mode to `"non-main"`
|
|
||||||
- Create an explicit command allowlist (block `rm -rf`, `sudo`, `chmod`)
|
|
||||||
- Rotate API keys every 90 days
|
|
||||||
- Never add the bot to group chats
|
|
||||||
|
|
||||||
### Budget Option ($0)
|
|
||||||
Per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Want me to help you with a specific part of the setup, or do you have a particular use case in mind (home automation, CRM, coding assistant, etc.)?
|
|
||||||
@@ -1,332 +0,0 @@
|
|||||||
---
|
|
||||||
name: last30days
|
|
||||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
|
||||||
argument-hint: '"[topic] for [tool]" or "[topic]"'
|
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
|
||||||
---
|
|
||||||
|
|
||||||
# last30days: Research Any Topic from the Last 30 Days
|
|
||||||
|
|
||||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
|
||||||
|
|
||||||
## CRITICAL: Parse User Intent
|
|
||||||
|
|
||||||
Before doing anything, parse the user's input for:
|
|
||||||
|
|
||||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
|
||||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
|
||||||
3. **QUERY TYPE**: What kind of research they want:
|
|
||||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
|
||||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
|
||||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
|
||||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
|
||||||
|
|
||||||
Common patterns:
|
|
||||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
|
||||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
|
||||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
|
||||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
|
||||||
- If tool is specified in the query, use it
|
|
||||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
|
||||||
|
|
||||||
**Store these variables:**
|
|
||||||
- `TOPIC = [extracted topic]`
|
|
||||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
|
||||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
|
||||||
|
|
||||||
**DISPLAY your parsing to the user.** Before running any tools, output a single line:
|
|
||||||
|
|
||||||
🔍 **{TOPIC}** · {QUERY_TYPE}
|
|
||||||
Searching Reddit, X, and the web for {natural language description of what you'll look for}...
|
|
||||||
|
|
||||||
Example outputs:
|
|
||||||
- 🔍 **kanye west** · News — Searching Reddit, X, and the web for the latest kanye west news and discussions...
|
|
||||||
- 🔍 **best MCP servers** · Recommendations — Searching Reddit, X, and the web for the most recommended MCP servers...
|
|
||||||
- 🔍 **nano banana pro prompting** · Prompting — Searching Reddit, X, and the web for nano banana pro prompting techniques and tips...
|
|
||||||
- 🔍 **open claw** · General — Searching Reddit, X, and the web for what people are saying about open claw...
|
|
||||||
|
|
||||||
If TARGET_TOOL is known, mention it: "...for nano banana pro prompting techniques to use in ChatGPT..."
|
|
||||||
|
|
||||||
This text MUST appear before you call any tools. It confirms to the user that you understood their request.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Research Execution
|
|
||||||
|
|
||||||
**Step 1: Run the research script**
|
|
||||||
```bash
|
|
||||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
|
||||||
```
|
|
||||||
|
|
||||||
The script will automatically:
|
|
||||||
- Detect available API keys
|
|
||||||
- Run Reddit/X searches if keys exist
|
|
||||||
- Signal if WebSearch is needed
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## STEP 2: DO WEBSEARCH WHILE SCRIPT RUNS
|
|
||||||
|
|
||||||
The script auto-detects sources (Bird CLI, API keys, etc). While waiting for it, do WebSearch.
|
|
||||||
|
|
||||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
|
||||||
|
|
||||||
Choose search queries based on QUERY_TYPE:
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
|
||||||
- Search for: `best {TOPIC} recommendations`
|
|
||||||
- Search for: `{TOPIC} list examples`
|
|
||||||
- Search for: `most popular {TOPIC}`
|
|
||||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
|
||||||
|
|
||||||
**If NEWS** ("what's happening with X", "X news"):
|
|
||||||
- Search for: `{TOPIC} news 2026`
|
|
||||||
- Search for: `{TOPIC} announcement update`
|
|
||||||
- Goal: Find current events and recent developments
|
|
||||||
|
|
||||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
|
||||||
- Search for: `{TOPIC} prompts examples 2026`
|
|
||||||
- Search for: `{TOPIC} techniques tips`
|
|
||||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
|
||||||
|
|
||||||
**If GENERAL** (default):
|
|
||||||
- Search for: `{TOPIC} 2026`
|
|
||||||
- Search for: `{TOPIC} discussion`
|
|
||||||
- Goal: Find what people are actually saying
|
|
||||||
|
|
||||||
For ALL query types:
|
|
||||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
|
||||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
|
||||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
|
||||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
|
||||||
|
|
||||||
**Depth options** (passed through from user's command):
|
|
||||||
- `--quick` → Faster, fewer sources (8-12 each)
|
|
||||||
- (default) → Balanced (20-30 each)
|
|
||||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Judge Agent: Synthesize All Sources
|
|
||||||
|
|
||||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
|
||||||
|
|
||||||
The Judge Agent must:
|
|
||||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
|
||||||
2. Weight WebSearch sources LOWER (no engagement data)
|
|
||||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
|
||||||
4. Note any contradictions between sources
|
|
||||||
5. Extract the top 3-5 actionable insights
|
|
||||||
|
|
||||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FIRST: Internalize the Research
|
|
||||||
|
|
||||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
|
||||||
|
|
||||||
Read the research output carefully. Pay attention to:
|
|
||||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
|
||||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
|
||||||
- **What the sources actually say**, not what you assume the topic is about
|
|
||||||
|
|
||||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
|
||||||
|
|
||||||
### If QUERY_TYPE = RECOMMENDATIONS
|
|
||||||
|
|
||||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
|
||||||
|
|
||||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
|
||||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
|
||||||
- Count how many times each is mentioned
|
|
||||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
|
||||||
- List them by popularity/mention count
|
|
||||||
|
|
||||||
**BAD synthesis for "best Claude Code skills":**
|
|
||||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
|
||||||
|
|
||||||
**GOOD synthesis for "best Claude Code skills":**
|
|
||||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
|
||||||
|
|
||||||
### For all QUERY_TYPEs
|
|
||||||
|
|
||||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
|
||||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords?
|
|
||||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
|
||||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
|
||||||
- Common pitfalls mentioned BY THE SOURCES
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## THEN: Show Summary + Invite Vision
|
|
||||||
|
|
||||||
**Display in this EXACT sequence:**
|
|
||||||
|
|
||||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
|
||||||
|
|
||||||
**If RECOMMENDATIONS** - Show specific things mentioned with sources:
|
|
||||||
```
|
|
||||||
🏆 Most mentioned:
|
|
||||||
|
|
||||||
[Tool Name] - {n}x mentions
|
|
||||||
Use Case: [what it does]
|
|
||||||
Sources: @handle1, @handle2, r/sub, blog.com
|
|
||||||
|
|
||||||
[Tool Name] - {n}x mentions
|
|
||||||
Use Case: [what it does]
|
|
||||||
Sources: @handle3, r/sub2, Complex
|
|
||||||
|
|
||||||
Notable mentions: [other specific things with 1-2 mentions]
|
|
||||||
```
|
|
||||||
|
|
||||||
**CRITICAL for RECOMMENDATIONS:**
|
|
||||||
- Each item MUST have a "Sources:" line with actual @handles from X posts (e.g., @LONGLIVE47, @ByDobson)
|
|
||||||
- Include subreddit names (r/hiphopheads) and web sources (Complex, Variety)
|
|
||||||
- Parse @handles from research output and include the highest-engagement ones
|
|
||||||
- Format naturally - tables work well for wide terminals, stacked cards for narrow
|
|
||||||
|
|
||||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
|
||||||
|
|
||||||
CITATION RULE: Cite sources sparingly to prove research is real.
|
|
||||||
- In the "What I learned" intro: cite 1-2 top sources total, not every sentence
|
|
||||||
- In KEY PATTERNS: cite 1 source per pattern, short format: "per @handle" or "per r/sub"
|
|
||||||
- Do NOT include engagement metrics in citations (likes, upvotes) - save those for stats box
|
|
||||||
- Do NOT chain multiple citations: "per @x, @y, @z" is too much. Pick the strongest one.
|
|
||||||
|
|
||||||
**BAD:** "His album is set for March 20 (per @cocoabutterbf; Rolling Stone; HotNewHipHop; Complex)."
|
|
||||||
**GOOD:** "His album BULLY is set for March 20 via Gamma, per Rolling Stone."
|
|
||||||
|
|
||||||
```
|
|
||||||
What I learned:
|
|
||||||
|
|
||||||
**{Topic 1}** — [1-2 sentences about this storyline, per source]
|
|
||||||
|
|
||||||
**{Topic 2}** — [1-2 sentences, per source]
|
|
||||||
|
|
||||||
**{Topic 3}** — [1-2 sentences, per source]
|
|
||||||
|
|
||||||
KEY PATTERNS from the research:
|
|
||||||
1. [Pattern] — per @handle
|
|
||||||
2. [Pattern] — per r/sub
|
|
||||||
3. [Pattern] — per source
|
|
||||||
```
|
|
||||||
|
|
||||||
**THEN - Stats (right before invitation):**
|
|
||||||
|
|
||||||
**CRITICAL: Calculate actual totals from the research output.**
|
|
||||||
- Count posts/threads from each section
|
|
||||||
- Sum engagement: parse `[Xlikes, Yrt]` from each X post, `[Xpts, Ycmt]` from Reddit
|
|
||||||
- Identify top voices: highest-engagement @handles from X, most active subreddits
|
|
||||||
|
|
||||||
**Copy this EXACTLY, replacing only the {placeholders}:**
|
|
||||||
|
|
||||||
```
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments
|
|
||||||
├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts (via Bird/xAI)
|
|
||||||
├─ 🌐 Web: {N} pages │ {domain1}, {domain2}, {domain3}
|
|
||||||
└─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2}
|
|
||||||
---
|
|
||||||
```
|
|
||||||
|
|
||||||
If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)"
|
|
||||||
NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji.
|
|
||||||
|
|
||||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
|
||||||
|
|
||||||
**LAST - Invitation:**
|
|
||||||
```
|
|
||||||
---
|
|
||||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WAIT FOR USER'S VISION
|
|
||||||
|
|
||||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
|
||||||
|
|
||||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
|
||||||
|
|
||||||
### CRITICAL: Match the FORMAT the research recommends
|
|
||||||
|
|
||||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT.**
|
|
||||||
|
|
||||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
|
||||||
|
|
||||||
### Quality Checklist (run before delivering):
|
|
||||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
|
||||||
- [ ] Directly addresses what the user said they want to create
|
|
||||||
- [ ] Uses specific patterns/keywords discovered in research
|
|
||||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
|
||||||
- [ ] Appropriate length and style for TARGET_TOOL
|
|
||||||
|
|
||||||
### Output Format:
|
|
||||||
|
|
||||||
```
|
|
||||||
Here's your prompt for {TARGET_TOOL}:
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS]
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
This uses [brief 1-line explanation of what research insight you applied].
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## IF USER ASKS FOR MORE OPTIONS
|
|
||||||
|
|
||||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
|
||||||
|
|
||||||
After delivering a prompt, offer to write more:
|
|
||||||
|
|
||||||
> Want another prompt? Just tell me what you're creating next.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## CONTEXT MEMORY
|
|
||||||
|
|
||||||
For the rest of this conversation, remember:
|
|
||||||
- **TOPIC**: {topic}
|
|
||||||
- **TARGET_TOOL**: {tool}
|
|
||||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
|
||||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
|
||||||
|
|
||||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
|
||||||
|
|
||||||
When the user asks follow-up questions:
|
|
||||||
- **DO NOT run new WebSearches** - you already have the research
|
|
||||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
|
||||||
- **If they ask for a prompt** - write one using your expertise
|
|
||||||
|
|
||||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Output Summary Footer (After Each Prompt)
|
|
||||||
|
|
||||||
After delivering a prompt, end with:
|
|
||||||
|
|
||||||
```
|
|
||||||
---
|
|
||||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
|
||||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
|
||||||
|
|
||||||
Want another prompt? Just tell me what you're creating next.
|
|
||||||
```
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
Here's what I found:
|
|
||||||
|
|
||||||
## What I learned:
|
|
||||||
|
|
||||||
**BULLY Album — March 20, 2026 via Gamma** — After years of delays (first announced September 2024), Kanye's 12th studio album finally has a firm release date. The 13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign. Sonically it recalls *808s & Heartbreak* and *MBDTF* — Ye mostly sings rather than raps. Notably, earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded with his real voice for the official release, per Rolling Stone.
|
|
||||||
|
|
||||||
**Public Apology for Antisemitism** — On January 26, Ye took out a full-page Wall Street Journal ad titled "To Those I've Hurt," publicly apologizing for his antisemitic remarks. He disclosed a previously undiagnosed brain injury from his 2002 car accident and attributed his behavior to a four-month manic episode fueled by bipolar disorder, psychosis, and paranoia. "I lost touch with reality," he wrote. The apology also extended to the Black community, per The Washington Post.
|
|
||||||
|
|
||||||
**Hellwatt Festival in Italy** — Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard.
|
|
||||||
|
|
||||||
**Health Concerns** — A rare January 2026 outing in LA with Bianca Censori reignited concern about Ye's physical appearance. Insiders point to medication side effects, frequent travel, and inconsistent routines, per AllHipHop.
|
|
||||||
|
|
||||||
**Grammys Ban** — Ye is reportedly not welcome at the 2026 Grammy Awards after clashing with organizers last year over his invitation terms, per The News International.
|
|
||||||
|
|
||||||
**Kim & Lewis Hamilton Buzz** — X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton, with users contrasting her new relationship against her marriage to Ye.
|
|
||||||
|
|
||||||
---
|
|
||||||
✅ All agents reported back!
|
|
||||||
├─ 🟠 Reddit: 0 threads (no results this cycle)
|
|
||||||
├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)
|
|
||||||
├─ 🌐 Web: 30+ pages │ rollingstone.com, washingtonpost.com, complex.com, billboard.com, npr.org
|
|
||||||
└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex
|
|
||||||
---
|
|
||||||
|
|
||||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice.
|
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/community-more-posts/top/?name=technology&t=week on 2026-05-29; trimmed to 5 post cards (start-tag attrs only). -->
|
||||||
|
<div id="feed">
|
||||||
|
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tq0zk7/the_netherlands_just_blocked_a_us_company_from/" content-href="https://www.techspot.com/news/112552-netherlands-blocked-us-company-buying-app-dutch-citizens.html" view-context="SubredditFeed" comment-count="1743" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="0" award-count="23" award-id="award_obsessed_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/Obsessed_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-28T11:37:01.506000+0000" domain="techspot.com" id="t3_1tq0zk7" post-title="The Netherlands just blocked a US company from buying the app Dutch citizens use for everything" post-language="en" post-type="link" score="52692" upvote-ratio="0.9606269354736776" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||||
|
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1toe7m2/erin_brockovich_launches_map_of_over_4200_data/" content-href="https://www.newsweek.com/erin-brockovich-asks-americans-for-help-as-she-launches-data-center-map-11989813" view-context="SubredditFeed" comment-count="673" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="2" award-count="6" award-id="award_this_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/this_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T17:39:43.272000+0000" domain="newsweek.com" id="t3_1toe7m2" post-title="Erin Brockovich launches map of over 4,200 data centres in the US, appeals for local communities to report environmental impact and other costs" post-language="en" post-type="link" score="33567" upvote-ratio="0.973297166968053" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_fj9vsvfd" author="marketrent" icon="https://www.redditstatic.com/avatars/defaults/v2/avatar_default_1.png" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||||
|
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tollgz/majority_of_americans_support_ban_on_surveillance/" content-href="https://gizmodo.com/majority-of-americans-support-ban-on-surveillance-pricing-and-electronic-shelf-labels-2000762717" view-context="SubredditFeed" comment-count="1043" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="3" award-count="7" award-id="award_free_bravo" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/bravo_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T21:55:07.322000+0000" domain="gizmodo.com" id="t3_1tollgz" post-title="Majority of Americans Support Ban on Surveillance Pricing and Electronic Shelf Labels" post-language="en" post-type="link" score="29791" upvote-ratio="0.9815063671850003" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_98wao505" author="Plastic_Ninja_9014" icon="https://preview.redd.it/snoovatar/avatars/69af2b53-b0a1-4ab6-b119-d90f21c423fe-headshot.png?width=64&height=64&crop=smart&auto=webp&s=f3661eb511798004968f8b115a689dcee30f1428" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||||
|
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tp5qz2/tech_ceos_are_apparently_suffering_from_ai/" content-href="https://techcrunch.com/2026/05/27/tech-ceos-are-apparently-suffering-from-ai-psychosis/" view-context="SubredditFeed" comment-count="1653" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="4" award-count="6" award-id="award_free_regret_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/regret_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-27T13:33:49.280000+0000" domain="techcrunch.com" id="t3_1tp5qz2" post-title="Tech CEOs are apparently suffering from AI psychosis" post-language="en" post-type="link" score="26419" upvote-ratio="0.9605741880002646" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||||
|
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tn5g7s/pope_leo_issues_ai_encyclical_warning_that_opaque/" content-href="https://variety.com/2026/biz/global/pope-leo-ai-encyclical-algorithms-threaten-dehumanisation-1236758186/" view-context="SubredditFeed" comment-count="608" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="6" award-count="7" award-id="award_hooray_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/FTUE_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-25T10:45:04.093000+0000" domain="variety.com" id="t3_1tn5g7s" post-title="Pope Leo Issues AI Encyclical Warning That ‘Opaque Algorithms’ Controlled by a ‘Few’ Companies Can Bring ‘New Forms of Dehumanisation’" post-language="en" post-type="link" score="25835" upvote-ratio="0.9760626539506095" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_1i1zizibn9" author="yourfavchoom" icon="https://styles.redditmedia.com/t5_dgdrt8/styles/profileIcon_k9x929ihm8rg1.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=2e8a5042cccc4555167f98d28bc0de4e13fd3ca5" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||||
|
</div>
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!-- FIXTURE: captured live from reddit.com/r/Rakuten/top.rss on 2026-05-29; trimmed to 5 entries. Atom shape identical to search.rss. --><feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/"><category term="Rakuten" label="r/Rakuten"/><updated>2026-05-29T14:14:32+00:00</updated><icon>https://www.redditstatic.com/icon.png/</icon><id>/r/Rakuten/top.rss?t=month</id><link rel="self" href="https://www.reddit.com/r/Rakuten/top.rss?t=month" type="application/atom+xml" /><link rel="alternate" href="https://www.reddit.com/r/Rakuten/top?t=month" type="text/html" /><subtitle>This is an unofficial subreddit for Rakuten Rewards, the cash back website. We are not affiliated with, endorsed by, or sponsored by Rakuten or any of its subsidiaries.</subtitle><title>top scoring links : Rakuten</title><entry><author><name>/u/InternetUser52</name><uri>https://www.reddit.com/user/InternetUser52</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>I&#39;m rich!!</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/InternetUser52"> /u/InternetUser52 </a> <br/> <span><a href="https://i.redd.it/q8fgmxs29c2h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/">[comments]</a></span></content><id>t3_1tiv013</id><link href="https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/" /><updated>2026-05-20T18:48:31+00:00</updated><published>2026-05-20T18:48:31+00:00</published><title>LETS GOO! $0.02!!!</title></entry>
|
||||||
|
<entry><author><name>/u/Immediate-Duck-6351</name><uri>https://www.reddit.com/user/Immediate-Duck-6351</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>I don’t travel and I’m buying a house in a few weeks so cash back is amazing 🙌 hoping to keep the pace in the next quarter so I can buy new kitchen appliances lol. </p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/Immediate-Duck-6351"> /u/Immediate-Duck-6351 </a> <br/> <span><a href="https://i.redd.it/d2a4s0ipvb1h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/">[comments]</a></span></content><id>t3_1te1fp8</id><link href="https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/" /><updated>2026-05-15T16:29:28+00:00</updated><published>2026-05-15T16:29:28+00:00</published><title>So excited 🥳</title></entry>
|
||||||
|
<entry><author><name>/u/gnibgnib</name><uri>https://www.reddit.com/user/gnibgnib</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>128k for the May transfer</p> <p>41k pending for August </p> <p>Got another 9k at Asics not showing but overall pretty happy with Rakuten</p> <p>P2 was able to secure 85k for May transfer</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/gnibgnib"> /u/gnibgnib </a> <br/> <span><a href="https://www.reddit.com/gallery/1tb8674">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/">[comments]</a></span></content><id>t3_1tb8674</id><link href="https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/" /><updated>2026-05-12T17:17:19+00:00</updated><published>2026-05-12T17:17:19+00:00</published><title>Had a great run so far this year thanks to this sub!</title></entry>
|
||||||
|
<entry><author><name>/u/TravelVet93</name><uri>https://www.reddit.com/user/TravelVet93</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&#32; submitted by &#32; <a href="https://www.reddit.com/user/TravelVet93"> /u/TravelVet93 </a> <br/> <span><a href="https://i.redd.it/x6b9whvupb1h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/">[comments]</a></span></content><id>t3_1te0hom</id><link href="https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/" /><updated>2026-05-15T15:56:40+00:00</updated><published>2026-05-15T15:56:40+00:00</published><title>My best payout so far</title></entry>
|
||||||
|
<entry><author><name>/u/Beautiful-Piece-4252</name><uri>https://www.reddit.com/user/Beautiful-Piece-4252</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>The amount of $$ available in sign up bonuses is amazing. It&#39;s kind of a part time job ensuring Rakuten captures everything, but my August and November payout should be sizeable. I&#39;m new to this and it always seemed like a lot of work for little reward. I know it&#39;s not sustainable, but wow!</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/Beautiful-Piece-4252"> /u/Beautiful-Piece-4252 </a> <br/> <span><a href="https://i.redd.it/1vqvajsci42h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/">[comments]</a></span></content><id>t3_1thsnm1</id><link href="https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/" /><updated>2026-05-19T16:46:17+00:00</updated><published>2026-05-19T16:46:17+00:00</published><title>How can this be real?</title></entry>
|
||||||
|
</feed>
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/comments/r/Rakuten/t3_1taeiw0 on 2026-05-29;
|
||||||
|
trimmed to 6 real comment elements (real attrs + real bodies) + 2 synthetic edge cases. -->
|
||||||
|
<shreddit-comment-tree-stats total-comments="14"></shreddit-comment-tree-stats>
|
||||||
|
<shreddit-comment-tree id="comment-tree" post-id="t3_1taeiw0">
|
||||||
|
<shreddit-comment created="2026-05-11T20:16:57.590000+0000" author="Obvious_Painting_881" thingId="t1_ol8tp8n" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8tp8n/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_ol8tp8n-comment-rtjson-content" slot="comment"><div id="t1_ol8tp8n-post-rtjson-content" dir="auto"><p dir="auto">Where do you find $750? The highest available package for Total was $284.99 when I did the lifelock promotion. I did get the full 284.99 from Rakuten.</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
<shreddit-comment created="2026-05-12T12:26:14.973000+0000" author="Stormtrooper149" thingId="t1_olcy1iv" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/olcy1iv/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_olcy1iv-comment-rtjson-content" slot="comment"><div id="t1_olcy1iv-post-rtjson-content" dir="auto"><p dir="auto">It went to pending ($712.49)</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
<shreddit-comment created="2026-05-19T01:43:48.026000+0000" author="heythereyou01" thingId="t1_omlbiqg" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/omlbiqg/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_omlbiqg-comment-rtjson-content" slot="comment"><div id="t1_omlbiqg-post-rtjson-content" dir="auto"><p dir="auto">Hey I PM’d. can I get the screenshot ?</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
<shreddit-comment created="2026-05-11T20:21:16.398000+0000" author="Stormtrooper149" thingId="t1_ol8undb" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8undb/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_ol8undb-comment-rtjson-content" slot="comment"><div id="t1_ol8undb-post-rtjson-content" dir="auto"><p dir="auto">Family plan</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
<shreddit-comment created="2026-05-11T20:28:33.803000+0000" author="Obvious_Painting_881" thingId="t1_ol8w8w6" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8w8w6/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_ol8w8w6-comment-rtjson-content" slot="comment"><div id="t1_ol8w8w6-post-rtjson-content" dir="auto"><p dir="auto">Price seems to change every time I go to the page but I see only 249.99-369.99 for Total/Advanced. No where near your $750. Just saying the Total plan for 299.99 worked for me and I got 284.99 which is 95%.</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
<shreddit-comment created="2026-05-12T02:33:48.200000+0000" author="jwegener" thingId="t1_olaqzjk" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/olaqzjk/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_olaqzjk-comment-rtjson-content" slot="comment"><div id="t1_olaqzjk-post-rtjson-content" dir="auto"><p dir="auto">I did that one. Let’s pray</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
<shreddit-comment created="2026-05-13T10:00:00.000000+0000" author="[deleted]" thingId="t1_synthdel" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/synthdel/" score="5" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_synthdel-comment-rtjson-content" slot="comment"><div id="t1_synthdel-post-rtjson-content" dir="auto"><p dir="auto">[removed]</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
<shreddit-comment created="2026-05-13T11:00:00.000000+0000" author="NegScoreUser" thingId="t1_synthneg" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/synthneg/" score="-7" postId="t3_1taeiw0" content-type="text">
|
||||||
|
<div id="t1_synthneg-comment-rtjson-content" slot="comment"><div id="t1_synthneg-post-rtjson-content" dir="auto"><p dir="auto">A downvoted but real reply with negative score for edge-case coverage.</p></div></div>
|
||||||
|
</shreddit-comment>
|
||||||
|
</shreddit-comment-tree>
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days-skill",
|
"name": "last30days-skill",
|
||||||
"version": "3.0.5",
|
"version": "3.3.2",
|
||||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||||
"settings": [
|
"settings": [
|
||||||
{
|
{
|
||||||
|
|||||||
+1
-2
@@ -6,8 +6,7 @@
|
|||||||
"hooks": [
|
"hooks": [
|
||||||
{
|
{
|
||||||
"type": "command",
|
"type": "command",
|
||||||
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-config.sh",
|
"command": "bash \"${CLAUDE_PLUGIN_ROOT:-${extensionPath:-.}}/hooks/scripts/check-config.sh\""
|
||||||
"timeout": 5
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,8 +33,13 @@ load_env_vars() {
|
|||||||
[[ -z "$key" ]] && continue
|
[[ -z "$key" ]] && continue
|
||||||
key=$(echo "$key" | xargs)
|
key=$(echo "$key" | xargs)
|
||||||
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||||
|
# Strip inline comments (# preceded by whitespace) to prevent
|
||||||
|
# command substitution in backtick-containing comments
|
||||||
|
value="${value%%[[:space:]]#*}"
|
||||||
if [[ -n "$key" && -n "$value" ]]; then
|
if [[ -n "$key" && -n "$value" ]]; then
|
||||||
eval "ENV_${key}=\"${value}\""
|
# printf -v writes via assignment semantics (global from inside a
|
||||||
|
# function), works on macOS's /bin/bash 3.2 — `declare -g` is 4.2+.
|
||||||
|
printf -v "ENV_${key}" '%s' "$value"
|
||||||
fi
|
fi
|
||||||
done < "$file"
|
done < "$file"
|
||||||
fi
|
fi
|
||||||
@@ -58,14 +63,53 @@ fi
|
|||||||
# Check SETUP_COMPLETE (from file or env)
|
# Check SETUP_COMPLETE (from file or env)
|
||||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||||
|
|
||||||
|
# Compute last-run summary line (if last-run.json exists)
|
||||||
|
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
||||||
|
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
||||||
|
LAST_RUN_FILE="$LAST30DAYS_CONFIG_DIR/last-run.json"
|
||||||
|
else
|
||||||
|
LAST_RUN_FILE=""
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
|
||||||
|
fi
|
||||||
|
LAST_RUN_LINE=""
|
||||||
|
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
|
||||||
|
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 - <<'PY' 2>/dev/null || true
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
path = os.environ["LAST_RUN_FILE"]
|
||||||
|
try:
|
||||||
|
with open(path) as fh:
|
||||||
|
d = json.load(fh)
|
||||||
|
topic = (d.get("topic") or "?")[:60]
|
||||||
|
ts = d.get("timestamp", "")
|
||||||
|
dt = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||||
|
delta = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
|
||||||
|
if delta < 60: ago = f"{int(delta)}s ago"
|
||||||
|
elif delta < 3600: ago = f"{int(delta//60)}m ago"
|
||||||
|
elif delta < 86400: ago = f"{int(delta//3600)}h ago"
|
||||||
|
else: ago = f"{int(delta//86400)}d ago"
|
||||||
|
total = d.get("total", 0)
|
||||||
|
print(f" Last run: \"{topic}\" · {ago} · {total} results")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
# If setup has never been run, show welcome message for new users
|
# If setup has never been run, show welcome message for new users
|
||||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||||
|
Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).
|
||||||
|
|
||||||
Reddit, Hacker News, and Polymarket work out of the box.
|
Reddit, Hacker News, and Polymarket work out of the box.
|
||||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||||
EOF
|
EOF
|
||||||
|
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -116,9 +160,13 @@ fi
|
|||||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||||
# Fully configured — compact ready message
|
# Fully configured — compact ready message
|
||||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||||
|
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||||
|
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||||
else
|
else
|
||||||
# Setup done but missing ScrapeCreators — recommend it
|
# Setup done but missing ScrapeCreators — recommend it
|
||||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||||
|
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||||
|
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
echo " 100 free credits, no credit card — scrapecreators.com"
|
||||||
echo " last30days has no affiliation with any API provider."
|
echo " last30days has no affiliation with any API provider."
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "last30days-skill"
|
name = "last30days-skill"
|
||||||
version = "3.2.4"
|
version = "3.3.2"
|
||||||
description = "Multi-source last-30-days research skill"
|
description = "Multi-source last-30-days research skill"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
|
|||||||
@@ -1,86 +0,0 @@
|
|||||||
The AI world reinvents itself every month. This skill keeps you current.
|
|
||||||
|
|
||||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
|
||||||
|
|
||||||
## v3 is the intelligent search release
|
|
||||||
|
|
||||||
v3 is a ground-up engine rewrite by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
|
||||||
|
|
||||||
Type "OpenClaw" and v3 resolves @steipete, r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags before a single API call fires. Type "Peter Steinberger" and it resolves his X handle and GitHub profile, switches to person mode, and shows what he shipped this month at 85% merge rate across 22 PRs. None of that was on Google.
|
|
||||||
|
|
||||||
## Headline features
|
|
||||||
|
|
||||||
### Intelligent pre-research
|
|
||||||
|
|
||||||
The killer feature. A new Python pre-research brain resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags, all resolved before a single API call.
|
|
||||||
|
|
||||||
### Best Takes
|
|
||||||
|
|
||||||
A second LLM judge scores every result for humor, wit, and virality alongside relevance. Every brief now ends with a Best Takes section surfacing the cleverest one-liners and most viral quotes. The Reddit and X people are funny, and the old engine buried their best stuff.
|
|
||||||
|
|
||||||
### Cross-source cluster merging
|
|
||||||
|
|
||||||
When the same story hits Reddit, X, and YouTube, v3 merges them into one cluster instead of three duplicates. Entity-based overlap detection catches matches even when the titles use different words.
|
|
||||||
|
|
||||||
### Single-pass comparisons
|
|
||||||
|
|
||||||
"X vs Y" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides at once. Same depth, 3 minutes.
|
|
||||||
|
|
||||||
### GitHub person-mode and project-mode
|
|
||||||
|
|
||||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. PR velocity, top repos by stars, release notes for what shipped this month, woven into the narrative alongside X posts and Reddit threads.
|
|
||||||
|
|
||||||
When the topic is a project, it pulls live star counts, READMEs, releases, and top issues from the GitHub API. No stale blog posts.
|
|
||||||
|
|
||||||
### ELI5 mode
|
|
||||||
|
|
||||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations, just clearer. Say "eli5 off" to go back.
|
|
||||||
|
|
||||||
### 13+ sources
|
|
||||||
|
|
||||||
v3 adds Threads, Pinterest, Perplexity, Bluesky, and Parallel AI grounding to the existing Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and Web lineup. Perplexity Deep Research (`--deep-research`) gives you 50+ citation reports for serious investigation.
|
|
||||||
|
|
||||||
### Per-author cap and entity disambiguation
|
|
||||||
|
|
||||||
Max 3 items per author prevents single-voice dominance. Synthesis trusts resolved handles over fuzzy keyword matches.
|
|
||||||
|
|
||||||
## Install
|
|
||||||
|
|
||||||
Claude Code:
|
|
||||||
|
|
||||||
```
|
|
||||||
/plugin marketplace add mvanhorn/last30days-skill
|
|
||||||
```
|
|
||||||
|
|
||||||
OpenClaw:
|
|
||||||
|
|
||||||
```
|
|
||||||
clawhub install last30days-official
|
|
||||||
```
|
|
||||||
|
|
||||||
OpenAI Codex CLI: install the repo as a local Codex marketplace/plugin. The plugin manifest lives at `.codex-plugin/plugin.json`, and the canonical skill payload is `skills/last30days/SKILL.md`.
|
|
||||||
|
|
||||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
|
||||||
|
|
||||||
## v3 Community
|
|
||||||
|
|
||||||
v3 was shaped by community contributors whose PRs and issues inspired core features. Their code wasn't merged directly (v3 was a ground-up rewrite), but their ideas drove what shipped.
|
|
||||||
|
|
||||||
Thanks to @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, and @hnshah. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list.
|
|
||||||
|
|
||||||
Contributors who shaped the release itself:
|
|
||||||
|
|
||||||
- @Jah-yee (#153) surfaced the need for a real Codex CLI integration, which shipped in #219
|
|
||||||
- @Cody-Coyote (#204) reported the marketplace validation bug that needed fixing before v3 could ship cleanly
|
|
||||||
- @dannyshmueli pushed for v3 and Codex family support publicly on X
|
|
||||||
|
|
||||||
Full Added / Changed / Fixed detail lives in [CHANGELOG.md](CHANGELOG.md) under `[3.0.0]`.
|
|
||||||
|
|
||||||
## Earlier contributors
|
|
||||||
|
|
||||||
From the v1 and v2 lineage:
|
|
||||||
|
|
||||||
- [@galligan](https://github.com/galligan) for marketplace plugin inspiration
|
|
||||||
- [@hutchins](https://x.com/hutchins) for pushing the YouTube feature
|
|
||||||
|
|
||||||
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
name: last30days
|
name: last30days
|
||||||
version: "3.2.4"
|
version: "3.3.2"
|
||||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||||
@@ -243,7 +243,7 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# last30days v3.2.4: Research Any Topic from the Last 30 Days
|
# last30days v3.3.2: Research Any Topic from the Last 30 Days
|
||||||
|
|
||||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||||
|
|
||||||
@@ -596,8 +596,8 @@ When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipe
|
|||||||
# the Read tool result. Examples:
|
# the Read tool result. Examples:
|
||||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days/SKILL.md
|
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days/SKILL.md
|
||||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days
|
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days
|
||||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||||
# packages SKILL.md and scripts/ as siblings).
|
# packages SKILL.md and scripts/ as siblings).
|
||||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||||
@@ -641,9 +641,11 @@ Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-r
|
|||||||
|
|
||||||
**Then do WebSearch supplements** for: `{TOPIC_A} vs {TOPIC_B} comparison {YEAR}` and `{TOPIC_A} vs {TOPIC_B} which is better` — these catch rivalry articles that per-entity passes might not surface.
|
**Then do WebSearch supplements** for: `{TOPIC_A} vs {TOPIC_B} comparison {YEAR}` and `{TOPIC_A} vs {TOPIC_B} which is better` — these catch rivalry articles that per-entity passes might not surface.
|
||||||
|
|
||||||
|
**Use `RESOLVED_POSITIONING` per entity (Step 0.55 item 6) in two ways.** First, ground each entity's `What it is` cell in its CURRENT fetched pitch - describe the entity as it pitches itself today, never from memory. Second, if an entity's month of evidence directly bears on its pitch - SUPPORTS a specific claim, CUTS AGAINST one, or the conversation is squarely ABOUT the pitched ground - say so in ONE prose sentence inside that entity's section of the comparison synthesis (right after the Community Sentiment line - the template marks the slot), anchored to the real item with its engagement. When the pulse is orthogonal to the pitch (on-entity but about something the pitch doesn't speak to), say NOTHING about the pitch: omission is the correct output, and a manufactured connection is worse than silence. Match altitude: test SPECIFIC claims ("zero-config", "fastest", an uptime number) against specific threads; never grade a broad tagline ("financial infrastructure") against an individual thread - it is too broad to hit or miss. Keep claims windowed - "this month's conversation" - never trend verbs like "losing the narrative" that one 30-day window cannot support. If positioning was not actually fetched this run for an entity, skip both uses for that entity - never supply a pitch from memory.
|
||||||
|
|
||||||
**Skip the normal Step 1 below** - go directly to the comparison synthesis format (see "If QUERY_TYPE = COMPARISON" in the synthesis section).
|
**Skip the normal Step 1 below** - go directly to the comparison synthesis format (see "If QUERY_TYPE = COMPARISON" in the synthesis section).
|
||||||
|
|
||||||
**COMPARISON TABLE SCAFFOLD (engine-emitted, pass through verbatim):** For comparison topics, the engine's compact output includes a `## Head-to-Head` block with an empty markdown table (columns = entities, rows = axes like "What it is", "Community sentiment", "Trajectory"). Your synthesis MUST include this block verbatim with filled cells, positioned between the narrative and the emoji-tree footer. Keep each cell to 5-15 words. Use ' - ' (hyphen with spaces) not em-dashes inside cells.
|
**COMPARISON TABLE SCAFFOLD (engine-emitted, pass through verbatim):** For comparison topics, the engine's compact output includes a `## Head-to-Head` block with an empty markdown table (columns = entities, rows = axes like "What it is", "Philosophy", "Best for"). Your synthesis MUST include this block verbatim with filled cells, positioned between the narrative and the emoji-tree footer. Keep each cell to 5-15 words. Use ' - ' (hyphen with spaces) not em-dashes inside cells.
|
||||||
|
|
||||||
### Competitor mode (`--competitors`)
|
### Competitor mode (`--competitors`)
|
||||||
|
|
||||||
@@ -748,6 +750,8 @@ Store as `RESOLVED_IG_CREATORS`.
|
|||||||
|
|
||||||
Store as `RESOLVED_YT_QUERIES`.
|
Store as `RESOLVED_YT_QUERIES`.
|
||||||
|
|
||||||
|
**6. First-party positioning** - **MANDATORY when WebSearch is available, for company / product / service topics.** If the topic (or, in a vs-run, an entity) is a company, product, or service with a public presence, fetch its CURRENT stated positioning. Do **NOT** rely on memory - homepages and positioning go stale as companies rewrite copy and pivot, and a stale claim produces a false gap. Anchor on first-party sources: the homepage tagline, docs, pricing, or a "compare/why-us" page. Fold this into the per-entity passes above where you can (e.g. add `official site` to a query); otherwise run one focused search per entity (`{TOPIC} official site`, `{TOPIC} pricing`). Capture the one-line value prop and any explicit claims ("zero-config", "fastest", "open source"). Store as `RESOLVED_POSITIONING`. This is what the entity *pitches*; the engine's community data is what people *actually talk about*. Use it three ways: ground `What it is` descriptions (describe the entity as it pitches itself TODAY, not as remembered), help reject unrelated brand-name noise (knowing what the entity is makes off-brand matches obvious), and feed the pitch-vs-pulse synthesis beat - a PROSE note that fires only when the month's evidence directly supports, cuts against, or is squarely about the pitch (see the synthesis section; orthogonal evidence gets silence, not a verdict). Skip (and omit `RESOLVED_POSITIONING`) for people, events, abstract concepts, and ownerless topics - they make no comparable public claim. The test is an identifiable first party with a fetchable pitch, and people NEVER pass it - not even founders/creators whose companies would qualify. The lens can apply to MrBeast (a company) but never to Jimmy Donaldson (a person); a person-vs-person run ("Garry Tan vs Sam Altman") gets no positioning research at all. Ownerless topics fail the same test: Bitcoin has no authoritative first party, and a foundation or fan site does not count.
|
||||||
|
|
||||||
**Concrete examples:**
|
**Concrete examples:**
|
||||||
|
|
||||||
| Topic | WebSearches needed | Reddit subs | TikTok hashtags | TikTok creators | IG creators | YT queries |
|
| Topic | WebSearches needed | Reddit subs | TikTok hashtags | TikTok creators | IG creators | YT queries |
|
||||||
@@ -800,9 +804,10 @@ Resolved:
|
|||||||
- Reddit: r/{sub1}, r/{sub2}, r/{sub3}, r/{peer1}, r/{peer2} (+ {category_id} peers)
|
- Reddit: r/{sub1}, r/{sub2}, r/{sub3}, r/{peer1}, r/{peer2} (+ {category_id} peers)
|
||||||
- TikTok: #{hashtag1}, #{hashtag2}
|
- TikTok: #{hashtag1}, #{hashtag2}
|
||||||
- YouTube: {query1}, {query2}
|
- YouTube: {query1}, {query2}
|
||||||
|
- Positioning: "{one-line stated value prop}" (first-party)
|
||||||
```
|
```
|
||||||
|
|
||||||
Only show lines for platforms where something was resolved. Skip empty lines. On the Reddit line, the trailing `(+ {category_id} peers)` annotation appears when Step 0.55 Section 2a added category-peer subs. Omit the annotation when the topic had no matching category. This display replaces the old "Parsed intent" block with something more useful.
|
Only show lines for platforms where something was resolved. Skip empty lines. On the Reddit line, the trailing `(+ {category_id} peers)` annotation appears when Step 0.55 Section 2a added category-peer subs. Omit the annotation when the topic had no matching category. The `Positioning:` line appears for company / product / service topics (from Step 0.55 item 6); omit it for people, events, abstract concepts, and ownerless topics. This display replaces the old "Parsed intent" block with something more useful.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -914,8 +919,8 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
|||||||
# the Read tool result. Examples:
|
# the Read tool result. Examples:
|
||||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days/SKILL.md
|
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days/SKILL.md
|
||||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.2.4/skills/last30days
|
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days
|
||||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||||
# packages SKILL.md and scripts/ as siblings).
|
# packages SKILL.md and scripts/ as siblings).
|
||||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||||
@@ -1268,6 +1273,8 @@ Voice contract LAWs 1, 3, 5 apply to comparisons unchanged (no `Sources:` block,
|
|||||||
|
|
||||||
**Community Sentiment:** [Positive / Mixed / Negative / Enthusiastic / Security-concerned / etc.] ({N}+ mentions across {source list})
|
**Community Sentiment:** [Positive / Mixed / Negative / Enthusiastic / Security-concerned / etc.] ({N}+ mentions across {source list})
|
||||||
|
|
||||||
|
[Optional pitch-vs-pulse sentence - ONLY if `RESOLVED_POSITIONING` was captured for this entity AND the month's evidence directly supports a specific claim, cuts against one, or is squarely about the pitched ground: one windowed prose sentence anchored to a real item with engagement. Otherwise omit entirely - silence, not a placeholder.]
|
||||||
|
|
||||||
**Strengths (what people love)**
|
**Strengths (what people love)**
|
||||||
- [Specific strength with `per <source>` attribution]
|
- [Specific strength with `per <source>` attribution]
|
||||||
- [Specific strength with `per <source>` attribution]
|
- [Specific strength with `per <source>` attribution]
|
||||||
@@ -1299,7 +1306,7 @@ Voice contract LAWs 1, 3, 5 apply to comparisons unchanged (no `Sources:` block,
|
|||||||
| Best for | ... | ... | ... |
|
| Best for | ... | ... | ... |
|
||||||
| Install | ... | ... | ... |
|
| Install | ... | ... | ... |
|
||||||
|
|
||||||
(Engine emits this scaffold; fill the cells with 5-15 words each. If an axis does not apply to the topic class, write "N/A" or a topic-appropriate substitute rather than inventing data.)
|
(Engine emits this scaffold; fill the cells with 5-15 words each. If an axis does not apply to the topic class, write "N/A" or a topic-appropriate substitute rather than inventing data. Ground the `What it is` row in `RESOLVED_POSITIONING` when captured - each entity described as it pitches itself today, fetched this run, never from memory.)
|
||||||
|
|
||||||
## The Bottom Line
|
## The Bottom Line
|
||||||
|
|
||||||
@@ -1441,6 +1448,8 @@ At render time the `@handle`, `r/sub`, and publication-name placeholders become
|
|||||||
|
|
||||||
Headlines should be specific and newsy ("BULLY dropped and it's dominating", "Europe is banning him one country at a time"), not generic ("Album release", "Tour updates").
|
Headlines should be specific and newsy ("BULLY dropped and it's dominating", "Europe is banning him one country at a time"), not generic ("Album release", "Tour updates").
|
||||||
|
|
||||||
|
**Pitch-vs-pulse beat (company / product / service topics).** If you captured `RESOLVED_POSITIONING` in Step 0.55 AND the month's evidence directly bears on it, work in ONE bold-lead-in paragraph saying how. Three cases qualify: the pulse SUPPORTS a specific claim (e.g. `**"Zero-config" is holding up** - this month's top deploy thread is devs praising the no-setup flow, 800 upvotes`), CUTS AGAINST one (e.g. `**Stripe's fraud-fighting pitch took a direct hit** - the loudest thread this month argues it is friendly to "friendly fraud", 323pt HN`), or the conversation is squarely ABOUT the pitched ground. Always anchor to the real top item with its engagement, and keep claims windowed - "this month's conversation" - never trend verbs like "losing the narrative" that one 30-day window cannot support. If the month's conversation is orthogonal to the pitch - on-entity but about something the pitch doesn't speak to - write NOTHING about the pitch: omission is the correct output, and a manufactured connection is worse than silence. Match altitude: test SPECIFIC claims ("zero-config", "fastest", an uptime number) against specific threads; never grade a broad tagline against an individual thread. Keep it a normal newsy bold-lead-in paragraph, NOT a new `##` section (LAW 4 still holds). Skip silently for people (always - the beat can cover MrBeast the company, never Jimmy Donaldson the person), events, abstract concepts, and ownerless topics (Bitcoin), and whenever positioning was not actually fetched this run - never supply a pitch from memory.
|
||||||
|
|
||||||
**THEN - Quality Nudge (if present in the output):**
|
**THEN - Quality Nudge (if present in the output):**
|
||||||
|
|
||||||
If the research output contains a `**🔍 Research Coverage:**` block, render it verbatim right before the stats block. This tells the user which core sources are missing and how to unlock them. Do NOT render this block if it is absent from the output (100% coverage = no nudge).
|
If the research output contains a `**🔍 Research Coverage:**` block, render it verbatim right before the stats block. This tells the user which core sources are missing and how to unlock them. Do NOT render this block if it is absent from the output (100% coverage = no nudge).
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import atexit
|
import atexit
|
||||||
|
import datetime
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
@@ -62,7 +63,10 @@ def _cleanup_children() -> None:
|
|||||||
pids = list(_child_pids)
|
pids = list(_child_pids)
|
||||||
for pid in pids:
|
for pid in pids:
|
||||||
try:
|
try:
|
||||||
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
if hasattr(os, "killpg"):
|
||||||
|
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||||
|
else:
|
||||||
|
os.kill(pid, signal.SIGTERM)
|
||||||
except (ProcessLookupError, PermissionError, OSError):
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -195,9 +199,9 @@ def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str)
|
|||||||
try:
|
try:
|
||||||
home = _Path.home().resolve()
|
home = _Path.home().resolve()
|
||||||
relative = raw.relative_to(home)
|
relative = raw.relative_to(home)
|
||||||
return f"~/{relative}"
|
return f"~/{relative.as_posix()}"
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return str(raw)
|
return raw.as_posix()
|
||||||
|
|
||||||
|
|
||||||
def read_synthesis_file(path: str) -> str:
|
def read_synthesis_file(path: str) -> str:
|
||||||
@@ -387,7 +391,7 @@ def subrun_kwargs_for(
|
|||||||
|
|
||||||
subreddits = _choose("subreddits", "subreddits")
|
subreddits = _choose("subreddits", "subreddits")
|
||||||
if isinstance(subreddits, list):
|
if isinstance(subreddits, list):
|
||||||
subreddits = [s.strip().lstrip("r/") for s in subreddits if s.strip()] or None
|
subreddits = [s.strip().removeprefix("r/") for s in subreddits if s.strip()] or None
|
||||||
|
|
||||||
x_related = plan_entry.get("x_related")
|
x_related = plan_entry.get("x_related")
|
||||||
if isinstance(x_related, list):
|
if isinstance(x_related, list):
|
||||||
@@ -531,6 +535,24 @@ def _show_runtime_ui(
|
|||||||
progress.show_promo(promo, diag=diag)
|
progress.show_promo(promo, diag=diag)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_last_run(topic: str, report: "schema.Report") -> None:
|
||||||
|
try:
|
||||||
|
if env.CONFIG_DIR is None:
|
||||||
|
return
|
||||||
|
target = env.CONFIG_DIR
|
||||||
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
|
counts = {source: len(items) for source, items in report.items_by_source.items()}
|
||||||
|
payload = {
|
||||||
|
"topic": topic,
|
||||||
|
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||||
|
"sources": counts,
|
||||||
|
"total": sum(counts.values()),
|
||||||
|
}
|
||||||
|
(target / "last-run.json").write_text(json.dumps(payload, indent=2))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = build_parser()
|
parser = build_parser()
|
||||||
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
||||||
@@ -606,7 +628,7 @@ def main() -> int:
|
|||||||
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
||||||
try:
|
try:
|
||||||
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
||||||
subreddits = [s.strip().lstrip("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
subreddits = [s.strip().removeprefix("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||||
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
||||||
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
||||||
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
||||||
@@ -625,6 +647,7 @@ def main() -> int:
|
|||||||
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
||||||
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
||||||
# without WebSearch (OpenClaw, Codex, raw CLI).
|
# without WebSearch (OpenClaw, Codex, raw CLI).
|
||||||
|
repos_from_auto_resolve = False
|
||||||
if args.auto_resolve and not external_plan:
|
if args.auto_resolve and not external_plan:
|
||||||
from lib import resolve
|
from lib import resolve
|
||||||
resolution = resolve.auto_resolve(topic, config)
|
resolution = resolve.auto_resolve(topic, config)
|
||||||
@@ -639,6 +662,9 @@ def main() -> int:
|
|||||||
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
||||||
if resolution.get("github_repos") and not args.github_repo:
|
if resolution.get("github_repos") and not args.github_repo:
|
||||||
args.github_repo = ",".join(resolution["github_repos"])
|
args.github_repo = ",".join(resolution["github_repos"])
|
||||||
|
# auto_resolve already canonicalized via canonicalize_github_repos(cap=5);
|
||||||
|
# mark so we don't re-canonicalize below and clobber its relevance order.
|
||||||
|
repos_from_auto_resolve = True
|
||||||
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
||||||
if resolution.get("context"):
|
if resolution.get("context"):
|
||||||
# Inject context into external_plan metadata for the planner to use
|
# Inject context into external_plan metadata for the planner to use
|
||||||
@@ -651,6 +677,20 @@ def main() -> int:
|
|||||||
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
||||||
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
||||||
|
|
||||||
|
# Only canonicalize when repos came from a user-supplied --github-repo flag.
|
||||||
|
# When repos_from_auto_resolve is True, auto_resolve already ran
|
||||||
|
# canonicalize_github_repos(cap=5) and ranked by relevance; re-running here
|
||||||
|
# with cap=None can re-sort by topic-slug match and lose that ordering.
|
||||||
|
if github_repos and not repos_from_auto_resolve:
|
||||||
|
from lib import resolve as resolve_lib
|
||||||
|
original_github_repos = github_repos[:]
|
||||||
|
github_repos = resolve_lib.canonicalize_github_repos(topic, github_repos, cap=None)
|
||||||
|
if github_repos != original_github_repos:
|
||||||
|
sys.stderr.write(
|
||||||
|
"[GitHub] Canonicalized repos: "
|
||||||
|
f"{','.join(original_github_repos)} -> {','.join(github_repos)}\n"
|
||||||
|
)
|
||||||
|
|
||||||
# --deep-research: auto-enable perplexity source and set deep flag
|
# --deep-research: auto-enable perplexity source and set deep flag
|
||||||
if args.deep_research:
|
if args.deep_research:
|
||||||
if not config.get("OPENROUTER_API_KEY"):
|
if not config.get("OPENROUTER_API_KEY"):
|
||||||
@@ -870,7 +910,18 @@ def main() -> int:
|
|||||||
report, progress, diag,
|
report, progress, diag,
|
||||||
suppress_web_promo=bool(external_plan or comp_plan),
|
suppress_web_promo=bool(external_plan or comp_plan),
|
||||||
)
|
)
|
||||||
if args.store:
|
_write_last_run(topic, report)
|
||||||
|
# LAST30DAYS_STORE env var = persistence default-on. Read both os.environ
|
||||||
|
# (for shell-exported users) and config (for users who set it in
|
||||||
|
# ~/.config/last30days/.env, which env.py loads but does not propagate
|
||||||
|
# to os.environ). Mirrors the LAST30DAYS_DEBUG / LAST30DAYS_SKIP_PREFLIGHT
|
||||||
|
# convention; env-var or config wins, with `--store` flag still working.
|
||||||
|
_store_env = (
|
||||||
|
os.environ.get("LAST30DAYS_STORE")
|
||||||
|
or config.get("LAST30DAYS_STORE")
|
||||||
|
or ""
|
||||||
|
).lower()
|
||||||
|
if args.store or _store_env in ("1", "true", "yes"):
|
||||||
counts = persist_report(report)
|
counts = persist_report(report)
|
||||||
sys.stderr.write(
|
sys.stderr.write(
|
||||||
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
||||||
@@ -880,7 +931,32 @@ def main() -> int:
|
|||||||
# Show quality nudge if applicable
|
# Show quality nudge if applicable
|
||||||
try:
|
try:
|
||||||
from lib import quality_nudge
|
from lib import quality_nudge
|
||||||
quality = quality_nudge.compute_quality_score(config, {})
|
# Populate transcript-fetch ratio so quality_nudge can detect the
|
||||||
|
# degraded-YouTube failure mode (videos returned but transcripts
|
||||||
|
# silently failed - typically a stale yt-dlp binary).
|
||||||
|
youtube_items = report.items_by_source.get("youtube") or []
|
||||||
|
instagram_items = report.items_by_source.get("instagram") or []
|
||||||
|
research_results = {
|
||||||
|
"youtube_videos_count": len(youtube_items),
|
||||||
|
"youtube_transcripts_count": sum(
|
||||||
|
1 for it in youtube_items
|
||||||
|
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
||||||
|
),
|
||||||
|
"youtube_error": report.errors_by_source.get("youtube"),
|
||||||
|
"x_error": report.errors_by_source.get("x"),
|
||||||
|
# Captions-disabled videos can never produce a transcript regardless
|
||||||
|
# of yt-dlp version; subtract them from the degraded-ratio
|
||||||
|
# denominator so a single uploader-disabled video does not trip the
|
||||||
|
# "stale yt-dlp" nudge.
|
||||||
|
"youtube_captions_disabled_count": sum(
|
||||||
|
1 for it in youtube_items if it.metadata.get("captions_disabled")
|
||||||
|
),
|
||||||
|
# Track Instagram returned-zero-items so quality_nudge can detect
|
||||||
|
# the silent-failure case (SC configured but the v2 reels endpoint
|
||||||
|
# 500'd through both the original query and the hashtag retry).
|
||||||
|
"instagram_items_count": len(instagram_items),
|
||||||
|
}
|
||||||
|
quality = quality_nudge.compute_quality_score(config, research_results)
|
||||||
if quality.get("nudge_text"):
|
if quality.get("nudge_text"):
|
||||||
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|||||||
@@ -1,10 +1,19 @@
|
|||||||
"""Bluesky search via AT Protocol (requires app password).
|
"""Bluesky search via AT Protocol (requires app password).
|
||||||
|
|
||||||
Uses bsky.social for auth and public.api.bsky.app for post search.
|
Uses bsky.social for auth and api.bsky.app for post search (the canonical
|
||||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars.
|
authenticated AppView). The previous default `public.api.bsky.app` is the
|
||||||
|
unauthenticated public mirror, which BunnyCDN now blocks for searchPosts
|
||||||
|
regardless of auth header (verified 2026-05-04). Override the search host
|
||||||
|
via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again.
|
||||||
|
|
||||||
|
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are
|
||||||
|
19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords.
|
||||||
|
The createSession endpoint accepts main-account passwords too, but they're
|
||||||
|
bad hygiene (no scope, can't revoke individually).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import math
|
import math
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
@@ -14,7 +23,64 @@ from typing import Any, Dict, List, Optional
|
|||||||
from . import http, log
|
from . import http, log
|
||||||
|
|
||||||
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
||||||
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
|
_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str:
|
||||||
|
"""Resolve the Bluesky search URL with BSKY_SEARCH_HOST override.
|
||||||
|
|
||||||
|
Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or
|
||||||
|
.env file. The project's env.py loads .env into config but not into
|
||||||
|
os.environ, so check both — same hybrid pattern as last30days.py for
|
||||||
|
LAST30DAYS_STORE.
|
||||||
|
|
||||||
|
Hardens user-supplied host values against three common mis-configurations:
|
||||||
|
whitespace (e.g. " api.bsky.app "), embedded path components (e.g.
|
||||||
|
"api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and
|
||||||
|
embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these
|
||||||
|
we log a warning and fall back to the default rather than building an
|
||||||
|
invalid URL with an opaque downstream error.
|
||||||
|
"""
|
||||||
|
config = config or {}
|
||||||
|
raw = (
|
||||||
|
os.environ.get("BSKY_SEARCH_HOST")
|
||||||
|
or config.get("BSKY_SEARCH_HOST")
|
||||||
|
or _DEFAULT_BSKY_SEARCH_HOST
|
||||||
|
)
|
||||||
|
host = raw.strip().rstrip("/")
|
||||||
|
# Strip embedded scheme so users who paste full URLs do not break the f-string.
|
||||||
|
for prefix in ("https://", "http://"):
|
||||||
|
if host.lower().startswith(prefix):
|
||||||
|
host = host[len(prefix):]
|
||||||
|
break
|
||||||
|
if not host or "/" in host or " " in host:
|
||||||
|
# Embedded path or whitespace remains — don't trust it. Default + log.
|
||||||
|
if raw != _DEFAULT_BSKY_SEARCH_HOST:
|
||||||
|
_log(
|
||||||
|
f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; "
|
||||||
|
f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}"
|
||||||
|
)
|
||||||
|
host = _DEFAULT_BSKY_SEARCH_HOST
|
||||||
|
return f"https://{host}/xrpc/app.bsky.feed.searchPosts"
|
||||||
|
|
||||||
|
|
||||||
|
# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric
|
||||||
|
# with three hyphens at fixed positions).
|
||||||
|
_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_app_password_format(value) -> bool:
|
||||||
|
"""Return True if value matches Bluesky's 19-char app-password format.
|
||||||
|
|
||||||
|
False for non-strings (None, int, list) so callers passing config dict
|
||||||
|
values directly don't crash. Detect-but-not-gate: the createSession
|
||||||
|
endpoint also accepts main-account passwords, so failing this check is
|
||||||
|
a hygiene smell, not a hard error.
|
||||||
|
"""
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return False
|
||||||
|
return bool(_APP_PASSWORD_RE.fullmatch(value))
|
||||||
|
|
||||||
|
|
||||||
DEPTH_CONFIG = {
|
DEPTH_CONFIG = {
|
||||||
"quick": 15,
|
"quick": 15,
|
||||||
@@ -144,6 +210,20 @@ def search_bluesky(
|
|||||||
if not handle or not app_password:
|
if not handle or not app_password:
|
||||||
return {"posts": [], "error": "Bluesky credentials not configured"}
|
return {"posts": [], "error": "Bluesky credentials not configured"}
|
||||||
|
|
||||||
|
# One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password
|
||||||
|
# form. createSession accepts main-account passwords too — but main
|
||||||
|
# passwords have no scope (full account access), can't be revoked
|
||||||
|
# individually, and rotating them breaks every service that holds them.
|
||||||
|
# We warn but do not gate, matching the project's detect-don't-block
|
||||||
|
# philosophy elsewhere.
|
||||||
|
if not _validate_app_password_format(app_password):
|
||||||
|
_log(
|
||||||
|
"BSKY_APP_PASSWORD does not look like an app password "
|
||||||
|
"(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main "
|
||||||
|
"account password — those work but are bad hygiene. Generate "
|
||||||
|
"an app password at https://bsky.app/settings/app-passwords"
|
||||||
|
)
|
||||||
|
|
||||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||||
core_topic = _extract_core_subject(topic)
|
core_topic = _extract_core_subject(topic)
|
||||||
|
|
||||||
@@ -155,7 +235,7 @@ def search_bluesky(
|
|||||||
"limit": str(min(count, 100)),
|
"limit": str(min(count, 100)),
|
||||||
"sort": "top",
|
"sort": "top",
|
||||||
}
|
}
|
||||||
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
|
url = f"{_resolve_search_url(config)}?{urlencode(params)}"
|
||||||
|
|
||||||
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
token = _create_session(handle, app_password)
|
token = _create_session(handle, app_password)
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
"""Chrome cookie extraction for macOS.
|
"""Chrome and Brave cookie extraction for macOS.
|
||||||
|
|
||||||
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
|
Extracts cookies from Chromium-based browser SQLite databases using only
|
||||||
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
|
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
|
||||||
|
dependencies.
|
||||||
|
|
||||||
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||||
|
Chrome and Brave share the same algorithm; only the DB path and Keychain
|
||||||
|
service name differ.
|
||||||
This is NOT affected by Windows App-Bound Encryption (v20).
|
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -18,10 +21,11 @@ from typing import Optional
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Chrome cookie DB location on macOS
|
# Cookie DB locations on macOS
|
||||||
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||||
|
BRAVE_BASE_DIR = Path.home() / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser"
|
||||||
|
|
||||||
# Chrome v10 encryption constants
|
# Chromium v10 encryption constants (shared by Chrome and Brave)
|
||||||
CHROME_SALT = b"saltysalt"
|
CHROME_SALT = b"saltysalt"
|
||||||
CHROME_PBKDF2_ITERATIONS = 1003
|
CHROME_PBKDF2_ITERATIONS = 1003
|
||||||
CHROME_KEY_LENGTH = 16
|
CHROME_KEY_LENGTH = 16
|
||||||
@@ -29,8 +33,8 @@ CHROME_KEY_LENGTH = 16
|
|||||||
CHROME_IV_HEX = "20" * 16
|
CHROME_IV_HEX = "20" * 16
|
||||||
|
|
||||||
|
|
||||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||||
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
|
"""Retrieve the encryption passphrase for a Chromium-based browser from macOS Keychain.
|
||||||
|
|
||||||
Calls `security find-generic-password` which may trigger a system dialog
|
Calls `security find-generic-password` which may trigger a system dialog
|
||||||
on first access.
|
on first access.
|
||||||
@@ -39,30 +43,34 @@ def _get_chrome_encryption_key() -> Optional[bytes]:
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
|
["security", "find-generic-password", "-w", "-s", service_name],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
|
logger.info("%s Keychain access denied or browser not installed: %s", service_name, result.stderr.strip())
|
||||||
return None
|
return None
|
||||||
passphrase = result.stdout.strip()
|
passphrase = result.stdout.strip()
|
||||||
if not passphrase:
|
if not passphrase:
|
||||||
logger.info("Chrome Keychain returned empty passphrase")
|
logger.info("%s Keychain returned empty passphrase", service_name)
|
||||||
return None
|
return None
|
||||||
return passphrase.encode("utf-8")
|
return passphrase.encode("utf-8")
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
logger.info("'security' command not found — not on macOS?")
|
logger.info("'security' command not found — not on macOS?")
|
||||||
return None
|
return None
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
logger.info("Chrome Keychain access timed out")
|
logger.info("%s Keychain access timed out", service_name)
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.info("Failed to get Chrome encryption key: %s", e)
|
logger.info("Failed to get %s encryption key: %s", service_name, e)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||||
|
return _get_chromium_encryption_key("Chrome Safe Storage")
|
||||||
|
|
||||||
|
|
||||||
def _derive_aes_key(passphrase: bytes) -> bytes:
|
def _derive_aes_key(passphrase: bytes) -> bytes:
|
||||||
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
||||||
return hashlib.pbkdf2_hmac(
|
return hashlib.pbkdf2_hmac(
|
||||||
@@ -165,36 +173,42 @@ def _get_db_version(cursor: sqlite3.Cursor) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
def _extract_chromium_cookies_macos(
|
||||||
"""Extract cookies from Chrome on macOS.
|
db_path: Path,
|
||||||
|
keychain_service: str,
|
||||||
|
domain: str,
|
||||||
|
cookie_names: list[str],
|
||||||
|
) -> Optional[dict[str, str]]:
|
||||||
|
"""Extract cookies from any Chromium-based browser on macOS.
|
||||||
|
|
||||||
Copies the locked Cookies database to a temp file, reads specified cookies,
|
Copies the locked Cookies database to a temp file, reads specified cookies,
|
||||||
and decrypts v10-encrypted values using the Keychain-stored key.
|
and decrypts v10-encrypted values using the Keychain-stored key.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
|
db_path: Path to the browser's Cookies SQLite file.
|
||||||
cookie_names: List of cookie names to extract
|
keychain_service: macOS Keychain service name (e.g. "Chrome Safe Storage").
|
||||||
|
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com").
|
||||||
|
cookie_names: List of cookie names to extract.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict mapping cookie name to decrypted value, or None on failure.
|
Dict mapping cookie name to decrypted value, or None on failure.
|
||||||
Only includes cookies that were successfully found and decrypted.
|
Only includes cookies that were successfully found and decrypted.
|
||||||
"""
|
"""
|
||||||
if not CHROME_COOKIES_DB.exists():
|
if not db_path.exists():
|
||||||
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
|
logger.info("%s cookies database not found at %s", keychain_service, db_path)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Get encryption key from Keychain
|
passphrase = _get_chromium_encryption_key(keychain_service)
|
||||||
passphrase = _get_chrome_encryption_key()
|
|
||||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||||
|
|
||||||
# Copy DB to temp file (Chrome locks the original)
|
# Copy DB to temp file (browser locks the original while running)
|
||||||
tmp_fd = None
|
tmp_fd = None
|
||||||
tmp_path = None
|
tmp_path = None
|
||||||
try:
|
try:
|
||||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||||
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
|
shutil.copy2(str(db_path), tmp_path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.info("Failed to copy Chrome cookies database: %s", e)
|
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
|
||||||
if tmp_path:
|
if tmp_path:
|
||||||
try:
|
try:
|
||||||
Path(tmp_path).unlink(missing_ok=True)
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
@@ -211,26 +225,22 @@ def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Option
|
|||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
|
|
||||||
db_version = _get_db_version(cursor)
|
db_version = _get_db_version(cursor)
|
||||||
logger.debug("Chrome cookie DB version: %d", db_version)
|
logger.debug("%s cookie DB version: %d", keychain_service, db_version)
|
||||||
|
|
||||||
# Build query with placeholders for cookie names
|
|
||||||
placeholders = ",".join("?" for _ in cookie_names)
|
placeholders = ",".join("?" for _ in cookie_names)
|
||||||
query = (
|
query = (
|
||||||
f"SELECT name, value, encrypted_value FROM cookies "
|
f"SELECT name, value, encrypted_value FROM cookies "
|
||||||
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
||||||
)
|
)
|
||||||
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
|
|
||||||
params = [f"%{domain}"] + list(cookie_names)
|
params = [f"%{domain}"] + list(cookie_names)
|
||||||
cursor.execute(query, params)
|
cursor.execute(query, params)
|
||||||
|
|
||||||
results: dict[str, str] = {}
|
results: dict[str, str] = {}
|
||||||
for name, value, encrypted_value in cursor.fetchall():
|
for name, value, encrypted_value in cursor.fetchall():
|
||||||
# Prefer unencrypted value if present
|
|
||||||
if value:
|
if value:
|
||||||
results[name] = value
|
results[name] = value
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Handle encrypted value
|
|
||||||
if encrypted_value and encrypted_value[:3] == b"v10":
|
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||||
if aes_key is None:
|
if aes_key is None:
|
||||||
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||||
@@ -241,25 +251,72 @@ def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Option
|
|||||||
else:
|
else:
|
||||||
logger.debug("Failed to decrypt cookie %s", name)
|
logger.debug("Failed to decrypt cookie %s", name)
|
||||||
elif encrypted_value:
|
elif encrypted_value:
|
||||||
# Unknown encryption version
|
|
||||||
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
if not results:
|
if not results:
|
||||||
logger.info("No matching cookies found in Chrome for domain %s", domain)
|
logger.info("No matching cookies found in %s for domain %s", keychain_service, domain)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
except sqlite3.Error as e:
|
except sqlite3.Error as e:
|
||||||
logger.info("Failed to read Chrome cookies database: %s", e)
|
logger.info("Failed to read %s cookies database: %s", keychain_service, e)
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.info("Unexpected error reading Chrome cookies: %s", e)
|
logger.info("Unexpected error reading %s cookies: %s", keychain_service, e)
|
||||||
return None
|
return None
|
||||||
finally:
|
finally:
|
||||||
try:
|
try:
|
||||||
Path(tmp_path).unlink(missing_ok=True)
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||||
|
"""Extract cookies from Chrome on macOS."""
|
||||||
|
return _extract_chromium_cookies_macos(
|
||||||
|
CHROME_COOKIES_DB, "Chrome Safe Storage", domain, cookie_names
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_brave_cookies_db() -> Optional[Path]:
|
||||||
|
"""Find Brave's Cookies database on macOS.
|
||||||
|
|
||||||
|
Tries the Default profile first, then scans numbered Profile directories
|
||||||
|
by most-recently-modified. Brave creates extra profiles as "Profile 1",
|
||||||
|
"Profile 2", etc. alongside Default; the most recently used one is the
|
||||||
|
likeliest to hold current cookies. Lexicographic sort would visit
|
||||||
|
"Profile 10" before "Profile 2", which can return the wrong profile.
|
||||||
|
"""
|
||||||
|
default = BRAVE_BASE_DIR / "Default" / "Cookies"
|
||||||
|
if default.exists():
|
||||||
|
return default
|
||||||
|
|
||||||
|
try:
|
||||||
|
candidates = [
|
||||||
|
child for child in BRAVE_BASE_DIR.iterdir()
|
||||||
|
if child.is_dir() and child.name.startswith("Profile ")
|
||||||
|
]
|
||||||
|
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
||||||
|
candidate = child / "Cookies"
|
||||||
|
if candidate.exists():
|
||||||
|
return candidate
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||||
|
"""Extract cookies from Brave on macOS.
|
||||||
|
|
||||||
|
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
|
||||||
|
path and Keychain service name differ.
|
||||||
|
"""
|
||||||
|
db_path = _find_brave_cookies_db()
|
||||||
|
if db_path is None:
|
||||||
|
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
|
||||||
|
return None
|
||||||
|
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"""Browser cookie extraction for last30days.
|
"""Browser cookie extraction for last30days.
|
||||||
|
|
||||||
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
|
Extracts cookies from local browser databases (Firefox, Chrome, Brave, Safari)
|
||||||
to enable zero-config authentication for services like X/Twitter.
|
to enable zero-config authentication for services like X/Twitter.
|
||||||
|
|
||||||
Only uses Python stdlib — no external dependencies.
|
Only uses Python stdlib — no external dependencies.
|
||||||
@@ -255,6 +255,29 @@ def extract_chrome_cookies(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_brave_cookies(
|
||||||
|
domain: str, cookie_names: List[str]
|
||||||
|
) -> Optional[Dict[str, str]]:
|
||||||
|
"""Extract cookies from Brave for the given domain and cookie names.
|
||||||
|
|
||||||
|
macOS only — Brave uses the same v10 AES-128-CBC encryption as Chrome,
|
||||||
|
with a different DB path and Keychain service name ("Brave Safe Storage").
|
||||||
|
Tries the Default profile first, then scans numbered Profile directories.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||||
|
"""
|
||||||
|
if platform.system() != "Darwin":
|
||||||
|
logger.debug("Brave cookie extraction only supported on macOS")
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from .chrome_cookies import extract_brave_cookies_macos
|
||||||
|
return extract_brave_cookies_macos(domain, cookie_names)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Brave cookie extraction failed: %s", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def extract_safari_cookies(
|
def extract_safari_cookies(
|
||||||
domain: str, cookie_names: List[str]
|
domain: str, cookie_names: List[str]
|
||||||
) -> Optional[Dict[str, str]]:
|
) -> Optional[Dict[str, str]]:
|
||||||
@@ -282,9 +305,9 @@ def extract_cookies(
|
|||||||
"""Extract cookies from the specified browser.
|
"""Extract cookies from the specified browser.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||||
'auto' tries browsers in platform-appropriate order:
|
'auto' tries browsers in platform-appropriate order:
|
||||||
- macOS: Chrome -> Firefox -> Safari
|
- macOS: Chrome -> Brave -> Firefox -> Safari
|
||||||
- Linux: Firefox only
|
- Linux: Firefox only
|
||||||
domain: The cookie domain to match (e.g. ".x.com").
|
domain: The cookie domain to match (e.g. ".x.com").
|
||||||
cookie_names: List of cookie names to extract.
|
cookie_names: List of cookie names to extract.
|
||||||
@@ -333,7 +356,7 @@ def extract_cookies_with_source(
|
|||||||
so callers can track the source.
|
so callers can track the source.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||||
domain: The cookie domain to match (e.g. ".x.com").
|
domain: The cookie domain to match (e.g. ".x.com").
|
||||||
cookie_names: List of cookie names to extract.
|
cookie_names: List of cookie names to extract.
|
||||||
|
|
||||||
@@ -344,6 +367,7 @@ def extract_cookies_with_source(
|
|||||||
extractors = {
|
extractors = {
|
||||||
"firefox": extract_firefox_cookies,
|
"firefox": extract_firefox_cookies,
|
||||||
"chrome": extract_chrome_cookies,
|
"chrome": extract_chrome_cookies,
|
||||||
|
"brave": extract_brave_cookies,
|
||||||
"safari": extract_safari_cookies,
|
"safari": extract_safari_cookies,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -360,7 +384,7 @@ def extract_cookies_with_source(
|
|||||||
# Auto mode: try browsers in platform-appropriate order
|
# Auto mode: try browsers in platform-appropriate order
|
||||||
system = platform.system()
|
system = platform.system()
|
||||||
if system == "Darwin":
|
if system == "Darwin":
|
||||||
order = ["chrome", "firefox", "safari"]
|
order = ["chrome", "brave", "firefox", "safari"]
|
||||||
elif system == "Linux":
|
elif system == "Linux":
|
||||||
order = ["firefox"]
|
order = ["firefox"]
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ def _extract_subreddits(reddit_items: List[Dict[str, Any]]) -> List[str]:
|
|||||||
|
|
||||||
for item in reddit_items:
|
for item in reddit_items:
|
||||||
# Primary subreddit
|
# Primary subreddit
|
||||||
sub = item.get("subreddit", "").strip().lstrip("r/")
|
sub = item.get("subreddit", "").strip().removeprefix("r/")
|
||||||
if sub:
|
if sub:
|
||||||
sub_counts[sub] += 1
|
sub_counts[sub] += 1
|
||||||
|
|
||||||
|
|||||||
@@ -314,6 +314,7 @@ def get_config() -> dict[str, Any]:
|
|||||||
('LAST30DAYS_RERANK_MODEL', None),
|
('LAST30DAYS_RERANK_MODEL', None),
|
||||||
('LAST30DAYS_X_MODEL', None),
|
('LAST30DAYS_X_MODEL', None),
|
||||||
('LAST30DAYS_X_BACKEND', None),
|
('LAST30DAYS_X_BACKEND', None),
|
||||||
|
('LAST30DAYS_STORE', None),
|
||||||
('OPENAI_MODEL_PIN', None),
|
('OPENAI_MODEL_PIN', None),
|
||||||
('XAI_MODEL_PIN', None),
|
('XAI_MODEL_PIN', None),
|
||||||
('SCRAPECREATORS_API_KEY', None),
|
('SCRAPECREATORS_API_KEY', None),
|
||||||
@@ -322,6 +323,7 @@ def get_config() -> dict[str, Any]:
|
|||||||
('CT0', None),
|
('CT0', None),
|
||||||
('BSKY_HANDLE', None),
|
('BSKY_HANDLE', None),
|
||||||
('BSKY_APP_PASSWORD', None),
|
('BSKY_APP_PASSWORD', None),
|
||||||
|
('BSKY_SEARCH_HOST', None),
|
||||||
('TRUTHSOCIAL_TOKEN', None),
|
('TRUTHSOCIAL_TOKEN', None),
|
||||||
('BRAVE_API_KEY', None),
|
('BRAVE_API_KEY', None),
|
||||||
('EXA_API_KEY', None),
|
('EXA_API_KEY', None),
|
||||||
@@ -334,11 +336,31 @@ def get_config() -> dict[str, Any]:
|
|||||||
('INCLUDE_SOURCES', ''),
|
('INCLUDE_SOURCES', ''),
|
||||||
('EXCLUDE_SOURCES', ''),
|
('EXCLUDE_SOURCES', ''),
|
||||||
('LAST30DAYS_YOUTUBE_SSH_HOST', None),
|
('LAST30DAYS_YOUTUBE_SSH_HOST', None),
|
||||||
|
('LAST30DAYS_TRANSCRIPT_TIMEOUT', None),
|
||||||
]
|
]
|
||||||
|
|
||||||
for key, default in keys:
|
for key, default in keys:
|
||||||
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
||||||
|
|
||||||
|
# Backward-compat: ScrapeCreators' own examples and tutorials use the
|
||||||
|
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and
|
||||||
|
# CREATORS). Accept that form too so users who follow the vendor's docs
|
||||||
|
# don't silently end up with has_scrapecreators=False. Canonical name
|
||||||
|
# wins when both are set.
|
||||||
|
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||||
|
legacy = os.environ.get('SCRAPE_CREATORS_API_KEY') or merged_env.get('SCRAPE_CREATORS_API_KEY')
|
||||||
|
if legacy:
|
||||||
|
config['SCRAPECREATORS_API_KEY'] = legacy
|
||||||
|
|
||||||
|
# Multi-key rotation: comma-separated SCRAPECREATORS_API_KEY round-robins
|
||||||
|
# via random.choice per run. Originally added in #268, accidentally dropped
|
||||||
|
# in v3.0.6, restored here.
|
||||||
|
sc_key_raw = config.get('SCRAPECREATORS_API_KEY') or ''
|
||||||
|
if ',' in sc_key_raw:
|
||||||
|
import random
|
||||||
|
sc_keys = [k.strip() for k in sc_key_raw.split(',') if k.strip()]
|
||||||
|
config['SCRAPECREATORS_API_KEY'] = random.choice(sc_keys) if sc_keys else ''
|
||||||
|
|
||||||
# Track which config source was used (highest-priority file source wins
|
# Track which config source was used (highest-priority file source wins
|
||||||
# the label; keychain is only reported when nothing else is configured).
|
# the label; keychain is only reported when nothing else is configured).
|
||||||
if project_env_path:
|
if project_env_path:
|
||||||
|
|||||||
@@ -62,6 +62,17 @@ def _resolve_token(token: Optional[str] = None) -> Optional[str]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_token(token: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""Public alias for ``_resolve_token``.
|
||||||
|
|
||||||
|
The pipeline calls this once before ``search_github`` and
|
||||||
|
``enrich_with_comments`` so the ``gh auth token`` subprocess fallback
|
||||||
|
only fires once per query when ``GITHUB_TOKEN`` is unset, instead of
|
||||||
|
twice (once per call site).
|
||||||
|
"""
|
||||||
|
return _resolve_token(token)
|
||||||
|
|
||||||
|
|
||||||
def _fetch_json(
|
def _fetch_json(
|
||||||
url: str,
|
url: str,
|
||||||
token: Optional[str] = None,
|
token: Optional[str] = None,
|
||||||
@@ -142,8 +153,14 @@ def search_github(
|
|||||||
to_date: str,
|
to_date: str,
|
||||||
depth: str = "default",
|
depth: str = "default",
|
||||||
token: Optional[str] = None,
|
token: Optional[str] = None,
|
||||||
) -> List[Dict[str, Any]]:
|
) -> Dict[str, Any]:
|
||||||
"""Search GitHub Issues and PRs.
|
"""Search GitHub Issues and PRs (HTTP fetch only).
|
||||||
|
|
||||||
|
Returns a raw envelope shaped like every other adapter's ``search_X``:
|
||||||
|
``{"items": [raw GitHub API items], "context": {core, from_date,
|
||||||
|
to_date, count}}``. Normalization, date filtering, and sorting move
|
||||||
|
to ``parse_github_response``; comment enrichment moves to
|
||||||
|
``enrich_with_comments``.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
topic: Search topic
|
topic: Search topic
|
||||||
@@ -153,15 +170,23 @@ def search_github(
|
|||||||
token: Optional GitHub token (falls back to env/gh CLI)
|
token: Optional GitHub token (falls back to env/gh CLI)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of normalized item dicts. Empty list on any failure.
|
Dict envelope. Empty ``items`` list on any failure.
|
||||||
"""
|
"""
|
||||||
|
count = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
|
||||||
|
core = extract_core_subject(topic)
|
||||||
resolved_token = _resolve_token(token)
|
resolved_token = _resolve_token(token)
|
||||||
if not resolved_token:
|
if not resolved_token:
|
||||||
_log("No GitHub token available (set GITHUB_TOKEN or install gh CLI)")
|
_log("No GitHub token available (set GITHUB_TOKEN or install gh CLI)")
|
||||||
return []
|
return {
|
||||||
|
"items": [],
|
||||||
count = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
|
"error": "no token",
|
||||||
core = extract_core_subject(topic)
|
"context": {
|
||||||
|
"core": core,
|
||||||
|
"from_date": from_date,
|
||||||
|
"to_date": to_date,
|
||||||
|
"count": count,
|
||||||
|
},
|
||||||
|
}
|
||||||
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
||||||
|
|
||||||
# Build search query with date filter
|
# Build search query with date filter
|
||||||
@@ -176,12 +201,41 @@ def search_github(
|
|||||||
|
|
||||||
data = _fetch_json(url, token=resolved_token, timeout=30)
|
data = _fetch_json(url, token=resolved_token, timeout=30)
|
||||||
if not data:
|
if not data:
|
||||||
return []
|
return {"items": [], "context": {"core": core, "from_date": from_date,
|
||||||
|
"to_date": to_date, "count": count}}
|
||||||
|
|
||||||
raw_items = data.get("items", [])
|
raw_items = data.get("items", [])
|
||||||
_log(f"Found {len(raw_items)} issues/PRs")
|
_log(f"Found {len(raw_items)} issues/PRs")
|
||||||
|
|
||||||
items = []
|
return {
|
||||||
|
"items": raw_items,
|
||||||
|
"context": {
|
||||||
|
"core": core,
|
||||||
|
"from_date": from_date,
|
||||||
|
"to_date": to_date,
|
||||||
|
"count": count,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_github_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||||
|
"""Normalize a ``search_github`` envelope into the skill's item shape.
|
||||||
|
|
||||||
|
Pure function: no I/O, no token, no enrichment. Applies the date
|
||||||
|
filter using the search context and sorts by relevance.
|
||||||
|
"""
|
||||||
|
if not isinstance(response, dict):
|
||||||
|
return []
|
||||||
|
raw_items = response.get("items") or []
|
||||||
|
if not isinstance(raw_items, list):
|
||||||
|
return []
|
||||||
|
context = response.get("context") or {}
|
||||||
|
core = context.get("core") or ""
|
||||||
|
from_date = context.get("from_date") or ""
|
||||||
|
to_date = context.get("to_date") or ""
|
||||||
|
count = context.get("count") or DEPTH_LIMITS["default"]
|
||||||
|
|
||||||
|
items: List[Dict[str, Any]] = []
|
||||||
for i, item in enumerate(raw_items[:count]):
|
for i, item in enumerate(raw_items[:count]):
|
||||||
html_url = item.get("html_url", "")
|
html_url = item.get("html_url", "")
|
||||||
repo = _parse_repo_from_url(html_url)
|
repo = _parse_repo_from_url(html_url)
|
||||||
@@ -224,20 +278,34 @@ def search_github(
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
# Enrich top items with comments
|
|
||||||
items = _enrich_top_items(items, depth, resolved_token)
|
|
||||||
|
|
||||||
# Date filter
|
# Date filter
|
||||||
filtered = []
|
if from_date and to_date:
|
||||||
for item in items:
|
items = [
|
||||||
d = item.get("date")
|
item for item in items
|
||||||
if d is None or (from_date <= d <= to_date):
|
if item.get("date") is None or (from_date <= item["date"] <= to_date)
|
||||||
filtered.append(item)
|
]
|
||||||
|
|
||||||
# Sort by relevance
|
items.sort(key=lambda x: x.get("relevance", 0), reverse=True)
|
||||||
filtered.sort(key=lambda x: x.get("relevance", 0), reverse=True)
|
return items
|
||||||
|
|
||||||
return filtered
|
|
||||||
|
def enrich_with_comments(
|
||||||
|
items: List[Dict[str, Any]],
|
||||||
|
depth: str = "default",
|
||||||
|
token: Optional[str] = None,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Fetch top comments for top-K items by reactions and attach to metadata.
|
||||||
|
|
||||||
|
Mutates and returns ``items``. Resolves ``token`` via env/gh CLI when
|
||||||
|
not supplied, matching ``search_github``'s fallback chain.
|
||||||
|
"""
|
||||||
|
if not items:
|
||||||
|
return items
|
||||||
|
resolved_token = _resolve_token(token)
|
||||||
|
if not resolved_token:
|
||||||
|
_log("No GitHub token available for comment enrichment")
|
||||||
|
return items
|
||||||
|
return _enrich_top_items(items, depth, resolved_token)
|
||||||
|
|
||||||
|
|
||||||
def _enrich_top_items(
|
def _enrich_top_items(
|
||||||
|
|||||||
@@ -140,7 +140,10 @@ def parallel_search(
|
|||||||
data = http.request(
|
data = http.request(
|
||||||
"POST", "https://api.parallel.ai/v1/search",
|
"POST", "https://api.parallel.ai/v1/search",
|
||||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||||
json_data={"query": query, "max_results": count},
|
json_data={
|
||||||
|
"search_queries": [query],
|
||||||
|
"advanced_settings": {"max_results": count},
|
||||||
|
},
|
||||||
timeout=15,
|
timeout=15,
|
||||||
)
|
)
|
||||||
items = []
|
items = []
|
||||||
@@ -150,7 +153,7 @@ def parallel_search(
|
|||||||
url = r.get("url", "")
|
url = r.get("url", "")
|
||||||
if not url:
|
if not url:
|
||||||
continue
|
continue
|
||||||
raw_date = r.get("published_date") or ""
|
raw_date = r.get("publish_date") or ""
|
||||||
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
|
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
|
||||||
if not _in_date_range(pub_date, date_range):
|
if not _in_date_range(pub_date, date_range):
|
||||||
continue
|
continue
|
||||||
@@ -159,7 +162,7 @@ def parallel_search(
|
|||||||
"title": r.get("title", ""),
|
"title": r.get("title", ""),
|
||||||
"url": url,
|
"url": url,
|
||||||
"source_domain": _domain(url),
|
"source_domain": _domain(url),
|
||||||
"snippet": r.get("snippet", ""),
|
"snippet": ((r.get("excerpts") or [""])[0] or "")[:500],
|
||||||
"date": pub_date,
|
"date": pub_date,
|
||||||
"relevance": 0.8,
|
"relevance": 0.8,
|
||||||
"why_relevant": "Parallel AI web search",
|
"why_relevant": "Parallel AI web search",
|
||||||
|
|||||||
@@ -223,6 +223,53 @@ def post_raw(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, st
|
|||||||
return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs)
|
return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
BROWSER_USER_AGENT = (
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/124.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_text(
|
||||||
|
url: str,
|
||||||
|
timeout: int = DEFAULT_TIMEOUT,
|
||||||
|
retries: int = 2,
|
||||||
|
accept: str = "*/*",
|
||||||
|
headers: Optional[Dict[str, str]] = None,
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Fetch a URL and return decoded text, or None on any failure.
|
||||||
|
|
||||||
|
Keyless helper for Reddit RSS and shreddit HTML endpoints — the free path
|
||||||
|
that replaced the now-403 ``.json`` endpoints. Sends a browser User-Agent
|
||||||
|
and never raises: returns None on HTTP error, network failure, or timeout
|
||||||
|
so tiered callers can fall through to the next source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: Request URL
|
||||||
|
timeout: HTTP timeout per attempt in seconds
|
||||||
|
retries: Number of retries on failure (kept low — these tiers fail fast)
|
||||||
|
accept: Accept header value (e.g. "application/atom+xml", "text/html")
|
||||||
|
headers: Optional extra headers merged over the defaults
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Decoded response body as text, or None on failure.
|
||||||
|
"""
|
||||||
|
merged = {
|
||||||
|
"User-Agent": BROWSER_USER_AGENT,
|
||||||
|
"Accept": accept,
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
}
|
||||||
|
if headers:
|
||||||
|
merged.update(headers)
|
||||||
|
try:
|
||||||
|
return request(
|
||||||
|
"GET", url, headers=merged, timeout=timeout, retries=retries, raw=True
|
||||||
|
)
|
||||||
|
except HTTPError as e:
|
||||||
|
log(f"get_text failed ({e}): {url}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def scrapecreators_headers(token: str) -> Dict[str, str]:
|
def scrapecreators_headers(token: str) -> Dict[str, str]:
|
||||||
"""Build ScrapeCreators request headers (x-api-key + JSON content type)."""
|
"""Build ScrapeCreators request headers (x-api-key + JSON content type)."""
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -7,12 +7,14 @@ Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
|
|||||||
API docs: https://scrapecreators.com/docs
|
API docs: https://scrapecreators.com/docs
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set
|
||||||
|
|
||||||
from . import dates, http, log
|
from . import dates, http, log
|
||||||
|
from .relevance import token_overlap_relevance as _compute_relevance
|
||||||
|
|
||||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||||
|
|
||||||
@@ -26,7 +28,42 @@ DEPTH_CONFIG = {
|
|||||||
# Max words to keep from each caption
|
# Max words to keep from each caption
|
||||||
CAPTION_MAX_WORDS = 500
|
CAPTION_MAX_WORDS = 500
|
||||||
|
|
||||||
from .relevance import token_overlap_relevance as _compute_relevance
|
# Default transcript fetch timeout (seconds). SC's
|
||||||
|
# /v2/instagram/media/transcript regularly takes >15s on real workloads,
|
||||||
|
# so the default is generous; override via LAST30DAYS_TRANSCRIPT_TIMEOUT.
|
||||||
|
DEFAULT_TRANSCRIPT_TIMEOUT = 30
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_transcript_timeout(
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
config: Optional[Dict[str, Any]] = None,
|
||||||
|
) -> float:
|
||||||
|
"""Resolve the IG transcript-fetch timeout.
|
||||||
|
|
||||||
|
Priority (highest wins):
|
||||||
|
1. Explicit ``timeout`` kwarg
|
||||||
|
2. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in os.environ
|
||||||
|
3. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in caller-supplied config dict
|
||||||
|
4. ``DEFAULT_TRANSCRIPT_TIMEOUT`` (30s)
|
||||||
|
|
||||||
|
Mirrors the ``os.environ.get(X) or config.get(X)`` pattern used for
|
||||||
|
LAST30DAYS_STORE in last30days.py so the env var works whether it's
|
||||||
|
shell-exported or set in ~/.config/last30days/.env.
|
||||||
|
"""
|
||||||
|
if timeout is not None:
|
||||||
|
try:
|
||||||
|
return float(timeout)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
raw = os.environ.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||||
|
if not raw and config:
|
||||||
|
raw = config.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||||
|
if raw:
|
||||||
|
try:
|
||||||
|
return float(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
return float(DEFAULT_TRANSCRIPT_TIMEOUT)
|
||||||
|
|
||||||
|
|
||||||
def _extract_core_subject(topic: str) -> str:
|
def _extract_core_subject(topic: str) -> str:
|
||||||
@@ -44,6 +81,17 @@ def _extract_core_subject(topic: str) -> str:
|
|||||||
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_hashtag_form(query: str) -> str:
|
||||||
|
"""Collapse a multi-word query to hashtag form (no spaces, lowercase).
|
||||||
|
|
||||||
|
SC's /v2/instagram/reels/search wraps Google Search and is documented
|
||||||
|
to be flaky on multi-token queries. Single-token queries map to a
|
||||||
|
hashtag page lookup which is the stable path. Used as a 500-retry
|
||||||
|
fallback before the request bubbles up as a silent failure.
|
||||||
|
"""
|
||||||
|
return ''.join(query.split()).lower()
|
||||||
|
|
||||||
|
|
||||||
def _infer_query_intent(topic: str) -> str:
|
def _infer_query_intent(topic: str) -> str:
|
||||||
"""Tiny local intent classifier for Instagram query expansion."""
|
"""Tiny local intent classifier for Instagram query expansion."""
|
||||||
text = topic.lower().strip()
|
text = topic.lower().strip()
|
||||||
@@ -283,6 +331,26 @@ def search_instagram(
|
|||||||
timeout=30,
|
timeout=30,
|
||||||
retries=2,
|
retries=2,
|
||||||
)
|
)
|
||||||
|
except http.HTTPError as e:
|
||||||
|
# SC's v2 reels search wraps Google Search and 500s frequently on
|
||||||
|
# multi-token queries. Single tokens hit the stable hashtag-page
|
||||||
|
# path. Retry once with hashtag form before bubbling up.
|
||||||
|
if getattr(e, "status_code", None) == 500 and ' ' in core_topic:
|
||||||
|
_log(f"IG search 500 on '{core_topic}', retrying with hashtag form")
|
||||||
|
try:
|
||||||
|
data = http.get(
|
||||||
|
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||||
|
params={"query": _to_hashtag_form(core_topic)},
|
||||||
|
headers=http.scrapecreators_headers(token),
|
||||||
|
timeout=30,
|
||||||
|
retries=2,
|
||||||
|
)
|
||||||
|
except Exception as retry_e:
|
||||||
|
_log(f"IG search retry failed: {retry_e}")
|
||||||
|
return {"items": [], "error": f"{type(retry_e).__name__}: {retry_e}"}
|
||||||
|
else:
|
||||||
|
_log(f"ScrapeCreators error: {e}")
|
||||||
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"ScrapeCreators error: {e}")
|
_log(f"ScrapeCreators error: {e}")
|
||||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||||
@@ -317,6 +385,8 @@ def fetch_captions(
|
|||||||
video_items: List[Dict[str, Any]],
|
video_items: List[Dict[str, Any]],
|
||||||
token: str,
|
token: str,
|
||||||
depth: str = "default",
|
depth: str = "default",
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
config: Optional[Dict[str, Any]] = None,
|
||||||
) -> Dict[str, str]:
|
) -> Dict[str, str]:
|
||||||
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
||||||
|
|
||||||
@@ -328,12 +398,19 @@ def fetch_captions(
|
|||||||
video_items: Items from search_instagram()
|
video_items: Items from search_instagram()
|
||||||
token: ScrapeCreators API key
|
token: ScrapeCreators API key
|
||||||
depth: Depth level for caption limit
|
depth: Depth level for caption limit
|
||||||
|
timeout: Optional per-request transcript timeout in seconds. When
|
||||||
|
None, resolves from LAST30DAYS_TRANSCRIPT_TIMEOUT (env or
|
||||||
|
config), defaulting to DEFAULT_TRANSCRIPT_TIMEOUT (30s).
|
||||||
|
config: Optional config dict (from env.get_config()) used as a
|
||||||
|
fallback source for LAST30DAYS_TRANSCRIPT_TIMEOUT when the
|
||||||
|
value is not exported in os.environ.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict mapping video_id -> caption text (truncated to 500 words)
|
Dict mapping video_id -> caption text (truncated to 500 words)
|
||||||
"""
|
"""
|
||||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
depth_cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||||
max_captions = config["max_captions"]
|
max_captions = depth_cfg["max_captions"]
|
||||||
|
transcript_timeout = _resolve_transcript_timeout(timeout, config)
|
||||||
|
|
||||||
if not video_items or not token:
|
if not video_items or not token:
|
||||||
return {}
|
return {}
|
||||||
@@ -364,7 +441,7 @@ def fetch_captions(
|
|||||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||||
params={"url": url},
|
params={"url": url},
|
||||||
headers=http.scrapecreators_headers(token),
|
headers=http.scrapecreators_headers(token),
|
||||||
timeout=15,
|
timeout=transcript_timeout,
|
||||||
retries=1,
|
retries=1,
|
||||||
)
|
)
|
||||||
transcripts = data.get("transcripts") or []
|
transcripts = data.get("transcripts") or []
|
||||||
|
|||||||
@@ -251,6 +251,11 @@ def _normalize_youtube(
|
|||||||
metadata: dict[str, Any] = {}
|
metadata: dict[str, Any] = {}
|
||||||
if highlights:
|
if highlights:
|
||||||
metadata["transcript_highlights"] = highlights
|
metadata["transcript_highlights"] = highlights
|
||||||
|
if item.get("captions_disabled"):
|
||||||
|
# Surfaced for quality_nudge: uploader disabled captions, so this
|
||||||
|
# video should be subtracted from the degraded-transcript-ratio
|
||||||
|
# denominator (it was never going to produce a transcript).
|
||||||
|
metadata["captions_disabled"] = True
|
||||||
metadata["top_comments"] = _remap_comments(
|
metadata["top_comments"] = _remap_comments(
|
||||||
item.get("top_comments") or [],
|
item.get("top_comments") or [],
|
||||||
score_keys=("score", "likes"),
|
score_keys=("score", "likes"),
|
||||||
|
|||||||
@@ -79,6 +79,8 @@ MOCK_AVAILABLE_SOURCES = [
|
|||||||
"xiaohongshu",
|
"xiaohongshu",
|
||||||
"github",
|
"github",
|
||||||
"perplexity",
|
"perplexity",
|
||||||
|
"threads",
|
||||||
|
"pinterest",
|
||||||
"xquik",
|
"xquik",
|
||||||
"digg",
|
"digg",
|
||||||
]
|
]
|
||||||
@@ -118,7 +120,9 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
|||||||
available.append("grounding")
|
available.append("grounding")
|
||||||
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
|
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
|
||||||
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
|
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
|
||||||
if config.get("OPENROUTER_API_KEY") and "perplexity" in include_sources:
|
if config.get("OPENROUTER_API_KEY") and (
|
||||||
|
"perplexity" in include_sources or (requested_sources and "perplexity" in requested_sources)
|
||||||
|
):
|
||||||
available.append("perplexity")
|
available.append("perplexity")
|
||||||
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
|
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
|
||||||
available.append("xiaohongshu")
|
available.append("xiaohongshu")
|
||||||
@@ -203,7 +207,7 @@ def run(
|
|||||||
available = [source for source in available if source in requested_sources]
|
available = [source for source in available if source in requested_sources]
|
||||||
if web_backend == "none":
|
if web_backend == "none":
|
||||||
available = [s for s in available if s != "grounding"]
|
available = [s for s in available if s != "grounding"]
|
||||||
elif web_backend in ("brave", "exa", "serper") and "grounding" not in available:
|
elif web_backend in ("brave", "exa", "serper", "parallel") and "grounding" not in available:
|
||||||
available.append("grounding")
|
available.append("grounding")
|
||||||
if not available:
|
if not available:
|
||||||
raise RuntimeError("No sources are available for this run.")
|
raise RuntimeError("No sources are available for this run.")
|
||||||
@@ -1003,8 +1007,14 @@ def _retrieve_stream(
|
|||||||
result = polymarket.search_polymarket(subquery.search_query, from_date, to_date, depth=depth)
|
result = polymarket.search_polymarket(subquery.search_query, from_date, to_date, depth=depth)
|
||||||
return polymarket.parse_polymarket_response(result, topic=subquery.search_query), {}
|
return polymarket.parse_polymarket_response(result, topic=subquery.search_query), {}
|
||||||
if source == "github":
|
if source == "github":
|
||||||
result = github.search_github(subquery.search_query, from_date, to_date, depth=depth, token=config.get("GITHUB_TOKEN"))
|
# Resolve once at the pipeline boundary so search and enrich
|
||||||
return result, {}
|
# share the result; otherwise each call would re-run the env
|
||||||
|
# lookup and gh-CLI subprocess fallback (up to 5s timeout each).
|
||||||
|
token = github.resolve_token(config.get("GITHUB_TOKEN"))
|
||||||
|
response = github.search_github(subquery.search_query, from_date, to_date, depth=depth, token=token)
|
||||||
|
items = github.parse_github_response(response)
|
||||||
|
items = github.enrich_with_comments(items, depth=depth, token=token)
|
||||||
|
return items, {}
|
||||||
if source == "pinterest":
|
if source == "pinterest":
|
||||||
result = pinterest.search_pinterest(
|
result = pinterest.search_pinterest(
|
||||||
subquery.search_query, from_date, to_date,
|
subquery.search_query, from_date, to_date,
|
||||||
|
|||||||
@@ -19,14 +19,14 @@ ALLOWED_INTENTS = {
|
|||||||
}
|
}
|
||||||
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
|
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
|
||||||
QUICK_SOURCE_PRIORITY = {
|
QUICK_SOURCE_PRIORITY = {
|
||||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
"factual": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||||
"product": ["youtube", "reddit", "x", "tiktok"],
|
"product": ["youtube", "reddit", "x", "xquik", "tiktok"],
|
||||||
"concept": ["hackernews", "reddit", "x", "youtube"],
|
"concept": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||||
"opinion": ["reddit", "x", "youtube", "hackernews"],
|
"opinion": ["reddit", "x", "xquik", "youtube", "hackernews"],
|
||||||
"how_to": ["youtube", "reddit", "x", "hackernews"],
|
"how_to": ["youtube", "reddit", "x", "xquik", "hackernews"],
|
||||||
"comparison": ["reddit", "x", "hackernews", "youtube"],
|
"comparison": ["reddit", "x", "xquik", "hackernews", "youtube"],
|
||||||
"breaking_news": ["x", "reddit", "hackernews", "youtube", "polymarket"],
|
"breaking_news": ["x", "xquik", "reddit", "hackernews", "youtube", "polymarket"],
|
||||||
"prediction": ["polymarket", "x", "hackernews", "reddit", "youtube"],
|
"prediction": ["polymarket", "x", "xquik", "hackernews", "reddit", "youtube"],
|
||||||
}
|
}
|
||||||
SOURCE_PRIORITY = {
|
SOURCE_PRIORITY = {
|
||||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||||
@@ -60,6 +60,7 @@ INTENT_SOURCE_EXCLUSIONS: dict[str, set[str]] = {
|
|||||||
SOURCE_CAPABILITIES = {
|
SOURCE_CAPABILITIES = {
|
||||||
"reddit": {"discussion", "social"},
|
"reddit": {"discussion", "social"},
|
||||||
"x": {"discussion", "social"},
|
"x": {"discussion", "social"},
|
||||||
|
"xquik": {"discussion", "social"},
|
||||||
"youtube": {"video", "video_longform", "discussion"},
|
"youtube": {"video", "video_longform", "discussion"},
|
||||||
"tiktok": {"video", "video_shortform", "social"},
|
"tiktok": {"video", "video_shortform", "social"},
|
||||||
"instagram": {"video", "video_shortform", "social"},
|
"instagram": {"video", "video_shortform", "social"},
|
||||||
@@ -273,7 +274,15 @@ def _sanitize_plan(
|
|||||||
freshness_mode=freshness_mode,
|
freshness_mode=freshness_mode,
|
||||||
cluster_mode=cluster_mode,
|
cluster_mode=cluster_mode,
|
||||||
raw_topic=topic,
|
raw_topic=topic,
|
||||||
subqueries=_normalize_subquery_weights(_trim_subqueries_for_depth(subqueries, intent, depth, eligible_sources)),
|
subqueries=_normalize_subquery_weights(
|
||||||
|
_trim_subqueries_for_depth(
|
||||||
|
subqueries,
|
||||||
|
intent,
|
||||||
|
depth,
|
||||||
|
eligible_sources,
|
||||||
|
requested_sources=requested_sources,
|
||||||
|
)
|
||||||
|
),
|
||||||
source_weights=source_weights,
|
source_weights=source_weights,
|
||||||
notes=[str(note).strip() for note in raw.get("notes") or [] if str(note).strip()],
|
notes=[str(note).strip() for note in raw.get("notes") or [] if str(note).strip()],
|
||||||
)
|
)
|
||||||
@@ -306,6 +315,7 @@ def _trim_subqueries_for_depth(
|
|||||||
intent: str,
|
intent: str,
|
||||||
depth: str,
|
depth: str,
|
||||||
available_sources: list[str],
|
available_sources: list[str],
|
||||||
|
requested_sources: list[str] | None = None,
|
||||||
) -> list[schema.SubQuery]:
|
) -> list[schema.SubQuery]:
|
||||||
# At non-quick depth, expand sources: use capability routing for intents
|
# At non-quick depth, expand sources: use capability routing for intents
|
||||||
# that define it, or all available sources otherwise. The LLM planner may
|
# that define it, or all available sources otherwise. The LLM planner may
|
||||||
@@ -335,6 +345,15 @@ def _trim_subqueries_for_depth(
|
|||||||
for subquery in subqueries:
|
for subquery in subqueries:
|
||||||
if depth in {"quick", "default"}:
|
if depth in {"quick", "default"}:
|
||||||
preferred_sources = ranked_sources[:limit]
|
preferred_sources = ranked_sources[:limit]
|
||||||
|
if requested_sources:
|
||||||
|
requested = [
|
||||||
|
source
|
||||||
|
for source in requested_sources
|
||||||
|
if source in available_sources and source in subquery.sources
|
||||||
|
]
|
||||||
|
for source in requested:
|
||||||
|
if source not in preferred_sources:
|
||||||
|
preferred_sources.append(source)
|
||||||
else:
|
else:
|
||||||
preferred_sources = [source for source in ranked_sources if source in subquery.sources][:limit]
|
preferred_sources = [source for source in ranked_sources if source in subquery.sources][:limit]
|
||||||
if len(preferred_sources) < limit:
|
if len(preferred_sources) < limit:
|
||||||
@@ -427,7 +446,13 @@ def _fallback_plan(
|
|||||||
cluster_mode=_default_cluster_mode(intent),
|
cluster_mode=_default_cluster_mode(intent),
|
||||||
raw_topic=topic,
|
raw_topic=topic,
|
||||||
subqueries=_normalize_subquery_weights(
|
subqueries=_normalize_subquery_weights(
|
||||||
_trim_subqueries_for_depth(subqueries[:_max_subqueries(intent, topic)], intent, depth, list(source_weights))
|
_trim_subqueries_for_depth(
|
||||||
|
subqueries[:_max_subqueries(intent, topic)],
|
||||||
|
intent,
|
||||||
|
depth,
|
||||||
|
list(source_weights),
|
||||||
|
requested_sources=requested_sources,
|
||||||
|
)
|
||||||
),
|
),
|
||||||
source_weights=_normalize_weights(source_weights),
|
source_weights=_normalize_weights(source_weights),
|
||||||
notes=[note],
|
notes=[note],
|
||||||
|
|||||||
@@ -19,7 +19,11 @@ OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
|||||||
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
|
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
|
||||||
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
|
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
|
||||||
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||||
OPENROUTER_DEFAULT = "google/gemini-flash-2.0"
|
# OpenRouter routes the Gemini Flash Lite tier as the -preview slug; that is the
|
||||||
|
# stable form on that routing layer even though native Gemini's GEMINI_FLASH_LITE
|
||||||
|
# constant is suffix-free. If GEMINI_FLASH_LITE moves to a non-preview stable ID,
|
||||||
|
# double-check that OpenRouter's slug still maps to the same upstream model.
|
||||||
|
OPENROUTER_DEFAULT = "google/gemini-3.1-flash-lite-preview"
|
||||||
|
|
||||||
|
|
||||||
class ReasoningClient:
|
class ReasoningClient:
|
||||||
|
|||||||
@@ -45,26 +45,100 @@ def _is_youtube_active(config: dict, research_results: dict) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# Below this transcript-fetch ratio, YouTube is considered "degraded" rather
|
||||||
|
# than active. Picked at 50% so a single legitimate caption-disabled video in a
|
||||||
|
# multi-video result does not trip the nudge, but a stale-yt-dlp run that fails
|
||||||
|
# every transcript does. Tunable via DEGRADED_TRANSCRIPT_THRESHOLD env var if
|
||||||
|
# operators need to adjust without code changes.
|
||||||
|
DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD = 0.5
|
||||||
|
|
||||||
|
|
||||||
|
def _is_youtube_degraded(research_results: dict, threshold: float) -> bool:
|
||||||
|
"""YouTube is degraded when videos were returned but the transcript-fetch
|
||||||
|
ratio is below threshold. The canonical cause is a stale yt-dlp binary -
|
||||||
|
YouTube's caption format changes frequently and old binaries silently fail
|
||||||
|
every transcript while the search itself still succeeds.
|
||||||
|
|
||||||
|
Captions-disabled videos are subtracted from the denominator: an uploader
|
||||||
|
who turned off captions can never produce a transcript, so counting that
|
||||||
|
video toward "fetch failures" produces false positives. A single
|
||||||
|
captions-disabled video in a small result set was tripping the nudge.
|
||||||
|
"""
|
||||||
|
videos = int(research_results.get("youtube_videos_count") or 0)
|
||||||
|
transcripts = int(research_results.get("youtube_transcripts_count") or 0)
|
||||||
|
captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0)
|
||||||
|
if videos <= 0:
|
||||||
|
return False
|
||||||
|
eligible = videos - captions_disabled
|
||||||
|
if eligible <= 0:
|
||||||
|
# Every returned video had captions disabled - upstream content fact,
|
||||||
|
# not a yt-dlp problem. Don't flag.
|
||||||
|
return False
|
||||||
|
return (transcripts / eligible) < threshold
|
||||||
|
|
||||||
|
|
||||||
|
def _is_instagram_silent_failure(config: dict, research_results: dict) -> bool:
|
||||||
|
"""Instagram is silently failing when SC is configured but the source
|
||||||
|
returned zero items. The canonical cause is SC's v2 reels endpoint
|
||||||
|
500'ing on multi-token queries (it wraps Google Search and is documented
|
||||||
|
to be flaky there). Pre-fix the user got no signal at all - no Instagram
|
||||||
|
section in the brief, no error in the footer, just unexplained absence.
|
||||||
|
"""
|
||||||
|
if not config.get("SCRAPECREATORS_API_KEY"):
|
||||||
|
return False # not configured — not a silent failure
|
||||||
|
# Honor EXCLUDE_SOURCES: a user who set EXCLUDE_SOURCES=instagram
|
||||||
|
# intentionally turned the source off, so a zero-item count is
|
||||||
|
# expected, not a silent failure. Mirror the canonical parsing
|
||||||
|
# pattern from pipeline.available_sources().
|
||||||
|
excluded = {
|
||||||
|
s.strip().lower()
|
||||||
|
for s in (config.get("EXCLUDE_SOURCES") or "").split(",")
|
||||||
|
if s.strip()
|
||||||
|
}
|
||||||
|
# Symmetric case: INCLUDE_SOURCES is an opt-in allowlist. If it is
|
||||||
|
# non-empty and does not name instagram, the source was intentionally
|
||||||
|
# filtered out, so a zero-item count is expected — not a silent failure.
|
||||||
|
included = {
|
||||||
|
s.strip().lower()
|
||||||
|
for s in (config.get("INCLUDE_SOURCES") or "").split(",")
|
||||||
|
if s.strip()
|
||||||
|
}
|
||||||
|
if "instagram" in excluded or (included and "instagram" not in included):
|
||||||
|
return False
|
||||||
|
count = research_results.get("instagram_items_count")
|
||||||
|
if count is None:
|
||||||
|
return False # source not run this invocation
|
||||||
|
return int(count) == 0
|
||||||
|
|
||||||
|
|
||||||
def compute_quality_score(config: dict, research_results: dict) -> dict:
|
def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||||
"""Compute research quality score based on 5 core sources.
|
"""Compute research quality score based on 5 core sources.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Configuration dict from env.get_config()
|
config: Configuration dict from env.get_config()
|
||||||
research_results: Dict with keys like x_error, youtube_error,
|
research_results: Dict with keys like x_error, youtube_error,
|
||||||
reddit_error reflecting what happened this run.
|
reddit_error reflecting what happened this run. Optional keys
|
||||||
|
``youtube_videos_count`` and ``youtube_transcripts_count`` enable
|
||||||
|
degraded-YouTube detection (transcript-fetch ratio below threshold).
|
||||||
|
Optional key ``instagram_items_count`` enables silent-failure
|
||||||
|
detection for the bonus Instagram source.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
{
|
{
|
||||||
"score_pct": 40-100,
|
"score_pct": 40-100,
|
||||||
"core_active": ["hn", "polymarket", ...],
|
"core_active": ["hn", "polymarket", ...],
|
||||||
"core_missing": ["x", "youtube"],
|
"core_missing": ["x", "youtube"],
|
||||||
"core_errored": [], # configured but errored
|
"core_errored": [], # configured but errored at top level
|
||||||
"nudge_text": "..." or None if 100%
|
"core_degraded": [], # configured and returned items but quality below threshold
|
||||||
|
"bonus_errored": [], # bonus sources (Instagram, etc.) configured but silent
|
||||||
|
"nudge_text": "..." or None if all sources healthy
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
core_active: List[str] = []
|
core_active: List[str] = []
|
||||||
core_missing: List[str] = []
|
core_missing: List[str] = []
|
||||||
core_errored: List[str] = []
|
core_errored: List[str] = []
|
||||||
|
core_degraded: List[str] = []
|
||||||
|
bonus_errored: List[str] = []
|
||||||
|
|
||||||
# HN, Polymarket, and Reddit are always active
|
# HN, Polymarket, and Reddit are always active
|
||||||
core_active.append("hn")
|
core_active.append("hn")
|
||||||
@@ -84,6 +158,13 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
|
|||||||
yt_active = _is_youtube_active(config, research_results)
|
yt_active = _is_youtube_active(config, research_results)
|
||||||
if yt_active:
|
if yt_active:
|
||||||
core_active.append("youtube")
|
core_active.append("youtube")
|
||||||
|
# Active means yt-dlp is installed and search did not error at the top
|
||||||
|
# level. But search-success + transcript-failure is the canonical
|
||||||
|
# stale-binary failure mode that the footer used to hide. Flag as
|
||||||
|
# degraded so the user gets an actionable nudge to update the binary.
|
||||||
|
threshold = float(config.get("DEGRADED_TRANSCRIPT_THRESHOLD") or DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD)
|
||||||
|
if _is_youtube_degraded(research_results, threshold):
|
||||||
|
core_degraded.append("youtube")
|
||||||
else:
|
else:
|
||||||
core_missing.append("youtube")
|
core_missing.append("youtube")
|
||||||
# Check if configured but errored (yt-dlp installed but failed this run)
|
# Check if configured but errored (yt-dlp installed but failed this run)
|
||||||
@@ -95,28 +176,54 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
|
|||||||
if has_ytdlp and research_results.get("youtube_error"):
|
if has_ytdlp and research_results.get("youtube_error"):
|
||||||
core_errored.append("youtube")
|
core_errored.append("youtube")
|
||||||
|
|
||||||
|
# Bonus sources (Instagram, etc.): SC-key holders expect content from
|
||||||
|
# these but until now the pipeline fell silent on configured-but-zero.
|
||||||
|
if _is_instagram_silent_failure(config, research_results):
|
||||||
|
bonus_errored.append("instagram")
|
||||||
|
|
||||||
score_pct = int(len(core_active) / 5 * 100)
|
score_pct = int(len(core_active) / 5 * 100)
|
||||||
|
|
||||||
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||||
active_sources = research_results.get("active_sources") or []
|
active_sources = research_results.get("active_sources") or []
|
||||||
nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None
|
nudge_text = _build_nudge_text(
|
||||||
|
core_missing,
|
||||||
|
core_errored,
|
||||||
|
core_degraded,
|
||||||
|
research_results,
|
||||||
|
has_sc=has_sc,
|
||||||
|
active_sources=active_sources,
|
||||||
|
bonus_errored=bonus_errored,
|
||||||
|
) if (core_missing or core_degraded or bonus_errored) else None
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"score_pct": score_pct,
|
"score_pct": score_pct,
|
||||||
"core_active": core_active,
|
"core_active": core_active,
|
||||||
"core_missing": core_missing,
|
"core_missing": core_missing,
|
||||||
"core_errored": core_errored,
|
"core_errored": core_errored,
|
||||||
|
"core_degraded": core_degraded,
|
||||||
|
"bonus_errored": bonus_errored,
|
||||||
"nudge_text": nudge_text,
|
"nudge_text": nudge_text,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str:
|
def _build_nudge_text(
|
||||||
"""Build human-readable nudge text describing what was missed.
|
core_missing: List[str],
|
||||||
|
core_errored: List[str],
|
||||||
|
core_degraded: List[str] = None,
|
||||||
|
research_results: dict = None,
|
||||||
|
has_sc: bool = False,
|
||||||
|
active_sources: list = None,
|
||||||
|
bonus_errored: List[str] = None,
|
||||||
|
) -> str:
|
||||||
|
"""Build human-readable nudge text describing what was missed or degraded.
|
||||||
|
|
||||||
Prioritizes free suggestions. Optionally mentions bonus sources
|
Prioritizes free suggestions. Optionally mentions bonus sources
|
||||||
(TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured.
|
(TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured.
|
||||||
"""
|
"""
|
||||||
lines: List[str] = []
|
lines: List[str] = []
|
||||||
|
core_degraded = core_degraded or []
|
||||||
|
bonus_errored = bonus_errored or []
|
||||||
|
research_results = research_results or {}
|
||||||
|
|
||||||
# Describe what was missed
|
# Describe what was missed
|
||||||
missed_parts: List[str] = []
|
missed_parts: List[str] = []
|
||||||
@@ -129,7 +236,14 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc:
|
|||||||
|
|
||||||
active_count = 5 - len(core_missing)
|
active_count = 5 - len(core_missing)
|
||||||
lines.append(f"Research quality: {active_count}/5 core sources.")
|
lines.append(f"Research quality: {active_count}/5 core sources.")
|
||||||
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
if missed_parts:
|
||||||
|
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
||||||
|
if core_degraded:
|
||||||
|
degraded_labels = ", ".join(SOURCE_LABELS[s] for s in core_degraded)
|
||||||
|
lines.append(f"Degraded: {degraded_labels}.")
|
||||||
|
if bonus_errored:
|
||||||
|
bonus_labels = ", ".join(s.capitalize() for s in bonus_errored)
|
||||||
|
lines.append(f"Bonus source silent: {bonus_labels}.")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# Free suggestions
|
# Free suggestions
|
||||||
@@ -159,6 +273,35 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc:
|
|||||||
"explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
|
"explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if "youtube" in core_degraded:
|
||||||
|
videos = int(research_results.get("youtube_videos_count") or 0)
|
||||||
|
transcripts = int(research_results.get("youtube_transcripts_count") or 0)
|
||||||
|
captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0)
|
||||||
|
captions_note = ""
|
||||||
|
if captions_disabled > 0:
|
||||||
|
captions_note = (
|
||||||
|
f" ({captions_disabled} of those had captions disabled by the "
|
||||||
|
"uploader, which is a separate cause and not fixable on your end)"
|
||||||
|
)
|
||||||
|
free_suggestions.append(
|
||||||
|
f"YouTube returned {videos} videos but only {transcripts} transcripts "
|
||||||
|
f"captured{captions_note}. The most common remaining cause is a stale "
|
||||||
|
"yt-dlp binary - YouTube's caption format changes frequently and old "
|
||||||
|
"binaries silently fail every transcript. Update via your package "
|
||||||
|
"manager: scoop update yt-dlp (Windows), brew upgrade yt-dlp (macOS), "
|
||||||
|
"or pip install -U yt-dlp."
|
||||||
|
)
|
||||||
|
|
||||||
|
if "instagram" in bonus_errored:
|
||||||
|
free_suggestions.append(
|
||||||
|
"Instagram returned 0 reels despite SC being configured. SC's "
|
||||||
|
"v2 reels endpoint wraps Google Search and 500's frequently on "
|
||||||
|
"multi-token queries. The skill now retries with hashtag-form "
|
||||||
|
"automatically; if zero items still appear, the topic may have "
|
||||||
|
"no reel coverage on Instagram. Try a single-word topic like "
|
||||||
|
"the most distinctive noun in your query."
|
||||||
|
)
|
||||||
|
|
||||||
# Mention bonus opt-in sources when SC key is present
|
# Mention bonus opt-in sources when SC key is present
|
||||||
if has_sc:
|
if has_sc:
|
||||||
bonus_hints = []
|
bonus_hints = []
|
||||||
|
|||||||
@@ -334,7 +334,7 @@ def _global_search(
|
|||||||
)
|
)
|
||||||
return data.get("posts", data.get("data", []))
|
return data.get("posts", data.get("data", []))
|
||||||
except http.HTTPError as e:
|
except http.HTTPError as e:
|
||||||
if e.status_code in (401, 403):
|
if e.status_code in (401, 402, 403):
|
||||||
raise
|
raise
|
||||||
_log(f"Global search error: {e}")
|
_log(f"Global search error: {e}")
|
||||||
return []
|
return []
|
||||||
@@ -376,6 +376,11 @@ def _subreddit_search(
|
|||||||
retries=2,
|
retries=2,
|
||||||
)
|
)
|
||||||
return data.get("posts", data.get("data", []))
|
return data.get("posts", data.get("data", []))
|
||||||
|
except http.HTTPError as e:
|
||||||
|
if e.status_code in (401, 402, 403):
|
||||||
|
raise
|
||||||
|
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||||
|
return []
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||||
return []
|
return []
|
||||||
@@ -403,6 +408,11 @@ def fetch_post_comments(
|
|||||||
retries=2,
|
retries=2,
|
||||||
)
|
)
|
||||||
return data.get("comments", data.get("data", []))
|
return data.get("comments", data.get("data", []))
|
||||||
|
except http.HTTPError as e:
|
||||||
|
if e.status_code in (401, 402, 403):
|
||||||
|
raise
|
||||||
|
_log(f"Comment fetch error: {e}")
|
||||||
|
return []
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log(f"Comment fetch error: {e}")
|
_log(f"Comment fetch error: {e}")
|
||||||
return []
|
return []
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
"""Keyless Reddit pipeline: tiered free search + comment enrichment.
|
||||||
|
|
||||||
|
Replaces the dead ``.json`` free path. Discovery tiers, cheapest/most-likely
|
||||||
|
first; enrichment then runs on whatever was discovered:
|
||||||
|
|
||||||
|
Tier 0 one-shot legacy ``.json`` search — demoted. Datacenter IPs get 403,
|
||||||
|
but a residential machine (where the skill usually runs) may still
|
||||||
|
get 200, so it is worth one cheap try. Honors the "brute-force .json"
|
||||||
|
intent without depending on it.
|
||||||
|
Tier 1 RSS discovery (reddit_rss) — keyless, robust, the load-bearing path.
|
||||||
|
Tier 2 shreddit comment + count enrichment (reddit_shreddit) for top posts.
|
||||||
|
|
||||||
|
Returns ``[]`` (never raises) so ``pipeline.py`` can fall through to the
|
||||||
|
ScrapeCreators backup when every keyless tier comes up empty.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import concurrent.futures
|
||||||
|
import sys
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
from . import reddit_rss, reddit_shreddit, reddit_listing
|
||||||
|
|
||||||
|
ENRICH_LIMITS = reddit_shreddit.ENRICH_LIMITS
|
||||||
|
ENRICH_BUDGET = 45 # seconds total across all enrichment threads
|
||||||
|
MAX_ENRICH_WORKERS = 4
|
||||||
|
MAX_DERIVED_SUBS = 5 # subreddits derived from RSS results for score backfill
|
||||||
|
|
||||||
|
|
||||||
|
def _log(msg: str) -> None:
|
||||||
|
sys.stderr.write(f"[RedditKeyless] {msg}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _tier0_json(topic: str, depth: str) -> List[Dict[str, Any]]:
|
||||||
|
"""One cheap global ``.json`` discovery attempt. Returns [] on the 403 wall."""
|
||||||
|
try:
|
||||||
|
from . import reddit_public
|
||||||
|
return reddit_public.search(topic, depth=depth) or []
|
||||||
|
except Exception as e: # never let the demoted tier sink the run
|
||||||
|
_log(f"Tier 0 (.json) unavailable: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _top_subreddits(posts: List[Dict[str, Any]], limit: int = MAX_DERIVED_SUBS) -> List[str]:
|
||||||
|
"""Most frequent subreddits across discovered posts (for score backfill)."""
|
||||||
|
counts = Counter(p.get("subreddit", "") for p in posts if p.get("subreddit"))
|
||||||
|
return [sub for sub, _ in counts.most_common(limit)]
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_scores(post: Dict[str, Any], scored: Dict[str, int]) -> None:
|
||||||
|
post["score"] = scored["score"]
|
||||||
|
post["num_comments"] = scored["num_comments"]
|
||||||
|
post.setdefault("engagement", {})["score"] = scored["score"]
|
||||||
|
post["engagement"]["num_comments"] = scored["num_comments"]
|
||||||
|
|
||||||
|
|
||||||
|
def _discover(topic: str, depth: str, subreddits: Optional[List[str]]) -> List[Dict[str, Any]]:
|
||||||
|
# Tier 0: demoted one-shot .json (dead for normal users too, but free to try).
|
||||||
|
posts = _tier0_json(topic, depth)
|
||||||
|
if posts:
|
||||||
|
_log(f"Tier 0 (.json) returned {len(posts)} posts")
|
||||||
|
return posts
|
||||||
|
|
||||||
|
# Tier 1: keyless discovery. RSS gives breadth (incl. global keyword search);
|
||||||
|
# the listing partials give real upvote scores.
|
||||||
|
rss_posts = reddit_rss.search_rss(topic, depth=depth, subreddits=subreddits)
|
||||||
|
|
||||||
|
if subreddits:
|
||||||
|
# Targeted run: the caller chose these subreddits, so their listing cards
|
||||||
|
# are on-topic — include them as scored discovery AND as a score source.
|
||||||
|
listing_posts = reddit_listing.fetch_listings(subreddits, depth=depth, query=topic)
|
||||||
|
score_source = listing_posts
|
||||||
|
else:
|
||||||
|
# Bare global run: subreddits derived from noisy RSS results are NOT
|
||||||
|
# reliably on-topic, so their listings are used ONLY to backfill scores
|
||||||
|
# onto the keyword-matched RSS posts — never merged as discovery, which
|
||||||
|
# would flood results with high-upvote but irrelevant posts.
|
||||||
|
listing_posts = []
|
||||||
|
derived = _top_subreddits(rss_posts)
|
||||||
|
score_source = reddit_listing.fetch_listings(derived, depth=depth, query=topic)
|
||||||
|
_log(
|
||||||
|
f"Tier 1 (RSS) {len(rss_posts)} posts; "
|
||||||
|
f"{'listing discovery ' + str(len(listing_posts)) if subreddits else 'score-only'}; "
|
||||||
|
f"{len(score_source)} scored cards"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Score lookup by post id, from the scored listing cards.
|
||||||
|
score_map: Dict[str, Dict[str, int]] = {}
|
||||||
|
for p in score_source:
|
||||||
|
pid = p.get("metadata", {}).get("post_id", "")
|
||||||
|
if pid:
|
||||||
|
score_map[pid] = {"score": p["score"], "num_comments": p["num_comments"]}
|
||||||
|
|
||||||
|
# Merge: scored listing posts first (targeted only), then RSS breadth,
|
||||||
|
# backfilled with real scores where the post appears in a listing.
|
||||||
|
merged: List[Dict[str, Any]] = []
|
||||||
|
seen: set = set()
|
||||||
|
for p in listing_posts:
|
||||||
|
if p["url"] not in seen:
|
||||||
|
seen.add(p["url"])
|
||||||
|
merged.append(p)
|
||||||
|
for p in rss_posts:
|
||||||
|
if p["url"] in seen:
|
||||||
|
continue
|
||||||
|
pid = reddit_listing._post_id(p["url"])
|
||||||
|
if pid in score_map:
|
||||||
|
_apply_scores(p, score_map[pid])
|
||||||
|
seen.add(p["url"])
|
||||||
|
merged.append(p)
|
||||||
|
return merged
|
||||||
|
|
||||||
|
|
||||||
|
def _enrich_one(post: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Attach shreddit comments + real comment count. Never raises."""
|
||||||
|
try:
|
||||||
|
data = reddit_shreddit.fetch_comments(post.get("url", ""))
|
||||||
|
if data.get("top_comments"):
|
||||||
|
post["top_comments"] = data["top_comments"]
|
||||||
|
if data.get("comment_insights"):
|
||||||
|
post["comment_insights"] = data["comment_insights"]
|
||||||
|
num = data.get("num_comments")
|
||||||
|
if num is not None:
|
||||||
|
post["num_comments"] = num
|
||||||
|
post.setdefault("engagement", {})["num_comments"] = num
|
||||||
|
except Exception:
|
||||||
|
pass # keep the post with whatever discovery gave us
|
||||||
|
return post
|
||||||
|
|
||||||
|
|
||||||
|
def _enrich(posts: List[Dict[str, Any]], depth: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Enrich the top N posts with comments under a total time budget."""
|
||||||
|
limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])
|
||||||
|
to_enrich = posts[:limit]
|
||||||
|
rest = posts[limit:]
|
||||||
|
if not to_enrich:
|
||||||
|
return posts
|
||||||
|
|
||||||
|
result_map: Dict[int, Dict[str, Any]] = {}
|
||||||
|
try:
|
||||||
|
with ThreadPoolExecutor(max_workers=min(limit, MAX_ENRICH_WORKERS)) as executor:
|
||||||
|
futures = {
|
||||||
|
executor.submit(_enrich_one, post): i
|
||||||
|
for i, post in enumerate(to_enrich)
|
||||||
|
}
|
||||||
|
done, not_done = concurrent.futures.wait(futures, timeout=ENRICH_BUDGET)
|
||||||
|
for future in done:
|
||||||
|
idx = futures[future]
|
||||||
|
try:
|
||||||
|
result_map[idx] = future.result(timeout=0)
|
||||||
|
except Exception:
|
||||||
|
result_map[idx] = to_enrich[idx]
|
||||||
|
for future in not_done:
|
||||||
|
idx = futures[future]
|
||||||
|
result_map[idx] = to_enrich[idx]
|
||||||
|
future.cancel()
|
||||||
|
enriched = [result_map[i] for i in range(len(to_enrich))]
|
||||||
|
except Exception:
|
||||||
|
enriched = to_enrich
|
||||||
|
|
||||||
|
return enriched + rest
|
||||||
|
|
||||||
|
|
||||||
|
def _slot_priority(topic: str, posts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||||
|
"""Order posts for enrichment slots: entity-matching posts first.
|
||||||
|
|
||||||
|
Comment slots (ENRICH_LIMITS) are scarce; spending them on high-upvote
|
||||||
|
posts that rerank later demotes as entity misses starves the on-topic
|
||||||
|
posts the user actually sees (2026-06-06 "OpenClaw vs Hermes" run:
|
||||||
|
2,000+ upvote Gemma/GPU threads took every slot, then were demoted to
|
||||||
|
zero). Mirror rerank's demotion signal via the shared `_entity_grounded`
|
||||||
|
check (head token of the topic's stripped primary entity present in the
|
||||||
|
post text) so slots go to posts likely to survive final ranking — keying
|
||||||
|
on the same head token keeps the two paths from diverging. Falls back to
|
||||||
|
token-overlap relevance when the
|
||||||
|
topic yields no usable primary entity. Within each tier the incoming
|
||||||
|
(score-first) order is preserved. Never raises; on any failure the
|
||||||
|
incoming order is returned unchanged.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from . import relevance, rerank
|
||||||
|
|
||||||
|
def _post_text(post: Dict[str, Any]) -> str:
|
||||||
|
return f"{post.get('title') or ''} {post.get('selftext') or ''}"
|
||||||
|
|
||||||
|
entity = rerank._primary_entity(topic).lower()
|
||||||
|
if entity:
|
||||||
|
def _matches(post: Dict[str, Any]) -> bool:
|
||||||
|
return rerank._entity_grounded(_post_text(post).lower(), entity)
|
||||||
|
else:
|
||||||
|
prepared = relevance.PreparedQuery(topic)
|
||||||
|
|
||||||
|
def _matches(post: Dict[str, Any]) -> bool:
|
||||||
|
return relevance.token_overlap_relevance(prepared, _post_text(post)) > 0.24
|
||||||
|
|
||||||
|
matches: List[Dict[str, Any]] = []
|
||||||
|
misses: List[Dict[str, Any]] = []
|
||||||
|
for post in posts:
|
||||||
|
(matches if _matches(post) else misses).append(post)
|
||||||
|
return matches + misses
|
||||||
|
except Exception:
|
||||||
|
return posts
|
||||||
|
|
||||||
|
|
||||||
|
def search_and_enrich(
|
||||||
|
topic: str,
|
||||||
|
from_date: str,
|
||||||
|
to_date: str,
|
||||||
|
depth: str = "default",
|
||||||
|
subreddits: Optional[List[str]] = None,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Full keyless Reddit pipeline: discover (Tier 0/1) then enrich (Tier 2).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
topic: Search topic
|
||||||
|
from_date: Start date (YYYY-MM-DD)
|
||||||
|
to_date: End date (YYYY-MM-DD)
|
||||||
|
depth: 'quick', 'default', or 'deep'
|
||||||
|
subreddits: Optional pre-resolved subreddit names (without r/)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of normalized item dicts matching the reddit_public output shape,
|
||||||
|
with top_comments/comment_insights attached on enriched posts.
|
||||||
|
Empty list when all keyless tiers fail (so SC backup can engage).
|
||||||
|
"""
|
||||||
|
posts = _discover(topic, depth, subreddits)
|
||||||
|
if not posts:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Date filter: keep posts in range or with unknown dates (mirrors reddit_public).
|
||||||
|
posts = [
|
||||||
|
p for p in posts
|
||||||
|
if p.get("date") is None or (from_date <= p["date"] <= to_date)
|
||||||
|
]
|
||||||
|
|
||||||
|
# Rank by real upvote score (from listing cards / backfill), then query
|
||||||
|
# relevance, then recency. Posts without a recovered score sort by the
|
||||||
|
# latter two — same behavior as before scores were available.
|
||||||
|
posts.sort(
|
||||||
|
key=lambda p: (
|
||||||
|
p.get("engagement", {}).get("score", 0) or 0,
|
||||||
|
p.get("relevance", 0) or 0,
|
||||||
|
p.get("date") or "",
|
||||||
|
),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Enrichment slot selection is relevance-aware: entity-matching posts
|
||||||
|
# claim the scarce comment slots first (score order preserved within
|
||||||
|
# each tier). The score-first sort above still governs within-tier order.
|
||||||
|
posts = _enrich(_slot_priority(topic, posts), depth)
|
||||||
|
|
||||||
|
for i, post in enumerate(posts):
|
||||||
|
post["id"] = f"R{i + 1}"
|
||||||
|
|
||||||
|
return posts
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Keyless Reddit listing scrape via shreddit /svc partials — with real scores.
|
||||||
|
|
||||||
|
The subreddit listing partial
|
||||||
|
``/svc/shreddit/community-more-posts/{sort}/?name={sub}[&t={range}]`` serves
|
||||||
|
HTTP 200 with no API key and **server-renders each post's upvote score**, which
|
||||||
|
neither RSS nor the comments endpoint provides. Each post is a
|
||||||
|
``<shreddit-post>`` element whose start-tag attributes carry ``score``,
|
||||||
|
``comment-count``, ``post-title``, ``permalink``, ``author``, ``subreddit-name``
|
||||||
|
and ``created-timestamp``.
|
||||||
|
|
||||||
|
This is the keyless source of post-level upvotes. It works for normal users on
|
||||||
|
ordinary connections (verified), so reddit_keyless uses it both as a scored
|
||||||
|
discovery source and to backfill scores onto RSS-discovered posts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import html as _html
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from . import http
|
||||||
|
from .relevance import token_overlap_relevance
|
||||||
|
|
||||||
|
# Listing sorts pulled per subreddit, by depth.
|
||||||
|
LISTING_SORTS = {
|
||||||
|
"quick": ["top"],
|
||||||
|
"default": ["top", "hot"],
|
||||||
|
"deep": ["top", "hot", "new"],
|
||||||
|
}
|
||||||
|
DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50}
|
||||||
|
TIMEFRAME = "month"
|
||||||
|
MAX_WORKERS = 4
|
||||||
|
LISTING_TIMEOUT = 15
|
||||||
|
|
||||||
|
_POST_CARD = re.compile(r"<shreddit-post(?=[\s>])[^>]*>")
|
||||||
|
|
||||||
|
|
||||||
|
def _log(msg: str) -> None:
|
||||||
|
sys.stderr.write(f"[RedditListing] {msg}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _attr(tag: str, name: str) -> Optional[str]:
|
||||||
|
m = re.search(rf'\b{name}="([^"]*)"', tag)
|
||||||
|
return _html.unescape(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _to_date(value: Optional[str]) -> Optional[str]:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.strip()).date().isoformat()
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _to_epoch(value: Optional[str]) -> Optional[float]:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(value.strip())
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
dt = dt.replace(tzinfo=timezone.utc)
|
||||||
|
return dt.timestamp()
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _post_id(permalink: str) -> str:
|
||||||
|
m = re.search(r"/comments/([A-Za-z0-9]+)", permalink or "")
|
||||||
|
return m.group(1) if m else ""
|
||||||
|
|
||||||
|
|
||||||
|
def parse_cards(html_text: str, query: str = "") -> List[Dict[str, Any]]:
|
||||||
|
"""Parse <shreddit-post> cards into normalized post dicts with real scores."""
|
||||||
|
posts: List[Dict[str, Any]] = []
|
||||||
|
for m in _POST_CARD.finditer(html_text or ""):
|
||||||
|
tag = m.group(0)
|
||||||
|
permalink = _attr(tag, "permalink") or ""
|
||||||
|
if "/comments/" not in permalink:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
score = int(_attr(tag, "score") or 0)
|
||||||
|
except ValueError:
|
||||||
|
score = 0
|
||||||
|
try:
|
||||||
|
num_comments = int(_attr(tag, "comment-count") or 0)
|
||||||
|
except ValueError:
|
||||||
|
num_comments = 0
|
||||||
|
title = _attr(tag, "post-title") or ""
|
||||||
|
author = _attr(tag, "author") or "[deleted]"
|
||||||
|
subreddit = _attr(tag, "subreddit-name") or ""
|
||||||
|
created = _attr(tag, "created-timestamp")
|
||||||
|
url = f"https://www.reddit.com{permalink}"
|
||||||
|
|
||||||
|
posts.append({
|
||||||
|
"id": "",
|
||||||
|
"title": title,
|
||||||
|
"url": url,
|
||||||
|
"score": score,
|
||||||
|
"num_comments": num_comments,
|
||||||
|
"subreddit": subreddit,
|
||||||
|
"created_utc": _to_epoch(created),
|
||||||
|
"author": author if author not in ("[deleted]", "[removed]") else "[deleted]",
|
||||||
|
"selftext": "",
|
||||||
|
"date": _to_date(created),
|
||||||
|
"engagement": {
|
||||||
|
"score": score,
|
||||||
|
"num_comments": num_comments,
|
||||||
|
"upvote_ratio": None,
|
||||||
|
},
|
||||||
|
"relevance": round(token_overlap_relevance(query, title), 3) if query else 0.0,
|
||||||
|
"why_relevant": "Reddit listing",
|
||||||
|
"metadata": {"post_id": _post_id(permalink)},
|
||||||
|
})
|
||||||
|
return posts
|
||||||
|
|
||||||
|
|
||||||
|
def _listing_url(subreddit: str, sort: str) -> str:
|
||||||
|
sub = subreddit.removeprefix("r/").strip()
|
||||||
|
url = f"https://www.reddit.com/svc/shreddit/community-more-posts/{sort}/?name={sub}"
|
||||||
|
if sort == "top":
|
||||||
|
url += f"&t={TIMEFRAME}"
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_one(subreddit: str, sort: str, query: str) -> List[Dict[str, Any]]:
|
||||||
|
try:
|
||||||
|
text = http.get_text(_listing_url(subreddit, sort), timeout=LISTING_TIMEOUT,
|
||||||
|
accept="text/html")
|
||||||
|
return parse_cards(text, query) if text else []
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"listing fetch failed r/{subreddit} {sort}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_listings(
|
||||||
|
subreddits: List[str],
|
||||||
|
depth: str = "default",
|
||||||
|
query: str = "",
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Fetch scored post cards across subreddits × depth-appropriate sorts.
|
||||||
|
|
||||||
|
Returns deduped normalized posts (with real scores), unranked/unsliced —
|
||||||
|
the caller merges these with other sources, ranks, and slices.
|
||||||
|
"""
|
||||||
|
if not subreddits:
|
||||||
|
return []
|
||||||
|
sorts = LISTING_SORTS.get(depth, LISTING_SORTS["default"])
|
||||||
|
jobs = [(sub, sort) for sub in subreddits for sort in sorts]
|
||||||
|
all_posts: List[Dict[str, Any]] = []
|
||||||
|
with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs)) or 1) as executor:
|
||||||
|
futures = {executor.submit(_fetch_one, sub, sort, query): (sub, sort)
|
||||||
|
for sub, sort in jobs}
|
||||||
|
for future in futures:
|
||||||
|
try:
|
||||||
|
all_posts.extend(future.result(timeout=LISTING_TIMEOUT + 5))
|
||||||
|
except (Exception, FuturesTimeoutError) as e:
|
||||||
|
_log(f"listing future failed: {e}")
|
||||||
|
|
||||||
|
seen: set = set()
|
||||||
|
unique: List[Dict[str, Any]] = []
|
||||||
|
for p in all_posts:
|
||||||
|
if p["url"] not in seen:
|
||||||
|
seen.add(p["url"])
|
||||||
|
unique.append(p)
|
||||||
|
return unique
|
||||||
|
|
||||||
|
|
||||||
|
def score_index(subreddits: List[str], depth: str = "default") -> Dict[str, Dict[str, int]]:
|
||||||
|
"""Build a {post_id: {score, num_comments}} map from subreddit listings.
|
||||||
|
|
||||||
|
Used to backfill real scores onto posts discovered via RSS, which carries
|
||||||
|
no engagement numbers.
|
||||||
|
"""
|
||||||
|
index: Dict[str, Dict[str, int]] = {}
|
||||||
|
for p in fetch_listings(subreddits, depth=depth):
|
||||||
|
pid = p.get("metadata", {}).get("post_id") or _post_id(p["url"])
|
||||||
|
if pid:
|
||||||
|
index[pid] = {"score": p["score"], "num_comments": p["num_comments"]}
|
||||||
|
return index
|
||||||
@@ -1,9 +1,16 @@
|
|||||||
"""Standalone Reddit public JSON search module.
|
"""Reddit public ``.json`` search module (demoted to keyless Tier 0).
|
||||||
|
|
||||||
Searches Reddit using the free public JSON endpoints (no API key required).
|
Reddit's public ``.json`` endpoints now return HTTP 403 from most contexts
|
||||||
Promoted from last-resort fallback to robust primary free path.
|
(shreddit anti-bot), so this is no longer the primary free path. The keyless
|
||||||
|
pipeline (see reddit_keyless.py) still calls ``search`` as a cheap one-shot
|
||||||
|
Tier 0 attempt — a residential machine may occasionally get a 200 — before
|
||||||
|
falling through to RSS discovery (reddit_rss.py) and shreddit comment
|
||||||
|
enrichment (reddit_shreddit.py).
|
||||||
|
|
||||||
Endpoints:
|
``search_reddit_public`` is retained as a compatibility shim that delegates to
|
||||||
|
the keyless pipeline, so existing callers (pipeline.py) need no change.
|
||||||
|
|
||||||
|
Endpoints (Tier 0):
|
||||||
- Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit}
|
- Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit}
|
||||||
- Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month
|
- Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month
|
||||||
|
|
||||||
@@ -11,17 +18,21 @@ Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
|
|||||||
network timeouts, and missing subreddits.
|
network timeouts, and missing subreddits.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import gzip
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
USER_AGENT = "last30days/3.0 (research tool)"
|
USER_AGENT = (
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/124.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
|
||||||
# Depth-aware limits for thread counts
|
# Depth-aware limits for thread counts
|
||||||
DEPTH_LIMITS = {
|
DEPTH_LIMITS = {
|
||||||
@@ -30,13 +41,6 @@ DEPTH_LIMITS = {
|
|||||||
"deep": 50,
|
"deep": 50,
|
||||||
}
|
}
|
||||||
|
|
||||||
# How many top posts to enrich with comments, by depth
|
|
||||||
ENRICH_LIMITS = {
|
|
||||||
"quick": 3,
|
|
||||||
"default": 5,
|
|
||||||
"deep": 8,
|
|
||||||
}
|
|
||||||
|
|
||||||
MAX_RETRIES = 3
|
MAX_RETRIES = 3
|
||||||
BASE_BACKOFF = 2.0 # seconds
|
BASE_BACKOFF = 2.0 # seconds
|
||||||
|
|
||||||
@@ -60,6 +64,9 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
|
|||||||
headers = {
|
headers = {
|
||||||
"User-Agent": USER_AGENT,
|
"User-Agent": USER_AGENT,
|
||||||
"Accept": "application/json",
|
"Accept": "application/json",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
"Accept-Encoding": "gzip, deflate",
|
||||||
|
"Connection": "keep-alive",
|
||||||
}
|
}
|
||||||
req = urllib.request.Request(url, headers=headers)
|
req = urllib.request.Request(url, headers=headers)
|
||||||
|
|
||||||
@@ -71,7 +78,10 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
|
|||||||
_log(f"Anti-bot HTML response (Content-Type: {content_type})")
|
_log(f"Anti-bot HTML response (Content-Type: {content_type})")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
body = resp.read().decode("utf-8")
|
raw = resp.read()
|
||||||
|
if resp.headers.get("Content-Encoding", "").lower() == "gzip":
|
||||||
|
raw = gzip.decompress(raw)
|
||||||
|
body = raw.decode("utf-8")
|
||||||
return json.loads(body)
|
return json.loads(body)
|
||||||
|
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
@@ -198,7 +208,7 @@ def search(
|
|||||||
encoded_query = _url_encode(query)
|
encoded_query = _url_encode(query)
|
||||||
|
|
||||||
if subreddit:
|
if subreddit:
|
||||||
sub = subreddit.lstrip("r/").strip()
|
sub = subreddit.removeprefix("r/").strip()
|
||||||
url = (
|
url = (
|
||||||
f"https://www.reddit.com/r/{sub}/search.json"
|
f"https://www.reddit.com/r/{sub}/search.json"
|
||||||
f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
|
f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
|
||||||
@@ -226,78 +236,6 @@ def search(
|
|||||||
return unique[:limit]
|
return unique[:limit]
|
||||||
|
|
||||||
|
|
||||||
def _enrich_post(item: Dict[str, Any], timeout: int = 10) -> Dict[str, Any]:
|
|
||||||
"""Enrich a single post with top comments. Never raises."""
|
|
||||||
try:
|
|
||||||
from . import reddit_enrich
|
|
||||||
thread_data = reddit_enrich.fetch_thread_data(item["url"], timeout=timeout)
|
|
||||||
if not thread_data:
|
|
||||||
return item
|
|
||||||
parsed = reddit_enrich.parse_thread_data(thread_data)
|
|
||||||
comments = parsed.get("comments", [])
|
|
||||||
top = reddit_enrich.get_top_comments(comments)
|
|
||||||
item["top_comments"] = [
|
|
||||||
{
|
|
||||||
"score": c.get("score", 0),
|
|
||||||
"excerpt": (c.get("body") or "")[:200],
|
|
||||||
"author": c.get("author", ""),
|
|
||||||
}
|
|
||||||
for c in top[:10]
|
|
||||||
]
|
|
||||||
except Exception:
|
|
||||||
# Never discard — keep post with empty metadata
|
|
||||||
pass
|
|
||||||
return item
|
|
||||||
|
|
||||||
|
|
||||||
def _enrich_posts(posts: List[Dict[str, Any]], depth: str = "default") -> List[Dict[str, Any]]:
|
|
||||||
"""Enrich top N posts with comment data using threads. Total budget 45s."""
|
|
||||||
limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])
|
|
||||||
to_enrich = posts[:limit]
|
|
||||||
rest = posts[limit:]
|
|
||||||
|
|
||||||
if not to_enrich:
|
|
||||||
return posts
|
|
||||||
|
|
||||||
enriched = []
|
|
||||||
try:
|
|
||||||
with ThreadPoolExecutor(max_workers=min(limit, 4)) as executor:
|
|
||||||
futures = {
|
|
||||||
executor.submit(_enrich_post, post, 10): i
|
|
||||||
for i, post in enumerate(to_enrich)
|
|
||||||
}
|
|
||||||
# Collect results with 45s total budget
|
|
||||||
import concurrent.futures
|
|
||||||
done, not_done = concurrent.futures.wait(futures, timeout=45)
|
|
||||||
# Build result list preserving order
|
|
||||||
result_map: Dict[int, Dict[str, Any]] = {}
|
|
||||||
for future in done:
|
|
||||||
idx = futures[future]
|
|
||||||
try:
|
|
||||||
result_map[idx] = future.result(timeout=0)
|
|
||||||
except Exception:
|
|
||||||
result_map[idx] = to_enrich[idx]
|
|
||||||
# Any not-done futures: keep original post
|
|
||||||
for future in not_done:
|
|
||||||
idx = futures[future]
|
|
||||||
result_map[idx] = to_enrich[idx]
|
|
||||||
future.cancel()
|
|
||||||
enriched = [result_map[i] for i in range(len(to_enrich))]
|
|
||||||
except Exception:
|
|
||||||
enriched = to_enrich
|
|
||||||
|
|
||||||
return enriched + rest
|
|
||||||
|
|
||||||
|
|
||||||
def _search_subreddit(sub: str, topic: str, depth: str, timeout: int = 15) -> List[Dict[str, Any]]:
|
|
||||||
"""Search a single subreddit. Never raises."""
|
|
||||||
try:
|
|
||||||
return search(topic, depth=depth, subreddit=sub, timeout=timeout)
|
|
||||||
except Exception as e:
|
|
||||||
_log(f"Subreddit search failed for r/{sub}: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def search_reddit_public(
|
def search_reddit_public(
|
||||||
topic: str,
|
topic: str,
|
||||||
from_date: str,
|
from_date: str,
|
||||||
@@ -305,12 +243,17 @@ def search_reddit_public(
|
|||||||
depth: str = "default",
|
depth: str = "default",
|
||||||
subreddits: Optional[List[str]] = None,
|
subreddits: Optional[List[str]] = None,
|
||||||
) -> List[Dict[str, Any]]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""High-level Reddit public search matching the openai_reddit interface.
|
"""High-level free Reddit search + enrichment (keyless).
|
||||||
|
|
||||||
When subreddits are provided (from agent planning), searches each targeted
|
Thin compatibility shim over the tiered keyless pipeline: the legacy
|
||||||
sub first, then does global search, and deduplicates across both. This
|
``.json`` search/enrichment endpoints now return HTTP 403, so this delegates
|
||||||
mirrors the SC search_and_enrich() flow where pre-resolved subreddits get
|
to ``reddit_keyless.search_and_enrich`` (Tier 0 one-shot ``.json`` →
|
||||||
priority.
|
Tier 1 RSS discovery → Tier 2 shreddit comment enrichment). The name and
|
||||||
|
signature are preserved so ``pipeline.py`` and other callers need no change
|
||||||
|
and the ScrapeCreators backup still engages when this returns empty.
|
||||||
|
|
||||||
|
The module-level ``search`` / ``_parse_posts`` helpers remain in use as the
|
||||||
|
keyless pipeline's demoted Tier 0 ``.json`` attempt.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
topic: Search topic
|
topic: Search topic
|
||||||
@@ -321,57 +264,9 @@ def search_reddit_public(
|
|||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of normalized item dicts matching ScrapeCreators output format.
|
List of normalized item dicts matching ScrapeCreators output format.
|
||||||
|
Empty list on total failure (so SC backup can engage).
|
||||||
"""
|
"""
|
||||||
all_posts: List[Dict[str, Any]] = []
|
from . import reddit_keyless
|
||||||
|
return reddit_keyless.search_and_enrich(
|
||||||
# Phase 1: Search targeted subreddits in parallel (if provided)
|
topic, from_date, to_date, depth=depth, subreddits=subreddits
|
||||||
if subreddits:
|
|
||||||
_log(f"Searching {len(subreddits)} targeted subreddits: {subreddits}")
|
|
||||||
workers = min(4, len(subreddits))
|
|
||||||
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
||||||
futures = {
|
|
||||||
executor.submit(_search_subreddit, sub, topic, depth): sub
|
|
||||||
for sub in subreddits
|
|
||||||
}
|
|
||||||
for future in futures:
|
|
||||||
sub = futures[future]
|
|
||||||
try:
|
|
||||||
sub_posts = future.result(timeout=30)
|
|
||||||
_log(f" -> {len(sub_posts)} results from r/{sub}")
|
|
||||||
all_posts.extend(sub_posts)
|
|
||||||
except (Exception, FuturesTimeoutError) as e:
|
|
||||||
_log(f" -> r/{sub} failed: {e}")
|
|
||||||
|
|
||||||
# Phase 2: Global search
|
|
||||||
global_posts = search(topic, depth=depth)
|
|
||||||
all_posts.extend(global_posts)
|
|
||||||
|
|
||||||
# Deduplicate by URL (targeted results keep priority since they come first)
|
|
||||||
seen_urls: set = set()
|
|
||||||
results: List[Dict[str, Any]] = []
|
|
||||||
for post in all_posts:
|
|
||||||
if post["url"] not in seen_urls:
|
|
||||||
seen_urls.add(post["url"])
|
|
||||||
results.append(post)
|
|
||||||
|
|
||||||
# Date filter: keep posts in range or with unknown dates
|
|
||||||
filtered = []
|
|
||||||
for item in results:
|
|
||||||
d = item.get("date")
|
|
||||||
if d is None or (from_date <= d <= to_date):
|
|
||||||
filtered.append(item)
|
|
||||||
|
|
||||||
# Sort by engagement (score desc)
|
|
||||||
filtered.sort(
|
|
||||||
key=lambda x: x.get("engagement", {}).get("score", 0),
|
|
||||||
reverse=True,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Enrich top posts with comments
|
|
||||||
filtered = _enrich_posts(filtered, depth=depth)
|
|
||||||
|
|
||||||
# Re-index IDs
|
|
||||||
for i, item in enumerate(filtered):
|
|
||||||
item["id"] = f"R{i + 1}"
|
|
||||||
|
|
||||||
return filtered
|
|
||||||
|
|||||||
@@ -0,0 +1,224 @@
|
|||||||
|
"""Keyless Reddit discovery via public RSS/Atom feeds.
|
||||||
|
|
||||||
|
Reddit's ``.json`` search endpoints now return HTTP 403 (shreddit anti-bot).
|
||||||
|
RSS feeds still serve HTTP 200 with no API key, so this module uses them for
|
||||||
|
post discovery, replacing ``reddit_public.search`` as the free search path.
|
||||||
|
|
||||||
|
Two feed families are combined and deduped:
|
||||||
|
- search: /search.rss?q=... and /r/{sub}/search.rss?q=...&restrict_sr=on
|
||||||
|
- listing: /r/{sub}/{top,hot}.rss?t=month
|
||||||
|
|
||||||
|
RSS entries carry no engagement score, so ``score``/``num_comments`` start at 0
|
||||||
|
and are backfilled during shreddit enrichment (see reddit_shreddit.py). Output
|
||||||
|
dicts match the normalized shape emitted by ``reddit_public._parse_posts`` so
|
||||||
|
downstream code (pipeline, renderer) is unaffected.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
|
from . import http
|
||||||
|
from .relevance import token_overlap_relevance
|
||||||
|
|
||||||
|
ATOM = "{http://www.w3.org/2005/Atom}"
|
||||||
|
|
||||||
|
# Mirror reddit_public depth-aware limits so the two free paths behave alike.
|
||||||
|
DEPTH_LIMITS = {
|
||||||
|
"quick": 10,
|
||||||
|
"default": 25,
|
||||||
|
"deep": 50,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Listing sorts pulled per subreddit (in addition to search), for volume.
|
||||||
|
LISTING_SORTS = {
|
||||||
|
"quick": ["top"],
|
||||||
|
"default": ["top", "hot"],
|
||||||
|
"deep": ["top", "hot", "new"],
|
||||||
|
}
|
||||||
|
|
||||||
|
MAX_WORKERS = 4
|
||||||
|
FEED_TIMEOUT = 15
|
||||||
|
|
||||||
|
|
||||||
|
def _log(msg: str) -> None:
|
||||||
|
sys.stderr.write(f"[RedditRSS] {msg}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_to_date(value: Optional[str]) -> Optional[str]:
|
||||||
|
"""Parse an ISO-8601 timestamp (e.g. 2026-05-20T18:48:31+00:00) to YYYY-MM-DD."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(value.strip())
|
||||||
|
return dt.date().isoformat()
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_to_epoch(value: Optional[str]) -> Optional[float]:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(value.strip())
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
dt = dt.replace(tzinfo=timezone.utc)
|
||||||
|
return dt.timestamp()
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _subreddit_from(category: str, url: str) -> str:
|
||||||
|
"""Derive subreddit name from the entry category or, failing that, the URL."""
|
||||||
|
if category:
|
||||||
|
return category
|
||||||
|
# URL form: https://www.reddit.com/r/{sub}/comments/{id}/...
|
||||||
|
parts = url.split("/r/", 1)
|
||||||
|
if len(parts) == 2:
|
||||||
|
return parts[1].split("/", 1)[0]
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_feed(xml_text: str, query: str = "") -> List[Dict[str, Any]]:
|
||||||
|
"""Parse an Atom feed string into normalized post dicts. Never raises."""
|
||||||
|
if not xml_text:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(xml_text)
|
||||||
|
except ET.ParseError as e:
|
||||||
|
_log(f"feed parse error: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
posts: List[Dict[str, Any]] = []
|
||||||
|
for entry in root.iter(f"{ATOM}entry"):
|
||||||
|
link_el = entry.find(f"{ATOM}link")
|
||||||
|
url = link_el.get("href", "").strip() if link_el is not None else ""
|
||||||
|
if not url or "/comments/" not in url:
|
||||||
|
continue
|
||||||
|
|
||||||
|
title_el = entry.find(f"{ATOM}title")
|
||||||
|
title = (title_el.text or "").strip() if title_el is not None else ""
|
||||||
|
|
||||||
|
author = ""
|
||||||
|
author_el = entry.find(f"{ATOM}author/{ATOM}name")
|
||||||
|
if author_el is not None and author_el.text:
|
||||||
|
author = author_el.text.strip().removeprefix("/u/").removeprefix("u/")
|
||||||
|
if author in ("[deleted]", "[removed]", ""):
|
||||||
|
author = "[deleted]"
|
||||||
|
|
||||||
|
cat_el = entry.find(f"{ATOM}category")
|
||||||
|
category = cat_el.get("term", "").strip() if cat_el is not None else ""
|
||||||
|
subreddit = _subreddit_from(category, url)
|
||||||
|
|
||||||
|
updated_el = entry.find(f"{ATOM}updated")
|
||||||
|
updated = (updated_el.text or "").strip() if updated_el is not None else ""
|
||||||
|
|
||||||
|
content_el = entry.find(f"{ATOM}content")
|
||||||
|
selftext = ""
|
||||||
|
if content_el is not None and content_el.text:
|
||||||
|
# Strip the simplest HTML; renderer only needs an excerpt.
|
||||||
|
import re as _re
|
||||||
|
selftext = _re.sub(r"<[^>]+>", " ", content_el.text)
|
||||||
|
selftext = _re.sub(r"\s+", " ", selftext).strip()[:500]
|
||||||
|
|
||||||
|
relevance = round(token_overlap_relevance(query, title), 3) if query else 0.0
|
||||||
|
|
||||||
|
posts.append({
|
||||||
|
"id": "", # assigned after dedup
|
||||||
|
"title": title,
|
||||||
|
"url": url,
|
||||||
|
"score": 0, # backfilled by shreddit enrichment
|
||||||
|
"num_comments": 0, # backfilled by shreddit enrichment
|
||||||
|
"subreddit": subreddit,
|
||||||
|
"created_utc": _iso_to_epoch(updated),
|
||||||
|
"author": author,
|
||||||
|
"selftext": selftext,
|
||||||
|
"date": _iso_to_date(updated),
|
||||||
|
"engagement": {
|
||||||
|
"score": 0,
|
||||||
|
"num_comments": 0,
|
||||||
|
"upvote_ratio": None,
|
||||||
|
},
|
||||||
|
"relevance": relevance,
|
||||||
|
"why_relevant": "Reddit RSS",
|
||||||
|
"metadata": {},
|
||||||
|
})
|
||||||
|
|
||||||
|
return posts
|
||||||
|
|
||||||
|
|
||||||
|
def _build_urls(query: str, depth: str, subreddits: Optional[List[str]]) -> List[str]:
|
||||||
|
"""Build the keyless RSS feed URLs to fan out across."""
|
||||||
|
q = quote_plus(query)
|
||||||
|
urls: List[str] = [
|
||||||
|
f"https://www.reddit.com/search.rss?q={q}&sort=relevance&t=month"
|
||||||
|
]
|
||||||
|
for raw_sub in (subreddits or []):
|
||||||
|
sub = raw_sub.removeprefix("r/").strip()
|
||||||
|
if not sub:
|
||||||
|
continue
|
||||||
|
urls.append(
|
||||||
|
f"https://www.reddit.com/r/{sub}/search.rss"
|
||||||
|
f"?q={q}&restrict_sr=on&sort=relevance&t=month"
|
||||||
|
)
|
||||||
|
for sort in LISTING_SORTS.get(depth, LISTING_SORTS["default"]):
|
||||||
|
urls.append(f"https://www.reddit.com/r/{sub}/{sort}.rss?t=month")
|
||||||
|
return urls
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_feed(url: str, query: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Fetch and parse one feed. Never raises."""
|
||||||
|
try:
|
||||||
|
text = http.get_text(url, timeout=FEED_TIMEOUT, accept="application/atom+xml")
|
||||||
|
return _parse_feed(text, query) if text else []
|
||||||
|
except Exception as e: # defensive: a single bad feed must not sink the run
|
||||||
|
_log(f"feed fetch failed for {url}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def search_rss(
|
||||||
|
query: str,
|
||||||
|
depth: str = "default",
|
||||||
|
subreddits: Optional[List[str]] = None,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Discover Reddit posts for a query via keyless RSS feeds.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query string
|
||||||
|
depth: 'quick', 'default', or 'deep' — controls result limit and feeds
|
||||||
|
subreddits: Optional pre-resolved subreddit names (without r/) to target
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of normalized post dicts (deduped by URL, capped by depth),
|
||||||
|
with placeholder scores to be backfilled during enrichment.
|
||||||
|
Empty list on any failure.
|
||||||
|
"""
|
||||||
|
limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
|
||||||
|
urls = _build_urls(query, depth, subreddits)
|
||||||
|
|
||||||
|
all_posts: List[Dict[str, Any]] = []
|
||||||
|
workers = min(MAX_WORKERS, len(urls)) or 1
|
||||||
|
with ThreadPoolExecutor(max_workers=workers) as executor:
|
||||||
|
futures = {executor.submit(_fetch_feed, url, query): url for url in urls}
|
||||||
|
for future in futures:
|
||||||
|
try:
|
||||||
|
all_posts.extend(future.result(timeout=FEED_TIMEOUT + 5))
|
||||||
|
except (Exception, FuturesTimeoutError) as e:
|
||||||
|
_log(f"feed future failed: {e}")
|
||||||
|
|
||||||
|
# Dedupe by URL (first occurrence wins).
|
||||||
|
seen: set = set()
|
||||||
|
unique: List[Dict[str, Any]] = []
|
||||||
|
for post in all_posts:
|
||||||
|
if post["url"] not in seen:
|
||||||
|
seen.add(post["url"])
|
||||||
|
unique.append(post)
|
||||||
|
|
||||||
|
for i, post in enumerate(unique):
|
||||||
|
post["id"] = f"R{i + 1}"
|
||||||
|
|
||||||
|
return unique[:limit]
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
"""Keyless Reddit comment enrichment via shreddit /svc endpoints.
|
||||||
|
|
||||||
|
Reddit's ``{thread}.json`` endpoint now returns HTTP 403. The shreddit partial
|
||||||
|
endpoint ``/svc/shreddit/comments/r/{sub}/t3_{id}`` still serves HTTP 200 HTML
|
||||||
|
with no API key, embedding each comment as a ``<shreddit-comment>`` custom
|
||||||
|
element whose start-tag attributes carry ``score`` / ``author`` / ``created`` /
|
||||||
|
``permalink``, and whose body lives in a ``<div id="{thingId}-post-rtjson-content">``
|
||||||
|
block. This module parses that markup into top comments, matching the
|
||||||
|
``top_comments`` / ``comment_insights`` shape produced by ``reddit_enrich`` so
|
||||||
|
the renderer is unaffected.
|
||||||
|
|
||||||
|
Limitation: the comments endpoint carries the real comment count
|
||||||
|
(``total-comments``) but not the post's upvote score, so post-level ``score``
|
||||||
|
cannot be recovered keylessly here (ScrapeCreators backup still provides it).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import html as _html
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from . import http
|
||||||
|
from . import reddit_enrich
|
||||||
|
|
||||||
|
# Up to N posts enriched per run, by depth (mirrors reddit_public.ENRICH_LIMITS).
|
||||||
|
ENRICH_LIMITS = {
|
||||||
|
"quick": 3,
|
||||||
|
"default": 5,
|
||||||
|
"deep": 8,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Max comments returned per post (independent of how many posts get enriched).
|
||||||
|
MAX_COMMENTS = 10
|
||||||
|
|
||||||
|
SVC_TIMEOUT = 12
|
||||||
|
|
||||||
|
# Match the exact <shreddit-comment> element start tag, not <shreddit-comment-tree>
|
||||||
|
# or <shreddit-comment-tree-stats> (lookahead requires whitespace or '>').
|
||||||
|
_COMMENT_START = re.compile(r"<shreddit-comment(?=[\s>])[^>]*>")
|
||||||
|
_TOTAL_COMMENTS = re.compile(r'total-comments="(\d+)"')
|
||||||
|
_PARA = re.compile(r"<p[^>]*>(.*?)</p>", re.S)
|
||||||
|
_TAG = re.compile(r"<[^>]+>")
|
||||||
|
_WS = re.compile(r"\s+")
|
||||||
|
_NEXT_RTJSON = re.compile(r'id="t1_[A-Za-z0-9]+-(?:comment|post)-rtjson-content"')
|
||||||
|
|
||||||
|
|
||||||
|
def _log(msg: str) -> None:
|
||||||
|
sys.stderr.write(f"[RedditShreddit] {msg}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_post_ref(url: str) -> Optional[tuple]:
|
||||||
|
"""Return (subreddit, post_id) from a Reddit thread URL, or None."""
|
||||||
|
m = re.search(r"/r/([^/]+)/comments/([A-Za-z0-9]+)", url or "")
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
return m.group(1), m.group(2)
|
||||||
|
|
||||||
|
|
||||||
|
def _svc_url(subreddit: str, post_id: str) -> str:
|
||||||
|
# sort=top guarantees Reddit front-loads the highest-scored comments on the
|
||||||
|
# first page, so the true top comments are captured even on huge threads
|
||||||
|
# (we still re-sort by score locally as a backstop).
|
||||||
|
return (
|
||||||
|
f"https://www.reddit.com/svc/shreddit/comments/r/{subreddit}/t3_{post_id}"
|
||||||
|
f"?sort=top"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _attr(tag: str, name: str) -> str:
|
||||||
|
m = re.search(rf'\b{name}="([^"]*)"', tag)
|
||||||
|
return _html.unescape(m.group(1)) if m else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_to_date(value: str) -> Optional[str]:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.strip()).date().isoformat()
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _body_for(html_text: str, thing_id: str) -> str:
|
||||||
|
"""Extract a comment's text body, anchored on its unique thingId.
|
||||||
|
|
||||||
|
The body div id embeds the comment's thingId, so this assigns body→comment
|
||||||
|
correctly even for nested replies. The slice is bounded by the next
|
||||||
|
comment's rtjson anchor to avoid swallowing child-comment text.
|
||||||
|
"""
|
||||||
|
if not thing_id:
|
||||||
|
return ""
|
||||||
|
anchor = f'id="{thing_id}-post-rtjson-content"'
|
||||||
|
idx = html_text.find(anchor)
|
||||||
|
if idx == -1:
|
||||||
|
return ""
|
||||||
|
window = html_text[idx + len(anchor): idx + len(anchor) + 8000]
|
||||||
|
nxt = _NEXT_RTJSON.search(window)
|
||||||
|
if nxt:
|
||||||
|
window = window[: nxt.start()]
|
||||||
|
paras = _PARA.findall(window)
|
||||||
|
if not paras:
|
||||||
|
return ""
|
||||||
|
text = " ".join(_TAG.sub("", p) for p in paras)
|
||||||
|
return _WS.sub(" ", _html.unescape(text)).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_comments(html_text: str, limit: int = MAX_COMMENTS) -> List[Dict[str, Any]]:
|
||||||
|
"""Parse <shreddit-comment> elements into scored comment dicts (sorted desc)."""
|
||||||
|
comments: List[Dict[str, Any]] = []
|
||||||
|
for m in _COMMENT_START.finditer(html_text or ""):
|
||||||
|
tag = m.group(0)
|
||||||
|
author = _attr(tag, "author") or "[deleted]"
|
||||||
|
if author in ("[deleted]", "[removed]"):
|
||||||
|
continue
|
||||||
|
thing_id = _attr(tag, "thingId")
|
||||||
|
body = _body_for(html_text, thing_id)
|
||||||
|
if not body or body in ("[deleted]", "[removed]"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
score = int(_attr(tag, "score") or 0)
|
||||||
|
except ValueError:
|
||||||
|
score = 0
|
||||||
|
permalink = _attr(tag, "permalink")
|
||||||
|
comments.append({
|
||||||
|
"score": score,
|
||||||
|
"author": author,
|
||||||
|
"body": body[:300],
|
||||||
|
"excerpt": body[:200],
|
||||||
|
"permalink": permalink,
|
||||||
|
"date": _iso_to_date(_attr(tag, "created")),
|
||||||
|
"url": f"https://reddit.com{permalink}" if permalink else "",
|
||||||
|
})
|
||||||
|
|
||||||
|
comments.sort(key=lambda c: c.get("score", 0), reverse=True)
|
||||||
|
return comments[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _total_comments(html_text: str) -> Optional[int]:
|
||||||
|
m = _TOTAL_COMMENTS.search(html_text or "")
|
||||||
|
return int(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_comments(
|
||||||
|
post_url: str,
|
||||||
|
timeout: int = SVC_TIMEOUT,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Fetch and parse top comments for a Reddit post via the shreddit endpoint.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
post_url: Reddit thread URL (…/r/{sub}/comments/{id}/…)
|
||||||
|
timeout: HTTP timeout in seconds
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with 'top_comments' (list, reddit_enrich shape), 'comment_insights'
|
||||||
|
(list[str]), and 'num_comments' (int or None). Empty/None on any
|
||||||
|
failure — never raises, so the caller can fall through to SC backup.
|
||||||
|
"""
|
||||||
|
ref = extract_post_ref(post_url)
|
||||||
|
if not ref:
|
||||||
|
return {"top_comments": [], "comment_insights": [], "num_comments": None}
|
||||||
|
sub, post_id = ref
|
||||||
|
|
||||||
|
html_text = http.get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html")
|
||||||
|
if not html_text:
|
||||||
|
return {"top_comments": [], "comment_insights": [], "num_comments": None}
|
||||||
|
|
||||||
|
comments = parse_comments(html_text, limit=MAX_COMMENTS)
|
||||||
|
insights = reddit_enrich.extract_comment_insights(comments)
|
||||||
|
return {
|
||||||
|
"top_comments": [
|
||||||
|
{
|
||||||
|
"score": c["score"],
|
||||||
|
"date": c["date"],
|
||||||
|
"author": c["author"],
|
||||||
|
"excerpt": c["excerpt"],
|
||||||
|
"url": c["url"],
|
||||||
|
}
|
||||||
|
for c in comments
|
||||||
|
],
|
||||||
|
"comment_insights": insights,
|
||||||
|
"num_comments": _total_comments(html_text),
|
||||||
|
}
|
||||||
@@ -533,7 +533,8 @@ def _render_comparison_scaffold(topic: str) -> list[str]:
|
|||||||
|
|
||||||
Axes match the April 9 launch-video exemplar (9 axes suited to AI-tool
|
Axes match the April 9 launch-video exemplar (9 axes suited to AI-tool
|
||||||
comparisons). For non-AI-tool comparisons, the synthesizer writes N/A
|
comparisons). For non-AI-tool comparisons, the synthesizer writes N/A
|
||||||
or topic-appropriate substitutes in irrelevant rows.
|
or topic-appropriate substitutes in irrelevant rows. The "What it is" row
|
||||||
|
grounds in first-party positioning fetched during the run when available.
|
||||||
"""
|
"""
|
||||||
entities = _parse_comparison_entities(topic)
|
entities = _parse_comparison_entities(topic)
|
||||||
if not entities:
|
if not entities:
|
||||||
@@ -558,10 +559,18 @@ def _render_comparison_scaffold(topic: str) -> list[str]:
|
|||||||
]
|
]
|
||||||
body = [f"| {axis} | " + " | ".join([" "] * len(entities)) + " |" for axis in axes]
|
body = [f"| {axis} | " + " | ".join([" "] * len(entities)) + " |" for axis in axes]
|
||||||
|
|
||||||
|
fill_instructions = (
|
||||||
|
"Fill each cell based on the research above. Keep cells short (5-15 words). "
|
||||||
|
"Use ' - ' (hyphen with spaces) not em-dashes. Write N/A for axes that do not apply to this topic class. "
|
||||||
|
"Ground the \"What it is\" row in first-party positioning fetched during this run's research when "
|
||||||
|
"available - describe each entity as it pitches itself today, never from memory. "
|
||||||
|
"This scaffold matches the April 9 launch-video exemplar shape."
|
||||||
|
)
|
||||||
|
|
||||||
return [
|
return [
|
||||||
"## Head-to-Head",
|
"## Head-to-Head",
|
||||||
"",
|
"",
|
||||||
"Fill each cell based on the research above. Keep cells short (5-15 words). Use ' - ' (hyphen with spaces) not em-dashes. Write N/A for axes that do not apply to this topic class. This scaffold matches the April 9 launch-video exemplar shape.",
|
fill_instructions,
|
||||||
"",
|
"",
|
||||||
header,
|
header,
|
||||||
separator,
|
separator,
|
||||||
@@ -1285,15 +1294,16 @@ def _build_source_footer_lines(report: schema.Report) -> list[str]:
|
|||||||
if total > 0:
|
if total > 0:
|
||||||
total_str = f"{total:,}" if total >= 1000 else str(total)
|
total_str = f"{total:,}" if total >= 1000 else str(total)
|
||||||
parts.append(f"{total_str} {word}")
|
parts.append(f"{total_str} {word}")
|
||||||
# YouTube: append "N with transcripts" instead of a third likes-based column.
|
# YouTube: always append "M/N with transcripts" so a zero-transcript run
|
||||||
# Transcripts are a more meaningful research-depth signal than likes.
|
# (typically caused by a stale yt-dlp binary) is visible at the conclusion
|
||||||
|
# surface. Hiding zero converts a problem signal into an absence; the very
|
||||||
|
# case that needs to be loud is the one previously omitted from the footer.
|
||||||
if source_key == "youtube":
|
if source_key == "youtube":
|
||||||
with_transcripts = sum(
|
with_transcripts = sum(
|
||||||
1 for it in items
|
1 for it in items
|
||||||
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
||||||
)
|
)
|
||||||
if with_transcripts > 0:
|
parts.append(f"{with_transcripts}/{len(items)} with transcripts")
|
||||||
parts.append(f"{with_transcripts} with transcripts")
|
|
||||||
stats = " │ ".join(parts)
|
stats = " │ ".join(parts)
|
||||||
out.append(_footer_line_for_source(emoji, label, len(items), item_word, stats))
|
out.append(_footer_line_for_source(emoji, label, len(items), item_word, stats))
|
||||||
|
|
||||||
|
|||||||
@@ -247,6 +247,29 @@ def _candidate_haystack(candidate: schema.Candidate) -> str:
|
|||||||
return " ".join(parts).lower()
|
return " ".join(parts).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _entity_grounded(haystack: str, primary_entity: str) -> bool:
|
||||||
|
"""True if the candidate text plausibly mentions the primary entity.
|
||||||
|
|
||||||
|
Grounds on the HEAD token of the primary entity (the brand / proper-noun
|
||||||
|
core), not the full multi-word phrase. Trailing tokens are usually category
|
||||||
|
descriptors the user/planner appended for search ("Stripe payments"), not
|
||||||
|
part of the entity, so requiring the whole phrase over-demotes on-entity
|
||||||
|
items that omit the descriptor. Items that never name the brand at all still
|
||||||
|
miss the head token and stay demoted.
|
||||||
|
|
||||||
|
Trade-off: a proper noun with a generic head ("New York Times" -> "new")
|
||||||
|
under-demotes rather than over-demotes - the safe direction, since the
|
||||||
|
observed harm was burying real high-engagement signal. Substring (not
|
||||||
|
word-boundary) matching is likewise deliberate: it catches plurals and
|
||||||
|
compounds ("stripes"), and vacuous matches from very short heads ("X",
|
||||||
|
"Go") merely disable the penalty rather than burying good items.
|
||||||
|
"""
|
||||||
|
tokens = primary_entity.lower().split()
|
||||||
|
if not tokens:
|
||||||
|
return True
|
||||||
|
return tokens[0] in haystack
|
||||||
|
|
||||||
|
|
||||||
def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]:
|
def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]:
|
||||||
score = (
|
score = (
|
||||||
(candidate.local_relevance * 100.0 * 0.7)
|
(candidate.local_relevance * 100.0 * 0.7)
|
||||||
@@ -254,17 +277,15 @@ def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") ->
|
|||||||
+ (candidate.source_quality * 100.0 * 0.1)
|
+ (candidate.source_quality * 100.0 * 0.1)
|
||||||
)
|
)
|
||||||
reason = "fallback-local-score"
|
reason = "fallback-local-score"
|
||||||
# Entity-grounding demotion: if the primary entity (topic minus intent
|
# Entity-grounding demotion: subtract ENTITY_MISS_PENALTY when the candidate
|
||||||
# modifier) is not present anywhere in the candidate's text surfaces
|
# never mentions the primary entity's head token, across all text surfaces
|
||||||
# (title, snippet, transcript, transcript highlights, top comments,
|
# (title, snippet, transcript, transcript highlights, top comments,
|
||||||
# insights), subtract ENTITY_MISS_PENALTY. Skip for candidates with
|
# insights). Skip for candidates with NO text anywhere (e.g. image-only
|
||||||
# NO text anywhere (e.g., image-only TikToks) to avoid penalizing
|
# TikToks) so thin-text sources aren't penalized unfairly. See
|
||||||
# thin-text sources unfairly. 2026-04-19 Nate Herk "Managed Agents"
|
# _entity_grounded for why grounding keys on the head token, not the phrase.
|
||||||
# video ranked #2 on a Hermes query despite zero Hermes mentions
|
|
||||||
# because the old haystack only checked title + snippet.
|
|
||||||
if primary_entity:
|
if primary_entity:
|
||||||
haystack = _candidate_haystack(candidate)
|
haystack = _candidate_haystack(candidate)
|
||||||
if haystack.strip() and primary_entity.lower() not in haystack:
|
if haystack.strip() and not _entity_grounded(haystack, primary_entity):
|
||||||
score -= ENTITY_MISS_PENALTY
|
score -= ENTITY_MISS_PENALTY
|
||||||
reason = "fallback-local-score (entity-miss demotion)"
|
reason = "fallback-local-score (entity-miss demotion)"
|
||||||
return max(0.0, min(100.0, score)), reason
|
return max(0.0, min(100.0, score)), reason
|
||||||
|
|||||||
@@ -160,6 +160,93 @@ def _extract_github_repos(items: list[dict]) -> list[str]:
|
|||||||
return repos[:5] # cap at 5 repos
|
return repos[:5] # cap at 5 repos
|
||||||
|
|
||||||
|
|
||||||
|
_INTEGRATION_SUFFIX_KEYWORDS: dict[str, set[str]] = {
|
||||||
|
"-action": {"action", "actions", "workflow", "workflows"},
|
||||||
|
"-sdk": {"sdk", "client", "library"},
|
||||||
|
"-plugin": {"plugin", "plugins", "extension", "extensions"},
|
||||||
|
"-plugins": {"plugin", "plugins", "extension", "extensions"},
|
||||||
|
"-docs": {"docs", "documentation"},
|
||||||
|
"-examples": {"example", "examples", "sample", "samples"},
|
||||||
|
"-template": {"template", "templates", "starter", "boilerplate"},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _topic_tokens(topic: str) -> set[str]:
|
||||||
|
return set(re.findall(r"[a-z0-9]+", (topic or "").lower()))
|
||||||
|
|
||||||
|
|
||||||
|
def _topic_entity_slugs(topic: str) -> list[str]:
|
||||||
|
entities = re.split(r"\b(?:vs|versus)\b", (topic or "").lower())
|
||||||
|
slugs: list[str] = []
|
||||||
|
for entity in entities:
|
||||||
|
tokens = re.findall(r"[a-z0-9]+", entity)
|
||||||
|
if tokens:
|
||||||
|
slugs.append("-".join(tokens))
|
||||||
|
return slugs
|
||||||
|
|
||||||
|
|
||||||
|
def _repo_slug(repo: str) -> str:
|
||||||
|
parts = repo.split("/", 1)
|
||||||
|
if len(parts) != 2:
|
||||||
|
return ""
|
||||||
|
return parts[1].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _canonicalize_integration_repo(topic: str, repo: str) -> str:
|
||||||
|
"""Map integration repos back to canonical product repos when intent allows.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
anthropics/claude-code-action -> anthropics/claude-code
|
||||||
|
unless topic explicitly asks for "action"/"workflow".
|
||||||
|
"""
|
||||||
|
parts = repo.split("/", 1)
|
||||||
|
if len(parts) != 2:
|
||||||
|
return repo
|
||||||
|
owner, name = parts[0], parts[1]
|
||||||
|
lower_name = name.lower()
|
||||||
|
topic_words = _topic_tokens(topic)
|
||||||
|
for suffix, intent_words in _INTEGRATION_SUFFIX_KEYWORDS.items():
|
||||||
|
if not lower_name.endswith(suffix):
|
||||||
|
continue
|
||||||
|
if topic_words.intersection(intent_words):
|
||||||
|
return repo
|
||||||
|
base = name[: -len(suffix)]
|
||||||
|
if base:
|
||||||
|
return f"{owner}/{base}"
|
||||||
|
return repo
|
||||||
|
|
||||||
|
|
||||||
|
def canonicalize_github_repos(topic: str, repos: list[str], *, cap: int | None = 5) -> list[str]:
|
||||||
|
"""Normalize/priority-sort GitHub repos for the current topic.
|
||||||
|
|
||||||
|
- Rewrites common integration suffixes to canonical product repos when
|
||||||
|
topic intent does not mention those integrations.
|
||||||
|
- Promotes exact topic slug matches (e.g., `claude-code`) over partials.
|
||||||
|
"""
|
||||||
|
canonicalized: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for repo in repos:
|
||||||
|
candidate = _canonicalize_integration_repo(topic, repo.strip())
|
||||||
|
if "/" not in candidate:
|
||||||
|
continue
|
||||||
|
key = candidate.lower()
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
canonicalized.append(candidate)
|
||||||
|
|
||||||
|
topic_slugs = set(_topic_entity_slugs(topic))
|
||||||
|
if topic_slugs:
|
||||||
|
exact = [r for r in canonicalized if _repo_slug(r) in topic_slugs]
|
||||||
|
prefixed = [r for r in canonicalized if any(_repo_slug(r).startswith(f"{slug}-") for slug in topic_slugs) and r not in exact]
|
||||||
|
rest = [r for r in canonicalized if r not in exact and r not in prefixed]
|
||||||
|
canonicalized = exact + prefixed + rest
|
||||||
|
|
||||||
|
if cap is not None:
|
||||||
|
return canonicalized[:cap]
|
||||||
|
return canonicalized
|
||||||
|
|
||||||
|
|
||||||
def _build_context_summary(items: list[dict]) -> str:
|
def _build_context_summary(items: list[dict]) -> str:
|
||||||
"""Build a 1-2 sentence current events summary from news search results."""
|
"""Build a 1-2 sentence current events summary from news search results."""
|
||||||
snippets: list[str] = []
|
snippets: list[str] = []
|
||||||
@@ -240,7 +327,7 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
|||||||
subreddits = _extract_subreddits(results.get("subreddit", []))
|
subreddits = _extract_subreddits(results.get("subreddit", []))
|
||||||
x_handle = _extract_x_handle(results.get("x_handle", []))
|
x_handle = _extract_x_handle(results.get("x_handle", []))
|
||||||
github_user = _extract_github_user(results.get("github", []))
|
github_user = _extract_github_user(results.get("github", []))
|
||||||
github_repos = _extract_github_repos(results.get("github", []))
|
github_repos = canonicalize_github_repos(topic, _extract_github_repos(results.get("github", [])))
|
||||||
context = _build_context_summary(results.get("news", []))
|
context = _build_context_summary(results.get("news", []))
|
||||||
|
|
||||||
subreddits, category = _merge_category_peers(topic, subreddits)
|
subreddits, category = _merge_category_peers(topic, subreddits)
|
||||||
|
|||||||
@@ -107,7 +107,18 @@ def extract_safari_cookies_macos(
|
|||||||
if sys.platform != "darwin":
|
if sys.platform != "darwin":
|
||||||
return None
|
return None
|
||||||
|
|
||||||
cookie_path = Path.home() / "Library" / "Cookies" / "Cookies.binarycookies"
|
cookie_paths = [
|
||||||
|
Path.home()
|
||||||
|
/ "Library"
|
||||||
|
/ "Containers"
|
||||||
|
/ "com.apple.Safari"
|
||||||
|
/ "Data"
|
||||||
|
/ "Library"
|
||||||
|
/ "Cookies"
|
||||||
|
/ "Cookies.binarycookies",
|
||||||
|
Path.home() / "Library" / "Cookies" / "Cookies.binarycookies",
|
||||||
|
]
|
||||||
|
cookie_path = next((path for path in cookie_paths if path.exists()), cookie_paths[0])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
raw = cookie_path.read_bytes()
|
raw = cookie_path.read_bytes()
|
||||||
|
|||||||
@@ -175,16 +175,24 @@ def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|||||||
break
|
break
|
||||||
|
|
||||||
if not output_text:
|
if not output_text:
|
||||||
return items
|
response_preview = str(response)[:200] if response else "(empty)"
|
||||||
|
raise http.HTTPError(
|
||||||
|
f"xAI API returned empty response (no output text found; response preview: {response_preview})"
|
||||||
|
)
|
||||||
|
|
||||||
# Extract JSON from the response
|
# Extract JSON from the response
|
||||||
json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
|
json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
|
||||||
if json_match:
|
if not json_match:
|
||||||
try:
|
raise http.HTTPError(
|
||||||
data = json.loads(json_match.group())
|
f"xAI API returned output without valid JSON items structure (output: {output_text[:200]})"
|
||||||
items = data.get("items", [])
|
)
|
||||||
except json.JSONDecodeError:
|
try:
|
||||||
_log(f"Failed to parse xAI response JSON: {output_text[:200]}")
|
data = json.loads(json_match.group())
|
||||||
|
items = data.get("items", [])
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
raise http.HTTPError(
|
||||||
|
f"xAI API returned valid output but invalid JSON structure (output: {output_text[:200]})"
|
||||||
|
)
|
||||||
|
|
||||||
# Validate and clean items
|
# Validate and clean items
|
||||||
clean_items = []
|
clean_items = []
|
||||||
|
|||||||
@@ -385,7 +385,11 @@ def _clean_vtt(vtt_text: str) -> str:
|
|||||||
_YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
_YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||||
|
|
||||||
|
|
||||||
def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]:
|
def _fetch_transcript_direct(
|
||||||
|
video_id: str,
|
||||||
|
timeout: int = 30,
|
||||||
|
status: Optional[Dict[str, Any]] = None,
|
||||||
|
) -> Optional[str]:
|
||||||
"""Fetch YouTube transcript via direct HTTP without yt-dlp.
|
"""Fetch YouTube transcript via direct HTTP without yt-dlp.
|
||||||
|
|
||||||
Scrapes the watch page HTML for the captions track URL in
|
Scrapes the watch page HTML for the captions track URL in
|
||||||
@@ -394,6 +398,9 @@ def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]:
|
|||||||
Args:
|
Args:
|
||||||
video_id: YouTube video ID
|
video_id: YouTube video ID
|
||||||
timeout: HTTP request timeout in seconds
|
timeout: HTTP request timeout in seconds
|
||||||
|
status: Optional dict mutated to record per-video signals. Sets
|
||||||
|
``status["no_caption_tracks"] = True`` when the player response
|
||||||
|
confirms the uploader has no caption tracks (vs. fetch failure).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Raw VTT text, or None if captions are unavailable.
|
Raw VTT text, or None if captions are unavailable.
|
||||||
@@ -442,6 +449,8 @@ def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]:
|
|||||||
|
|
||||||
if not caption_tracks:
|
if not caption_tracks:
|
||||||
_log(f"Direct transcript: no caption tracks for {video_id}")
|
_log(f"Direct transcript: no caption tracks for {video_id}")
|
||||||
|
if status is not None:
|
||||||
|
status["no_caption_tracks"] = True
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Find English track (prefer exact 'en', then any en variant, then first track)
|
# Find English track (prefer exact 'en', then any en variant, then first track)
|
||||||
@@ -527,7 +536,11 @@ def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
def fetch_transcript(
|
||||||
|
video_id: str,
|
||||||
|
temp_dir: str,
|
||||||
|
status: Optional[Dict[str, Any]] = None,
|
||||||
|
) -> Optional[str]:
|
||||||
"""Fetch auto-generated transcript for a YouTube video.
|
"""Fetch auto-generated transcript for a YouTube video.
|
||||||
|
|
||||||
Uses yt-dlp when available (preferred, more robust). Falls back to
|
Uses yt-dlp when available (preferred, more robust). Falls back to
|
||||||
@@ -536,6 +549,10 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
|||||||
Args:
|
Args:
|
||||||
video_id: YouTube video ID
|
video_id: YouTube video ID
|
||||||
temp_dir: Temporary directory for subtitle files
|
temp_dir: Temporary directory for subtitle files
|
||||||
|
status: Optional dict mutated by the direct-HTTP path to record
|
||||||
|
per-video signals like ``no_caption_tracks``. Used to surface a
|
||||||
|
captions-disabled count so the quality nudge avoids false-positive
|
||||||
|
"stale yt-dlp" flags.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Plaintext transcript string, or None if no captions available.
|
Plaintext transcript string, or None if no captions available.
|
||||||
@@ -551,13 +568,13 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
|||||||
raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
|
raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
|
||||||
if not raw_vtt:
|
if not raw_vtt:
|
||||||
_log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback")
|
_log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback")
|
||||||
raw_vtt = _fetch_transcript_direct(video_id)
|
raw_vtt = _fetch_transcript_direct(video_id, status=status)
|
||||||
else:
|
else:
|
||||||
if ssh_host:
|
if ssh_host:
|
||||||
_log("SSH-routing active, using direct HTTP transcript fetch")
|
_log("SSH-routing active, using direct HTTP transcript fetch")
|
||||||
else:
|
else:
|
||||||
_log("yt-dlp not installed, using direct HTTP transcript fetch")
|
_log("yt-dlp not installed, using direct HTTP transcript fetch")
|
||||||
raw_vtt = _fetch_transcript_direct(video_id)
|
raw_vtt = _fetch_transcript_direct(video_id, status=status)
|
||||||
|
|
||||||
if not raw_vtt:
|
if not raw_vtt:
|
||||||
_log(f"No transcript available for {video_id} (no captions found)")
|
_log(f"No transcript available for {video_id} (no captions found)")
|
||||||
@@ -576,12 +593,16 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
|||||||
def fetch_transcripts_parallel(
|
def fetch_transcripts_parallel(
|
||||||
video_ids: List[str],
|
video_ids: List[str],
|
||||||
max_workers: int = 5,
|
max_workers: int = 5,
|
||||||
|
out_captions_disabled: Optional[Set[str]] = None,
|
||||||
) -> Dict[str, Optional[str]]:
|
) -> Dict[str, Optional[str]]:
|
||||||
"""Fetch transcripts for multiple videos in parallel.
|
"""Fetch transcripts for multiple videos in parallel.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
video_ids: List of YouTube video IDs
|
video_ids: List of YouTube video IDs
|
||||||
max_workers: Max parallel fetches
|
max_workers: Max parallel fetches
|
||||||
|
out_captions_disabled: Optional set mutated to record video_ids whose
|
||||||
|
uploader confirmed no caption tracks (vs. transient fetch failures).
|
||||||
|
Backward-compatible: callers that don't care can omit.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict mapping video_id to transcript text (or None).
|
Dict mapping video_id to transcript text (or None).
|
||||||
@@ -592,10 +613,11 @@ def fetch_transcripts_parallel(
|
|||||||
_log(f"Fetching transcripts for {len(video_ids)} videos")
|
_log(f"Fetching transcripts for {len(video_ids)} videos")
|
||||||
|
|
||||||
results = {}
|
results = {}
|
||||||
|
statuses: Dict[str, Dict[str, Any]] = {vid: {} for vid in video_ids}
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||||
futures = {
|
futures = {
|
||||||
executor.submit(fetch_transcript, vid, temp_dir): vid
|
executor.submit(fetch_transcript, vid, temp_dir, statuses[vid]): vid
|
||||||
for vid in video_ids
|
for vid in video_ids
|
||||||
}
|
}
|
||||||
for future in as_completed(futures):
|
for future in as_completed(futures):
|
||||||
@@ -609,6 +631,11 @@ def fetch_transcripts_parallel(
|
|||||||
_log(f"Unexpected transcript error for {vid}: {type(exc).__name__}: {exc}")
|
_log(f"Unexpected transcript error for {vid}: {type(exc).__name__}: {exc}")
|
||||||
results[vid] = None
|
results[vid] = None
|
||||||
|
|
||||||
|
if out_captions_disabled is not None:
|
||||||
|
for vid, st in statuses.items():
|
||||||
|
if st.get("no_caption_tracks"):
|
||||||
|
out_captions_disabled.add(vid)
|
||||||
|
|
||||||
got = sum(1 for v in results.values() if v)
|
got = sum(1 for v in results.values() if v)
|
||||||
errors = sum(1 for v in results.values() if v is None)
|
errors = sum(1 for v in results.values() if v is None)
|
||||||
_log(f"Got transcripts for {got}/{len(video_ids)} videos ({errors} failed)")
|
_log(f"Got transcripts for {got}/{len(video_ids)} videos ({errors} failed)")
|
||||||
@@ -659,15 +686,21 @@ def search_and_transcribe(
|
|||||||
# good chance of reaching the target number of successful transcripts.
|
# good chance of reaching the target number of successful transcripts.
|
||||||
transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
|
transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
|
||||||
transcripts: Dict[str, Optional[str]] = {}
|
transcripts: Dict[str, Optional[str]] = {}
|
||||||
|
captions_disabled_ids: Set[str] = set()
|
||||||
if transcript_limit > 0:
|
if transcript_limit > 0:
|
||||||
attempt_count = min(len(items), transcript_limit * 3)
|
attempt_count = min(len(items), transcript_limit * 3)
|
||||||
candidate_ids = [item["video_id"] for item in items[:attempt_count]]
|
candidate_ids = [item["video_id"] for item in items[:attempt_count]]
|
||||||
_log(f"Fetching transcripts for up to {attempt_count} videos (target: {transcript_limit}): {candidate_ids}")
|
_log(f"Fetching transcripts for up to {attempt_count} videos (target: {transcript_limit}): {candidate_ids}")
|
||||||
transcripts = fetch_transcripts_parallel(candidate_ids)
|
transcripts = fetch_transcripts_parallel(
|
||||||
|
candidate_ids, out_captions_disabled=captions_disabled_ids,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
_log(f"Transcript limit is 0 for depth={depth}, skipping transcript fetch")
|
_log(f"Transcript limit is 0 for depth={depth}, skipping transcript fetch")
|
||||||
|
|
||||||
# Step 3: Attach transcripts and extract highlights
|
# Step 3: Attach transcripts and extract highlights. Mark captions_disabled
|
||||||
|
# so quality_nudge can subtract those videos from the degraded-ratio
|
||||||
|
# denominator (uploader-disabled captions can never produce a transcript;
|
||||||
|
# counting them was producing false-positive stale-yt-dlp nudges).
|
||||||
core_topic = _extract_core_subject(topic)
|
core_topic = _extract_core_subject(topic)
|
||||||
for item in items:
|
for item in items:
|
||||||
vid = item["video_id"]
|
vid = item["video_id"]
|
||||||
@@ -676,6 +709,7 @@ def search_and_transcribe(
|
|||||||
item["transcript_highlights"] = extract_transcript_highlights(
|
item["transcript_highlights"] = extract_transcript_highlights(
|
||||||
transcript or "", core_topic,
|
transcript or "", core_topic,
|
||||||
)
|
)
|
||||||
|
item["captions_disabled"] = vid in captions_disabled_ids
|
||||||
|
|
||||||
return {"items": items}
|
return {"items": items}
|
||||||
|
|
||||||
|
|||||||
@@ -360,6 +360,22 @@ def update_run(run_id: int, **kwargs):
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def get_latest_completed_runs(topic_id: int, limit: int = 2) -> List[Dict[str, Any]]:
|
||||||
|
"""Return newest completed runs for a topic."""
|
||||||
|
conn = _connect()
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""SELECT * FROM research_runs
|
||||||
|
WHERE topic_id = ? AND status = 'completed'
|
||||||
|
ORDER BY datetime(run_date) DESC, id DESC
|
||||||
|
LIMIT ?""",
|
||||||
|
(topic_id, limit),
|
||||||
|
).fetchall()
|
||||||
|
return [dict(r) for r in rows]
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
# --- Findings ---
|
# --- Findings ---
|
||||||
|
|
||||||
|
|
||||||
@@ -536,6 +552,88 @@ def get_sightings_for_run(topic_id: int, run_id: int) -> List[Dict[str, Any]]:
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def compute_topic_delta(topic_id: int) -> Dict[str, Any]:
|
||||||
|
"""Compare the latest completed watchlist run with the previous run."""
|
||||||
|
runs = get_latest_completed_runs(topic_id, limit=2)
|
||||||
|
topic = _get_topic_by_id(topic_id)
|
||||||
|
topic_name = topic["name"] if topic else str(topic_id)
|
||||||
|
if len(runs) < 2:
|
||||||
|
return {
|
||||||
|
"topic": topic_name,
|
||||||
|
"status": "insufficient_history",
|
||||||
|
"message": "Need at least two completed runs to compute a delta.",
|
||||||
|
}
|
||||||
|
|
||||||
|
current_run, previous_run = runs[0], runs[1]
|
||||||
|
current = _sightings_by_url(get_sightings_for_run(topic_id, current_run["id"]))
|
||||||
|
previous = _sightings_by_url(get_sightings_for_run(topic_id, previous_run["id"]))
|
||||||
|
|
||||||
|
current_urls = set(current)
|
||||||
|
previous_urls = set(previous)
|
||||||
|
new_urls = sorted(current_urls - previous_urls)
|
||||||
|
continued_urls = sorted(current_urls & previous_urls)
|
||||||
|
dropped_urls = sorted(previous_urls - current_urls)
|
||||||
|
|
||||||
|
findings = {
|
||||||
|
"new": [current[url] for url in new_urls],
|
||||||
|
"continued": [current[url] for url in continued_urls],
|
||||||
|
"dropped": [previous[url] for url in dropped_urls],
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"topic": topic_name,
|
||||||
|
"status": "ok",
|
||||||
|
"current_run_id": current_run["id"],
|
||||||
|
"previous_run_id": previous_run["id"],
|
||||||
|
"new": len(new_urls),
|
||||||
|
"continued": len(continued_urls),
|
||||||
|
"dropped": len(dropped_urls),
|
||||||
|
"sources": _delta_source_counts(findings),
|
||||||
|
"findings": findings,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _get_topic_by_id(topic_id: int) -> Optional[Dict[str, Any]]:
|
||||||
|
conn = _connect()
|
||||||
|
try:
|
||||||
|
row = conn.execute("SELECT * FROM topics WHERE id = ?", (topic_id,)).fetchone()
|
||||||
|
return dict(row) if row else None
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _sightings_by_url(sightings: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
|
||||||
|
"""Index sightings by stable URL identity for run-to-run delta comparisons.
|
||||||
|
|
||||||
|
URL-less sightings are intentionally excluded because there is no stable
|
||||||
|
cross-run identity to classify them as new, continued, or dropped.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
sighting["source_url"]: sighting
|
||||||
|
for sighting in sightings
|
||||||
|
if sighting.get("source_url")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _delta_source_counts(
|
||||||
|
findings: Dict[str, List[Dict[str, Any]]]
|
||||||
|
) -> Dict[str, Dict[str, int]]:
|
||||||
|
sources = sorted({
|
||||||
|
finding.get("source") or "unknown"
|
||||||
|
for group in findings.values()
|
||||||
|
for finding in group
|
||||||
|
})
|
||||||
|
counts = {
|
||||||
|
source: {"new": 0, "continued": 0, "dropped": 0}
|
||||||
|
for source in sources
|
||||||
|
}
|
||||||
|
for group_name, group in findings.items():
|
||||||
|
for finding in group:
|
||||||
|
source = finding.get("source") or "unknown"
|
||||||
|
counts[source][group_name] += 1
|
||||||
|
return counts
|
||||||
|
|
||||||
|
|
||||||
def get_new_findings(
|
def get_new_findings(
|
||||||
topic_id: int,
|
topic_id: int,
|
||||||
since: Optional[str] = None,
|
since: Optional[str] = None,
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ set -euo pipefail
|
|||||||
# using `claude --print` to capture real end-to-end output.
|
# using `claude --print` to capture real end-to-end output.
|
||||||
|
|
||||||
SKILL_DIR="$HOME/.claude/skills/last30days"
|
SKILL_DIR="$HOME/.claude/skills/last30days"
|
||||||
REPO_DIR="/Users/mvanhorn/last30days-skill"
|
REPO_DIR="${REPO_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
|
||||||
|
CLAUDE="${CLAUDE:-$(command -v claude || echo claude)}"
|
||||||
|
|
||||||
# Safety: always restore V2 SKILL.md on exit/crash
|
# Safety: always restore V2 SKILL.md on exit/crash
|
||||||
cleanup() {
|
cleanup() {
|
||||||
@@ -101,7 +102,7 @@ run_version() {
|
|||||||
|
|
||||||
# Run claude --print with the skill invocation
|
# Run claude --print with the skill invocation
|
||||||
# No timeout — claude --print exits on its own; kill manually if stuck
|
# No timeout — claude --print exits on its own; kill manually if stuck
|
||||||
if /Users/mvanhorn/.local/bin/claude --print \
|
if "$CLAUDE" --print \
|
||||||
"/last30days $query" \
|
"/last30days $query" \
|
||||||
> "$outfile" 2>"$errfile"; then
|
> "$outfile" 2>"$errfile"; then
|
||||||
local end_time
|
local end_time
|
||||||
|
|||||||
@@ -111,6 +111,14 @@ def cmd_list(args):
|
|||||||
}, default=str))
|
}, default=str))
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_delta(args):
|
||||||
|
topic = store.get_topic(args.topic)
|
||||||
|
if not topic:
|
||||||
|
print(json.dumps({"error": f'Topic not found: "{args.topic}"'}))
|
||||||
|
sys.exit(1)
|
||||||
|
print(json.dumps(store.compute_topic_delta(topic["id"]), default=str))
|
||||||
|
|
||||||
|
|
||||||
def cmd_run_one(args):
|
def cmd_run_one(args):
|
||||||
topic = store.get_topic(args.topic)
|
topic = store.get_topic(args.topic)
|
||||||
if not topic:
|
if not topic:
|
||||||
@@ -252,6 +260,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
list_parser = sub.add_parser("list")
|
list_parser = sub.add_parser("list")
|
||||||
list_parser.set_defaults(func=cmd_list)
|
list_parser.set_defaults(func=cmd_list)
|
||||||
|
|
||||||
|
delta = sub.add_parser("delta")
|
||||||
|
delta.add_argument("topic")
|
||||||
|
delta.set_defaults(func=cmd_delta)
|
||||||
|
|
||||||
run_one = sub.add_parser("run-one")
|
run_one = sub.add_parser("run-one")
|
||||||
run_one.add_argument("topic")
|
run_one.add_argument("topic")
|
||||||
run_one.set_defaults(func=cmd_run_one)
|
run_one.set_defaults(func=cmd_run_one)
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "skills" / "last30days" / "scripts"))
|
||||||
@@ -5,11 +5,7 @@ comparisons, 'difference between X and Y' phrasing, trailing context
|
|||||||
leaking into entities, degenerate inputs, and false-positive resistance.
|
leaking into entities, degenerate inputs, and false-positive resistance.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import planner
|
from lib import planner
|
||||||
|
|
||||||
@@ -193,6 +189,5 @@ class TestNoiseWordEntities(unittest.TestCase):
|
|||||||
entities = planner._comparison_entities("Swift vs Rust vs Go")
|
entities = planner._comparison_entities("Swift vs Rust vs Go")
|
||||||
self.assertTrue(any("Go" in e for e in entities))
|
self.assertTrue(any("Go" in e for e in entities))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -2,16 +2,12 @@ import json
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
|
||||||
import textwrap
|
import textwrap
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib.bird_x import parse_bird_response
|
from lib.bird_x import parse_bird_response
|
||||||
|
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
VENDORED_BIRD = REPO_ROOT / "skills" / "last30days" / "scripts" / "lib" / "vendor" / "bird-search" / "bird-search.mjs"
|
VENDORED_BIRD = REPO_ROOT / "skills" / "last30days" / "scripts" / "lib" / "vendor" / "bird-search" / "bird-search.mjs"
|
||||||
|
|
||||||
@@ -31,7 +27,6 @@ class TestBirdXEngagementZero(unittest.TestCase):
|
|||||||
self.assertEqual(0, items[0]["engagement"]["likes"])
|
self.assertEqual(0, items[0]["engagement"]["likes"])
|
||||||
self.assertEqual(5, items[0]["engagement"]["reposts"])
|
self.assertEqual(5, items[0]["engagement"]["reposts"])
|
||||||
|
|
||||||
|
|
||||||
@unittest.skipUnless(shutil.which("node"), "node is required for vendored Bird tests")
|
@unittest.skipUnless(shutil.which("node"), "node is required for vendored Bird tests")
|
||||||
class TestVendoredBirdRuntime(unittest.TestCase):
|
class TestVendoredBirdRuntime(unittest.TestCase):
|
||||||
def test_check_uses_env_credentials_without_browser_cookie_dependency(self):
|
def test_check_uses_env_credentials_without_browser_cookie_dependency(self):
|
||||||
@@ -305,6 +300,5 @@ class TestRunBirdSearchJsonDecodeRetry(unittest.TestCase):
|
|||||||
self.assertEqual(response, timeout_error)
|
self.assertEqual(response, timeout_error)
|
||||||
mock_sleep.assert_not_called()
|
mock_sleep.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+146
-3
@@ -1,11 +1,9 @@
|
|||||||
"""Tests for bluesky module."""
|
"""Tests for bluesky module."""
|
||||||
|
|
||||||
import sys
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import patch, MagicMock
|
from unittest.mock import patch, MagicMock
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "last30days" / "scripts"))
|
|
||||||
from lib import bluesky
|
from lib import bluesky
|
||||||
|
|
||||||
|
|
||||||
@@ -211,5 +209,150 @@ class TestSearchBlueskyAuth(unittest.TestCase):
|
|||||||
self.assertEqual(mock_request.call_args_list[3].kwargs.get("headers", {}), {"Authorization": "Bearer tok-new"})
|
self.assertEqual(mock_request.call_args_list[3].kwargs.get("headers", {}), {"Authorization": "Bearer tok-new"})
|
||||||
|
|
||||||
|
|
||||||
|
class TestSearchEndpointHostResolution(unittest.TestCase):
|
||||||
|
"""The default search host moved from `public.api.bsky.app` (the
|
||||||
|
unauthenticated public mirror, now BunnyCDN-blocked for searchPosts) to
|
||||||
|
`api.bsky.app` (the canonical authenticated AppView). BSKY_SEARCH_HOST
|
||||||
|
env var or config value can override the default if Bluesky migrates
|
||||||
|
infrastructure again. Same os.environ-or-config hybrid pattern as
|
||||||
|
LAST30DAYS_STORE.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# Snapshot env so per-test overrides don't leak
|
||||||
|
self._saved_env = os.environ.pop("BSKY_SEARCH_HOST", None)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
if self._saved_env is not None:
|
||||||
|
os.environ["BSKY_SEARCH_HOST"] = self._saved_env
|
||||||
|
else:
|
||||||
|
os.environ.pop("BSKY_SEARCH_HOST", None)
|
||||||
|
|
||||||
|
def test_resolver_default_uses_canonical_appview(self):
|
||||||
|
# Regression guard against the public mirror reappearing as the default.
|
||||||
|
# Anchored at the resolver because that is the code path search_bluesky
|
||||||
|
# actually calls; a module-level constant would not catch a resolver
|
||||||
|
# regression.
|
||||||
|
self.assertIn("api.bsky.app", bluesky._resolve_search_url())
|
||||||
|
|
||||||
|
def test_resolver_default_does_not_use_public_mirror(self):
|
||||||
|
# Hard regression guard — the exact host that BunnyCDN was blocking.
|
||||||
|
# Asserted at the resolver level (the runtime path) so a default-host
|
||||||
|
# regression in _resolve_search_url is actually caught.
|
||||||
|
self.assertNotIn("public.api.bsky.app", bluesky._resolve_search_url())
|
||||||
|
|
||||||
|
def test_resolver_default_when_no_override(self):
|
||||||
|
self.assertEqual(
|
||||||
|
bluesky._resolve_search_url(),
|
||||||
|
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolver_env_var_override(self):
|
||||||
|
os.environ["BSKY_SEARCH_HOST"] = "staging.bsky.app"
|
||||||
|
self.assertEqual(
|
||||||
|
bluesky._resolve_search_url(),
|
||||||
|
"https://staging.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolver_config_dict_override(self):
|
||||||
|
# User has BSKY_SEARCH_HOST only in .env file (project loads .env into
|
||||||
|
# config, not os.environ). Resolver must read both.
|
||||||
|
url = bluesky._resolve_search_url({"BSKY_SEARCH_HOST": "pds.example.com"})
|
||||||
|
self.assertEqual(url, "https://pds.example.com/xrpc/app.bsky.feed.searchPosts")
|
||||||
|
|
||||||
|
def test_resolver_env_var_wins_over_config(self):
|
||||||
|
# When both are set, os.environ takes precedence (matches LAST30DAYS_STORE)
|
||||||
|
os.environ["BSKY_SEARCH_HOST"] = "shell-host.example"
|
||||||
|
url = bluesky._resolve_search_url({"BSKY_SEARCH_HOST": "config-host.example"})
|
||||||
|
self.assertIn("shell-host.example", url)
|
||||||
|
self.assertNotIn("config-host.example", url)
|
||||||
|
|
||||||
|
def test_resolver_output_does_not_use_public_mirror(self):
|
||||||
|
# Regression guard at the resolver level (not just the constant) —
|
||||||
|
# this is what runtime actually calls. The constant-level guard
|
||||||
|
# above doesn't catch a regression where the resolver reverts.
|
||||||
|
self.assertNotIn("public.api.bsky.app", bluesky._resolve_search_url())
|
||||||
|
|
||||||
|
def test_resolver_strips_surrounding_whitespace(self):
|
||||||
|
# Pre-fix: " api.bsky.app " produced "https:// api.bsky.app /xrpc/..."
|
||||||
|
# which urllib raises ValueError on with no hint the env var caused it.
|
||||||
|
os.environ["BSKY_SEARCH_HOST"] = " api.bsky.app "
|
||||||
|
self.assertEqual(
|
||||||
|
bluesky._resolve_search_url(),
|
||||||
|
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolver_rejects_embedded_path(self):
|
||||||
|
# "my-proxy.com/xrpc/prefix" would have doubled the /xrpc/ segment.
|
||||||
|
# We fall back to the default to avoid a guaranteed 404.
|
||||||
|
os.environ["BSKY_SEARCH_HOST"] = "my-proxy.example.com/xrpc/prefix"
|
||||||
|
self.assertEqual(
|
||||||
|
bluesky._resolve_search_url(),
|
||||||
|
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolver_strips_embedded_scheme(self):
|
||||||
|
# Users who paste a full URL get a sane outcome, not a malformed URL.
|
||||||
|
os.environ["BSKY_SEARCH_HOST"] = "https://api.bsky.app"
|
||||||
|
self.assertEqual(
|
||||||
|
bluesky._resolve_search_url(),
|
||||||
|
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolver_empty_string_falls_back_to_default(self):
|
||||||
|
os.environ["BSKY_SEARCH_HOST"] = ""
|
||||||
|
self.assertEqual(
|
||||||
|
bluesky._resolve_search_url(),
|
||||||
|
"https://api.bsky.app/xrpc/app.bsky.feed.searchPosts",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestAppPasswordFormat(unittest.TestCase):
|
||||||
|
"""Bluesky app passwords are 19-char xxxx-xxxx-xxxx-xxxx (lowercase
|
||||||
|
alphanumeric, three hyphens at fixed positions). Main-account passwords
|
||||||
|
are accepted by createSession but are bad hygiene. The validator detects
|
||||||
|
the format mismatch without gating any caller.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_accepts_valid_app_password_form(self):
|
||||||
|
# Use a fake example — never a real password
|
||||||
|
self.assertTrue(bluesky._validate_app_password_format("wfwp-cq7o-5six-7wy5"))
|
||||||
|
|
||||||
|
def test_rejects_length_15_string(self):
|
||||||
|
# The exact failure mode that triggered the 2026-05-04 investigation:
|
||||||
|
# user stored their main login password (15 chars) in BSKY_APP_PASSWORD
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format("mainpassword123"))
|
||||||
|
|
||||||
|
def test_rejects_16_char_no_hyphen_string(self):
|
||||||
|
# Hex-style API key shape — common confusion with other services
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format("abcdef0123456789"))
|
||||||
|
|
||||||
|
def test_rejects_uppercase_letters(self):
|
||||||
|
# Bluesky app passwords are all-lowercase by spec
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format("WFWP-cq7o-5six-7wy5"))
|
||||||
|
|
||||||
|
def test_rejects_underscore_separator(self):
|
||||||
|
# Wrong separator
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format("wfwp_cq7o_5six_7wy5"))
|
||||||
|
|
||||||
|
def test_rejects_special_chars_in_groups(self):
|
||||||
|
# Special characters are not part of the alphanumeric class
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format("wfwp-cq7o-5six-7wy@"))
|
||||||
|
|
||||||
|
def test_rejects_empty_string(self):
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format(""))
|
||||||
|
|
||||||
|
def test_rejects_none(self):
|
||||||
|
# Callers may pass config.get('BSKY_APP_PASSWORD') which is None when unset
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format(None))
|
||||||
|
|
||||||
|
def test_rejects_integer(self):
|
||||||
|
# Defensive: don't crash if a numeric value sneaks in
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format(123456789012345))
|
||||||
|
|
||||||
|
def test_rejects_list(self):
|
||||||
|
# Defensive: don't crash on iterables
|
||||||
|
self.assertFalse(bluesky._validate_app_password_format(["wfwp", "cq7o", "5six", "7wy5"]))
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
import sys
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
import briefing
|
import briefing
|
||||||
import store
|
import store
|
||||||
|
|
||||||
@@ -52,6 +49,5 @@ class BriefingV3Tests(unittest.TestCase):
|
|||||||
finally:
|
finally:
|
||||||
briefing.BRIEFS_DIR = old_briefs_dir
|
briefing.BRIEFS_DIR = old_briefs_dir
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -7,11 +7,7 @@ where prompting techniques actually live.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import categories
|
from lib import categories
|
||||||
from lib.categories import CATEGORY_PEERS, detect_category, peer_subs_for
|
from lib.categories import CATEGORY_PEERS, detect_category, peer_subs_for
|
||||||
@@ -149,6 +145,5 @@ class CategoryMapInvariants(unittest.TestCase):
|
|||||||
self.assertGreaterEqual(len(CATEGORY_PEERS), 8)
|
self.assertGreaterEqual(len(CATEGORY_PEERS), 8)
|
||||||
self.assertLessEqual(len(CATEGORY_PEERS), 20)
|
self.assertLessEqual(len(CATEGORY_PEERS), 20)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -13,17 +13,12 @@ Fixture reference: `tests/fixtures/prompting-gpt-image-2-resolved-block.md`.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from contextlib import redirect_stderr
|
from contextlib import redirect_stderr
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import resolve
|
from lib import resolve
|
||||||
|
|
||||||
|
|
||||||
OPENAI_BRAND_SUBREDDIT_RESULTS = [
|
OPENAI_BRAND_SUBREDDIT_RESULTS = [
|
||||||
{
|
{
|
||||||
"title": "r/OpenAI community hub",
|
"title": "r/OpenAI community hub",
|
||||||
@@ -140,6 +135,5 @@ class PromptingGptImage2RegressionGuard(unittest.TestCase):
|
|||||||
self.assertIsNone(result["category"])
|
self.assertIsNone(result["category"])
|
||||||
self.assertNotIn("Matched category=", buf.getvalue())
|
self.assertNotIn("Matched category=", buf.getvalue())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,19 +1,15 @@
|
|||||||
"""Tests for Chrome cookie extraction on macOS."""
|
"""Tests for Chrome cookie extraction on macOS."""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days"))
|
from lib.chrome_cookies import (
|
||||||
|
|
||||||
from scripts.lib.chrome_cookies import (
|
|
||||||
CHROME_COOKIES_DB,
|
CHROME_COOKIES_DB,
|
||||||
CHROME_IV_HEX,
|
CHROME_IV_HEX,
|
||||||
CHROME_KEY_LENGTH,
|
CHROME_KEY_LENGTH,
|
||||||
@@ -27,7 +23,6 @@ from scripts.lib.chrome_cookies import (
|
|||||||
extract_chrome_cookies_macos,
|
extract_chrome_cookies_macos,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Helpers — create real encrypted cookie values using known key + system openssl
|
# Helpers — create real encrypted cookie values using known key + system openssl
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -112,11 +107,11 @@ def _create_chrome_cookies_db(path: str, cookies: list[tuple], db_version: int =
|
|||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# PKCS7 padding tests
|
# PKCS7 padding tests
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestPkcs7Padding:
|
class TestPkcs7Padding:
|
||||||
def test_valid_padding_1(self):
|
def test_valid_padding_1(self):
|
||||||
# 1 byte of padding
|
# 1 byte of padding
|
||||||
@@ -143,11 +138,11 @@ class TestPkcs7Padding:
|
|||||||
def test_empty_data(self):
|
def test_empty_data(self):
|
||||||
assert _remove_pkcs7_padding(b"") is None
|
assert _remove_pkcs7_padding(b"") is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Key derivation test
|
# Key derivation test
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestKeyDerivation:
|
class TestKeyDerivation:
|
||||||
def test_derive_aes_key_deterministic(self):
|
def test_derive_aes_key_deterministic(self):
|
||||||
key1 = _derive_aes_key(b"my_passphrase")
|
key1 = _derive_aes_key(b"my_passphrase")
|
||||||
@@ -160,11 +155,11 @@ class TestKeyDerivation:
|
|||||||
key2 = _derive_aes_key(b"passphrase_b")
|
key2 = _derive_aes_key(b"passphrase_b")
|
||||||
assert key1 != key2
|
assert key1 != key2
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Decryption test (real openssl, known key)
|
# Decryption test (real openssl, known key)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestDecryption:
|
class TestDecryption:
|
||||||
def test_decrypt_v10_roundtrip(self):
|
def test_decrypt_v10_roundtrip(self):
|
||||||
"""Encrypt then decrypt — verifies the full pipeline works."""
|
"""Encrypt then decrypt — verifies the full pipeline works."""
|
||||||
@@ -197,28 +192,28 @@ class TestDecryption:
|
|||||||
"""v10 prefix with no ciphertext should return None."""
|
"""v10 prefix with no ciphertext should return None."""
|
||||||
assert _decrypt_v10_value(b"v10", KNOWN_AES_KEY, db_version=20) is None
|
assert _decrypt_v10_value(b"v10", KNOWN_AES_KEY, db_version=20) is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Chrome not installed → returns None
|
# Chrome not installed → returns None
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestChromeNotInstalled:
|
class TestChromeNotInstalled:
|
||||||
def test_db_not_found(self):
|
def test_db_not_found(self):
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
"scripts.lib.chrome_cookies.CHROME_COOKIES_DB",
|
"lib.chrome_cookies.CHROME_COOKIES_DB",
|
||||||
Path("/nonexistent/path/Cookies"),
|
Path("/nonexistent/path/Cookies"),
|
||||||
):
|
):
|
||||||
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Keychain access denied → returns None
|
# Keychain access denied → returns None
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestKeychainDenied:
|
class TestKeychainDenied:
|
||||||
def test_security_command_fails(self):
|
def test_security_command_fails(self):
|
||||||
with mock.patch("scripts.lib.chrome_cookies.subprocess.run") as mock_run:
|
with mock.patch("lib.chrome_cookies.subprocess.run") as mock_run:
|
||||||
mock_run.return_value = subprocess.CompletedProcess(
|
mock_run.return_value = subprocess.CompletedProcess(
|
||||||
args=[], returncode=44, stdout="", stderr="security: SecKeychainSearchCopyNext: The specified item could not be found in the keychain."
|
args=[], returncode=44, stdout="", stderr="security: SecKeychainSearchCopyNext: The specified item could not be found in the keychain."
|
||||||
)
|
)
|
||||||
@@ -226,27 +221,27 @@ class TestKeychainDenied:
|
|||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
def test_security_command_not_found(self):
|
def test_security_command_not_found(self):
|
||||||
with mock.patch("scripts.lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
|
with mock.patch("lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
|
||||||
result = _get_chrome_encryption_key()
|
result = _get_chrome_encryption_key()
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# openssl not found → returns None
|
# openssl not found → returns None
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestOpensslNotFound:
|
class TestOpensslNotFound:
|
||||||
def test_openssl_missing(self):
|
def test_openssl_missing(self):
|
||||||
encrypted = _encrypt_value_v10("test", KNOWN_AES_KEY)
|
encrypted = _encrypt_value_v10("test", KNOWN_AES_KEY)
|
||||||
with mock.patch("scripts.lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
|
with mock.patch("lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
|
||||||
result = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=20)
|
result = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=20)
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Unencrypted cookie values → returned as-is
|
# Unencrypted cookie values → returned as-is
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestUnencryptedCookies:
|
class TestUnencryptedCookies:
|
||||||
def test_plain_value_returned(self, tmp_path):
|
def test_plain_value_returned(self, tmp_path):
|
||||||
"""Unencrypted cookies (value column populated) returned without decryption."""
|
"""Unencrypted cookies (value column populated) returned without decryption."""
|
||||||
@@ -256,18 +251,18 @@ class TestUnencryptedCookies:
|
|||||||
(".x.com", "ct0", "plain_ct0_value", b""),
|
(".x.com", "ct0", "plain_ct0_value", b""),
|
||||||
])
|
])
|
||||||
|
|
||||||
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
with mock.patch("lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
# No keychain needed for unencrypted values
|
# No keychain needed for unencrypted values
|
||||||
with mock.patch("scripts.lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
|
with mock.patch("lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
|
||||||
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
assert result == {"auth_token": "plain_token_value", "ct0": "plain_ct0_value"}
|
assert result == {"auth_token": "plain_token_value", "ct0": "plain_ct0_value"}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Full integration: mock DB with real v10 encryption, mock Keychain
|
# Full integration: mock DB with real v10 encryption, mock Keychain
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestFullExtraction:
|
class TestFullExtraction:
|
||||||
def test_encrypted_cookies_extracted(self, tmp_path):
|
def test_encrypted_cookies_extracted(self, tmp_path):
|
||||||
"""End-to-end: create DB with real v10-encrypted values, extract them."""
|
"""End-to-end: create DB with real v10-encrypted values, extract them."""
|
||||||
@@ -284,9 +279,9 @@ class TestFullExtraction:
|
|||||||
(".other.com", "other", "", b""), # unrelated cookie
|
(".other.com", "other", "", b""), # unrelated cookie
|
||||||
])
|
])
|
||||||
|
|
||||||
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
with mock.patch("lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
|
"lib.chrome_cookies._get_chromium_encryption_key",
|
||||||
return_value=KNOWN_PASSPHRASE,
|
return_value=KNOWN_PASSPHRASE,
|
||||||
):
|
):
|
||||||
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
|
||||||
@@ -301,8 +296,8 @@ class TestFullExtraction:
|
|||||||
(".other.com", "session", "val", b""),
|
(".other.com", "session", "val", b""),
|
||||||
])
|
])
|
||||||
|
|
||||||
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
with mock.patch("lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
with mock.patch("scripts.lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
|
with mock.patch("lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
|
||||||
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
||||||
|
|
||||||
assert result is None
|
assert result is None
|
||||||
@@ -317,9 +312,9 @@ class TestFullExtraction:
|
|||||||
(".x.com", "auth_token", "", encrypted_auth),
|
(".x.com", "auth_token", "", encrypted_auth),
|
||||||
], db_version=24)
|
], db_version=24)
|
||||||
|
|
||||||
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
with mock.patch("lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
|
"lib.chrome_cookies._get_chromium_encryption_key",
|
||||||
return_value=KNOWN_PASSPHRASE,
|
return_value=KNOWN_PASSPHRASE,
|
||||||
):
|
):
|
||||||
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
||||||
@@ -327,11 +322,11 @@ class TestFullExtraction:
|
|||||||
assert result is not None
|
assert result is not None
|
||||||
assert result["auth_token"] == auth_val
|
assert result["auth_token"] == auth_val
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# DB version detection
|
# DB version detection
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestDbVersion:
|
class TestDbVersion:
|
||||||
def test_reads_version_from_meta(self, tmp_path):
|
def test_reads_version_from_meta(self, tmp_path):
|
||||||
db_path = str(tmp_path / "test.db")
|
db_path = str(tmp_path / "test.db")
|
||||||
|
|||||||
@@ -1,16 +1,10 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
"""CLI parsing and validation for --competitors / --competitors-list."""
|
"""CLI parsing and validation for --competitors / --competitors-list."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from contextlib import redirect_stderr
|
from contextlib import redirect_stderr
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
import last30days as cli
|
import last30days as cli
|
||||||
|
|
||||||
@@ -132,6 +126,5 @@ class CompetitorsCliTests(unittest.TestCase):
|
|||||||
cli.resolve_competitors_args(args)
|
cli.resolve_competitors_args(args)
|
||||||
self.assertEqual(cm.exception.code, 2)
|
self.assertEqual(cm.exception.code, 2)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+91
-5
@@ -1,6 +1,6 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
import json
|
import json
|
||||||
import io
|
import io
|
||||||
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -10,13 +10,11 @@ from contextlib import redirect_stderr, redirect_stdout
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
import last30days as cli
|
import last30days as cli
|
||||||
from lib import schema
|
from lib import schema
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
class CliV3Tests(unittest.TestCase):
|
class CliV3Tests(unittest.TestCase):
|
||||||
def make_report(self) -> schema.Report:
|
def make_report(self) -> schema.Report:
|
||||||
@@ -71,6 +69,26 @@ class CliV3Tests(unittest.TestCase):
|
|||||||
cli.parse_search_flag("web, reddit, hn, web"),
|
cli.parse_search_flag("web, reddit, hn, web"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_parse_search_flag_accepts_optional_social_sources(self):
|
||||||
|
self.assertEqual(
|
||||||
|
["threads", "pinterest"],
|
||||||
|
cli.parse_search_flag("threads, pinterest"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_explicit_threads_search_uses_scrapecreators_key_without_include_sources(self):
|
||||||
|
available = cli.pipeline.available_sources(
|
||||||
|
{"SCRAPECREATORS_API_KEY": "test-key", "INCLUDE_SOURCES": ""},
|
||||||
|
requested_sources=["threads"],
|
||||||
|
)
|
||||||
|
self.assertIn("threads", available)
|
||||||
|
|
||||||
|
def test_explicit_perplexity_search_uses_openrouter_key_without_include_sources(self):
|
||||||
|
available = cli.pipeline.available_sources(
|
||||||
|
{"OPENROUTER_API_KEY": "test-key", "INCLUDE_SOURCES": ""},
|
||||||
|
requested_sources=["perplexity"],
|
||||||
|
)
|
||||||
|
self.assertIn("perplexity", available)
|
||||||
|
|
||||||
def test_parse_search_flag_rejects_invalid_or_empty_inputs(self):
|
def test_parse_search_flag_rejects_invalid_or_empty_inputs(self):
|
||||||
with self.assertRaises(SystemExit):
|
with self.assertRaises(SystemExit):
|
||||||
cli.parse_search_flag("unknown")
|
cli.parse_search_flag("unknown")
|
||||||
@@ -143,6 +161,30 @@ class CliV3Tests(unittest.TestCase):
|
|||||||
_, kwargs = write_text.call_args
|
_, kwargs = write_text.call_args
|
||||||
self.assertEqual("utf-8", kwargs.get("encoding"))
|
self.assertEqual("utf-8", kwargs.get("encoding"))
|
||||||
|
|
||||||
|
def test_compute_save_path_display_uses_posix_slashes_under_home(self):
|
||||||
|
# Regression: f"~/{relative}" stringified pathlib.Path with the
|
||||||
|
# OS-native separator, producing "~/Documents\\Last30Days\\..." on
|
||||||
|
# Windows that no shell or File Explorer could open. The fix is
|
||||||
|
# f"~/{relative.as_posix()}" which forces forward slashes regardless
|
||||||
|
# of host OS. On POSIX hosts this asserts the contract for
|
||||||
|
# cross-platform safety; on Windows hosts it would fail without the fix.
|
||||||
|
real_home = Path.home()
|
||||||
|
tmp_under_home = Path(tempfile.mkdtemp(prefix="l30d_save_path_", dir=str(real_home)))
|
||||||
|
try:
|
||||||
|
save_dir = tmp_under_home / "Documents" / "Last30Days"
|
||||||
|
save_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
display = cli.compute_save_path_display(
|
||||||
|
str(save_dir), "british airways middle east", "v3", "compact"
|
||||||
|
)
|
||||||
|
self.assertTrue(display.startswith("~/"), f"Expected '~/' prefix, got: {display}")
|
||||||
|
self.assertNotIn("\\", display, f"Backslash leaked into display: {display}")
|
||||||
|
self.assertTrue(
|
||||||
|
display.endswith("british-airways-middle-east-raw-v3.md"),
|
||||||
|
f"Expected slug+suffix at end, got: {display}",
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(tmp_under_home, ignore_errors=True)
|
||||||
|
|
||||||
def test_persist_report_updates_run_status_on_success_and_failure(self):
|
def test_persist_report_updates_run_status_on_success_and_failure(self):
|
||||||
report = self.make_report()
|
report = self.make_report()
|
||||||
|
|
||||||
@@ -215,6 +257,50 @@ class CliV3Tests(unittest.TestCase):
|
|||||||
fake_progress.show_promo.assert_called_once_with("both", diag=diag)
|
fake_progress.show_promo.assert_called_once_with("both", diag=diag)
|
||||||
self.assertIn("# rendered", stdout.getvalue())
|
self.assertIn("# rendered", stdout.getvalue())
|
||||||
|
|
||||||
|
def test_main_canonicalizes_explicit_github_repo_flags(self):
|
||||||
|
report = self.make_report()
|
||||||
|
diag = {
|
||||||
|
"available_sources": ["grounding"],
|
||||||
|
"providers": {"google": True, "openai": False, "xai": False},
|
||||||
|
"x_backend": None,
|
||||||
|
"bird_installed": True,
|
||||||
|
"bird_authenticated": False,
|
||||||
|
"bird_username": None,
|
||||||
|
"native_web_backend": "brave",
|
||||||
|
}
|
||||||
|
with mock.patch.object(cli.env, "get_config", return_value={}), \
|
||||||
|
mock.patch.object(cli.pipeline, "diagnose", return_value=diag), \
|
||||||
|
mock.patch.object(cli.pipeline, "run", return_value=report) as run_mock, \
|
||||||
|
mock.patch.object(cli, "emit_output", return_value="# rendered"), \
|
||||||
|
mock.patch.object(sys, "argv", [
|
||||||
|
"last30days.py",
|
||||||
|
"claude",
|
||||||
|
"code",
|
||||||
|
"vs",
|
||||||
|
"codex",
|
||||||
|
"--github-repo",
|
||||||
|
"openai/codex,anthropics/claude-code-action",
|
||||||
|
]):
|
||||||
|
stdout = io.StringIO()
|
||||||
|
stderr = io.StringIO()
|
||||||
|
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||||
|
rc = cli.main()
|
||||||
|
self.assertEqual(0, rc)
|
||||||
|
# In vs-mode main + competitors run in parallel via ThreadPoolExecutor,
|
||||||
|
# so the order of pipeline.run invocations is non-deterministic. Find
|
||||||
|
# the main runner's call by predicate on the canonicalized github_repos
|
||||||
|
# rather than by index.
|
||||||
|
expected_repos = ["openai/codex", "anthropics/claude-code"]
|
||||||
|
main_call = next(
|
||||||
|
(c for c in run_mock.call_args_list if c.kwargs.get("github_repos") == expected_repos),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(
|
||||||
|
main_call,
|
||||||
|
f"No pipeline.run call had github_repos={expected_repos}; "
|
||||||
|
f"saw {[c.kwargs.get('github_repos') for c in run_mock.call_args_list]}",
|
||||||
|
)
|
||||||
|
self.assertIn("[GitHub] Canonicalized repos:", stderr.getvalue())
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,8 +1,4 @@
|
|||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import cluster, schema
|
from lib import cluster, schema
|
||||||
|
|
||||||
@@ -196,6 +192,5 @@ class TestClusterUncertainty(unittest.TestCase):
|
|||||||
result = cluster._cluster_uncertainty(candidates)
|
result = cluster._cluster_uncertainty(candidates)
|
||||||
self.assertEqual("thin-evidence", result)
|
self.assertEqual("thin-evidence", result)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,20 +1,14 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
"""Tests for scripts/lib/fanout.run_competitor_fanout."""
|
"""Tests for scripts/lib/fanout.run_competitor_fanout."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import sys
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
from contextlib import redirect_stderr
|
from contextlib import redirect_stderr
|
||||||
from pathlib import Path
|
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import fanout
|
from lib import fanout
|
||||||
|
|
||||||
|
|
||||||
@@ -155,6 +149,5 @@ class FanoutOrchestratorTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual([label for label, _ in results], ["OpenAI"])
|
self.assertEqual([label for label, _ in results], ["OpenAI"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
"""Regression tests: main-topic flags must not leak into competitor sub-runs.
|
"""Regression tests: main-topic flags must not leak into competitor sub-runs.
|
||||||
|
|
||||||
Based on 2026-04-22 Kanye West --competitors receipt where Drake and
|
Based on 2026-04-22 Kanye West --competitors receipt where Drake and
|
||||||
@@ -10,15 +9,10 @@ via closure capture, config mutation, or any other path.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from contextlib import redirect_stderr
|
from contextlib import redirect_stderr
|
||||||
from pathlib import Path
|
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
|
|
||||||
def _fake_report(topic: str):
|
def _fake_report(topic: str):
|
||||||
class _R:
|
class _R:
|
||||||
@@ -191,6 +185,5 @@ class SubRunIsolationTests(unittest.TestCase):
|
|||||||
by_topic["Kendrick Lamar"]["config"].get("_auto_resolve_context", ""),
|
by_topic["Kendrick Lamar"]["config"].get("_auto_resolve_context", ""),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,18 +1,12 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
"""Tests for scripts/lib/competitors.discover_competitors."""
|
"""Tests for scripts/lib/competitors.discover_competitors."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from contextlib import redirect_stderr
|
from contextlib import redirect_stderr
|
||||||
from pathlib import Path
|
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import competitors
|
from lib import competitors
|
||||||
|
|
||||||
|
|
||||||
@@ -23,7 +17,6 @@ def _serp(items: list[tuple[str, str]]) -> list[dict]:
|
|||||||
for title, snippet in items
|
for title, snippet in items
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
OPENAI_SERP = _serp(
|
OPENAI_SERP = _serp(
|
||||||
[
|
[
|
||||||
("OpenAI vs Anthropic vs xAI: which is better?", "xAI and Anthropic now compete directly with OpenAI."),
|
("OpenAI vs Anthropic vs xAI: which is better?", "xAI and Anthropic now compete directly with OpenAI."),
|
||||||
@@ -147,6 +140,5 @@ class CompetitorDiscoveryTests(unittest.TestCase):
|
|||||||
results = self._run(OPENAI_SERP, "OpenAI", count=0)
|
results = self._run(OPENAI_SERP, "OpenAI", count=0)
|
||||||
self.assertEqual(results, [])
|
self.assertEqual(results, [])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,20 +1,15 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
"""Tests for --competitors-plan JSON parsing and per-entity kwargs threading."""
|
"""Tests for --competitors-plan JSON parsing and per-entity kwargs threading."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import sys
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from contextlib import redirect_stderr
|
from contextlib import redirect_stderr
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
import last30days as cli
|
import last30days as cli
|
||||||
|
|
||||||
|
|
||||||
@@ -175,6 +170,5 @@ class SubrunKwargsForTests(unittest.TestCase):
|
|||||||
kwargs = cli.subrun_kwargs_for("X", {}, resolved=resolved)
|
kwargs = cli.subrun_kwargs_for("X", {}, resolved=resolved)
|
||||||
self.assertEqual(kwargs["_context"], "Resolved context")
|
self.assertEqual(kwargs["_context"], "Resolved context")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
"""Integration tests for per-entity Step 0.55 resolution inside competitor fan-out."""
|
"""Integration tests for per-entity Step 0.55 resolution inside competitor fan-out."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -7,12 +6,8 @@ import io
|
|||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
from contextlib import redirect_stderr
|
from contextlib import redirect_stderr
|
||||||
from pathlib import Path
|
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
|
|
||||||
def _fake_report(topic: str):
|
def _fake_report(topic: str):
|
||||||
"""Minimal Report stand-in for runner return values."""
|
"""Minimal Report stand-in for runner return values."""
|
||||||
@@ -326,6 +321,5 @@ class PerEntityResolveTests(unittest.TestCase):
|
|||||||
|
|
||||||
return [runner(c) for c in competitors]
|
return [runner(c) for c in competitors]
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -2,24 +2,19 @@
|
|||||||
|
|
||||||
import configparser
|
import configparser
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import sys
|
|
||||||
import textwrap
|
import textwrap
|
||||||
from pathlib import Path
|
|
||||||
from typing import Dict, List, Optional, Tuple
|
from typing import Dict, List, Optional, Tuple
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days"))
|
from lib.cookie_extract import (
|
||||||
|
|
||||||
from scripts.lib.cookie_extract import (
|
|
||||||
extract_cookies,
|
extract_cookies,
|
||||||
extract_firefox_cookies,
|
extract_firefox_cookies,
|
||||||
_find_default_profile,
|
_find_default_profile,
|
||||||
_get_firefox_profiles_dir,
|
_get_firefox_profiles_dir,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def mock_firefox_env(tmp_path):
|
def mock_firefox_env(tmp_path):
|
||||||
"""Create a mock Firefox profiles directory with cookies.sqlite.
|
"""Create a mock Firefox profiles directory with cookies.sqlite.
|
||||||
@@ -102,7 +97,7 @@ class TestExtractFirefoxCookies:
|
|||||||
profiles_dir = mock_firefox_env()
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
):
|
):
|
||||||
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
@@ -142,7 +137,7 @@ class TestExtractFirefoxCookies:
|
|||||||
)
|
)
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
):
|
):
|
||||||
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
@@ -154,10 +149,10 @@ class TestExtractFirefoxCookies:
|
|||||||
def test_firefox_not_installed(self):
|
def test_firefox_not_installed(self):
|
||||||
"""Returns None when Firefox profiles directory doesn't exist."""
|
"""Returns None when Firefox profiles directory doesn't exist."""
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=None,
|
return_value=None,
|
||||||
), patch(
|
), patch(
|
||||||
"scripts.lib.cookie_extract._is_wsl",
|
"lib.cookie_extract._is_wsl",
|
||||||
return_value=False,
|
return_value=False,
|
||||||
):
|
):
|
||||||
result = extract_firefox_cookies(".x.com", ["auth_token"])
|
result = extract_firefox_cookies(".x.com", ["auth_token"])
|
||||||
@@ -171,10 +166,10 @@ class TestExtractFirefoxCookies:
|
|||||||
)
|
)
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
), patch(
|
), patch(
|
||||||
"scripts.lib.cookie_extract._is_wsl",
|
"lib.cookie_extract._is_wsl",
|
||||||
return_value=False,
|
return_value=False,
|
||||||
):
|
):
|
||||||
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
@@ -192,10 +187,10 @@ class TestExtractFirefoxCookies:
|
|||||||
)
|
)
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
), patch(
|
), patch(
|
||||||
"scripts.lib.cookie_extract._is_wsl",
|
"lib.cookie_extract._is_wsl",
|
||||||
return_value=False,
|
return_value=False,
|
||||||
):
|
):
|
||||||
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
@@ -214,7 +209,7 @@ class TestExtractFirefoxCookies:
|
|||||||
)
|
)
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
):
|
):
|
||||||
result = extract_firefox_cookies(".x.com", ["auth_token"])
|
result = extract_firefox_cookies(".x.com", ["auth_token"])
|
||||||
@@ -231,17 +226,17 @@ class TestExtractCookiesAuto:
|
|||||||
profiles_dir = mock_firefox_env()
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch("scripts.lib.cookie_extract.platform.system", return_value="Darwin"),
|
patch("lib.cookie_extract.platform.system", return_value="Darwin"),
|
||||||
patch(
|
patch(
|
||||||
"scripts.lib.cookie_extract.extract_chrome_cookies",
|
"lib.cookie_extract.extract_chrome_cookies",
|
||||||
return_value=None,
|
return_value=None,
|
||||||
),
|
),
|
||||||
patch(
|
patch(
|
||||||
"scripts.lib.cookie_extract.extract_safari_cookies",
|
"lib.cookie_extract.extract_safari_cookies",
|
||||||
return_value=None,
|
return_value=None,
|
||||||
),
|
),
|
||||||
patch(
|
patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
@@ -257,9 +252,9 @@ class TestExtractCookiesAuto:
|
|||||||
profiles_dir = mock_firefox_env()
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch("scripts.lib.cookie_extract.platform.system", return_value="Linux"),
|
patch("lib.cookie_extract.platform.system", return_value="Linux"),
|
||||||
patch(
|
patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
@@ -273,7 +268,7 @@ class TestExtractCookiesAuto:
|
|||||||
profiles_dir = mock_firefox_env()
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
"lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
return_value=profiles_dir,
|
return_value=profiles_dir,
|
||||||
):
|
):
|
||||||
result = extract_cookies("firefox", ".x.com", ["auth_token"])
|
result = extract_cookies("firefox", ".x.com", ["auth_token"])
|
||||||
@@ -289,7 +284,7 @@ class TestExtractCookiesAuto:
|
|||||||
def test_chrome_delegates_to_chrome_module(self):
|
def test_chrome_delegates_to_chrome_module(self):
|
||||||
"""Chrome extraction delegates to chrome_cookies module."""
|
"""Chrome extraction delegates to chrome_cookies module."""
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract.extract_chrome_cookies",
|
"lib.cookie_extract.extract_chrome_cookies",
|
||||||
return_value={"auth_token": "chrome_tok"},
|
return_value={"auth_token": "chrome_tok"},
|
||||||
):
|
):
|
||||||
result = extract_cookies("chrome", ".x.com", ["auth_token"])
|
result = extract_cookies("chrome", ".x.com", ["auth_token"])
|
||||||
@@ -298,7 +293,7 @@ class TestExtractCookiesAuto:
|
|||||||
def test_safari_delegates_to_safari_module(self):
|
def test_safari_delegates_to_safari_module(self):
|
||||||
"""Safari extraction delegates to safari_cookies module."""
|
"""Safari extraction delegates to safari_cookies module."""
|
||||||
with patch(
|
with patch(
|
||||||
"scripts.lib.cookie_extract.extract_safari_cookies",
|
"lib.cookie_extract.extract_safari_cookies",
|
||||||
return_value={"auth_token": "safari_tok"},
|
return_value={"auth_token": "safari_tok"},
|
||||||
):
|
):
|
||||||
result = extract_cookies("safari", ".x.com", ["auth_token"])
|
result = extract_cookies("safari", ".x.com", ["auth_token"])
|
||||||
|
|||||||
@@ -1,12 +1,9 @@
|
|||||||
"""Tests for dates module."""
|
"""Tests for dates module."""
|
||||||
|
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
# Add lib to path
|
# Add lib to path
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import dates
|
from lib import dates
|
||||||
|
|
||||||
@@ -109,6 +106,5 @@ class TestRecencyScore(unittest.TestCase):
|
|||||||
result = dates.recency_score(None)
|
result = dates.recency_score(None)
|
||||||
self.assertEqual(result, 0)
|
self.assertEqual(result, 0)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,9 +1,5 @@
|
|||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import dates
|
from lib import dates
|
||||||
|
|
||||||
@@ -58,6 +54,5 @@ class DatesV3Tests(unittest.TestCase):
|
|||||||
self.assertEqual(100, dates.recency_score(future))
|
self.assertEqual(100, dates.recency_score(future))
|
||||||
self.assertEqual(0, dates.recency_score(None))
|
self.assertEqual(0, dates.recency_score(None))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+7
-12
@@ -1,10 +1,6 @@
|
|||||||
"""Unit tests for dedupe.py: text normalization, similarity metrics, and deduplication."""
|
"""Unit tests for dedupe.py: text normalization, similarity metrics, and deduplication."""
|
||||||
|
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import dedupe
|
from lib import dedupe
|
||||||
from lib.schema import SourceItem
|
from lib.schema import SourceItem
|
||||||
@@ -16,11 +12,11 @@ def _item(title: str, body: str = "", source: str = "reddit", item_id: str = "t1
|
|||||||
url="https://example.com", engagement={}, metadata={},
|
url="https://example.com", engagement={}, metadata={},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# normalize_text
|
# normalize_text
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestNormalizeText(unittest.TestCase):
|
class TestNormalizeText(unittest.TestCase):
|
||||||
|
|
||||||
def test_lowercases(self):
|
def test_lowercases(self):
|
||||||
@@ -35,11 +31,11 @@ class TestNormalizeText(unittest.TestCase):
|
|||||||
def test_empty_string(self):
|
def test_empty_string(self):
|
||||||
self.assertEqual(dedupe.normalize_text(""), "")
|
self.assertEqual(dedupe.normalize_text(""), "")
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# get_ngrams
|
# get_ngrams
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestGetNgrams(unittest.TestCase):
|
class TestGetNgrams(unittest.TestCase):
|
||||||
|
|
||||||
def test_simple_trigrams(self):
|
def test_simple_trigrams(self):
|
||||||
@@ -58,11 +54,11 @@ class TestGetNgrams(unittest.TestCase):
|
|||||||
ngrams = dedupe.get_ngrams("A!B")
|
ngrams = dedupe.get_ngrams("A!B")
|
||||||
self.assertEqual(ngrams, {"a b"})
|
self.assertEqual(ngrams, {"a b"})
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# jaccard_similarity
|
# jaccard_similarity
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestJaccardSimilarity(unittest.TestCase):
|
class TestJaccardSimilarity(unittest.TestCase):
|
||||||
|
|
||||||
def test_identical_sets(self):
|
def test_identical_sets(self):
|
||||||
@@ -81,11 +77,11 @@ class TestJaccardSimilarity(unittest.TestCase):
|
|||||||
def test_both_empty(self):
|
def test_both_empty(self):
|
||||||
self.assertAlmostEqual(dedupe.jaccard_similarity(set(), set()), 0.0)
|
self.assertAlmostEqual(dedupe.jaccard_similarity(set(), set()), 0.0)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# token_jaccard
|
# token_jaccard
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestTokenJaccard(unittest.TestCase):
|
class TestTokenJaccard(unittest.TestCase):
|
||||||
|
|
||||||
def test_identical_texts(self):
|
def test_identical_texts(self):
|
||||||
@@ -105,11 +101,11 @@ class TestTokenJaccard(unittest.TestCase):
|
|||||||
# "am" is len 2, "great"/"terrible" are content
|
# "am" is len 2, "great"/"terrible" are content
|
||||||
self.assertGreater(result, 0.0)
|
self.assertGreater(result, 0.0)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# hybrid_similarity
|
# hybrid_similarity
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestHybridSimilarity(unittest.TestCase):
|
class TestHybridSimilarity(unittest.TestCase):
|
||||||
|
|
||||||
def test_identical_texts(self):
|
def test_identical_texts(self):
|
||||||
@@ -131,11 +127,11 @@ class TestHybridSimilarity(unittest.TestCase):
|
|||||||
max(ngram_sim, token_sim),
|
max(ngram_sim, token_sim),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# item_text
|
# item_text
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestItemText(unittest.TestCase):
|
class TestItemText(unittest.TestCase):
|
||||||
|
|
||||||
def test_combines_fields(self):
|
def test_combines_fields(self):
|
||||||
@@ -159,11 +155,11 @@ class TestItemText(unittest.TestCase):
|
|||||||
self.assertIn("john", text)
|
self.assertIn("john", text)
|
||||||
self.assertIn("r/python", text)
|
self.assertIn("r/python", text)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# dedupe_items
|
# dedupe_items
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestDedupeItems(unittest.TestCase):
|
class TestDedupeItems(unittest.TestCase):
|
||||||
|
|
||||||
def test_keeps_unique_items(self):
|
def test_keeps_unique_items(self):
|
||||||
@@ -212,6 +208,5 @@ class TestDedupeItems(unittest.TestCase):
|
|||||||
result_loose = dedupe.dedupe_items(items, threshold=0.3)
|
result_loose = dedupe.dedupe_items(items, threshold=0.3)
|
||||||
self.assertEqual(len(result_loose), 1)
|
self.assertEqual(len(result_loose), 1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+11
-16
@@ -5,21 +5,17 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "last30days" / "scripts"))
|
from lib import digg
|
||||||
|
from lib import subproc
|
||||||
from lib import digg # noqa: E402
|
|
||||||
from lib import subproc # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
# === Helpers ===
|
# === Helpers ===
|
||||||
|
|
||||||
|
|
||||||
def _cluster(
|
def _cluster(
|
||||||
cluster_url_id: str = "abc123xy",
|
cluster_url_id: str = "abc123xy",
|
||||||
title: str = "Sample cluster",
|
title: str = "Sample cluster",
|
||||||
@@ -66,9 +62,9 @@ def _post(
|
|||||||
def _stdout_for(payload: dict) -> subproc.SubprocResult:
|
def _stdout_for(payload: dict) -> subproc.SubprocResult:
|
||||||
return subproc.SubprocResult(returncode=0, stdout=json.dumps(payload), stderr="")
|
return subproc.SubprocResult(returncode=0, stdout=json.dumps(payload), stderr="")
|
||||||
|
|
||||||
|
|
||||||
# === _parse_first_post_age ===
|
# === _parse_first_post_age ===
|
||||||
|
|
||||||
|
|
||||||
def test_parse_first_post_age_days():
|
def test_parse_first_post_age_days():
|
||||||
today = datetime(2026, 5, 9, tzinfo=timezone.utc)
|
today = datetime(2026, 5, 9, tzinfo=timezone.utc)
|
||||||
assert digg._parse_first_post_age("5d", today=today) == "2026-05-04"
|
assert digg._parse_first_post_age("5d", today=today) == "2026-05-04"
|
||||||
@@ -104,9 +100,9 @@ def test_parse_first_post_age_invalid():
|
|||||||
assert digg._parse_first_post_age("d") is None
|
assert digg._parse_first_post_age("d") is None
|
||||||
assert digg._parse_first_post_age("-3d") is None
|
assert digg._parse_first_post_age("-3d") is None
|
||||||
|
|
||||||
|
|
||||||
# === parse_digg_response ===
|
# === parse_digg_response ===
|
||||||
|
|
||||||
|
|
||||||
def test_parse_response_happy_path():
|
def test_parse_response_happy_path():
|
||||||
response = {
|
response = {
|
||||||
"results": [
|
"results": [
|
||||||
@@ -193,9 +189,9 @@ def test_parse_response_engagement_rank_score():
|
|||||||
assert by_id["top"]["engagement"]["rank_score"] == 50.0
|
assert by_id["top"]["engagement"]["rank_score"] == 50.0
|
||||||
assert by_id["off-leaderboard"]["engagement"]["rank_score"] == 0.0
|
assert by_id["off-leaderboard"]["engagement"]["rank_score"] == 0.0
|
||||||
|
|
||||||
|
|
||||||
# === _parse_post ===
|
# === _parse_post ===
|
||||||
|
|
||||||
|
|
||||||
def test_parse_post_happy():
|
def test_parse_post_happy():
|
||||||
out = digg._parse_post(_post(username="adam", body="Hello world"))
|
out = digg._parse_post(_post(username="adam", body="Hello world"))
|
||||||
assert out is not None
|
assert out is not None
|
||||||
@@ -210,9 +206,9 @@ def test_parse_post_drops_missing_body_or_handle_or_url():
|
|||||||
assert digg._parse_post({"author": {"username": "x"}, "body": "txt", "xUrl": ""}) is None
|
assert digg._parse_post({"author": {"username": "x"}, "body": "txt", "xUrl": ""}) is None
|
||||||
assert digg._parse_post(None) is None # type: ignore[arg-type]
|
assert digg._parse_post(None) is None # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
# === _run_cli / search_digg with stubbed subprocess ===
|
# === _run_cli / search_digg with stubbed subprocess ===
|
||||||
|
|
||||||
|
|
||||||
def test_search_digg_binary_missing_returns_empty(monkeypatch):
|
def test_search_digg_binary_missing_returns_empty(monkeypatch):
|
||||||
monkeypatch.setattr(digg.shutil, "which", lambda _: None)
|
monkeypatch.setattr(digg.shutil, "which", lambda _: None)
|
||||||
out = digg.search_digg("anything", "2026-04-09", "2026-05-09")
|
out = digg.search_digg("anything", "2026-04-09", "2026-05-09")
|
||||||
@@ -280,9 +276,9 @@ def test_search_digg_empty_query_short_circuits(monkeypatch):
|
|||||||
assert out["results"] == []
|
assert out["results"] == []
|
||||||
called.assert_not_called()
|
called.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
# === enrich_with_top_posts ===
|
# === enrich_with_top_posts ===
|
||||||
|
|
||||||
|
|
||||||
def test_enrich_with_top_posts_attaches_posts(monkeypatch):
|
def test_enrich_with_top_posts_attaches_posts(monkeypatch):
|
||||||
monkeypatch.setattr(digg.shutil, "which", lambda _: "/fake/path")
|
monkeypatch.setattr(digg.shutil, "which", lambda _: "/fake/path")
|
||||||
|
|
||||||
@@ -357,9 +353,9 @@ def test_enrich_top_k_zero_skips_all(monkeypatch):
|
|||||||
digg.enrich_with_top_posts(items, top_k=0)
|
digg.enrich_with_top_posts(items, top_k=0)
|
||||||
fake.assert_not_called()
|
fake.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
# === enrich_source_items (post-dedupe path) ===
|
# === enrich_source_items (post-dedupe path) ===
|
||||||
|
|
||||||
|
|
||||||
class _FakeSourceItem:
|
class _FakeSourceItem:
|
||||||
def __init__(self, source, item_id, engagement, metadata):
|
def __init__(self, source, item_id, engagement, metadata):
|
||||||
self.source = source
|
self.source = source
|
||||||
@@ -410,14 +406,14 @@ def test_enrich_source_items_falls_back_to_item_id(monkeypatch):
|
|||||||
digg.enrich_source_items(items, top_k=1)
|
digg.enrich_source_items(items, top_k=1)
|
||||||
assert captured["cluster_id"] == "fallbackid"
|
assert captured["cluster_id"] == "fallbackid"
|
||||||
|
|
||||||
|
|
||||||
# === Live tests (opt-in) ===
|
# === Live tests (opt-in) ===
|
||||||
|
|
||||||
LIVE = os.environ.get("LAST30DAYS_DIGG_LIVE", "").lower() in ("1", "true", "yes")
|
LIVE = os.environ.get("LAST30DAYS_DIGG_LIVE", "").lower() in ("1", "true", "yes")
|
||||||
HAVE_BIN = shutil.which(digg.CLI_BIN) is not None
|
HAVE_BIN = shutil.which(digg.CLI_BIN) is not None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(not (LIVE and HAVE_BIN), reason="LAST30DAYS_DIGG_LIVE not set or digg-pp-cli missing")
|
@pytest.mark.skipif(not (LIVE and HAVE_BIN), reason="LAST30DAYS_DIGG_LIVE not set or digg-pp-cli missing")
|
||||||
|
|
||||||
|
|
||||||
class TestLiveDigg:
|
class TestLiveDigg:
|
||||||
def test_search_returns_clusters(self):
|
def test_search_returns_clusters(self):
|
||||||
out = digg.search_digg("claude code", "2026-04-09", "2026-05-09", depth="quick")
|
out = digg.search_digg("claude code", "2026-04-09", "2026-05-09", depth="quick")
|
||||||
@@ -451,6 +447,5 @@ class TestLiveDigg:
|
|||||||
posts = digg.fetch_top_posts("notarealclusterid", posts_per=2)
|
posts = digg.fetch_top_posts("notarealclusterid", posts_per=2)
|
||||||
assert posts == []
|
assert posts == []
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
pytest.main([__file__, "-v"])
|
pytest.main([__file__, "-v"])
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
"""Tests for entity_extract module."""
|
"""Tests for entity_extract module."""
|
||||||
|
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
# Add lib to path
|
# Add lib to path
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import entity_extract
|
from lib import entity_extract
|
||||||
|
|
||||||
@@ -162,6 +159,5 @@ class TestExtractEntities(unittest.TestCase):
|
|||||||
result = entity_extract.extract_entities([], [])
|
result = entity_extract.extract_entities([], [])
|
||||||
self.assertSetEqual(set(result.keys()), {"x_handles", "x_hashtags", "reddit_subreddits"})
|
self.assertSetEqual(set(result.keys()), {"x_handles", "x_hashtags", "reddit_subreddits"})
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,8 +1,4 @@
|
|||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import entity_extract
|
from lib import entity_extract
|
||||||
|
|
||||||
@@ -53,6 +49,5 @@ class TestExtractSubreddits(unittest.TestCase):
|
|||||||
def test_empty_items(self):
|
def test_empty_items(self):
|
||||||
self.assertEqual([], entity_extract._extract_subreddits([]))
|
self.assertEqual([], entity_extract._extract_subreddits([]))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,14 +1,10 @@
|
|||||||
"""Tests for browser cookie extraction integration in env.py."""
|
"""Tests for browser cookie extraction integration in env.py."""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib.env import extract_browser_credentials, COOKIE_DOMAINS
|
from lib.env import extract_browser_credentials, COOKIE_DOMAINS
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,4 @@
|
|||||||
import sys
|
from lib import env
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days"))
|
|
||||||
|
|
||||||
from scripts.lib import env
|
|
||||||
|
|
||||||
|
|
||||||
def test_include_sources_defaults_to_empty_string(monkeypatch, tmp_path):
|
def test_include_sources_defaults_to_empty_string(monkeypatch, tmp_path):
|
||||||
|
|||||||
@@ -12,19 +12,15 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
from lib import env
|
||||||
|
|
||||||
from lib import env # noqa: E402
|
|
||||||
|
|
||||||
SETUP_KEYCHAIN_SH = Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts" / "setup-keychain.sh"
|
SETUP_KEYCHAIN_SH = Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts" / "setup-keychain.sh"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _load_keychain unit tests
|
# _load_keychain unit tests
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -93,12 +89,10 @@ def test_load_keychain_skips_empty_stdout():
|
|||||||
mock.patch("subprocess.run", return_value=_run_result(0, "")):
|
mock.patch("subprocess.run", return_value=_run_result(0, "")):
|
||||||
assert env._load_keychain(["XAI_API_KEY"]) == {}
|
assert env._load_keychain(["XAI_API_KEY"]) == {}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# get_config integration tests
|
# get_config integration tests
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def clean_env(monkeypatch, tmp_path):
|
def clean_env(monkeypatch, tmp_path):
|
||||||
"""Hide every key get_config might touch and point CONFIG_FILE at a
|
"""Hide every key get_config might touch and point CONFIG_FILE at a
|
||||||
@@ -154,7 +148,6 @@ def test_get_config_openai_key_can_come_from_keychain(clean_env):
|
|||||||
assert cfg["OPENAI_API_KEY"] == "sk-from-kc"
|
assert cfg["OPENAI_API_KEY"] == "sk-from-kc"
|
||||||
assert cfg["OPENAI_AUTH_SOURCE"] == "api_key"
|
assert cfg["OPENAI_AUTH_SOURCE"] == "api_key"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Drift guard: lib/env.py KEYCHAIN_KEYS and setup-keychain.sh ALL_KEYS must
|
# Drift guard: lib/env.py KEYCHAIN_KEYS and setup-keychain.sh ALL_KEYS must
|
||||||
# stay in lockstep. A mismatch means users storing a key via the helper script
|
# stay in lockstep. A mismatch means users storing a key via the helper script
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import bird_x, env
|
from lib import bird_x, env
|
||||||
|
|
||||||
|
|
||||||
@@ -69,6 +66,5 @@ class ThreadsAvailabilityTests(unittest.TestCase):
|
|||||||
"INCLUDE_SOURCES": "",
|
"INCLUDE_SOURCES": "",
|
||||||
}))
|
}))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,13 +1,10 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
import evaluate_search_quality as evaluator
|
import evaluate_search_quality as evaluator
|
||||||
|
|
||||||
|
|
||||||
@@ -202,6 +199,5 @@ class EvaluatorV3Tests(unittest.TestCase):
|
|||||||
self.assertIn("| topic a | 0.10 | 0.30 |", summary)
|
self.assertIn("| topic a | 0.10 | 0.30 |", summary)
|
||||||
self.assertEqual("HEAD~1", metrics["baseline"])
|
self.assertEqual("HEAD~1", metrics["baseline"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
# ruff: noqa: E402
|
|
||||||
"""Tests for the BRAVE/SERPER web-promo suppression when hosting-model-driven."""
|
"""Tests for the BRAVE/SERPER web-promo suppression when hosting-model-driven."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -11,7 +10,6 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(REPO_ROOT / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
|
|
||||||
def _engine() -> Path:
|
def _engine() -> Path:
|
||||||
@@ -90,6 +88,5 @@ class FooterNudgeSuppressionTests(unittest.TestCase):
|
|||||||
msg="web promo should be suppressed when --plan is passed",
|
msg="web promo should be suppressed when --plan is passed",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,8 +1,4 @@
|
|||||||
import sys
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "last30days" / "scripts"))
|
|
||||||
|
|
||||||
from lib import fusion, schema
|
from lib import fusion, schema
|
||||||
|
|
||||||
@@ -52,7 +48,6 @@ class FusionV3Tests(unittest.TestCase):
|
|||||||
self.assertEqual({"reddit", "x"}, set(merged.sources))
|
self.assertEqual({"reddit", "x"}, set(merged.sources))
|
||||||
self.assertEqual(2, len(merged.source_items))
|
self.assertEqual(2, len(merged.source_items))
|
||||||
|
|
||||||
|
|
||||||
def test_diversify_pool_guarantees_min_per_qualifying_source(self):
|
def test_diversify_pool_guarantees_min_per_qualifying_source(self):
|
||||||
"""Every qualifying source (local_relevance >= 0.25) gets at least 2
|
"""Every qualifying source (local_relevance >= 0.25) gets at least 2
|
||||||
items in the fused pool.
|
items in the fused pool.
|
||||||
@@ -118,7 +113,6 @@ class FusionV3Tests(unittest.TestCase):
|
|||||||
f"Source '{src}' has {source_counts.get(src, 0)} items, expected >= 2",
|
f"Source '{src}' has {source_counts.get(src, 0)} items, expected >= 2",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_diversify_pool_denies_slots_for_low_relevance_source(self):
|
def test_diversify_pool_denies_slots_for_low_relevance_source(self):
|
||||||
"""Sources with best local_relevance < 0.25 do not get reserved slots.
|
"""Sources with best local_relevance < 0.25 do not get reserved slots.
|
||||||
|
|
||||||
@@ -439,6 +433,5 @@ class TestUrlNormalization(unittest.TestCase):
|
|||||||
_normalize_url("https://reddit.com/r/test"),
|
_normalize_url("https://reddit.com/r/test"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user