Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Matt Van Horn
|
5864c687a3 | ||
|
Matt Van Horn
|
790e5bc26a |
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"interface": {
|
||||
"displayName": "Last 30 Days"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Research"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,17 +1,16 @@
|
||||
{
|
||||
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
||||
"name": "last30days-skill",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"owner": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Marketplace hosting the Last 30 Days research plugin."
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "last30days",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"version": "3.3.2",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, and 5+ more sources.",
|
||||
"version": "3.0.9",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
"url": "https://github.com/mvanhorn"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days",
|
||||
"version": "3.3.2",
|
||||
"version": "3.0.9",
|
||||
"description": "Research any topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources. AI agent scores by upvotes, likes, and real money - not editors.",
|
||||
"author": {
|
||||
"name": "Matt Van Horn",
|
||||
@@ -10,5 +10,6 @@
|
||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||
"license": "MIT",
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"]
|
||||
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket", "github", "perplexity", "threads", "pinterest", "eli5", "hacker-news"],
|
||||
"hooks": {}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"name": "last30days"
|
||||
}
|
||||
+6
-4
@@ -1,6 +1,5 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by skills/last30days/scripts/build-skill.sh to produce a
|
||||
# claude.ai-upload-ready .skill file from the canonical skills/last30days tree.
|
||||
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||
# See docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
@@ -23,16 +22,19 @@ assets/ export-ignore
|
||||
# claude.ai-bundle-specific exclusions live in scripts/build-skill.sh.
|
||||
|
||||
# Historical + repo-only manifests
|
||||
SKILL-original.md export-ignore
|
||||
SPEC.md export-ignore
|
||||
TASKS.md export-ignore
|
||||
test-run.log export-ignore
|
||||
CONTRIBUTORS.md export-ignore
|
||||
HERMES_SETUP.md export-ignore
|
||||
release-notes.md export-ignore
|
||||
CHANGELOG.md export-ignore
|
||||
uv.lock export-ignore
|
||||
|
||||
# Platform adapters are kept in git archives because Claude Code and Codex
|
||||
# plugin installs use the same repository archive as their source payload.
|
||||
# Platform adapters - skill-upload path is platform-agnostic
|
||||
.agents/ export-ignore
|
||||
.codex-plugin/ export-ignore
|
||||
.hermes-plugin/ export-ignore
|
||||
|
||||
# CI workflows - repo-only, not needed at skill runtime
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
name: Bug Report
|
||||
description: Report a bug or unexpected behavior
|
||||
labels: [bug]
|
||||
body:
|
||||
- type: textarea
|
||||
id: summary
|
||||
attributes:
|
||||
label: Summary
|
||||
description: What happened?
|
||||
placeholder: Describe the bug in 1-2 sentences.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: How can we reproduce this?
|
||||
placeholder: |
|
||||
1. Run `python3 skills/last30days/scripts/last30days.py "topic" --emit=compact`
|
||||
2. ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected Behavior
|
||||
description: What should have happened?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: traceback
|
||||
attributes:
|
||||
label: Error / Traceback
|
||||
description: Paste the full traceback or error output.
|
||||
render: text
|
||||
- type: dropdown
|
||||
id: install
|
||||
attributes:
|
||||
label: Install Method
|
||||
options:
|
||||
- Claude Code plugin
|
||||
- Gemini CLI extension
|
||||
- Codex plugin
|
||||
- Hermes skill
|
||||
- Manual (git clone)
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: OS
|
||||
placeholder: macOS 15.4, Ubuntu 24.04, Windows 11, etc.
|
||||
@@ -1,24 +0,0 @@
|
||||
name: Feature Request
|
||||
description: Suggest a new feature or improvement
|
||||
labels: [enhancement]
|
||||
body:
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem
|
||||
description: What problem does this solve?
|
||||
placeholder: When I try to ..., I can't ...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: How should this work?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
description: Other approaches you thought of (optional).
|
||||
@@ -1,19 +0,0 @@
|
||||
## Summary
|
||||
|
||||
<!-- What does this PR do? 1-3 sentences. -->
|
||||
|
||||
## Changes
|
||||
|
||||
<!-- Bullet list of what changed. Reference files if helpful. -->
|
||||
|
||||
-
|
||||
|
||||
## Testing
|
||||
|
||||
<!-- How did you verify this works? -->
|
||||
|
||||
- [ ] Ran `uv run python -m pytest -q --tb=short`
|
||||
|
||||
## Related Issues
|
||||
|
||||
<!-- Link issues: Fixes #123 or Relates to #456 -->
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash skills/last30days/scripts/build-skill.sh
|
||||
bash scripts/build-skill.sh
|
||||
test -f dist/last30days.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
name: Security
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
name: Dependency audit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Export locked dependency set
|
||||
run: |
|
||||
uv export \
|
||||
--locked \
|
||||
--all-groups \
|
||||
--no-hashes \
|
||||
--format requirements.txt \
|
||||
--output-file /tmp/last30days-requirements.txt
|
||||
|
||||
# Advisory-first: visibility before enforcement. This repo handles API keys,
|
||||
# cookies, browser tokens, and local env files, so dependency CVEs should be
|
||||
# visible in CI logs even before the project has a clean blocking baseline.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
- name: Run pip-audit against locked dependencies
|
||||
continue-on-error: true
|
||||
run: uvx --python 3.12 pip-audit -r /tmp/last30days-requirements.txt --progress-spinner=off
|
||||
|
||||
secret-scan:
|
||||
name: Secret scan
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout full history for diff-aware scanning
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Advisory-first: this reports verified secrets in pull requests and pushes to
|
||||
# main, but does not block merges until maintainers confirm a clean baseline.
|
||||
# The TruffleHog action automatically scans the PR range for pull_request
|
||||
# events and the pushed commit range for push events.
|
||||
# Set continue-on-error: false once a clean baseline run is confirmed.
|
||||
# Contributor policy: never commit real secrets in fixtures, tests, docs, or
|
||||
# examples; use obvious dummy values and env-based auth patterns instead.
|
||||
- name: Run TruffleHog OSS secret scan
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
uses: trufflesecurity/trufflehog@v3.95.2
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: ./
|
||||
version: v3.95.2
|
||||
extra_args: --only-verified
|
||||
@@ -1,26 +0,0 @@
|
||||
name: Validate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.12
|
||||
|
||||
- name: Run test suite
|
||||
run: uv run pytest
|
||||
@@ -28,7 +28,3 @@ htmlcov/
|
||||
|
||||
# Internal planning docs (ce:plan output) — keep local, don't publish
|
||||
docs/plans/
|
||||
.context/
|
||||
|
||||
/work
|
||||
/print
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
# last30days Skill
|
||||
|
||||
Agent Skills package for researching any topic across Reddit, X, YouTube, and web. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ other [Agent Skills](https://agentskills.io) hosts. Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `skills/last30days/SKILL.md` — canonical skill definition / runtime spec the model reads when the slash command fires
|
||||
- `skills/last30days/scripts/last30days.py` — main research engine
|
||||
- `skills/last30days/scripts/lib/` — search, enrichment, rendering modules
|
||||
- `skills/last30days/scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`)
|
||||
- `CONCEPTS.md` — shared domain vocabulary (Skill, Engine, Harness, Beta channel) — relevant when orienting to the codebase or discussing project terminology
|
||||
- `CONFIGURATION.md` — user-facing knobs (env vars, flags, per-host install patterns); keep in sync per the rules below
|
||||
- `CHANGELOG.md` — structured release history (launch copy lives in GitHub Releases)
|
||||
- `HERMES_SETUP.md` — install instructions for the Hermes harness specifically
|
||||
|
||||
## Orientation
|
||||
- This is an Agent Skills package, not a CLI tool. The product is the slash-command-invoked skill (`/last30days <topic>` in most harnesses); `scripts/last30days.py` is implementation. Claude Code is the most common host but not the only one — features must work across every harness the skill installs into.
|
||||
- Feature design starts from the slash-command UX. A new engine flag with no SKILL.md integration is incomplete — the model invoking the skill won't know the flag exists.
|
||||
- README and PR examples show `/last30days <topic>` first. Direct CLI invocation (`python3 scripts/last30days.py ...`) is a fallback for scripting, cron, and dev-time engine testing; label it as such, never as the primary path.
|
||||
- Slash commands don't pass shell mechanics through. `/last30days OpenClaw --emit=html | pbcopy` is invalid in any harness — either use the slash form (no flags or pipes; let the model translate user intent into engine flags) or use the direct CLI form (full `python3 ...` with explicit flags and a real shell).
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
# Dev/fallback: direct engine invocation (scripting, cron, or engine testing only)
|
||||
python3 skills/last30days/scripts/last30days.py "test query" --emit=compact
|
||||
npx skills add . -g -y # copies skill into ~/.agents/skills/<name>/ (frozen at install time); re-run to sync working-tree edits — see Rules below
|
||||
|
||||
# Tests (pytest, ~89 files under tests/, configured in pyproject.toml)
|
||||
uv run pytest # full suite
|
||||
uv run pytest tests/test_dedupe_v3.py # single file
|
||||
uv run pytest tests/test_dedupe_v3.py -k some_case # single case
|
||||
uv run pytest --cov # with coverage (skips lib/vendor/)
|
||||
```
|
||||
|
||||
Python 3.12+ required. Use `uv` for the env; the venv lives at `.venv/`.
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- One-time setup: `npx skills add . -g -y` copies the skill into `~/.agents/skills/<name>/` (real directory) and, for harnesses that support symlinked skill dirs, drops a per-host symlink pointing at that copy. **Working-tree edits do NOT propagate automatically** — the `~/.agents/skills/<name>/` copy is frozen at install time. To sync after edits, re-run `npx skills add . -g -y`. For live-edit on a dev machine, replace the install copy with a symlink to the working tree: `ln -sfn "$PWD/skills/last30days" ~/.agents/skills/last30days` (run from the repo root).
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Security hygiene
|
||||
- Never commit real API keys, browser cookies, auth tokens, app passwords, access tokens, or `.env` contents.
|
||||
- Use the env-based auth patterns in `skills/last30days/scripts/lib/env.py`; tests and fixtures must use obvious dummy values only.
|
||||
- Keep examples safe by redacting secrets and avoiding copy/pasteable live credentials in docs, fixtures, and test data.
|
||||
- Do not weaken or disable the advisory security workflow (`.github/workflows/security.yml`) without explaining why in the PR description or review thread.
|
||||
|
||||
## Maintaining CONFIGURATION.md
|
||||
|
||||
`CONFIGURATION.md` is the user-facing configuration reference — save paths, per-source API keys, web-search backend priority, trend-monitoring stack, per-client install patterns. Distinct from `SKILL.md` (the canonical runtime spec).
|
||||
|
||||
Update `CONFIGURATION.md` when:
|
||||
|
||||
- adding a new env var (e.g. `LAST30DAYS_*`, `BSKY_*`, `*_API_KEY`)
|
||||
- adding a new CLI flag that affects configuration (e.g. `--store`, `--web-backend`)
|
||||
- adding a new per-client install pattern (Claude Code, Gemini, Codex, Cursor, Hermes…)
|
||||
- adding a new optional source that requires its own credential
|
||||
- changing the priority order of config layers (per-run flag > env > `.env` file > defaults)
|
||||
|
||||
Keep the existing structure organized by how often each layer is touched: per-run flags → env vars / `.env` → optional trend-monitoring stack → per-client patterns. Add new content into the right section rather than appending at the end.
|
||||
|
||||
When a new config concept lands in `SKILL.md` or `AGENTS.md`, mirror the user-facing knob in `CONFIGURATION.md` so non-agent readers can configure the skill without reverse-engineering it from the runtime spec.
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
+3
-266
@@ -5,269 +5,6 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **First-party positioning research + pitch-vs-pulse synthesis (company / product / service topics).** A new mandatory research step captures each entity's current stated positioning from first-party sources (homepage, docs, pricing) rather than from memory. The fetched pitch grounds `What it is` descriptions (entities described as they pitch themselves today), helps reject unrelated brand-name noise, and feeds an evidence-triggered prose beat: when the month's conversation directly supports a specific claim, cuts against one, or is squarely about the pitched ground, the synthesis says so anchored to the top thread — and stays silent when the pulse is orthogonal to the pitch, because a manufactured connection is worse than omission. Claims are tested at matched altitude (specific claims against specific threads; broad taglines are never graded against individual items), and statements stay windowed to the 30 days — no trend verdicts. Scoped to entities with an identifiable first party: people are always excluded (even founders whose companies qualify), as are events, abstract concepts, and ownerless topics like Bitcoin; the beat requires positioning fetched during the run, never from memory.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Entity-grounding rerank demotion now keys on the head token of the primary entity instead of requiring the full multi-word phrase as a contiguous substring. A high-engagement on-entity item (e.g. a 323-pt HN thread titled "Stripe is friendly to 'friendly fraud'") is no longer demoted to score 0 on a `Stripe payments` query just because it lacks the trailing search-hint word. The intended demotion still fires for items that never name the brand at all. The keyless Reddit comment-enrichment slot selection (`_slot_priority`), which mirrors this signal, was updated to the same head-token grounding so the two paths stay consistent.
|
||||
|
||||
## [3.3.2] - 2026-06-06
|
||||
|
||||
### Fixed
|
||||
|
||||
- Keyless Reddit comment enrichment now spends its limited slots on entity-matching posts first (mirroring rerank's entity-miss demotion signal) instead of raw upvote order, so off-topic high-upvote threads from broad subreddits no longer consume the comment budget only to be demoted afterward ([#484](https://github.com/mvanhorn/last30days-skill/pull/484))
|
||||
|
||||
## [3.3.1] - 2026-05-30
|
||||
|
||||
### Fixed
|
||||
|
||||
- Removed the redundant `commands/last30days.md` wrapper so the plugin exposes only the skill ([#461](https://github.com/mvanhorn/last30days-skill/issues/461)). Previously the plugin shipped both a command wrapper and the skill under the same name, so `/last30` surfaced two `last30days` entries with two different descriptions. The skill already carries its own `argument-hint`, so the `/last30days <topic>` picker UX is unchanged.
|
||||
- Corrected the README install note that claimed Claude Code dedupes the slash command across install methods; it does not, so having both the marketplace plugin and the `npx skills` copy active shows two entries.
|
||||
|
||||
## [3.3.0] - 2026-05-17
|
||||
|
||||
A week-long shipping cycle: ~75 PRs merged plus 7 community fixes salvaged through PR triage. Big themes: install story modernized for the multi-harness world (Claude Code, Codex, Cursor, Gemini CLI, Copilot, Windsurf, and 50+ Agent Skills hosts), new emit and source modes, and a substantial reliability sweep across Reddit, X, Windows, YouTube, and the planner.
|
||||
|
||||
### Added
|
||||
|
||||
**Emit modes and sources**
|
||||
|
||||
- `--emit=html` for shareable, print-friendly HTML research briefs ([#332](https://github.com/mvanhorn/last30days-skill/pull/332)).
|
||||
- **Digg AI 1000 source**, auto-enabled when `digg-pp-cli` is on PATH ([#370](https://github.com/mvanhorn/last30days-skill/pull/370)). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief.
|
||||
|
||||
**Configuration knobs**
|
||||
|
||||
- `EXCLUDE_SOURCES` env var — the inverse of `INCLUDE_SOURCES`, honored in source count and pipeline filter ([#399](https://github.com/mvanhorn/last30days-skill/pull/399)).
|
||||
- `LAST30DAYS_YOUTUBE_SSH_HOST` — opt-in SSH routing for `yt-dlp` through a residential-IP host, for users on datacenter VPS hit by YouTube's bot-wall ([#376](https://github.com/mvanhorn/last30days-skill/pull/376)). Host validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection. Transcript path unchanged (uses HTTP fallback).
|
||||
- macOS Keychain as a credential source — reads from the system keychain when env vars and config files aren't set ([#407](https://github.com/mvanhorn/last30days-skill/pull/407)).
|
||||
- Configuration enablement: env-var defaults and source-resilience patterns across the config layer ([#344](https://github.com/mvanhorn/last30days-skill/pull/344)).
|
||||
|
||||
**Pipeline and storage**
|
||||
|
||||
- Reddit URL auto-enrichment from web search via the public JSON API ([#366](https://github.com/mvanhorn/last30days-skill/pull/366)).
|
||||
- Per-run finding sightings recorded in the SQLite store ([#373](https://github.com/mvanhorn/last30days-skill/pull/373)).
|
||||
- Brave browser support for X/Twitter cookie extraction ([#320](https://github.com/mvanhorn/last30days-skill/pull/320)).
|
||||
|
||||
**Tests and CI**
|
||||
|
||||
- Full pytest suite restored to CI; 13 rotted tests repaired ([#416](https://github.com/mvanhorn/last30days-skill/pull/416)).
|
||||
- `greptile.json` added with `triggerOnUpdates` + `statusCheck` ([#418](https://github.com/mvanhorn/last30days-skill/pull/418)).
|
||||
- Advisory security workflow ([#368](https://github.com/mvanhorn/last30days-skill/pull/368)).
|
||||
- Parallel grounding backend test coverage ([#355](https://github.com/mvanhorn/last30days-skill/pull/355)).
|
||||
|
||||
**Docs**
|
||||
|
||||
- New `CONFIGURATION.md` with README pointers ([#339](https://github.com/mvanhorn/last30days-skill/pull/339)).
|
||||
- `docs/solutions/` learning capture for release-time consistency-test cascades ([#413](https://github.com/mvanhorn/last30days-skill/pull/413)) and the eval-not-in-CI design decision ([#417](https://github.com/mvanhorn/last30days-skill/pull/417)).
|
||||
|
||||
### Changed
|
||||
|
||||
**Install story modernized**
|
||||
|
||||
- `npx skills add` is now the canonical install path for every harness ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). README and SKILL.md flipped to recommend `npx skills add . -g -y` over per-harness manual instructions. Surfaces Gemini CLI, Copilot, Windsurf, and 50+ other Agent Skills hosts that the install pattern reaches.
|
||||
- README dropped the Gemini CLI native-extension install path (now covered by `npx skills add`).
|
||||
- `hooks.json` made polyglot for Gemini CLI + Claude Code compatibility ([#318](https://github.com/mvanhorn/last30days-skill/pull/318)).
|
||||
|
||||
**Skill semantics and multi-harness reframe**
|
||||
|
||||
- `AGENTS.md` is now canonical; `CLAUDE.md` points at it ([#410](https://github.com/mvanhorn/last30days-skill/pull/410)). Reframes the project as a multi-harness Agent Skills package rather than a Claude-Code-specific tool.
|
||||
- SKILL.md path resolution rewritten: STEP 0 narrows to a Claude-Code-marketplaces-only stale-clone guard; Step 1 walks a single `SKILL_DIR` substitution pattern ([#400](https://github.com/mvanhorn/last30days-skill/pull/400), [#409](https://github.com/mvanhorn/last30days-skill/pull/409)). Removes ~80 lines of bash and fixes a real spec-vs-engine divergence where the previous resolver could pick a different install than the SKILL.md the model loaded from.
|
||||
- SKILL.md version regex consolidated into `lib/skill_meta.py` ([#412](https://github.com/mvanhorn/last30days-skill/pull/412)).
|
||||
- `--plan` / `--competitors-plan` invocation templates switched from inline single-quoted JSON to heredoc-written tmpfiles ([#404](https://github.com/mvanhorn/last30days-skill/pull/404), fixes [#403](https://github.com/mvanhorn/last30days-skill/issues/403)). Apostrophes in resolved context strings ("McDonald's", "people's choice") no longer break shell parsing.
|
||||
- `POSTS_PER_CLUSTER` raised 3→5 and render-side display limit 2→3 to match the per-source enrichment caps used by Reddit, HN, YouTube, TikTok, and GitHub. The previous caps routinely truncated cluster context.
|
||||
- Digg AI 1000 renamed to "Digg" in user-facing output ([#372](https://github.com/mvanhorn/last30days-skill/pull/372)) — footer line, source label, inline-quote suffix, why_relevant, container attribution. Internal references retain the upstream product name.
|
||||
- GitHub repo resolution canonicalized for ambiguous product comparisons ([#302](https://github.com/mvanhorn/last30days-skill/pull/302)).
|
||||
|
||||
**Dependencies and tooling**
|
||||
|
||||
- Dropped `requests` runtime dependency. All providers route through stdlib `urllib` via the `lib/http` wrapper ([#393](https://github.com/mvanhorn/last30days-skill/pull/393)).
|
||||
- Migrated to `gemini-3.1-flash-lite` GA model ([#378](https://github.com/mvanhorn/last30days-skill/pull/378)).
|
||||
- Aligned Codex/Claude plugin manifests + added Codex `AGENTS.md` ([#321](https://github.com/mvanhorn/last30days-skill/pull/321)).
|
||||
- pytest dev dep bumped 9.0.2 → 9.0.3 ([#414](https://github.com/mvanhorn/last30days-skill/pull/414)).
|
||||
|
||||
### Removed
|
||||
|
||||
- **BREAKING for Codex native-plugin users:** `.codex-plugin/plugin.json` and the matching SKILL_ROOT resolver branch in SKILL.md Step 1 ([#400](https://github.com/mvanhorn/last30days-skill/pull/400)). Codex users should install via `npx skills add mvanhorn/last30days-skill` or copy the skill to `~/.codex/skills/last30days/`.
|
||||
- **`skills/last30days/scripts/sync.sh`** — maintainer dev-deploy script ([#405](https://github.com/mvanhorn/last30days-skill/pull/405)). Replaced by `npx skills add . -g -y` (live-symlink into every detected harness's skill dir — better than sync.sh's copy model since edits propagate live). Hermes uses `hermes skills install mvanhorn/last30days-skill --force`; OpenClaw uses `clawhub install last30days-official`.
|
||||
- Orphaned `SPEC.md` and `TASKS.md` ([#419](https://github.com/mvanhorn/last30days-skill/pull/419)).
|
||||
|
||||
### Fixed
|
||||
|
||||
**Reddit**
|
||||
|
||||
- `lstrip("r/")` mangled subreddits starting with `r` (`r/robotics` → `obotics`, `r/ruby` → `uby`); replaced with `removeprefix("r/")` at 4 sites (Alex Key, salvaged from #288).
|
||||
- Browser-like User-Agent + `Accept-Language`/`Accept-Encoding`/`Connection` headers + gzip decompression to fix `urllib` 403s on Reddit's public JSON endpoint (Franco Carballar, salvaged from #199).
|
||||
- HTTP 402 re-raised across all three ScrapeCreators paths (`_global_search`, `_subreddit_search`, `fetch_post_comments`) so the OpenAI/public-JSON fallback chain triggers when credits are exhausted (Jonathan Oppenheim, salvaged from #170).
|
||||
|
||||
**Authentication and credentials**
|
||||
|
||||
- Restored multi-key rotation for `SCRAPECREATORS_API_KEY` accidentally dropped in v3.0.6 (Eric Oberhofer, salvaged from #287). Comma-separated keys round-robin via `random.choice` per run.
|
||||
|
||||
**Windows compatibility**
|
||||
|
||||
- `os.killpg` in `_cleanup_children()` guarded with `hasattr(os, "killpg")`, falls back to `os.kill(SIGTERM)` (gujishh, salvaged from #226).
|
||||
- POSIX-style secret-permission warning skipped on Windows ([#357](https://github.com/mvanhorn/last30days-skill/pull/357)).
|
||||
- Render uses forward slashes in save-path footer for Windows ([#338](https://github.com/mvanhorn/last30days-skill/pull/338)).
|
||||
|
||||
**xAI / X / xurl**
|
||||
|
||||
- `parse_x_response` now raises `http.HTTPError` on empty output, missing JSON, or decode failure — surfaces in `errors_by_source` instead of silently returning an empty result list (Kaustav Mishra, salvaged from #155).
|
||||
- `xurl` treats `PermissionError` from PATH lookup as unavailable ([#322](https://github.com/mvanhorn/last30days-skill/pull/322)).
|
||||
|
||||
**YouTube**
|
||||
|
||||
- SC YouTube + multi-token HN searches unblocked ([#388](https://github.com/mvanhorn/last30days-skill/pull/388)).
|
||||
- Transcript-fetch ratio surfaced + degraded-run nudge for stale `yt-dlp` ([#340](https://github.com/mvanhorn/last30days-skill/pull/340)).
|
||||
|
||||
**bird_x / HTTP**
|
||||
|
||||
- Subprocess retry on non-JSON stdout to handle X anti-bot HTML interstitials ([#383](https://github.com/mvanhorn/last30days-skill/pull/383)).
|
||||
- HTTP retry budget expanded + exponential backoff on DNS resolution failure ([#382](https://github.com/mvanhorn/last30days-skill/pull/382)).
|
||||
- Parallel AI search aligned with current API schema ([#341](https://github.com/mvanhorn/last30days-skill/pull/341)).
|
||||
- Parallel web backend routed through grounding ([#354](https://github.com/mvanhorn/last30days-skill/pull/354)).
|
||||
|
||||
**Planner and sources**
|
||||
|
||||
- `xquik` registered in `SOURCE_CAPABILITIES` ([#336](https://github.com/mvanhorn/last30days-skill/pull/336), fixes [#319](https://github.com/mvanhorn/last30days-skill/issues/319)).
|
||||
- Honor explicit optional source requests ([#356](https://github.com/mvanhorn/last30days-skill/pull/356)).
|
||||
- ScrapeCreators source-gating aligned between code and docs ([#415](https://github.com/mvanhorn/last30days-skill/pull/415)).
|
||||
- OpenClaw works without ScrapeCreators key ([#392](https://github.com/mvanhorn/last30days-skill/pull/392), by @thinkun).
|
||||
|
||||
**Render, version display, hosting paths**
|
||||
|
||||
- Hardcoded `v3.0.0` in render replaced with dynamic `_skill_version()` ([#365](https://github.com/mvanhorn/last30days-skill/pull/365)).
|
||||
- Comparison HTML artifacts saved correctly ([#389](https://github.com/mvanhorn/last30days-skill/pull/389)).
|
||||
- `OPENROUTER_DEFAULT` model ID corrected ([#323](https://github.com/mvanhorn/last30days-skill/pull/323)).
|
||||
- OpenClaw poll-timing initialized once ([#358](https://github.com/mvanhorn/last30days-skill/pull/358)).
|
||||
- Prefer sandboxed Safari cookie path ([#343](https://github.com/mvanhorn/last30days-skill/pull/343)).
|
||||
- Preserve clean mode for last-run state ([#334](https://github.com/mvanhorn/last30days-skill/pull/334)).
|
||||
- Replaced hardcoded `/Users/mvanhorn/...` paths in `test-v1-vs-v2.sh` with portable env-var overrides (Dave Morin, salvaged from #297).
|
||||
|
||||
**Hooks**
|
||||
|
||||
- `check-config.sh` path-quoting fix for paths with spaces ([#337](https://github.com/mvanhorn/last30days-skill/pull/337)).
|
||||
- Replaced unsafe `eval` with `declare` in `check-config.sh` ([#364](https://github.com/mvanhorn/last30days-skill/pull/364)).
|
||||
|
||||
**Sync and version metadata**
|
||||
|
||||
- `sync.sh` pointed at this repo's plugin cache, not the private repo's ([#402](https://github.com/mvanhorn/last30days-skill/pull/402)).
|
||||
- Sync cache target bumped to 3.2.1 to match SKILL.md ([#397](https://github.com/mvanhorn/last30days-skill/pull/397)).
|
||||
- ScrapeCreators free-tier credit count corrected to 100 in docs ([#369](https://github.com/mvanhorn/last30days-skill/pull/369), fixes [#367](https://github.com/mvanhorn/last30days-skill/issues/367)).
|
||||
- Gemini extension version synced ([#349](https://github.com/mvanhorn/last30days-skill/pull/349)).
|
||||
- Various stale path/link fixes ([#345](https://github.com/mvanhorn/last30days-skill/pull/345), [#346](https://github.com/mvanhorn/last30days-skill/pull/346), [#347](https://github.com/mvanhorn/last30days-skill/pull/347), [#348](https://github.com/mvanhorn/last30days-skill/pull/348), [#351](https://github.com/mvanhorn/last30days-skill/pull/351)).
|
||||
|
||||
### Contributors
|
||||
|
||||
First-time contributors whose fixes shipped in this release (most via PR triage salvage — fix re-applied directly to main with co-author credit when path migration made the original branch un-rebaseable):
|
||||
|
||||
- Dave Morin — portable test-harness paths
|
||||
- Alex Key — `removeprefix("r/")` for subreddit names
|
||||
- Eric Oberhofer — multi-key rotation restored
|
||||
- gujishh — Windows process cleanup
|
||||
- Franco Carballar — Reddit browser-like headers
|
||||
- Jonathan Oppenheim — Reddit 402 fallback chain
|
||||
- Kaustav Mishra — xAI error surfacing
|
||||
- [@thinkun](https://github.com/thinkun) ([#363](https://github.com/mvanhorn/last30days-skill/pull/363)) — OpenClaw ScrapeCreators-key-optional fix
|
||||
|
||||
Full PR list at [github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0](https://github.com/mvanhorn/last30days-skill/releases/tag/v3.3.0).
|
||||
|
||||
## [3.2.0] - 2026-05-09
|
||||
|
||||
### Added
|
||||
|
||||
- Add `--emit=html` for shareable, print-friendly HTML research briefs.
|
||||
- **Digg AI 1000 source** (auto-enabled when `digg-pp-cli` is on PATH). Surfaces curated story clusters from the AI 1000 leaderboard and pulls attributable X-post quotes into the brief as `[@handle](xUrl) via Digg AI 1000: ...` lines. Footer line: `⛏️ Digg AI 1000: N clusters │ K posts │ M authors`. No X auth required for the inline quotes since they flow through Digg's read-only endpoints.
|
||||
|
||||
## [3.1.1] - 2026-04-24
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Codex plugin layout.** Move the canonical runtime payload under `skills/last30days/` and update Codex/Claude plugin metadata and tests for the relocated engine path.
|
||||
- **Claude Code cache resolution.** Resolve Claude plugin installs to `skills/last30days/scripts/last30days.py` after the plugin-layout restructure.
|
||||
|
||||
## [3.1.0] - 2026-04-22
|
||||
|
||||
Consolidates the 3.0.10 to 3.0.14 dev cycle (commenter handles, `--competitors`, per-entity Step 0.55, vs-mode N passes, comparison title attribution) and republishes the OpenClaw bundle, which had been frozen on ClawHub at `3.0.0-open` since April 8.
|
||||
|
||||
### Added
|
||||
|
||||
- **OpenClaw republish.** `clawhub install last30days-official` now resolves to `3.1.0-open`, matching current main. Closes [#307](https://github.com/mvanhorn/last30days-skill/issues/307), [#195](https://github.com/mvanhorn/last30days-skill/issues/195), [#236](https://github.com/mvanhorn/last30days-skill/issues/236). The ClawHub bundle had shipped a broken `env.py get_config()` and stale SKILL.md path references since April; both are fixed at source on main and the republish carries the fixes to installers.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Claude Code plugin manifest path-escape.** The `.claude-plugin/plugin.json` `skills` key was removed in commit `93fbed2` but never shipped in a tagged release. Installing via `/plugin install last30days-skill` could hit `/doctor`'s `Path escapes plugin directory: ./ (skills)` error. This release ships the fix. Closes [#306](https://github.com/mvanhorn/last30days-skill/issues/306).
|
||||
- **Broken README link.** The README's "source of truth" link pointed at root `SKILL.md`, which is no longer maintained after the plugin-layout restructure. Fixed to point at `skills/last30days/SKILL.md`.
|
||||
|
||||
### Dev cycle journal (3.0.10 - 3.0.14, not separately tagged)
|
||||
|
||||
Individual changelog entries for 3.0.10 through 3.0.14 below document the incremental work consolidated into this release.
|
||||
|
||||
## [3.0.14] - 2026-04-22
|
||||
|
||||
### Changed
|
||||
|
||||
- **Comparison-mode title attribution.** The synthesis title for vs-mode and `--competitors` outputs changes from `What the Community Says (Last 30 Days)` to `What the Community Says (/Last30Days)`. Surfaces the slash-command identity instead of restating the date range. Three SKILL.md occurrences updated; pure documentation change.
|
||||
|
||||
## [3.0.13] - 2026-04-22
|
||||
|
||||
### Changed
|
||||
|
||||
- **vs mode runs N full passes in parallel, one per entity.** Architectural revert of the 3-pass → 1-pass latency optimization from an earlier version. `/last30days "OpenAI vs Anthropic vs xAI"` now runs three full `pipeline.run()` calls in parallel via the same fanout `--competitors` uses, producing three `*-raw.md` save files plus a merged comparison output. Each entity gets its own Step 0.55-grade targeting, own primary X handle weight, own subreddit scoping — apples-to-apples depth instead of the one-pool merged retrieval the single-pass path produced. Parallel execution keeps wall clock ≈ single pass.
|
||||
- **`--competitors` is now a SKILL.md-level shortcut for vs-mode with auto-discovery.** The hosting reasoning model (Claude Code, Codex, Hermes, Gemini, any agent with WebSearch) performs discovery and Step 0.55 per entity via its own WebSearch tool, then invokes the engine with a vs-topic and `--competitors-plan` JSON. The engine flag remains for headless/cron use with BRAVE/EXA/SERPER/PARALLEL/OPENROUTER keys (engine-internal `auto_resolve` stays as fallback).
|
||||
- **LAW 7-style stderr for `--competitors` with no backend** now leads with the hosting-model path (WebSearch + Step 0.55 + `--competitors-plan`) instead of `BRAVE_API_KEY`. API-key framing moved to a secondary "headless" section.
|
||||
|
||||
### Added
|
||||
|
||||
- **`--competitors-plan` JSON flag** for per-entity Step 0.55 targeting. Schema: `{entity_name: {x_handle?, x_related?, subreddits?, github_user?, github_repos?, context?}}`. Accepts inline JSON or a file path (matches `--plan`). When present for an entity, skips engine-internal `auto_resolve` and uses the provided values; missing fields fall back to `auto_resolve` (if backend) or planner defaults. Case-insensitive entity matching. The `subrun_kwargs_for` helper is the single source of truth for per-entity kwargs — no closure-default fallthrough from main scope.
|
||||
- **Per-entity save files** when `--save-dir` is set on a vs-mode or `--competitors` run. Each entity's sub-run produces its own `{slug}-raw.md` with a single-row Resolved Entities block — matches historical vs-mode behavior (N passes → N save files).
|
||||
- **`--polymarket-keywords "kw1,kw2"`** to filter Polymarket matches for ambiguous single-token topics (e.g., "Warriors" → `nba,gsw,golden-state` kills Glasgow Warriors rugby and Honor of Kings Rogue Warriors noise).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **BRAVE/SERPER footer nudge suppressed** when `--plan` or `--competitors-plan` is present. The nudge told Claude Code users to set an API key when they already have WebSearch via the hosting model. Nudge still fires for true headless runs (no `--plan`, no backend) where the advice is correct.
|
||||
- **Override-leak regression testing.** 3.0.12 already fixed the main-topic `--subreddits` / `--x-handle` / `--github-*` from leaking into peer sub-runs via explicit per-entity kwargs scrubbing. This release adds a 4-test regression suite (`test_competitor_subrun_isolation.py`) locking in the invariant.
|
||||
|
||||
## [3.0.12] - 2026-04-22
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Per-entity Step 0.55 resolution for competitor sub-runs.** In 3.0.11, only the main topic got X handle / subreddit / GitHub resolution; competitor sub-runs ran with planner defaults and produced visibly thinner evidence (Reddit 403 fallbacks, single-word queries). Each competitor sub-run now calls `resolve.auto_resolve()` inside `fanout.run_competitor_fanout` when a web backend is available, mirroring the main topic's pre-flight resolution. Per-entity X handle, subreddit list, GitHub user/repos, and news context are threaded into each sub-run's `pipeline.run()` call. Deep-copied config per sub-run prevents `_auto_resolve_context` cross-leak. Surfaces in a new `## Resolved Entities` output block so the resolution coverage is visible without reading stderr.
|
||||
- **LAW 7 false-positive on internal fan-out sub-runs.** Each competitor sub-run was emitting the `[Planner] No --plan passed... YOU ARE the planner` stderr warning. LAW 7 targets the hosting-reasoning-model path, not engine-internal fan-out. New `internal_subrun=True` keyword on `planner.plan_query` and `pipeline.run` suppresses the warning for sub-runs only; the default path is unchanged.
|
||||
- **Marketplace-stale SKILL.md trap.** Added a STEP 0 canonical-path self-check at the top of SKILL.md. Two of three 2026-04-22 test runs loaded SKILL.md from `plugins/marketplaces/last30days-skill/` (Claude-Code-managed git clone pinned to origin/main, lagging the versioned cache), then ran `--help` against the same stale path, did not see `--competitors`, and fell back to a manual comparison plan. The STEP 0 block forces any reader to verify they loaded from `plugins/cache/last30days-skill/last30days/{VERSION}/SKILL.md` and re-read from the versioned cache if not.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Default `--competitors` count is now 2 (3-way total: original + 2 peers).** Previously 3. `--competitors=N` still customizes (range 1..6). Matches the feature description's canonical example (`Kanye vs Drake vs Kendrick`).
|
||||
|
||||
### Added
|
||||
|
||||
- **`## Resolved Entities` block** in `render_comparison_multi` output. Shows per-entity X handle, subreddits, GitHub user/repos, and truncated context for every entity in the comparison. Block is omitted entirely when no entity has a resolved payload (mock mode, no backend).
|
||||
|
||||
## [3.0.11] - 2026-04-22
|
||||
|
||||
### Added
|
||||
|
||||
- **`--competitors` flag for auto-discovered comparison fan-out.** Pass `--competitors` on a single-entity topic and the engine discovers 2-6 peer entities via web search, then runs the full pipeline on each in parallel and emits one N-way comparison. `last30days Kanye West --competitors` resolves Drake, Kendrick Lamar, and one more peer. `last30days OpenAI --competitors` resolves Anthropic, xAI, Google Gemini. `--competitors=N` controls count, `--competitors-list="A,B,C"` skips discovery and uses the explicit list. Discovery mirrors the `auto_resolve` pattern (Brave / Exa / Serper / Parallel) with deterministic text extraction - no internal LLM call. Sub-runs inherit the main `--quick`/`--deep`/`--days`, run in a `ThreadPoolExecutor`, and degrade gracefully when at least 2 entities survive. Output reuses the existing 9-axis `## Head-to-Head` scaffold.
|
||||
|
||||
## [3.0.10] - 2026-04-21
|
||||
|
||||
### Added
|
||||
|
||||
- **Commenter handles on evidence lines.** Top-comment rendering now includes the commenter's handle - `u/author` for Reddit, `@handle` for TikTok/YouTube/Instagram/Bluesky/X/Threads. The enrichment adapters already captured `author`; the render layer just was not using it. Evidence lines change from `- Comment (6822 upvotes): Finally, John Apple` to `- u/Cyrisaurus (6822 upvotes): Finally, John Apple`. Person-level citations make synthesis-side inline markdown links per LAW 8 much more natural. Both the compact and full render paths are covered.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **TikTok author preference.** `_fetch_post_comments` in `scripts/lib/tiktok.py` preferred `user.nickname` over `user.unique_id`, so the engine captured display names ("Moosa Noormahomed") instead of @handles ("moosanoormahomed"). Flipped to prefer `unique_id`. Nickname still wins as a fallback when `unique_id` is missing. Display names can contain emoji, spaces, and non-Latin characters that do not round-trip to a profile URL; the @handle is the stable identifier.
|
||||
- **Single plugin payload layout.** The canonical runtime moved to `skills/last30days/` for both Claude Code and Codex plugin loading. Root-level `SKILL.md`, `scripts/`, `agents/`, and `assets/` are no longer maintained as duplicate copies.
|
||||
|
||||
### Behavior fallback
|
||||
|
||||
- When an author is empty, `[deleted]`, or `[removed]`, the render falls back to the legacy `Comment (...)` shape - no `u/` or `@` prefix with an empty handle is ever emitted.
|
||||
|
||||
## [3.0.9] - 2026-04-18 - The Self-Debug Release
|
||||
|
||||
### Highlights
|
||||
@@ -465,15 +202,15 @@ Intelligent search, fun judge, cross-source cluster merging, single-pass compari
|
||||
|
||||
### Highlights
|
||||
|
||||
Auto-save research briefings to the default memory directory as topic-named .md files. Every run now builds a personal research library automatically - no more manual copy-paste.
|
||||
Auto-save research briefings to `~/Documents/Last30Days/` as topic-named .md files. Every run now builds a personal research library automatically - no more manual copy-paste.
|
||||
|
||||
### Added
|
||||
|
||||
- Auto-save complete research briefings (synthesis, stats, follow-up suggestions) to the default memory directory after every run
|
||||
- Auto-save complete research briefings (synthesis, stats, follow-up suggestions) to `~/Documents/Last30Days/{topic-slug}.md` after every run
|
||||
- Kebab-case filename generation from topic (e.g., "Claude Code skills" -> `claude-code-skills.md`)
|
||||
- Duplicate topic handling: appends date suffix instead of overwriting (e.g., `claude-code-skills-2026-03-05.md`)
|
||||
- Agent mode (`--agent`) also saves research files
|
||||
- Brief confirmation after save with the saved file path
|
||||
- Brief confirmation after save: "Saved to ~/Documents/Last30Days/{slug}.md"
|
||||
|
||||
### Credits
|
||||
|
||||
|
||||
@@ -1 +1,25 @@
|
||||
@AGENTS.md
|
||||
# last30days Skill
|
||||
|
||||
Claude Code skill for researching any topic across Reddit, X, YouTube, and web.
|
||||
Python scripts with multi-source search aggregation.
|
||||
|
||||
## Structure
|
||||
- `scripts/last30days.py` — main research engine
|
||||
- `scripts/lib/` — search, enrichment, rendering modules
|
||||
- `scripts/lib/vendor/bird-search/` — vendored X search client
|
||||
- `SKILL.md` — skill definition (deployed to ~/.claude/skills/last30days/)
|
||||
|
||||
## Commands
|
||||
```bash
|
||||
python3 scripts/last30days.py "test query" --emit=compact # Run research
|
||||
bash scripts/sync.sh # Deploy to ~/.claude, ~/.agents, ~/.codex
|
||||
```
|
||||
|
||||
## Rules
|
||||
- `lib/__init__.py` must be bare package marker (comment only, NO eager imports)
|
||||
- After edits: run `bash scripts/sync.sh` to deploy
|
||||
- Git remote: origin = public (`mvanhorn/last30days-skill`)
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental changes get tested on `mvanhorn/last30days-skill-private`, which installs as a parallel `/last30days-beta` slash command. Beta-only changes never ship to public without a review PR here. Workflow guide lives at `BETA.md` in the private repo. Plan that established this setup: `docs/plans/2026-04-17-005-feat-beta-skill-from-private-repo-plan.md`.
|
||||
|
||||
-47
@@ -1,47 +0,0 @@
|
||||
# Concepts
|
||||
|
||||
Shared vocabulary for `last30days-skill`. Terms here have a precise project-specific meaning — distinct enough from their general technical sense that a new contributor would need them defined to follow conversations, PR descriptions, or the SKILL.md contract.
|
||||
|
||||
## The package
|
||||
|
||||
### Skill
|
||||
|
||||
A self-contained agent-instructions package consisting of a `SKILL.md` prose contract plus a sibling `scripts/` directory containing the executable code the SKILL.md invokes. The package conforms to the [Agent Skills](https://agentskills.io) open format and installs across every major harness (Claude Code, Codex, Cursor, GitHub Copilot, Gemini CLI, and 50+ others) via `npx skills add`, harness-native plugin installers, or per-harness skill directories. A Skill is the unit of distribution; the Skill is the product.
|
||||
|
||||
### Engine
|
||||
|
||||
The Python script (`scripts/last30days.py`) the Skill's SKILL.md invokes to do the actual research work. The Engine and SKILL.md have a contract: SKILL.md tells the model which flags to pass (`--plan`, `--competitors-plan`, `--x-handle`, `--subreddits`, `--emit=compact`, etc.), and the Engine produces a specific output shape (badge line, ranked evidence clusters, emoji-tree footer) that the model is contractually required to pass through. The Engine is implementation; the SKILL.md prose is the agent-facing surface.
|
||||
|
||||
### Harness
|
||||
|
||||
The agent runtime that loads Skills and invokes them on the user's behalf. Claude Code is the most common Harness for this Skill but not the only one — Codex, Cursor, GitHub Copilot, Gemini CLI, and the rest of the Agent Skills ecosystem also count. "Multi-harness" describes a Skill that works correctly across every Harness it installs into; features written without multi-harness awareness (e.g., engine flags with no SKILL.md integration, or paths hardcoded to one Harness's install layout) regress on Harnesses other than the one they were tested against.
|
||||
|
||||
## Research pipeline
|
||||
|
||||
### Primary entity
|
||||
|
||||
The brand or proper-noun core of a research topic — the topic with its Intent modifier stripped. It is what the research is *about*, as distinct from how the user phrased the search.
|
||||
|
||||
### Intent modifier
|
||||
|
||||
A trailing word or phrase in a topic that expresses what the user wants to know rather than what the topic is ("review", "use cases", "pricing"). Stripped when deriving the Primary entity.
|
||||
|
||||
### Entity grounding
|
||||
|
||||
The check that a candidate item plausibly mentions the Primary entity before final ranking. Grounding keys on the head token (first word) of the Primary entity rather than the full phrase — trailing words are usually search descriptors, so requiring them falsely demotes on-entity items.
|
||||
|
||||
An item that fails grounding receives a decisive entity-miss demotion, designed so engagement cannot rescue off-entity content. Because the demotion is decisive, the grounding bar is deliberately conservative: its failure modes degrade toward "no penalty," never toward burying on-entity signal.
|
||||
|
||||
### Keyless path
|
||||
|
||||
The research flow available with no API keys: source data is gathered by scraping and RSS rather than authenticated APIs, and ranking falls back to local scoring instead of LLM-based reranking. This is the free tier of the Skill; lexical quality safeguards like Entity grounding matter most here, because no LLM is available to judge relevance semantically.
|
||||
|
||||
### Comment-enrichment slots
|
||||
|
||||
The small, depth-dependent budget of Reddit posts whose comments get fetched in the Keyless path. Slot selection is relevance-aware: posts that pass Entity grounding claim slots first, so the budget is not spent on high-engagement posts that final ranking will demote anyway.
|
||||
|
||||
## Distribution
|
||||
|
||||
### Beta channel
|
||||
|
||||
A parallel install of the Skill, sourced from the private `mvanhorn/last30days-skill-private` repo and installed as `/last30days-beta` rather than `/last30days`. The Beta channel exists so experimental changes can be tested by real users before they ship to the public `/last30days`. Promotion from Beta to public happens via a review PR against this (public) repo — Beta-only changes never ship to public without that PR. The Beta channel workflow guide lives in `BETA.md` in the private repo.
|
||||
@@ -1,268 +0,0 @@
|
||||
# Configuration
|
||||
|
||||
Everything you can tune in `/last30days` without editing the engine source.
|
||||
Three layers, in order of how often you'll touch them:
|
||||
|
||||
1. **Per-run flags** - what you pass on the command line.
|
||||
2. **Environment variables and `.env`** - what's enabled across all runs.
|
||||
3. **Optional trend-monitoring stack** - SQLite store, watchlist, briefings.
|
||||
|
||||
Per-client patterns and the experimental beta channel are at the bottom.
|
||||
|
||||
> Skip ahead: [Where output is saved](#where-output-is-saved) - [API keys](#api-keys-env) - [Reasoning provider](#reasoning-provider-priority) - [Web search backend](#web-search-backend-priority) - [Trend monitoring](#trend-monitoring-store--watchlist--briefings) - [Per-client patterns](#per-client-patterns) - [Beta channel](#beta-channel)
|
||||
|
||||
## Why this document exists
|
||||
|
||||
This is a focused **configuration reference** maintained alongside the engine. The runtime contract (the voice rules, the planner protocol, the LAWs the synthesizing model follows) lives in [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md) - that file is authoritative when the two ever differ. This file's job is narrower: surface every knob a user or operator can turn, in one place, kept current with the code so client-facing setups stay reliable. New configuration knobs added to the engine should be reflected here in the same PR.
|
||||
|
||||
---
|
||||
|
||||
## Where output is saved
|
||||
|
||||
| Platform | Default path | Override |
|
||||
|---|---|---|
|
||||
| Linux / macOS | `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` | set `LAST30DAYS_MEMORY_DIR=/path` |
|
||||
| Windows | `LAST30DAYS_MEMORY_DIR` defaults to `C:\Users\<you>\Documents\Last30Days\` | set `LAST30DAYS_MEMORY_DIR=C:\path` |
|
||||
|
||||
Each run produces one file per topic, slug-named:
|
||||
`<slug>-raw[-suffix].md`. Same topic + same suffix on the same day overwrites; same topic + same suffix on different days appends a date stamp.
|
||||
|
||||
**Per-run overrides:**
|
||||
- `--save-dir <path>` - one-off output location.
|
||||
- `--save-suffix <name>` - distinguish runs of the same topic (e.g. per client: `--save-suffix=acme`).
|
||||
|
||||
The footer line `📎 Raw results saved to ${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}/<slug>-raw.md` is the canonical pointer; if it shows backslashes on Windows update past v3.1.1.
|
||||
|
||||
---
|
||||
|
||||
## API keys (`.env`)
|
||||
|
||||
The skill reads keys from a `.env` file. Two locations are supported, in priority order:
|
||||
|
||||
1. **`.claude/last30days.env`** in the current project directory (project-scoped) - takes precedence when present.
|
||||
2. **`~/.config/last30days/.env`** at the user level (global default) - the fallback.
|
||||
|
||||
Override the global location with `LAST30DAYS_CONFIG_DIR=/path` (or `LAST30DAYS_CONFIG_DIR=""` for no-config mode). File permissions should be `600` on POSIX hosts - the engine warns on every run if they aren't.
|
||||
|
||||
The project-scoped file is the cleanest pattern for **per-client setups**: drop a `.claude/last30days.env` into each client folder (`SCRAPECREATORS_API_KEY`, `INCLUDE_SOURCES`, `LAST30DAYS_MEMORY_DIR`, `BSKY_HANDLE`, etc), `cd` into that folder, and the skill picks up that client's configuration automatically. No wrapper scripts needed for the common case.
|
||||
|
||||
**Source-by-source** - what each key unlocks:
|
||||
|
||||
| Source | Key(s) | Required for | Free tier |
|
||||
|---|---|---|---|
|
||||
| Reddit (public) | none | always on | yes |
|
||||
| Hacker News | none | always on | yes |
|
||||
| Polymarket | none | always on | yes |
|
||||
| GitHub | `gh` CLI installed (uses your GitHub auth) | always on if `gh` present | yes |
|
||||
| YouTube | `yt-dlp` CLI installed | always on if `yt-dlp` present | yes |
|
||||
| X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; xAI / ScrapeCreators = paid |
|
||||
| TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls |
|
||||
| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network |
|
||||
| Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls |
|
||||
| Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls |
|
||||
| Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) |
|
||||
| TruthSocial | `TRUTHSOCIAL_TOKEN` | TruthSocial items | yes |
|
||||
| Web search | one of: `BRAVE_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PARALLEL_API_KEY` | `--auto-resolve` and Step 2 supplements | Brave has a free tier; native WebSearch on Claude Code / Codex / Gemini works as a fallback |
|
||||
| Perplexity Deep Research | `OPENROUTER_API_KEY` | `--deep-research` flag (~$0.90/query) | no |
|
||||
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
|
||||
|
||||
**Example `.env` skeleton** (placeholders only - replace with your own values):
|
||||
|
||||
```bash
|
||||
# Reasoning + planning (one provider; see priority below)
|
||||
GOOGLE_API_KEY=<your-gemini-key>
|
||||
|
||||
# Web search backend (one is enough; Brave is the cheapest)
|
||||
BRAVE_API_KEY=<your-brave-key>
|
||||
|
||||
# Optional sources
|
||||
SCRAPECREATORS_API_KEY=<your-scrapecreators-key>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
|
||||
# X authentication (one option only)
|
||||
XAI_API_KEY=<your-xai-key>
|
||||
# OR cookie-jar (no key needed; logs in via your browser session)
|
||||
# FROM_BROWSER=firefox
|
||||
|
||||
# Bluesky
|
||||
BSKY_HANDLE=<your-handle>.bsky.social
|
||||
BSKY_APP_PASSWORD=<your-app-password>
|
||||
```
|
||||
|
||||
After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last30days.env` if using the project-scoped variant).
|
||||
|
||||
**Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search.
|
||||
|
||||
### Bluesky app-password format and search host
|
||||
|
||||
`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at <https://bsky.app/settings/app-passwords>. The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually.
|
||||
|
||||
The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`:
|
||||
|
||||
```bash
|
||||
BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Reasoning provider priority
|
||||
|
||||
`/last30days` needs one reasoning model for planning + reranking when you don't pass `--plan` yourself. Auto-detect priority (set `LAST30DAYS_REASONING_PROVIDER=<name>` to pin one):
|
||||
|
||||
1. **Gemini** - `GOOGLE_API_KEY` / `GEMINI_API_KEY` / `GOOGLE_GENAI_API_KEY`
|
||||
2. **OpenAI** - `OPENAI_API_KEY` (or Codex auth at `~/.codex/auth.json`)
|
||||
3. **xAI** - `XAI_API_KEY`
|
||||
4. **OpenRouter** - `OPENROUTER_API_KEY` (also unlocks `--deep-research`)
|
||||
5. **Local / deterministic** - always available, lowest quality
|
||||
|
||||
When you invoke `/last30days` from Claude Code, Codex, or Gemini, the host model **is** the reasoning provider for plan + synthesis - you don't need any of the keys above unless you also run the script headlessly (cron, CI, watchlist).
|
||||
|
||||
---
|
||||
|
||||
## Web search backend priority
|
||||
|
||||
Used by `--auto-resolve` (when WebSearch isn't available from the host) and Step 2 supplements. Auto-detect priority (override per-run with `--web-backend=<name>`):
|
||||
|
||||
1. **Brave** - `BRAVE_API_KEY`
|
||||
2. **Exa** - `EXA_API_KEY`
|
||||
3. **Serper** - `SERPER_API_KEY`
|
||||
4. **Parallel** - `PARALLEL_API_KEY`
|
||||
5. **Host's native WebSearch** - Claude Code, Codex, Gemini all have one built in
|
||||
|
||||
Visible quality difference between hosts with vs without a configured backend. If your client setup produces thinner results than yours, this is usually why.
|
||||
|
||||
---
|
||||
|
||||
## Trend monitoring (`--store` + watchlist + briefings)
|
||||
|
||||
The default behavior - one slug-named file per topic, overwritten on rerun - is the snapshot mode. For continuous monitoring, the repo ships three components most users miss:
|
||||
|
||||
### `--store` flag
|
||||
|
||||
Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate.
|
||||
|
||||
**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`.
|
||||
|
||||
Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py).
|
||||
|
||||
### `watchlist.py` - recurring topics
|
||||
|
||||
[`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) manages topics that should be researched on a schedule. Subcommands: `add`, `remove`, `list`, `run-one`, `run-all`, `config`. Built-in delivery to Slack incoming webhooks (`hooks.slack.com/...`) or any HTTPS endpoint, fired only when new findings appear.
|
||||
|
||||
Two-step flow (the watchlist holds the topic; an external scheduler invokes the run):
|
||||
|
||||
```bash
|
||||
# 1. Add the topic to the watchlist
|
||||
# Default schedule daily 8am; --weekly switches to Mondays 8am
|
||||
python3 scripts/watchlist.py add "british airways middle east" --weekly
|
||||
|
||||
# 2. Configure delivery and budget (optional)
|
||||
python3 scripts/watchlist.py config delivery "https://hooks.slack.com/services/..."
|
||||
python3 scripts/watchlist.py config budget 5.00
|
||||
|
||||
# 3. Trigger via cron / Task Scheduler / GitHub Actions
|
||||
python3 scripts/watchlist.py run-one "british airways middle east"
|
||||
# or run every enabled topic, gated by daily_budget
|
||||
python3 scripts/watchlist.py run-all
|
||||
```
|
||||
|
||||
The schedule field stored on each topic is metadata - the actual cron / Task Scheduler invocation is your responsibility. Watchlist runs hardcode `--quick` and `--lookback-days 90` when spawning the underlying engine.
|
||||
|
||||
### `briefing.py` - daily / weekly digests
|
||||
|
||||
[`scripts/briefing.py`](skills/last30days/scripts/briefing.py) reads the SQLite store and emits structured data the agent then synthesizes into prose. Modes: `generate` (daily), `generate --weekly`, `show [--date DATE]` (display a saved briefing). Briefs save to `~/.local/share/last30days/briefs/`.
|
||||
|
||||
### Recommended cadence pattern
|
||||
|
||||
| Step | Cadence | Command |
|
||||
|---|---|---|
|
||||
| Baseline | one-time per topic | `/last30days "<topic>" --days=30 --store` |
|
||||
| Add to watchlist | one-time per topic | `python3 scripts/watchlist.py add "<topic>" --weekly` |
|
||||
| Recurring run | daily or weekly (external scheduler) | `python3 scripts/watchlist.py run-all` |
|
||||
| Digest | weekly | `python3 scripts/briefing.py generate --weekly` |
|
||||
|
||||
---
|
||||
|
||||
## Per-client patterns
|
||||
|
||||
The skill is built to flex around different client environments. Four patterns that compose well:
|
||||
|
||||
### 1. Per-client `.claude/last30days.env` (preferred when you cd into client folders)
|
||||
|
||||
The simplest pattern when each client has its own working directory: drop a `.claude/last30days.env` into the client folder. The skill picks it up automatically (see [API keys](#api-keys-env) for the lookup priority). Typical contents:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_MEMORY_DIR=C:\Users\<you>\Clients\acme\Research\Last30Days
|
||||
SCRAPECREATORS_API_KEY=<acme-scoped-key-or-shared>
|
||||
INCLUDE_SOURCES=tiktok,instagram
|
||||
BSKY_HANDLE=<acme-bluesky-handle>.bsky.social
|
||||
```
|
||||
|
||||
`cd` into the client folder, run `/last30days <topic>` as normal, no flags or wrappers. Combine with `--save-suffix=<client-slug>` per run if you also need to differentiate filenames within that folder.
|
||||
|
||||
### 2. Per-client save dir + suffix wrapper
|
||||
|
||||
For workflows where you don't `cd` into a client folder (running from anywhere, scripted batches), a tiny shell function isolates each client's research without engine changes.
|
||||
|
||||
PowerShell example:
|
||||
|
||||
```powershell
|
||||
function Run-L30D-Client {
|
||||
param([string]$ClientSlug, [Parameter(ValueFromRemainingArguments=$true)]$Args)
|
||||
$env:LAST30DAYS_MEMORY_DIR = "C:\Users\$env:USERNAME\Clients\$ClientSlug\Research\Last30Days"
|
||||
/last30days @Args --save-suffix=$ClientSlug
|
||||
}
|
||||
# Usage: Run-L30D-Client acme "british airways middle east"
|
||||
```
|
||||
|
||||
Bash example:
|
||||
|
||||
```bash
|
||||
l30d-client() {
|
||||
local client=$1; shift
|
||||
LAST30DAYS_MEMORY_DIR="$HOME/Clients/$client/Research/Last30Days" \
|
||||
/last30days "$@" --save-suffix="$client"
|
||||
}
|
||||
# Usage: l30d-client acme "british airways middle east"
|
||||
```
|
||||
|
||||
### 3. Custom category-peer subreddits
|
||||
|
||||
[`scripts/lib/categories.py`](skills/last30days/scripts/lib/categories.py) holds a table of `(category_id, trigger_keywords, peer_subreddits)`. If a client lives in a vertical that isn't covered (legal-tech, real-estate-tech, B2B HR SaaS), add a row. Pure data, no logic.
|
||||
|
||||
Section 2a of `SKILL.md` documents the merging rule the skill applies when your topic matches a category.
|
||||
|
||||
### 4. Pre-built `--competitors-plan` JSON
|
||||
|
||||
For competitor-vs-comparisons that recur, a pre-written JSON skeleton per client industry saves real time:
|
||||
|
||||
```json
|
||||
{
|
||||
"Competitor B": {
|
||||
"x_handle": "competitor_b_handle",
|
||||
"subreddits": ["sub1", "sub2"],
|
||||
"github_user": "competitor-b-org",
|
||||
"context": "Founded 2019, focused on ..."
|
||||
},
|
||||
"Competitor C": { ... }
|
||||
}
|
||||
```
|
||||
|
||||
Pass as `--competitors-plan @client/competitors-plan.json` (or as a string). See `SKILL.md` section "If QUERY_TYPE = COMPARISON" for the full schema.
|
||||
|
||||
---
|
||||
|
||||
## Beta channel
|
||||
|
||||
Experimental customizations live on a private companion repo (`mvanhorn/last30days-skill-private`) installed as `/last30days-beta`. Never ship beta-only changes to the public marketplace without a review PR against the public repo. Workflow guide: `BETA.md` in the private repo.
|
||||
|
||||
This is the right home for client-specific changes you don't intend to upstream - custom category rows, internal subreddit lists, per-vertical plan templates.
|
||||
|
||||
---
|
||||
|
||||
## Cross-references
|
||||
|
||||
- The CLI flag surface: `python3 scripts/last30days.py --help`
|
||||
- The skill contract (voice, LAWs, pre-flight protocol): [`skills/last30days/SKILL.md`](skills/last30days/SKILL.md)
|
||||
- Engine spec (some sections stale; SKILL.md wins on conflicts): [`SPEC.md`](SPEC.md)
|
||||
- Contributor guidance: [`CONTRIBUTORS.md`](CONTRIBUTORS.md)
|
||||
+1
-1
@@ -23,7 +23,7 @@ v3 has full GitHub search: issues, PRs, person-mode profiles, project-mode repos
|
||||
### @thinkun
|
||||
[PR #116](https://github.com/mvanhorn/last30days-skill/pull/116) - Resilient Reddit, prevent enrichment timeout from discarding results
|
||||
v3 has parallel enrichment with per-item timeouts. No results are ever dropped.
|
||||
> Thinker, technologist, AI expert, music-tinkerer. Founder of [Thinkun](https://thinkun.com). [@thinkun on GitHub](https://github.com/thinkun) · [@unthink on X](https://x.com/unthink)
|
||||
> _Add your bio, website, or anything you'd like here._
|
||||
|
||||
### @thomasmktong
|
||||
[PR #124](https://github.com/mvanhorn/last30days-skill/pull/124) - Pure Python Reddit fallback
|
||||
|
||||
+23
-13
@@ -10,20 +10,28 @@ This guide covers installing last30days on Hermes AI Agent.
|
||||
|
||||
## Installation
|
||||
|
||||
### Option 1: Via sync.sh (Recommended)
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
# Clone the repo
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
cd last30days-skill
|
||||
|
||||
# Run the sync script
|
||||
bash scripts/sync.sh
|
||||
```
|
||||
|
||||
This pulls the latest release from GitHub and deploys to `~/.hermes/skills/research/last30days/`. `--force` reinstalls over any existing copy.
|
||||
This will auto-detect Hermes and deploy to `~/.hermes/skills/research/last30days/`
|
||||
|
||||
### Developer / live-edit alternative
|
||||
|
||||
If you're hacking on the skill locally and want edits to propagate to Hermes without re-installing, symlink your working tree:
|
||||
### Option 2: Manual Copy
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
mkdir -p ~/.hermes/skills/research
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.hermes/skills/research/last30days
|
||||
# Create directory
|
||||
mkdir -p ~/.hermes/skills/research/last30days
|
||||
|
||||
# Copy files
|
||||
cp -r scripts ~/.hermes/skills/research/last30days/
|
||||
cp .hermes-plugin/SKILL.md ~/.hermes/skills/research/last30days/
|
||||
```
|
||||
|
||||
## Usage
|
||||
@@ -51,7 +59,7 @@ On first run, the skill will guide you through setup:
|
||||
|
||||
2. **Optional: ScrapeCreators**
|
||||
- Adds TikTok, Instagram, Reddit backup
|
||||
- 100 free credits (no expiration)
|
||||
- 10,000 free API calls
|
||||
- Sign up at scrapecreators.com
|
||||
|
||||
3. **Optional: API Keys**
|
||||
@@ -98,11 +106,13 @@ python3.12 scripts/last30days.py --diagnose
|
||||
|
||||
## Updating
|
||||
|
||||
```bash
|
||||
hermes skills install mvanhorn/last30days-skill --force
|
||||
```
|
||||
To update to the latest version:
|
||||
|
||||
If you symlinked your working tree (developer alternative above), just `git pull` in the repo — edits propagate live, no re-install step.
|
||||
```bash
|
||||
cd last30days-skill
|
||||
git pull
|
||||
bash scripts/sync.sh
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
|
||||
@@ -14,19 +14,21 @@
|
||||
|
||||
This README tracks the current v3 pipeline. The runtime skill spec lives in [skills/last30days/SKILL.md](skills/last30days/SKILL.md), which is the source of truth for the latest command and setup behavior.
|
||||
|
||||
**Claude Code (recommended — auto-updates via marketplace):**
|
||||
Claude Code:
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
/plugin install last30days
|
||||
```
|
||||
|
||||
**Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:**
|
||||
OpenClaw:
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
clawhub install last30days-official
|
||||
```
|
||||
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
|
||||
|
||||
More install options (claude.ai web, OpenClaw, manual) in the [Install](#install) section below.
|
||||
Hermes:
|
||||
```
|
||||
# The skill auto-deploys when you run sync.sh
|
||||
# Or manually copy to ~/.hermes/skills/research/last30days/
|
||||
```
|
||||
|
||||
Zero config. Reddit, HN, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
@@ -66,7 +68,6 @@ If you're meeting with a CEO, have you read all their tweets and YouTube transcr
|
||||
| **Hacker News** | The developer consensus. 825 points, 899 comments. Where technical people actually argue. |
|
||||
| **Polymarket** | Not opinions. Odds. Backed by real money. 96% confidence on album sales. 4% on an acquisition. |
|
||||
| **GitHub** | For people: PR velocity, top repos by stars, release notes. For topics: issues and discussions. |
|
||||
| **Digg** | Curated story clusters from Digg's AI 1000 leaderboard (~1000 high-signal AI accounts on X), with attributable inline quotes (no X auth required). Auto-enabled when `digg-pp-cli` is on PATH. |
|
||||
| **Threads** | The post-Twitter text layer. Conversations from creators and brands. |
|
||||
| **Pinterest** | Visual discovery. Pins, saves, and comments on products and ideas. |
|
||||
| **Bluesky** | The decentralized social layer. AT Protocol posts from the post-Twitter migration. |
|
||||
@@ -95,28 +96,6 @@ The synthesis ranks by what real people actually engaged with. Social relevancy,
|
||||
|
||||
## What v3 Changed
|
||||
|
||||
### Shareable HTML briefs
|
||||
|
||||
Ask for an HTML brief and the skill saves a self-contained, dark-mode, print-friendly file you can drop into Slack, email, or Notion. No raw markdown leaks. Inline CSS, system-font fallbacks behind Inter and JetBrains Mono. No JavaScript. Works offline.
|
||||
|
||||
```
|
||||
/last30days OpenClaw --emit=html
|
||||
```
|
||||
|
||||
or just ask in plain language:
|
||||
|
||||
```
|
||||
/last30days OpenClaw, give me a shareable HTML brief
|
||||
/last30days Cursor IDE for slack
|
||||
/last30days Anthropic earnings export as html
|
||||
```
|
||||
|
||||
The skill emits the synthesis in chat as usual AND saves a brief to `${LAST30DAYS_MEMORY_DIR}/{topic}-brief.html` (defaults to `~/Documents/Last30Days/`). The chat response ends with the file path so you can `open` it or drag it into a message.
|
||||
|
||||
What's in the file: badge, inline metadata line, the model's synthesis verbatim with all citations, the engine footer (✅ All agents reported back! tree), and a colophon noting the topic + how to re-run. Data quality warnings (degraded run, thin evidence, etc.) stay in the engine's stderr logs; they never leak into the shareable artifact.
|
||||
|
||||
For direct CLI use without the model in the loop, the engine also accepts `--synthesis-file PATH` to convert any markdown synthesis to HTML.
|
||||
|
||||
### Intelligent search: the killer feature
|
||||
|
||||
The v3 engine doesn't just search for your topic. It figures out *where* to search before the search begins. Type "OpenClaw" and the engine resolves @steipete (Peter Steinberger, the creator), r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags - all via a new Python pre-research brain built by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
@@ -135,10 +114,6 @@ When the same story appears on Reddit, X, and YouTube, v3 merges them into one c
|
||||
|
||||
"CLI vs MCP" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides simultaneously. Same depth, 3 minutes.
|
||||
|
||||
### Auto-discovered competitor comparisons
|
||||
|
||||
`/last30days OpenAI --competitors` tells the hosting reasoning model to discover the top 2 peers via WebSearch (Anthropic, xAI), run Step 0.55 per entity, and invoke the engine with `"OpenAI vs Anthropic vs xAI"` and a per-entity `--competitors-plan` JSON. The engine fans out 3 full pipelines in parallel, saves a `*-raw.md` file per entity, and merges them into a 3-way comparison. Same mechanics power `/last30days "OpenAI vs Anthropic vs xAI"` directly.
|
||||
|
||||
### GitHub person-mode
|
||||
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. Instead of "who mentioned this name in an issue body," it answers: what are they shipping and where is it landing?
|
||||
@@ -153,10 +128,8 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
|
||||
- **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works.
|
||||
- **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions.
|
||||
- **TikTok, Instagram, Threads.** All three activate automatically once `SCRAPECREATORS_API_KEY` is set — same key, same per-call cost. Suppress any of them with `EXCLUDE_SOURCES=tiktok,instagram,threads` (any comma-separated subset).
|
||||
- **Pinterest.** Per-query opt-in (visual pins, narrow utility): the model passes `--search=pinterest` for the runs that need it. Requires `SCRAPECREATORS_API_KEY`.
|
||||
- **YouTube + TikTok comments.** Persistent opt-in via `INCLUDE_SOURCES=youtube_comments,tiktok_comments` because each video pulls N extra ScrapeCreators calls on top of the base search. Surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` and `INCLUDE_SOURCES=perplexity` (it's a separate paid API — opt-in keeps you from being surprise-billed).
|
||||
- **Threads, Pinterest, YouTube + TikTok comments.** Opt-in sources via ScrapeCreators. Set `INCLUDE_SOURCES=tiktok,instagram` and add threads, pinterest, youtube_comments, tiktok_comments for more. `youtube_comments` and `tiktok_comments` surface top comments with vote counts the same way Reddit does.
|
||||
- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` to unlock.
|
||||
- **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?"
|
||||
- **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run.
|
||||
- **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section.
|
||||
@@ -168,61 +141,12 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language.
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install | Updates |
|
||||
|---------|---------|---------|
|
||||
| **Claude Code** (recommended) | `/plugin marketplace add mvanhorn/last30days-skill` | Auto via marketplace, or `claude plugin update last30days@last30days-skill` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, GitHub Copilot, or any of 50+ [Agent Skills](https://agentskills.io) hosts** | `npx skills add mvanhorn/last30days-skill -g` | `npx skills update last30days -g` |
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + | Re-download and re-upload |
|
||||
| **OpenClaw** | `clawhub install last30days-official` | `clawhub update last30days-official` |
|
||||
|
||||
### Claude Code (recommended)
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Recommended because the Claude Code marketplace handles updates for you — the plugin cache is versioned and auto-refreshes when a new release publishes. Run `claude plugin update last30days@last30days-skill` to force a check.
|
||||
|
||||
If you'd rather use the agent-skills install path on Claude Code, that's also supported:
|
||||
|
||||
```
|
||||
npx skills add mvanhorn/last30days-skill -g -a claude-code
|
||||
```
|
||||
|
||||
The native plugin and the `npx skills` install can coexist. Note that Claude Code does not dedupe across install methods: if you have both the marketplace plugin and the `npx skills` copy active, `/last30days` will show two entries. Use one install method per machine.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI, and other Agent Skills hosts
|
||||
|
||||
Install via the open [Agent Skills](https://agentskills.io) CLI — supports 50+ harnesses including `codex`, `cursor`, `github-copilot`, `gemini-cli`, `claude-code`, `windsurf`, `cline`, `continue`, `roo`, `aider-desk`, `opencode`, `goose`, and more (full list on the [vercel-labs/skills repo](https://github.com/vercel-labs/skills)).
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g
|
||||
```
|
||||
|
||||
The `-g` (global) flag installs to your user directory so the skill is available across all projects. Without `-g`, `npx skills` installs project-locally into `./.skills/` (committed with the repo). For a research-the-world tool, global is what you want.
|
||||
|
||||
By default this installs for whichever harness `npx skills` detects. To target a specific one (or multiple):
|
||||
|
||||
```bash
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex
|
||||
npx skills add mvanhorn/last30days-skill -g -a cursor
|
||||
npx skills add mvanhorn/last30days-skill -g -a gemini-cli
|
||||
npx skills add mvanhorn/last30days-skill -g -a codex -a cursor
|
||||
```
|
||||
|
||||
Update later with:
|
||||
|
||||
```bash
|
||||
npx skills update last30days -g
|
||||
```
|
||||
|
||||
Or update everything you've installed globally via `npx skills`:
|
||||
|
||||
```bash
|
||||
npx skills update -g
|
||||
```
|
||||
|
||||
List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| **claude.ai** (web) | [Download `last30days.skill`](https://github.com/mvanhorn/last30days-skill/releases/latest/download/last30days.skill) and upload via Settings > Capabilities > Skills > + |
|
||||
| **Claude Code** | `/plugin marketplace add mvanhorn/last30days-skill` |
|
||||
| **OpenClaw** | `clawhub install last30days-official` |
|
||||
| **Gemini CLI** | Clone then `gemini extensions install ./last30days-skill` (see below) |
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
@@ -230,7 +154,15 @@ List and remove with `npx skills list -g` and `npx skills remove last30days -g`.
|
||||
2. Go to [claude.ai Settings > Capabilities > Skills](https://claude.ai/settings/capabilities)
|
||||
3. Click the `+` button in the Skills panel and drop the file in
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — skills won't run without it.
|
||||
Enable "Code execution and file creation" under Capabilities first - skills won't run without it.
|
||||
|
||||
### Claude Code
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
Update later with `claude plugin update last30days@last30days-skill`.
|
||||
|
||||
### OpenClaw
|
||||
|
||||
@@ -238,14 +170,22 @@ Enable "Code execution and file creation" under Capabilities first — skills wo
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
### Gemini CLI
|
||||
|
||||
Gemini CLI v0.9.0 has an upstream installer bug that can fail with `Configuration file not found at /tmp/gemini-extensionXXXXXX/gemini-extension.json` ([upstream issue](https://github.com/google-gemini/gemini-cli/issues/11452)). Workaround:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill
|
||||
gemini extensions install ./last30days-skill
|
||||
```
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git
|
||||
ln -s "$(pwd)/last30days-skill/skills/last30days" ~/.claude/skills/last30days
|
||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||
```
|
||||
|
||||
The symlink keeps the install in sync with your working tree as you edit — no re-copy needed. For `claude.ai`, build the `.skill` file from source: `bash skills/last30days/scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
Or build the claude.ai `.skill` file from source: `bash scripts/build-skill.sh` produces `dist/last30days.skill`.
|
||||
|
||||
Reddit (with comments), Hacker News, Polymarket, and GitHub work immediately. Zero configuration. Run `/last30days` once and the setup wizard unlocks more sources in 30 seconds.
|
||||
|
||||
@@ -259,40 +199,10 @@ These platforms don't have relationships with each other. X doesn't know what Re
|
||||
| X / Twitter | Log into x.com in any browser | Free |
|
||||
| YouTube | `brew install yt-dlp` | Free |
|
||||
| Bluesky | App password from bsky.app | Free |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 100 free credits, then PAYG |
|
||||
| TikTok + Instagram + Threads + Pinterest + YouTube comments | ScrapeCreators key | 10,000 free calls |
|
||||
| Perplexity Sonar | OpenRouter key | Pay as you go |
|
||||
| Web search | Brave Search key | 2,000 free queries/month |
|
||||
|
||||
### macOS Keychain (optional)
|
||||
|
||||
On macOS you can store keys in the system Keychain instead of a `.env` file. The skill picks them up automatically as the lowest-priority source — `.env` files and process environment still win on collision.
|
||||
|
||||
```bash
|
||||
# Interactive setup — prompts for each known key, skip with empty input
|
||||
skills/last30days/scripts/setup-keychain.sh
|
||||
|
||||
# Or store a single key by hand
|
||||
security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."
|
||||
|
||||
# Inspect / clean up
|
||||
skills/last30days/scripts/setup-keychain.sh --list
|
||||
skills/last30days/scripts/setup-keychain.sh --delete XAI_API_KEY
|
||||
```
|
||||
|
||||
Items are stored under service name `last30days-<KEY>` for the current user. On non-Darwin platforms the loader is a no-op, so there is no behaviour change for Linux/Windows users.
|
||||
|
||||
See [CONFIGURATION.md](CONFIGURATION.md) for the full per-source key matrix, reasoning provider priority, and web-search backend priority.
|
||||
|
||||
## Configuration
|
||||
|
||||
Two things you'll likely want to know on day one:
|
||||
|
||||
**Where research files are saved.** `LAST30DAYS_MEMORY_DIR` defaults to `~/Documents/Last30Days/` (Windows: `C:\Users\<you>\Documents\Last30Days\`). Override by setting that env var to any path in your shell, or `--save-dir <path>` per run. Use `--save-suffix=<name>` to keep multiple variations of the same topic separate (e.g. per client). Each run produces `<slug>-raw[-suffix].md`.
|
||||
|
||||
**Trend monitoring across runs.** The default mode produces a fresh markdown snapshot per run. To accumulate findings over time, add `--store` to persist into a SQLite database, then use [`scripts/watchlist.py`](skills/last30days/scripts/watchlist.py) for scheduled runs (with optional Slack / webhook delivery on new findings) and [`scripts/briefing.py`](skills/last30days/scripts/briefing.py) for daily / weekly digests. The full cadence pattern is in [CONFIGURATION.md](CONFIGURATION.md#trend-monitoring-store--watchlist--briefings).
|
||||
|
||||
Per-client wrapper scripts, custom category-peer subreddits, and the experimental beta channel for in-progress customizations are also documented in [CONFIGURATION.md](CONFIGURATION.md).
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You type a topic.** Person, company, product, technology, "X vs Y." Anything.
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
context: fork
|
||||
agent: Explore
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: last30days
|
||||
version: "3.3.2"
|
||||
version: "3.0.1"
|
||||
description: "Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web."
|
||||
argument-hint: 'last30days nvidia earnings reaction | last30days AI video tools | last30days what users want in react'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
@@ -13,9 +13,9 @@ metadata:
|
||||
openclaw:
|
||||
emoji: "📰"
|
||||
requires:
|
||||
env: []
|
||||
optionalEnv:
|
||||
env:
|
||||
- SCRAPECREATORS_API_KEY
|
||||
optionalEnv:
|
||||
- OPENAI_API_KEY
|
||||
- XAI_API_KEY
|
||||
- OPENROUTER_API_KEY
|
||||
@@ -45,7 +45,6 @@ metadata:
|
||||
- instagram
|
||||
- hackernews
|
||||
- polymarket
|
||||
- digg
|
||||
- bluesky
|
||||
- truthsocial
|
||||
- trends
|
||||
@@ -60,35 +59,6 @@ metadata:
|
||||
- clawhub
|
||||
---
|
||||
|
||||
# STEP 0: STALE-CLONE SELF-CHECK — RUN BEFORE READING BELOW
|
||||
|
||||
Before reading anything else in this file, check whether you loaded SKILL.md from the one known stale-clone location: Claude Code's marketplaces directory.
|
||||
|
||||
**Run this check:**
|
||||
|
||||
```bash
|
||||
CLAUDE_CACHE_LATEST=$(find "$HOME/.claude/plugins/cache/last30days-skill/last30days" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -1)
|
||||
# Two cache layouts ship in the wild — nested ({version}/skills/last30days/SKILL.md)
|
||||
# and flat ({version}/SKILL.md). Resolve to whichever shape actually exists.
|
||||
CLAUDE_CACHE_SKILL_MD=""
|
||||
if [ -n "$CLAUDE_CACHE_LATEST" ]; then
|
||||
if [ -f "$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md" ]; then
|
||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/skills/last30days/SKILL.md"
|
||||
elif [ -f "$CLAUDE_CACHE_LATEST/SKILL.md" ]; then
|
||||
CLAUDE_CACHE_SKILL_MD="$CLAUDE_CACHE_LATEST/SKILL.md"
|
||||
fi
|
||||
fi
|
||||
echo "CLAUDE_CACHE_SKILL_MD=$CLAUDE_CACHE_SKILL_MD"
|
||||
```
|
||||
|
||||
If the SKILL.md path you just Read contains `/.claude/plugins/marketplaces/` AND `$CLAUDE_CACHE_SKILL_MD` is non-empty, STOP and re-read `$CLAUDE_CACHE_SKILL_MD` before proceeding. Otherwise the SKILL.md you have is fine — continue.
|
||||
|
||||
**Why this specific check:** `~/.claude/plugins/marketplaces/last30days-skill/` is a git clone Claude Code auto-restores to `origin/main` on session start. It can lag the versioned cache by one or more releases. Three 2026-04-22 test runs (Linear, Coinbase) loaded SKILL.md from `marketplaces/`, ran `--help` from the same stale path, did not see the `--competitors` flag that existed in the cache, and fell back to a manual comparison plan. Result: 2 of 3 windows never invoked the feature they were asked to test. STEP 0 defends against that one Claude Code-specific bug.
|
||||
|
||||
**Other install paths are fine:** `~/.codex/skills/`, `~/.agents/skills/`, an `npx skills add` install dir, or a repo checkout are all valid load points - the resolver in Step 1 picks them up. Do NOT abort or hop on those paths.
|
||||
|
||||
---
|
||||
|
||||
# SKILL CONTRACT — READ BEFORE ANY TOOL CALL
|
||||
|
||||
You are inside the `/last30days` SKILL. This is a specific research tool with a 1400+ line instruction contract (the rest of this file) that defines EXACTLY how to produce the research output. It is not a generic "last 30 days of X" research prompt. Do NOT treat `/last30days` as a search keyword you can improvise against.
|
||||
@@ -97,7 +67,7 @@ You are inside the `/last30days` SKILL. This is a specific research tool with a
|
||||
|
||||
**How v3.0.7 fixes it:** three structural anchors.
|
||||
1. **The MANDATORY first-line badge** (`🌐 last30days v{VERSION} · synced {YYYY-MM-DD}`) at the top of every response is the LAW 2 / LAW 4 enforcement anchor. See "BADGE (MANDATORY, FIRST LINE OF OUTPUT)" in the synthesis section.
|
||||
2. **The SKILL_DIR substitution** in the engine Bash calls uses the directory of the SKILL.md the model just Read — no resolver list, no precedence walk. Whichever install the harness loaded SKILL.md from is the install whose engine runs. Aligns spec-with-code and works for any harness without enumerating its install path.
|
||||
2. **The pinned SKILL_ROOT resolution** in the engine Bash calls always points to the public plugin cache, never `~/.openclaw/` or other stale copies.
|
||||
3. **This preface** tells you plainly: do NOT improvise. Follow SKILL.md top to bottom.
|
||||
|
||||
If you catch yourself about to write a `##` section header in a GENERAL-query body, a custom title line, a `Sources:` bullet list, a `for dir in ...` path-discovery loop, or a bare `python3 scripts/last30days.py "{TOPIC}"` engine call with no pre-flight flags — stop. Those are the exact failure modes the LAWs and this contract exist to prevent. The 10/10 beta validation from 2026-04-18 and the 0/8 public v3.0.6 regression from the same day had THE SAME MODEL and SIMILAR SKILL.md CONTENT; the delta is the three anchors this release restores. Read SKILL.md top to bottom before emitting your first response.
|
||||
@@ -114,13 +84,13 @@ These anchors used to live at line 1094 of this file. Three independent Opus 4.7
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
```
|
||||
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_DIR/../../.claude-plugin/plugin.json" 2>/dev/null || awk '/^version:/{gsub(/"/,"",$2); print $2; exit}' "$SKILL_DIR/SKILL.md"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
Replace `{VERSION}` with the installed plugin version (`jq -r '.version' "$SKILL_ROOT/.claude-plugin/plugin.json"`) and `{YYYY-MM-DD}` with today's date. No other text on this line. One blank line after, then the synthesis begins.
|
||||
|
||||
**Why the badge is MANDATORY:** it is the structural anchor for the canonical output shape. Without it the model drifts into blog-post narrative format with `##` section headers and invented titles, violating LAW 2 and LAW 4. The 2026-04-18 public v3.0.6 0/8 regression produced outputs with section headers like "The headline", "Why he is everywhere", "1. gstack dominates", "The 'Homecoming' peak". Direct cause: this anchor was absent. Do NOT skip the badge. Do NOT describe it. Do NOT paraphrase it. Emit it verbatim as line 1.
|
||||
|
||||
**Placement by query type:**
|
||||
- GENERAL / NEWS / PROMPTING / RECOMMENDATIONS: badge on line 1, blank line 2, `What I learned:` on line 3, then bold-lead-in paragraphs
|
||||
- COMPARISON: badge on line 1, blank line 2, `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (/Last30Days)` on line 3, then Quick Verdict section
|
||||
- COMPARISON: badge on line 1, blank line 2, `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (Last 30 Days)` on line 3, then Quick Verdict section
|
||||
|
||||
---
|
||||
|
||||
@@ -138,7 +108,7 @@ These LAWs dominate every other rule in this file. If you find yourself about to
|
||||
|
||||
**LAW 2 - NO INVENTED TITLE LINE (with COMPARISON exception).** For QUERY_TYPE GENERAL, NEWS, PROMPTING, RECOMMENDATIONS: the first line of your synthesis body (after the badge and one blank line) is the prose label `What I learned:` on its own line. Not `What I learned about {Topic}`, not `{Topic} - Last 30 Days`, not `{Topic}: What People Are Saying`, not `# {Topic}`, not `The headline`, not `Why he is everywhere this month`. Nothing above `What I learned:` except the badge. If you are tempted to write a title or a `##`-prefixed section name, the rule is: the badge IS the title, and section headers are forbidden (see LAW 4).
|
||||
|
||||
**COMPARISON exception:** For QUERY_TYPE=COMPARISON (topics containing `vs` or `versus`), the title `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (/Last30Days)` is REQUIRED, not a violation. Comparison queries do NOT use the `What I learned:` prose label at all.
|
||||
**COMPARISON exception:** For QUERY_TYPE=COMPARISON (topics containing `vs` or `versus`), the title `# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (Last 30 Days)` is REQUIRED, not a violation. Comparison queries do NOT use the `What I learned:` prose label at all.
|
||||
|
||||
**Global-preference override:** The skill-authored template for GENERAL / NEWS / PROMPTING / RECOMMENDATIONS queries uses `**bold**` for KEY PATTERNS items and for mid-paragraph lead-ins. Do NOT strip this bold on the grounds of a personal "no bold" memory. The skill's voice contract is the formatting authority here.
|
||||
|
||||
@@ -186,13 +156,13 @@ The self-evolving loop is the sticky use case. Every 15 tool calls Hermes pauses
|
||||
Cron-scheduled autonomous briefings are the most-cited concrete workflow. r/TunisiaTech's "Use cases of OpenClaw, Hermes Agent" thread says it plainly: "Currently I have daily cron jobs for news briefing, but I know there's much more I can do."
|
||||
```
|
||||
|
||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan "$QUERY_PLAN_FILE"` (the path to a tmpfile you wrote via heredoc — see Step 1 for the pattern; never inline `--plan '$JSON'`, apostrophes in search/ranking strings break shell parsing).
|
||||
**LAW 7 - YOU ARE THE PLANNER. `--plan` IS MANDATORY ON NAMED-ENTITY TOPICS.** If you are the reasoning model hosting this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime that invoked `/last30days`), YOU generate the JSON query plan. You do not need an API key, "LLM provider" credentials, or an external planning service - you ARE the LLM. The `--plan` flag exists precisely so a reasoning model generates its own plan upstream and passes it to the engine. The engine's internal planner and deterministic fallback are headless/cron paths only; on any reasoning-model path, bypass them by passing `--plan '$JSON'`.
|
||||
|
||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan "$QUERY_PLAN_FILE"` (or any path the engine can read). A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||
Named-entity topics (capitalized proper nouns, product names, person names, project names, or any topic that would benefit from handle resolution in Step 0.55) REQUIRE `--plan`. Your invocation of `scripts/last30days.py` MUST contain `--plan '$JSON'`. A bare `python3 scripts/last30days.py "$TOPIC" --emit=compact` on a named-entity topic is a LAW 7 violation. Before you invoke Bash, self-check: does my command contain `--plan`? If no, STOP and generate a plan first (see Step 0.75 for the schema).
|
||||
|
||||
**Observed LAW 7 violation (2026-04-19, Hermes Agent Use Cases Run 1):** the model called the engine bare with no `--plan`, no pre-flight handle resolution. The engine emitted a stderr warning ("No --plan and no LLM provider configured. Using deterministic fallback...") which the model read as a capability constraint ("I don't have a key, I can't do LLM stuff") instead of as what it actually was: a reminder that the reasoning model skipped its own planning step. The misread came from the word "provider" - the engine uses "provider" to mean "the key for the engine's INTERNAL planner," but the model parsed it as "I need a provider to plan at all." You do not. You ARE the provider. Run 2 of the same topic (2026-04-19, framed as "best workflows") with the same model and same cache generated the plan itself via `--plan` and produced clean results - the delta was this step.
|
||||
|
||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan "$QUERY_PLAN_FILE"` (or another path the engine can read)? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan, then write it to a tmpfile per the Step 1 pattern. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||
**Self-check before Bash:** re-read your pending `scripts/last30days.py` command. Does it contain `--plan '$JSON'`? If no, and the topic is a named entity, STOP. Return to Step 0.75 and generate the plan. Do not interpret the word "provider" in any engine message as "you need credentials" - you are the provider.
|
||||
|
||||
**LAW 8 - EVERY CITATION IN THE NARRATIVE IS AN INLINE MARKDOWN LINK `[name](url)`. NEVER A RAW URL STRING. NEVER A PLAIN NAME WHEN A URL IS AVAILABLE.** Applies to every query type. In the "What I learned:" narrative, in KEY PATTERNS, and in the COMPARISON body sections, every cited @handle, r/subreddit, publication, YouTube channel, TikTok creator, Instagram creator, and Polymarket market is wrapped as `[name](url)` at first mention. The URL comes from the raw research dump — every engine item carries a URL; WebSearch supplements carry URLs in their own output. Claude Code renders `[text](url)` as blue CMD-clickable text; the URL is hidden in the rendering, only the link text shows. The stats footer (emoji-tree block) is engine-emitted per LAW 5 and passes through verbatim — do NOT reformat its links yourself.
|
||||
|
||||
@@ -243,9 +213,9 @@ If your Bash call to `last30days.py` does NOT include the FULL pre-flight checkl
|
||||
|
||||
---
|
||||
|
||||
# last30days v3.3.2: Research Any Topic from the Last 30 Days
|
||||
# last30days v3.0.1: Research Any Topic from the Last 30 Days
|
||||
|
||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||
|
||||
Research ANY topic across Reddit, X, YouTube, and other sources. Surface what people are actually discussing, recommending, betting on, and debating right now.
|
||||
|
||||
@@ -265,14 +235,8 @@ if [ -z "${LAST30DAYS_PYTHON:-}" ]; then
|
||||
echo "ERROR: last30days v3 requires Python 3.12+. Install python3.12 or python3.13 and rerun." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LAST30DAYS_MEMORY_DIR="${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}"
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Set `LAST30DAYS_MEMORY_DIR` before invoking the skill to choose where raw research files are saved. If it is not set, the skill defaults to `~/Documents/Last30Days`.
|
||||
|
||||
## Step 0: First-Run Setup Wizard
|
||||
|
||||
Before proceeding to Step 1, handle first-run setup.
|
||||
@@ -327,14 +291,14 @@ Common patterns:
|
||||
|
||||
- Always active: Reddit, Hacker News, Polymarket
|
||||
- If gh CLI is installed (check `which gh`): add GitHub
|
||||
- If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg
|
||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X
|
||||
- If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set: add X
|
||||
- If yt-dlp is installed (check `which yt-dlp`): add YouTube
|
||||
- If SCRAPECREATORS_API_KEY is set: add TikTok, Instagram, Threads (suppress any of these via EXCLUDE_SOURCES)
|
||||
- If SCRAPECREATORS_API_KEY is set and the user explicitly requested pinterest for this query (e.g. via `--search=pinterest`): add Pinterest
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains threads: add Threads
|
||||
- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest
|
||||
- If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky
|
||||
- If OPENROUTER_API_KEY is set and INCLUDE_SOURCES contains perplexity: add Perplexity
|
||||
- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying
|
||||
- If OPENROUTER_API_KEY is set: add Perplexity
|
||||
|
||||
Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma):
|
||||
|
||||
@@ -558,7 +522,7 @@ If `--agent` appears in ARGUMENTS (e.g., `/last30days plaud granola --agent`):
|
||||
5. **Skip** the follow-up invitation ("I'm now an expert on X...")
|
||||
6. **Output** the complete research report and stop - do not wait for further input
|
||||
|
||||
Agent mode saves raw research data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) automatically via `--save-dir` (handled by the script, no extra tool calls).
|
||||
Agent mode saves raw research data to `~/Documents/Last30Days/` automatically via `--save-dir` (handled by the script, no extra tool calls).
|
||||
|
||||
Agent mode report format:
|
||||
|
||||
@@ -580,95 +544,28 @@ Generated: {date} | Sources: Reddit, X, Bluesky, YouTube, TikTok, HN, Polymarket
|
||||
|
||||
## If QUERY_TYPE = COMPARISON
|
||||
|
||||
When the user asks "X vs Y" (or "X vs Y vs Z"), the engine fans out N full `pipeline.run()` calls in parallel — one per entity — each with its own Step 0.55-grade targeting. This restored the old N-pass architecture (reverted the one-pass latency optimization that removed per-entity depth); parallel execution keeps wall clock ≈ a single pass.
|
||||
When the user asks "X vs Y", run ONE research pass with a comparison-optimized plan that covers both entities AND their rivalry. This replaces the old 3-pass approach (which took 13+ minutes and produced tangential content).
|
||||
|
||||
**MANDATORY per-entity resolution.** For each entity, resolve the full Step 0.55 stack (X handle, subreddits, GitHub user/repos, news context). Then assemble a `--competitors-plan` JSON mapping each entity to its targeting, and invoke the engine ONCE with the vs-topic string.
|
||||
**IMPORTANT: Include BOTH X handles (`--x-handle={TOPIC_A_HANDLE} --x-related={TOPIC_B_HANDLE},{COMPANY_HANDLES},{COMMENTATOR_HANDLES}`), `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||
|
||||
**Output shape per run:**
|
||||
- Main topic saves to `{main-slug}-raw.md`.
|
||||
- Each peer saves to `{peer-slug}-raw.md`.
|
||||
- Stdout shows a merged comparison with the `## Head-to-Head` scaffold + per-entity Resolved Entities block.
|
||||
|
||||
**Invocation:**
|
||||
**Single pass with entity-aware subqueries:**
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Write the per-entity plan to a tmpfile and pass the path to the engine.
|
||||
# The engine's parse_competitors_plan() reads file paths transparently. This
|
||||
# avoids the inline-single-quoted-JSON apostrophe trap (resolved context
|
||||
# strings like "people's choice" or "McDonald's" otherwise close the outer
|
||||
# single-quote and break shell parsing before the engine is even invoked).
|
||||
# Trailing XXXXXX (no .json suffix) so BSD/macOS mktemp works the same as
|
||||
# GNU; BSD only substitutes X's at the end of the template.
|
||||
COMPETITORS_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-competitors.XXXXXX")
|
||||
trap 'rm -f "$COMPETITORS_PLAN_FILE"' EXIT
|
||||
cat > "$COMPETITORS_PLAN_FILE" <<'PLAN_EOF'
|
||||
{
|
||||
"{TOPIC_B}": {"x_handle":"{TOPIC_B_HANDLE}","subreddits":["{TOPIC_B_SUB_1}","{TOPIC_B_SUB_2}"],"github_user":"{TOPIC_B_GH}","context":"{TOPIC_B_CONTEXT}"},
|
||||
"{TOPIC_C}": {"x_handle":"{TOPIC_C_HANDLE}","subreddits":["{TOPIC_C_SUB_1}"],"github_user":"{TOPIC_C_GH}","context":"{TOPIC_C_CONTEXT}"}
|
||||
}
|
||||
PLAN_EOF
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B} vs {TOPIC_C}" \
|
||||
--emit=compact \
|
||||
--save-dir="${LAST30DAYS_MEMORY_DIR}" \
|
||||
--save-suffix=v3 \
|
||||
--x-handle={TOPIC_A_HANDLE} \
|
||||
--subreddits={TOPIC_A_SUBS} \
|
||||
--competitors-plan "$COMPETITORS_PLAN_FILE"
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" "{TOPIC_A} vs {TOPIC_B}" --emit=compact --save-dir=~/Documents/Last30Days --save-suffix=v3 --plan 'COMPARISON_PLAN_JSON' --x-handle={TOPIC_A_HANDLE} --x-related={TOPIC_B_HANDLE},{COMPANY_A_HANDLE},{COMPANY_B_HANDLE},{COMMENTATOR_HANDLES} --subreddits={RESOLVED_SUBREDDITS} --tiktok-hashtags={RESOLVED_HASHTAGS} --tiktok-creators={RESOLVED_TIKTOK_CREATORS} --ig-creators={RESOLVED_IG_CREATORS}
|
||||
```
|
||||
|
||||
**The quoted heredoc marker `'PLAN_EOF'` is load-bearing** — quoting suppresses shell interpolation so apostrophes, `$`, backticks, etc. pass through verbatim. If you ever switch to an unquoted `<<PLAN_EOF`, every variable reference and apostrophe inside the JSON becomes a parse hazard.
|
||||
**The `--plan` JSON for comparisons should include 3-4 subqueries:**
|
||||
1. **Head-to-head:** `"{TOPIC_A} vs {TOPIC_B}"` - catches rivalry content, direct comparisons
|
||||
2. **Entity A news:** `"{TOPIC_A} news {MONTH} {YEAR}"` - catches entity-specific developments
|
||||
3. **Entity B news:** `"{TOPIC_B} news {MONTH} {YEAR}"` - catches entity-specific developments
|
||||
4. (Optional) **Domain context:** `"{COMPANY_A} {COMPANY_B} {DOMAIN} news"` - catches industry context (e.g., "OpenAI Anthropic AI news")
|
||||
|
||||
Topic A (the main topic, first in the vs-string) uses outer `--x-handle`, `--x-related`, `--subreddits`, `--github-user`, `--github-repo`, `--tiktok-*`, `--ig-creators` as usual. Topics B and C get their targeting from `--competitors-plan` entries (keyed by entity name, case-insensitive).
|
||||
ALL subqueries include ALL sources. The fusion engine handles deduplication across subqueries. **At least one subquery MUST include YouTube-specific search terms** (e.g., "{PERSON} interview 2026", "{PRODUCT_A} vs {PRODUCT_B} review") to ensure YouTube content is found. Without YouTube-specific terms, the engine may only find 0-1 videos for comparison queries.
|
||||
|
||||
**Step 0.55 for N entities.** The same pre-research protocol that applies to a single-entity topic applies to EACH entity in a vs-run. For N=3, that means 3 WebSearches for X handles, 3 for subreddits, 3 for GitHub, 3 for news context — or equivalent batched queries. A `## Resolved Entities` block with dashes for any entity means you skipped Step 0.55 for that one. Re-run with a corrected plan.
|
||||
|
||||
**Then do WebSearch supplements** for: `{TOPIC_A} vs {TOPIC_B} comparison {YEAR}` and `{TOPIC_A} vs {TOPIC_B} which is better` — these catch rivalry articles that per-entity passes might not surface.
|
||||
|
||||
**Use `RESOLVED_POSITIONING` per entity (Step 0.55 item 6) in two ways.** First, ground each entity's `What it is` cell in its CURRENT fetched pitch - describe the entity as it pitches itself today, never from memory. Second, if an entity's month of evidence directly bears on its pitch - SUPPORTS a specific claim, CUTS AGAINST one, or the conversation is squarely ABOUT the pitched ground - say so in ONE prose sentence inside that entity's section of the comparison synthesis (right after the Community Sentiment line - the template marks the slot), anchored to the real item with its engagement. When the pulse is orthogonal to the pitch (on-entity but about something the pitch doesn't speak to), say NOTHING about the pitch: omission is the correct output, and a manufactured connection is worse than silence. Match altitude: test SPECIFIC claims ("zero-config", "fastest", an uptime number) against specific threads; never grade a broad tagline ("financial infrastructure") against an individual thread - it is too broad to hit or miss. Keep claims windowed - "this month's conversation" - never trend verbs like "losing the narrative" that one 30-day window cannot support. If positioning was not actually fetched this run for an entity, skip both uses for that entity - never supply a pitch from memory.
|
||||
Then do WebSearch for: `{TOPIC_A} vs {TOPIC_B} comparison {YEAR}` and `{TOPIC_A} vs {TOPIC_B} which is better` and `{COMPANY_A} vs {COMPANY_B} news {MONTH} {YEAR}`.
|
||||
|
||||
**Skip the normal Step 1 below** - go directly to the comparison synthesis format (see "If QUERY_TYPE = COMPARISON" in the synthesis section).
|
||||
|
||||
**COMPARISON TABLE SCAFFOLD (engine-emitted, pass through verbatim):** For comparison topics, the engine's compact output includes a `## Head-to-Head` block with an empty markdown table (columns = entities, rows = axes like "What it is", "Philosophy", "Best for"). Your synthesis MUST include this block verbatim with filled cells, positioned between the narrative and the emoji-tree footer. Keep each cell to 5-15 words. Use ' - ' (hyphen with spaces) not em-dashes inside cells.
|
||||
|
||||
### Competitor mode (`--competitors`)
|
||||
|
||||
`--competitors` is a SKILL.md-level shortcut for vs-mode with auto-discovery. The engine flag itself just signals intent; YOU (the hosting reasoning model) do the discovery and Step 0.55 via your own WebSearch tool, then invoke the vs-topic path above.
|
||||
|
||||
**The four-step protocol:**
|
||||
1. **Discover peers** via WebSearch: `"{topic} competitors"` / `"{topic} alternatives"`. Pick N=2 by default (match the flag's default), N=argument value if the user passed `--competitors=N`.
|
||||
2. **Run Step 0.55 for the main topic AND each peer** — same protocol you use for a single-entity topic, just N times. X handle, subreddits, GitHub, news context, per entity.
|
||||
3. **Build the vs-topic string**: `"{main} vs {peer1} vs {peer2}"`.
|
||||
4. **Invoke the engine** with the vs-topic, `--competitors-plan` JSON covering both peers (and the main topic if you want to override the outer flags), and the outer `--x-handle`/`--subreddits`/`--github-*` for the main topic.
|
||||
|
||||
**Flag surface (engine):**
|
||||
- `--competitors` (bare) - signals the hosting model to discover 2 peers (3-way total).
|
||||
- `--competitors=N` - N peers (1..6; out-of-range clamps with stderr warning).
|
||||
- `--competitors-list="A,B,C"` - minimum escape hatch; names only, no per-entity targeting. Peer sub-runs fall back to planner defaults (visibly thinner data).
|
||||
- `--competitors-plan '{entity: {x_handle, subreddits, github_user, github_repos, context}}'` - full per-entity targeting; implies vs-mode; preferred.
|
||||
- `--polymarket-keywords "kw1,kw2"` - disambiguate Polymarket for ambiguous single-token topics ("Warriors" → `nba,gsw,golden-state`).
|
||||
|
||||
**Why --competitors-plan over --competitors-list:** without per-entity handles/subs, peer sub-runs run with deterministic single-word planner queries and produce visibly thinner evidence than the main topic. The Resolved Entities block in stdout makes the gap visible — dashes for a peer = you skipped its Step 0.55.
|
||||
|
||||
**Engine-internal auto-resolve (headless fallback):** if the engine detects BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / PARALLEL_API_KEY / OPENROUTER_API_KEY, it runs its own per-entity `resolve.auto_resolve()` before each sub-run. The hosting-model path does NOT need those keys — you are the WebSearch. The engine's auto-resolve is the cron/CI fallback for when no reasoning model is driving.
|
||||
|
||||
**Output:** one `{slug}-raw.md` per entity in `--save-dir` plus the merged comparison on stdout. Synthesis contract identical to the vs-mode protocol above.
|
||||
**COMPARISON TABLE SCAFFOLD (engine-emitted, pass through verbatim):** For comparison topics, the engine's compact output includes a `## Head-to-Head Comparison` block with an empty markdown table (columns = entities, rows = axes like "Core pitch", "Who it's for", "Community stance", "Trajectory") plus a "Choose X if / Choose Y if" prose block. Your synthesis MUST include this block verbatim with filled cells, positioned between the narrative and the emoji-tree footer. Keep each cell to 5-15 words. Use ' - ' (hyphen with spaces) not em-dashes inside cells. The block is the canonical comparison output shape - do not invent your own table structure.
|
||||
|
||||
---
|
||||
|
||||
@@ -693,43 +590,6 @@ The first search finds subreddits. The second gives you current events context (
|
||||
|
||||
Extract 3-5 subreddit names from the results. Store as `RESOLVED_SUBREDDITS` (comma-separated, no r/ prefix).
|
||||
|
||||
**2a. Category-peer expansion (MANDATORY for product topics).** If the topic is a product in a recognizable category (AI image generation, AI video generation, AI coding agents, AI music, AI chat models, SaaS screen recording, prediction markets, etc.), the brand-specific subreddits that WebSearch returned are INSUFFICIENT. Add 2-3 peer subreddits from the category. Peer subs are where cross-product technique discussion actually lives. Missing them is the 2026-04-22 `GPT Image 2` failure mode: the model resolved `r/OpenAI, r/ChatGPT, r/singularity, r/ChatGPTpromptengineering` (all OpenAI-brand) and missed `r/StableDiffusion, r/midjourney, r/dalle2, r/aiArt` where prompting techniques are actually shared. The user had to manually prompt "check image generation reddits too" to get a usable run.
|
||||
|
||||
Canonical category peers (single source of truth; `scripts/lib/categories.py` mirrors this for the `--auto-resolve` engine path):
|
||||
|
||||
| Category | Trigger keywords | Peer subs (priority order) |
|
||||
|----------|------------------|---------------------------|
|
||||
| `ai_image_generation` | image generation, text to image, GPT Image, Nano Banana, Midjourney, Stable Diffusion, DALL-E, Flux.1, Imagen, Seedance, Ideogram, Recraft | `StableDiffusion, midjourney, dalle2, aiArt, PromptEngineering, MediaSynthesis` |
|
||||
| `ai_video_generation` | video generation, text to video, Sora, Veo 3, Runway Gen, Kling, Pika Labs, Luma Dream Machine, Hailuo | `aivideo, StableDiffusion, runwayml, singularity, MediaSynthesis` |
|
||||
| `ai_music_generation` | music generation, ai music, Suno, Udio, Riffusion, Stable Audio | `SunoAI, udiomusic, aimusic, artificial` |
|
||||
| `ai_coding_agent` | Claude Code, Cursor IDE, GitHub Copilot, Windsurf, Aider, Cline, OpenClaw, Hermes Agent, Continue.dev, Codeium, Devin | `ChatGPTCoding, LocalLLaMA, singularity, PromptEngineering` |
|
||||
| `ai_agent_framework` | agent framework, LangChain, LangGraph, CrewAI, AutoGen, LlamaIndex, DSPy, smolagents | `LangChain, LocalLLaMA, AI_Agents, MachineLearning` |
|
||||
| `ai_chat_model` | GPT-5/4, Claude Opus/Sonnet/Haiku, Gemini Pro/Flash, Llama 3/4, DeepSeek, Qwen, Mistral Large, Grok | `LocalLLaMA, ChatGPT, ClaudeAI, singularity, artificial` |
|
||||
| `saas_screen_recording` | screen recording, screen recorder, Loom video, Tella screen, Vidyard | `SaaS, screenrecording, productivity, Entrepreneur` |
|
||||
| `saas_productivity` | Notion app, Obsidian, Linear app, Asana, ClickUp, productivity app | `productivity, SaaS, ObsidianMD, Notion` |
|
||||
| `prediction_markets` | Polymarket, Kalshi, prediction market, event contracts, Manifold Markets | `Polymarket, Kalshi, predictionmarkets` |
|
||||
| `crypto_defi` | DeFi protocol, yield farming, liquidity pool, stablecoin, layer 2, L2 rollup | `defi, ethfinance, CryptoCurrency, ethereum` |
|
||||
|
||||
**Merging rule.** Start with WebSearch-returned subs. Append 2-3 category peers in the priority order shown. Dedupe case-insensitively (don't list `midjourney` twice if WebSearch already returned it). Cap total at 10: if adding all peers would exceed the cap, keep every WebSearch-returned sub (they are the freshest signal) and drop peers from the end of the priority list.
|
||||
|
||||
**Extrapolation.** If the topic is a product in a category NOT listed in the table (new AI tool, niche SaaS), use the same spirit: pick the 2-3 most active cross-product communities where technique discussion happens. A new image-gen tool still gets `r/StableDiffusion, r/midjourney, r/aiArt`. A new code editor still gets `r/ChatGPTCoding, r/LocalLLaMA`.
|
||||
|
||||
**Worked example — the failing query.** Topic: `Prompting GPT Image 2`.
|
||||
|
||||
Before (the 2026-04-22 failure mode):
|
||||
```
|
||||
Resolved:
|
||||
- Reddit: r/OpenAI, r/ChatGPT, r/singularity, r/ChatGPTpromptengineering, r/artificial
|
||||
```
|
||||
|
||||
After (with category-peer expansion):
|
||||
```
|
||||
Resolved:
|
||||
- Reddit: r/OpenAI, r/ChatGPT, r/singularity, r/ChatGPTpromptengineering, r/StableDiffusion, r/midjourney, r/dalle2, r/aiArt (+ ai_image_generation peers)
|
||||
```
|
||||
|
||||
The parenthetical `(+ ai_image_generation peers)` is the observable contract of the new Resolved block format. See Step 0.55 self-check below.
|
||||
|
||||
**3. TikTok hashtags + creators** - **INFER these from your topic knowledge. Do NOT WebSearch for "{PERSON} TikTok account" - most people/CEOs don't have TikTok, and the search is wasted.**
|
||||
|
||||
- **Hashtags:** Infer 2-3 from the topic name + category. Examples: "Kanye West" → `kanyewest,ye,bully`. "Claude Code" → `claudecode,aiagent,aicoding`. "Sam Altman" → `samaltman,openai,chatgpt`.
|
||||
@@ -750,8 +610,6 @@ Store as `RESOLVED_IG_CREATORS`.
|
||||
|
||||
Store as `RESOLVED_YT_QUERIES`.
|
||||
|
||||
**6. First-party positioning** - **MANDATORY when WebSearch is available, for company / product / service topics.** If the topic (or, in a vs-run, an entity) is a company, product, or service with a public presence, fetch its CURRENT stated positioning. Do **NOT** rely on memory - homepages and positioning go stale as companies rewrite copy and pivot, and a stale claim produces a false gap. Anchor on first-party sources: the homepage tagline, docs, pricing, or a "compare/why-us" page. Fold this into the per-entity passes above where you can (e.g. add `official site` to a query); otherwise run one focused search per entity (`{TOPIC} official site`, `{TOPIC} pricing`). Capture the one-line value prop and any explicit claims ("zero-config", "fastest", "open source"). Store as `RESOLVED_POSITIONING`. This is what the entity *pitches*; the engine's community data is what people *actually talk about*. Use it three ways: ground `What it is` descriptions (describe the entity as it pitches itself TODAY, not as remembered), help reject unrelated brand-name noise (knowing what the entity is makes off-brand matches obvious), and feed the pitch-vs-pulse synthesis beat - a PROSE note that fires only when the month's evidence directly supports, cuts against, or is squarely about the pitch (see the synthesis section; orthogonal evidence gets silence, not a verdict). Skip (and omit `RESOLVED_POSITIONING`) for people, events, abstract concepts, and ownerless topics - they make no comparable public claim. The test is an identifiable first party with a fetchable pitch, and people NEVER pass it - not even founders/creators whose companies would qualify. The lens can apply to MrBeast (a company) but never to Jimmy Donaldson (a person); a person-vs-person run ("Garry Tan vs Sam Altman") gets no positioning research at all. Ownerless topics fail the same test: Bitcoin has no authoritative first party, and a foundation or fan site does not count.
|
||||
|
||||
**Concrete examples:**
|
||||
|
||||
| Topic | WebSearches needed | Reddit subs | TikTok hashtags | TikTok creators | IG creators | YT queries |
|
||||
@@ -794,20 +652,17 @@ Passing the resolved block visibly (per-entity, all 4 types each) is the observa
|
||||
|
||||
**If you can't infer targeting for a platform, skip that flag -- the Python engine will fall back to keyword search.**
|
||||
|
||||
**Step 0.55 self-check: category-peer coverage.** Before emitting the Resolved block, re-read your resolved subreddit list. Does the topic match any category in the Section 2a table (or fit the spirit of one — AI image gen, AI coding, AI music, etc.)? If YES: does your list include AT LEAST 2 peer subs from that category? If NO, widen the list NOW — do not run the engine yet. The observable contract is the `(+ {category_id} peers)` annotation on the Reddit line in the Resolved block. Its absence on a product-in-a-known-category topic is a Step 0.55 regression — the named 2026-04-22 failure mode. Person topics, music artists, news stories, and topics outside any category are exempt; omit the annotation.
|
||||
|
||||
**After resolving all handles and communities, display what you found before moving on.** This shows the user that intelligent pre-research happened:
|
||||
|
||||
```
|
||||
Resolved:
|
||||
- X: @{HANDLE} (+ @{COMPANY}, @{COMMENTATOR})
|
||||
- Reddit: r/{sub1}, r/{sub2}, r/{sub3}, r/{peer1}, r/{peer2} (+ {category_id} peers)
|
||||
- Reddit: r/{sub1}, r/{sub2}, r/{sub3}
|
||||
- TikTok: #{hashtag1}, #{hashtag2}
|
||||
- YouTube: {query1}, {query2}
|
||||
- Positioning: "{one-line stated value prop}" (first-party)
|
||||
```
|
||||
|
||||
Only show lines for platforms where something was resolved. Skip empty lines. On the Reddit line, the trailing `(+ {category_id} peers)` annotation appears when Step 0.55 Section 2a added category-peer subs. Omit the annotation when the topic had no matching category. The `Positioning:` line appears for company / product / service topics (from Step 0.55 item 6); omit it for people, events, abstract concepts, and ownerless topics. This display replaces the old "Parsed intent" block with something more useful.
|
||||
Only show lines for platforms where something was resolved. Skip empty lines. This display replaces the old "Parsed intent" block with something more useful.
|
||||
|
||||
---
|
||||
|
||||
@@ -871,7 +726,7 @@ Only show lines for platforms where something was resolved. Skip empty lines. On
|
||||
- For how_to: prioritize YouTube (tutorials) and Reddit (guides)
|
||||
- Primary subquery weight = 1.0, secondary = 0.6-0.8, peripheral = 0.3-0.5
|
||||
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key), digg (Digg clusters - only if `digg-pp-cli` is on PATH)
|
||||
**Available sources (include ALL in primary subquery):** reddit, x, youtube, tiktok, instagram, hackernews, polymarket. Optional: bluesky, truthsocial, threads, pinterest, grounding (web search - only if user has Brave/Exa/Serper key)
|
||||
|
||||
**Intent → freshness_mode mapping:**
|
||||
- breaking_news, prediction → `strict_recent`
|
||||
@@ -914,45 +769,34 @@ Store your plan as `QUERY_PLAN_JSON` - you'll pass it to the script in the next
|
||||
**IMPORTANT: Include `--x-handle={RESOLVED_HANDLE}` in the command. For comparison mode: Pass `--x-handle={TOPIC_A_HANDLE}` to the first pass, `--x-handle={TOPIC_B_HANDLE}` to the second pass, and both to the head-to-head pass. Also include `--subreddits={RESOLVED_SUBREDDITS}`, `--tiktok-hashtags={RESOLVED_HASHTAGS}`, `--tiktok-creators={RESOLVED_TIKTOK_CREATORS}`, and `--ig-creators={RESOLVED_IG_CREATORS}` from Step 0.55. Omit any flag where the value was not resolved (empty).**
|
||||
|
||||
```bash
|
||||
# SKILL_DIR = absolute path of the directory containing THIS SKILL.md you just Read.
|
||||
# Substitute the actual path below — your harness told you where this file lives via
|
||||
# the Read tool result. Examples:
|
||||
# Read ~/.claude/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.claude/skills/last30days
|
||||
# Read ~/.codex/skills/last30days/SKILL.md → SKILL_DIR=$HOME/.codex/skills/last30days
|
||||
# Read ~/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days/SKILL.md
|
||||
# → SKILL_DIR=$HOME/.claude/plugins/cache/last30days-skill/last30days/3.3.2/skills/last30days
|
||||
# scripts/last30days.py is always a direct child of SKILL_DIR (every install layout
|
||||
# packages SKILL.md and scripts/ as siblings).
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
# PIN SKILL_ROOT to the public plugin cache (highest-version dir wins on upgrade).
|
||||
# DO NOT write your own path-discovery loop. The 2026-04-18 Peter Steinberger run 1
|
||||
# regression was caused by a custom discovery loop landing on ~/.openclaw/skills/last30days/
|
||||
# (a stale copy from a private-repo sync pattern). That path contains a pre-plan-007
|
||||
# engine and produces non-canonical output. This pinned resolution ignores every stale
|
||||
# copy (~/.openclaw/, ~/.agents/, ~/.codex/) and picks the plugin cache exclusively.
|
||||
SKILL_ROOT="$(ls -d "$HOME/.claude/plugins/cache/last30days-skill/last30days/"*/ 2>/dev/null | sort -V | tail -1)"
|
||||
SKILL_ROOT="${SKILL_ROOT%/}"
|
||||
|
||||
if [ ! -f "$SKILL_DIR/scripts/last30days.py" ]; then
|
||||
echo "ERROR: scripts/last30days.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR above." >&2
|
||||
# Fallback for repo checkout / Gemini / Codex hosts where the plugin cache does not exist.
|
||||
# Only runs if the public plugin cache is missing entirely.
|
||||
if [ -z "$SKILL_ROOT" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
for dir in "." "${CLAUDE_PLUGIN_ROOT:-}" "${GEMINI_EXTENSION_DIR:-}"; do
|
||||
[ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "${SKILL_ROOT:-}" ] || [ ! -f "$SKILL_ROOT/scripts/last30days.py" ]; then
|
||||
echo "ERROR: Could not find scripts/last30days.py in public plugin cache or repo checkout" >&2
|
||||
echo "Expected: $HOME/.claude/plugins/cache/last30days-skill/last30days/{VERSION}/scripts/last30days.py" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_DIR}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir="${LAST30DAYS_MEMORY_DIR}" --save-suffix=v3
|
||||
"${LAST30DAYS_PYTHON}" "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact --save-dir=~/Documents/Last30Days --save-suffix=v3
|
||||
```
|
||||
|
||||
**If you ran Steps 0.55 and 0.75 (agent planning), pass the plan via a tmpfile and add the targeting flags:**
|
||||
|
||||
```bash
|
||||
# Write QUERY_PLAN_JSON to a tmpfile before the engine invocation above.
|
||||
# parse_plan() reads file paths transparently; this avoids inline-JSON
|
||||
# shell-quoting hazards (apostrophes in search_query / ranking_query
|
||||
# strings break single-quoted command-line JSON). Trailing XXXXXX (no
|
||||
# .json suffix) for BSD/macOS portability — BSD mktemp only substitutes
|
||||
# X's at the end of the template.
|
||||
QUERY_PLAN_FILE=$(mktemp "${TMPDIR:-/tmp}/last30days-plan.XXXXXX")
|
||||
trap 'rm -f "$QUERY_PLAN_FILE"' EXIT
|
||||
cat > "$QUERY_PLAN_FILE" <<'PLAN_EOF'
|
||||
{QUERY_PLAN_JSON_FROM_STEP_0.75}
|
||||
PLAN_EOF
|
||||
```
|
||||
|
||||
Then add to the engine command:
|
||||
|
||||
- `--plan "$QUERY_PLAN_FILE"` (path to the file you just wrote)
|
||||
**If you ran Steps 0.55 and 0.75 (agent planning), add these flags:**
|
||||
- `--plan 'QUERY_PLAN_JSON'` (replace with actual JSON from Step 0.75)
|
||||
- `--x-handle={RESOLVED_HANDLE}` (from Step 0.5)
|
||||
- `--subreddits={RESOLVED_SUBREDDITS}` (from Step 0.55)
|
||||
- `--tiktok-hashtags={RESOLVED_HASHTAGS}` (from Step 0.55)
|
||||
@@ -1042,7 +886,7 @@ For ALL query types:
|
||||
|
||||
## Step 2.5: Append WebSearch Results to Saved Raw File
|
||||
|
||||
**MANDATORY - do not skip this step.** Every post-engine WebSearch supplement you ran in Step 2 MUST be appended to the saved raw file under `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`). Skipping this step is a common Opus 4.7 failure mode: the saved file ends at `## Source Coverage` with no appendix, future sessions cannot see what blog/tutorial/news sources informed the synthesis, and the user cannot trace where specific claims came from.
|
||||
**MANDATORY - do not skip this step.** Every post-engine WebSearch supplement you ran in Step 2 MUST be appended to the saved raw file under `~/Documents/Last30Days/`. Skipping this step is a common Opus 4.7 failure mode: the saved file ends at `## Source Coverage` with no appendix, future sessions cannot see what blog/tutorial/news sources informed the synthesis, and the user cannot trace where specific claims came from.
|
||||
|
||||
**LAW 1 OVERRIDE (read before synthesizing):** the WebSearch tool description declares a "MANDATORY Sources section" in its own contract. That instruction applies to generic WebSearch usage. Inside `/last30days` it is SUPERSEDED. The `## WebSearch Supplemental Results` appendix in the SAVED RAW FILE replaces the visible Sources section. Never emit a visible `Sources:` bullet list to the user. Your user-facing response ends at the invitation block. The emoji-tree footer's `🌐 Web:` line is the only visible citation. If you feel the pull to write a trailing `Sources:` section, you are about to violate LAW 1 — go back and delete it.
|
||||
|
||||
@@ -1263,7 +1107,7 @@ Voice contract LAWs 1, 3, 5 apply to comparisons unchanged (no `Sources:` block,
|
||||
```
|
||||
🌐 last30days v{VERSION} · synced {YYYY-MM-DD}
|
||||
|
||||
# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (/Last30Days)
|
||||
# {TOPIC_A} vs {TOPIC_B} [vs {TOPIC_C}]: What the Community Says (Last 30 Days)
|
||||
|
||||
## Quick Verdict
|
||||
|
||||
@@ -1273,8 +1117,6 @@ Voice contract LAWs 1, 3, 5 apply to comparisons unchanged (no `Sources:` block,
|
||||
|
||||
**Community Sentiment:** [Positive / Mixed / Negative / Enthusiastic / Security-concerned / etc.] ({N}+ mentions across {source list})
|
||||
|
||||
[Optional pitch-vs-pulse sentence - ONLY if `RESOLVED_POSITIONING` was captured for this entity AND the month's evidence directly supports a specific claim, cuts against one, or is squarely about the pitched ground: one windowed prose sentence anchored to a real item with engagement. Otherwise omit entirely - silence, not a placeholder.]
|
||||
|
||||
**Strengths (what people love)**
|
||||
- [Specific strength with `per <source>` attribution]
|
||||
- [Specific strength with `per <source>` attribution]
|
||||
@@ -1306,7 +1148,7 @@ Voice contract LAWs 1, 3, 5 apply to comparisons unchanged (no `Sources:` block,
|
||||
| Best for | ... | ... | ... |
|
||||
| Install | ... | ... | ... |
|
||||
|
||||
(Engine emits this scaffold; fill the cells with 5-15 words each. If an axis does not apply to the topic class, write "N/A" or a topic-appropriate substitute rather than inventing data. Ground the `What it is` row in `RESOLVED_POSITIONING` when captured - each entity described as it pitches itself today, fetched this run, never from memory.)
|
||||
(Engine emits this scaffold; fill the cells with 5-15 words each. If an axis does not apply to the topic class, write "N/A" or a topic-appropriate substitute rather than inventing data.)
|
||||
|
||||
## The Bottom Line
|
||||
|
||||
@@ -1341,7 +1183,7 @@ I've compared {TOPIC_A} vs {TOPIC_B} [vs ...] using the latest community data. S
|
||||
- Fabricate a `## Notable Stats` block (the engine footer IS the stats block, LAW 5)
|
||||
- Produce section headers outside the six listed above (`## Quick Verdict`, `## {Entity}` per entity, `## Head-to-Head`, `## The Bottom Line`, `## The emerging stack` are the only allowed `##` headers per LAW 4 comparison exception)
|
||||
|
||||
**Reference exemplar:** `$LAST30DAYS_MEMORY_DIR/openclaw-vs-hermes-vs-paperclip-LAUNCH-VIDEO-april9-exemplar.md` preserves the April 9 canonical output with full structural analysis. Match this shape section-for-section.
|
||||
**Reference exemplar:** `~/Documents/Last30Days/openclaw-vs-hermes-vs-paperclip-LAUNCH-VIDEO-april9-exemplar.md` preserves the April 9 canonical output with full structural analysis. Match this shape section-for-section.
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
@@ -1448,8 +1290,6 @@ At render time the `@handle`, `r/sub`, and publication-name placeholders become
|
||||
|
||||
Headlines should be specific and newsy ("BULLY dropped and it's dominating", "Europe is banning him one country at a time"), not generic ("Album release", "Tour updates").
|
||||
|
||||
**Pitch-vs-pulse beat (company / product / service topics).** If you captured `RESOLVED_POSITIONING` in Step 0.55 AND the month's evidence directly bears on it, work in ONE bold-lead-in paragraph saying how. Three cases qualify: the pulse SUPPORTS a specific claim (e.g. `**"Zero-config" is holding up** - this month's top deploy thread is devs praising the no-setup flow, 800 upvotes`), CUTS AGAINST one (e.g. `**Stripe's fraud-fighting pitch took a direct hit** - the loudest thread this month argues it is friendly to "friendly fraud", 323pt HN`), or the conversation is squarely ABOUT the pitched ground. Always anchor to the real top item with its engagement, and keep claims windowed - "this month's conversation" - never trend verbs like "losing the narrative" that one 30-day window cannot support. If the month's conversation is orthogonal to the pitch - on-entity but about something the pitch doesn't speak to - write NOTHING about the pitch: omission is the correct output, and a manufactured connection is worse than silence. Match altitude: test SPECIFIC claims ("zero-config", "fastest", an uptime number) against specific threads; never grade a broad tagline against an individual thread. Keep it a normal newsy bold-lead-in paragraph, NOT a new `##` section (LAW 4 still holds). Skip silently for people (always - the beat can cover MrBeast the company, never Jimmy Donaldson the person), events, abstract concepts, and ownerless topics (Bitcoin), and whenever positioning was not actually fetched this run - never supply a pitch from memory.
|
||||
|
||||
**THEN - Quality Nudge (if present in the output):**
|
||||
|
||||
If the research output contains a `**🔍 Research Coverage:**` block, render it verbatim right before the stats block. This tells the user which core sources are missing and how to unlock them. Do NOT render this block if it is absent from the output (100% coverage = no nudge).
|
||||
@@ -1465,7 +1305,7 @@ Options:
|
||||
|
||||
**THEN - Engine footer pass-through (right before invitation):**
|
||||
|
||||
**The research output ENDS with a deterministic footer block bracketed by `---` lines, starting with `✅ All agents reported back!` and ending with `📎 Raw results saved to {resolved LAST30DAYS_MEMORY_DIR}/<slug>-raw.md`. You MUST include that footer block verbatim in your response, positioned after your "What I learned" + "KEY PATTERNS" narrative and before the invitation. Do not recompute the stats. Do not reformat the tree. Do not paraphrase. Do not skip it. Do not add your own source lines. Copy the exact bytes.**
|
||||
**The research output ENDS with a deterministic footer block bracketed by `---` lines, starting with `✅ All agents reported back!` and ending with `📎 Raw results saved to ~/Documents/Last30Days/<slug>-raw.md`. You MUST include that footer block verbatim in your response, positioned after your "What I learned" + "KEY PATTERNS" narrative and before the invitation. Do not recompute the stats. Do not reformat the tree. Do not paraphrase. Do not skip it. Do not add your own source lines. Copy the exact bytes.**
|
||||
|
||||
- The engine already omits zero-count sources. You do not need to filter them.
|
||||
- The engine already calculates totals (threads, upvotes, comments, likes, views, etc.). You do not need to add them up.
|
||||
@@ -1559,36 +1399,9 @@ Close with `I have all the links to the {N} {source list} I pulled from. Just as
|
||||
|
||||
---
|
||||
|
||||
## SHAREABLE HTML BRIEF (when the user asked for one)
|
||||
|
||||
**This section fires if EITHER trigger is true:**
|
||||
|
||||
- `$ARGUMENTS` contains `--emit=html`, `--emit:html`, or `--html` as a flag
|
||||
- The user's natural-language request asks for an HTML brief, shareable doc, or file for sharing (Slack, email, Notion, "export as HTML", etc). Use your judgment for phrasing variants.
|
||||
|
||||
**If neither trigger fires, skip this entire section and proceed to WAIT FOR USER'S RESPONSE.** No HTML save flow, no reference read needed.
|
||||
|
||||
**When triggered, you MUST:**
|
||||
|
||||
- Read `references/save-html-brief.md` BEFORE proceeding to WAIT FOR USER'S RESPONSE
|
||||
- Follow that file's instructions exactly - it is the canonical source for the save flow
|
||||
- Append the confirmation line (`📎 Shareable brief saved to <path>`) to your already-emitted chat response
|
||||
|
||||
**You MUST NOT:**
|
||||
|
||||
- Improvise the HTML save flow from memory or from instructions you've seen before
|
||||
- Skip the reference read because the steps "look familiar"
|
||||
- Save to a different path than the reference specifies
|
||||
- Add data quality warnings, debug headers, or safety notes to the saved HTML
|
||||
- Re-research the topic for the HTML render - the engine cache covers the second invocation
|
||||
|
||||
**Why the directive is forceful:** the reference file is the only source of truth for the save flow. Skipping it produces broken artifacts - wrong path conventions, missing synthesis content, leaked engine debug output, or warnings that don't belong in shareable docs.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S RESPONSE
|
||||
|
||||
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`) via `--save-dir`.
|
||||
**STOP and wait** for the user to respond. Do NOT call any tools after displaying the invitation. Do NOT append a `Sources:` section (see override above - WebSearch's mandate does not apply here). The research script already saved raw data to `~/Documents/Last30Days/` via `--save-dir`.
|
||||
|
||||
---
|
||||
|
||||
@@ -1693,15 +1506,15 @@ Want another prompt? Just tell me what you're creating next.
|
||||
**What this skill does:**
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, and as a Reddit backup when public Reddit is unavailable (requires SCRAPECREATORS_API_KEY)
|
||||
- Legacy: Sends search queries to OpenAI's Responses API (`api.openai.com`) for Reddit discovery (fallback if no SCRAPECREATORS_API_KEY)
|
||||
- Sends search queries to Twitter's GraphQL API (via optional user-provided AUTH_TOKEN/CT0 env vars - no browser session access), xAI's API (`api.x.ai`), or the official X API v2 via xurl CLI (OAuth2, auto-detected when installed and authenticated) for X search
|
||||
- Sends search queries to Twitter's GraphQL API (via optional user-provided AUTH_TOKEN/CT0 env vars - no browser session access) or xAI's API (`api.x.ai`) for X search
|
||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 100 free credits)
|
||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (PAYG after 10,000 free API calls)
|
||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||
- Stores research findings in local SQLite database (watchlist mode only)
|
||||
- Saves research briefings as .md files to `LAST30DAYS_MEMORY_DIR` (defaults to `~/Documents/Last30Days`)
|
||||
- Saves research briefings as .md files to ~/Documents/Last30Days/
|
||||
|
||||
**What this skill does NOT do:**
|
||||
- Does not post, like, or modify content on any platform
|
||||
@@ -1710,7 +1523,7 @@ Want another prompt? Just tell me what you're creating next.
|
||||
- Does not log, cache, or write API keys to output files
|
||||
- Does not send data to any endpoint not listed above
|
||||
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (100 free credits one-time, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (10,000 free API calls, then PAYG). Reddit uses ScrapeCreators only as a backup when public Reddit is unavailable.
|
||||
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
||||
|
||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||
@@ -0,0 +1,77 @@
|
||||
# last30days Skill Specification
|
||||
|
||||
## Overview
|
||||
|
||||
`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. OpenAI auth can come from `OPENAI_API_KEY` or Codex login credentials.
|
||||
|
||||
The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability.
|
||||
|
||||
## Architecture
|
||||
|
||||
The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`:
|
||||
|
||||
- **env.py**: Load API keys from `~/.config/last30days/.env` and Codex auth from `~/.codex/auth.json`
|
||||
- **dates.py**: Date range calculation and confidence scoring
|
||||
- **cache.py**: 24-hour TTL caching keyed by topic + date range
|
||||
- **http.py**: stdlib-only HTTP client with retry logic
|
||||
- **models.py**: Auto-selection of OpenAI/xAI models with 7-day caching
|
||||
- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit
|
||||
- **xai_x.py**: xAI Responses API + x_search for X
|
||||
- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics
|
||||
- **hackernews.py**: Hacker News search via Algolia API (free, no auth)
|
||||
- **polymarket.py**: Polymarket prediction market search via Gamma API (free, no auth)
|
||||
- **normalize.py**: Convert raw API responses to canonical schema
|
||||
- **score.py**: Compute popularity-aware scores (relevance + recency + engagement)
|
||||
- **dedupe.py**: Near-duplicate detection via text similarity
|
||||
- **render.py**: Generate markdown and JSON outputs
|
||||
- **schema.py**: Type definitions and validation
|
||||
|
||||
## Embedding in Other Skills
|
||||
|
||||
Other skills can import the research context in several ways:
|
||||
|
||||
### Inline Context Injection
|
||||
```markdown
|
||||
## Recent Research Context
|
||||
!python3 ~/.claude/skills/last30days/scripts/last30days.py "your topic" --emit=context
|
||||
```
|
||||
|
||||
### Read from File
|
||||
```markdown
|
||||
## Research Context
|
||||
!cat ~/.local/share/last30days/out/last30days.context.md
|
||||
```
|
||||
|
||||
### Get Path for Dynamic Loading
|
||||
```bash
|
||||
CONTEXT_PATH=$(python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=path)
|
||||
cat "$CONTEXT_PATH"
|
||||
```
|
||||
|
||||
### JSON for Programmatic Use
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=json > research.json
|
||||
```
|
||||
|
||||
## CLI Reference
|
||||
|
||||
```
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py <topic> [options]
|
||||
|
||||
Options:
|
||||
--refresh Bypass cache and fetch fresh data
|
||||
--mock Use fixtures instead of real API calls
|
||||
--emit=MODE Output mode: compact|json|md|context|path (default: compact)
|
||||
--sources=MODE Source selection: auto|reddit|x|both (default: auto)
|
||||
```
|
||||
|
||||
## Output Files
|
||||
|
||||
All outputs are written to `~/.local/share/last30days/out/`:
|
||||
|
||||
- `report.md` - Human-readable full report
|
||||
- `report.json` - Normalized data with scores
|
||||
- `last30days.context.md` - Compact reusable snippet for other skills
|
||||
- `raw_openai.json` - Raw OpenAI API response
|
||||
- `raw_xai.json` - Raw xAI API response
|
||||
- `raw_reddit_threads_enriched.json` - Enriched Reddit thread data
|
||||
@@ -0,0 +1,47 @@
|
||||
# last30days Implementation Tasks
|
||||
|
||||
## Setup & Configuration
|
||||
- [x] Create directory structure
|
||||
- [x] Write SPEC.md
|
||||
- [x] Write TASKS.md
|
||||
- [x] Write SKILL.md with proper frontmatter
|
||||
|
||||
## Core Library Modules
|
||||
- [x] scripts/lib/env.py - Environment and API key loading
|
||||
- [x] scripts/lib/dates.py - Date range and confidence utilities
|
||||
- [x] scripts/lib/cache.py - TTL-based caching
|
||||
- [x] scripts/lib/http.py - HTTP client with retry
|
||||
- [x] scripts/lib/models.py - Auto model selection
|
||||
- [x] scripts/lib/schema.py - Data structures
|
||||
- [x] scripts/lib/openai_reddit.py - OpenAI Responses API
|
||||
- [x] scripts/lib/xai_x.py - xAI Responses API
|
||||
- [x] scripts/lib/reddit_enrich.py - Reddit thread JSON fetcher
|
||||
- [x] scripts/lib/normalize.py - Schema normalization
|
||||
- [x] scripts/lib/score.py - Popularity scoring
|
||||
- [x] scripts/lib/dedupe.py - Near-duplicate detection
|
||||
- [x] scripts/lib/render.py - Output rendering
|
||||
|
||||
## Main Script
|
||||
- [x] scripts/last30days.py - CLI orchestrator
|
||||
|
||||
## Fixtures
|
||||
- [x] fixtures/openai_sample.json
|
||||
- [x] fixtures/xai_sample.json
|
||||
- [x] fixtures/reddit_thread_sample.json
|
||||
- [x] fixtures/models_openai_sample.json
|
||||
- [x] fixtures/models_xai_sample.json
|
||||
|
||||
## Tests
|
||||
- [x] tests/test_dates.py
|
||||
- [x] tests/test_cache.py
|
||||
- [x] tests/test_models.py
|
||||
- [x] tests/test_score.py
|
||||
- [x] tests/test_dedupe.py
|
||||
- [x] tests/test_normalize.py
|
||||
- [x] tests/test_render.py
|
||||
|
||||
## Validation
|
||||
- [x] Run tests in mock mode
|
||||
- [x] Demo --emit=compact
|
||||
- [x] Demo --emit=context
|
||||
- [x] Verify file tree
|
||||
|
Before Width: | Height: | Size: 2.7 MiB After Width: | Height: | Size: 2.7 MiB |
|
Before Width: | Height: | Size: 2.3 MiB After Width: | Height: | Size: 2.3 MiB |
|
Before Width: | Height: | Size: 3.8 MiB After Width: | Height: | Size: 3.8 MiB |
|
Before Width: | Height: | Size: 2.6 MiB After Width: | Height: | Size: 2.6 MiB |
@@ -0,0 +1,9 @@
|
||||
---
|
||||
description: Research what people actually say about any topic in the last 30 days across Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.
|
||||
argument-hint: <topic> — e.g. "nvidia earnings reaction" or "best noise cancelling headphones"
|
||||
allowed-tools: [Bash, Read, Write, AskUserQuestion, WebSearch]
|
||||
---
|
||||
|
||||
Invoke the `last30days` skill with the user's arguments: $ARGUMENTS
|
||||
|
||||
Use the skill's canonical pipeline (plan → retrieve → normalize → fuse → rerank → cluster → render). If the user provided no arguments, ask them for a topic before proceeding.
|
||||
+11
-12
@@ -142,7 +142,7 @@ The repo vendors a search-only subset of Bird's Twitter GraphQL client and shell
|
||||
| Likes/reposts | Real (X API) | Real (x_search tool) |
|
||||
| Replies/quotes | Real | Real |
|
||||
| Author handle | Real | Real |
|
||||
| Relevance score | Default 0.7 (re-ranked by relevance.py) | AI-assessed 0.0-1.0 |
|
||||
| Relevance score | Default 0.7 (re-ranked by score.py) | AI-assessed 0.0-1.0 |
|
||||
|
||||
### Depth settings
|
||||
|
||||
@@ -183,14 +183,13 @@ After both searches complete:
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `skills/last30days/scripts/last30days.py` | Main CLI entry point |
|
||||
| `skills/last30days/scripts/lib/pipeline.py` | Multi-source retrieval orchestration |
|
||||
| `skills/last30days/scripts/lib/reddit_public.py` | Reddit public JSON search |
|
||||
| `skills/last30days/scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `skills/last30days/scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `skills/last30days/scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `skills/last30days/scripts/lib/providers.py` | Reasoning provider and model selection |
|
||||
| `skills/last30days/scripts/lib/env.py` | API key loading, source detection |
|
||||
| `skills/last30days/scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `skills/last30days/scripts/lib/relevance.py` | Query matching and relevance scoring |
|
||||
| `skills/last30days/scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
| `scripts/last30days.py` | Main orchestrator, concurrent execution |
|
||||
| `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API |
|
||||
| `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API |
|
||||
| `scripts/lib/xai_x.py` | X search via xAI API |
|
||||
| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) |
|
||||
| `scripts/lib/models.py` | Auto-select best available model |
|
||||
| `scripts/lib/env.py` | API key loading, source detection |
|
||||
| `scripts/lib/http.py` | HTTP transport with retries |
|
||||
| `scripts/lib/score.py` | Relevance scoring |
|
||||
| `scripts/lib/dedupe.py` | URL-based deduplication |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Search Quality Eval
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
`scripts/evaluate_search_quality.py` is an optional local evaluation step for retrieval quality. It is not part of the user-facing runtime and does not need to run in CI by default.
|
||||
|
||||
What it does:
|
||||
|
||||
@@ -18,13 +18,13 @@ What it does:
|
||||
Recommended usage:
|
||||
|
||||
```bash
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py
|
||||
uv run python scripts/evaluate_search_quality.py
|
||||
```
|
||||
|
||||
Useful flags:
|
||||
|
||||
```bash
|
||||
uv run python skills/last30days/scripts/evaluate_search_quality.py \
|
||||
uv run python scripts/evaluate_search_quality.py \
|
||||
--baseline-rev origin/main \
|
||||
--candidate-rev HEAD \
|
||||
--no-default-topics \
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
title: Search-quality eval is manual by default, not a CI gate on every PR
|
||||
date: 2026-05-10
|
||||
category: docs/solutions/architecture
|
||||
module: skills/last30days/scripts/evaluate_search_quality.py
|
||||
problem_type: design_decision
|
||||
component: ci_policy
|
||||
severity: low
|
||||
applies_when:
|
||||
- a contributor proposes wiring search-quality eval into PR CI
|
||||
- a change affects retrieval, ranking, grounding, or synthesis quality and a reviewer asks "why aren't we testing this in CI?"
|
||||
- someone is deciding whether a new evaluator-style script belongs in the default CI workflow
|
||||
related_components:
|
||||
- search_quality_evaluation
|
||||
- ci_workflow
|
||||
- llm_judging
|
||||
tags:
|
||||
- ci-policy
|
||||
- eval
|
||||
- design-decision
|
||||
- cost-vs-signal
|
||||
- non-determinism
|
||||
- manual-gates
|
||||
---
|
||||
|
||||
# Search-quality eval is manual by default, not a CI gate on every PR
|
||||
|
||||
## Context
|
||||
|
||||
`skills/last30days/scripts/evaluate_search_quality.py` compares a baseline revision against a candidate revision across a fixed pool of reviewer topics. It produces two flavors of metrics: deterministic overlap (Jaccard, retention) and LLM-judged quality scores. The natural impulse on seeing an evaluator script is to wire it into CI on every PR — "regression catcher, run it automatically." We deliberately don't.
|
||||
|
||||
Three properties of this particular evaluator make CI-on-every-PR the wrong default:
|
||||
|
||||
1. **Live API access.** The candidate revision typically needs the engine to actually run, which means real ScrapeCreators calls, real reddit fetches, real YouTube searches. CI runs would either need production credentials or a record/replay fixture set that drifts almost immediately as external APIs change shape.
|
||||
|
||||
2. **Cost and latency.** A full eval pass runs the pipeline N times across reviewer topics. Multiplied by every PR (including doc-only PRs), the spend is meaningful and the wall-clock pushes CI from ~30s to many minutes.
|
||||
|
||||
3. **Non-determinism in the judging path.** The LLM-judged metrics are valuable for review but depend on judge-model behavior on a given day. A flaky eval that fails 1 PR in 20 because the judge re-scored an item differently is a worse CI signal than no eval at all — it teaches contributors to retry rather than read the result.
|
||||
|
||||
The deterministic overlap metrics are useful regression signals but they are not the same as user-facing correctness. A change that improves overlap can degrade synthesis quality; a change that drops overlap can be a deliberate improvement. So even the deterministic side isn't safe to auto-fail on.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Keep search-quality eval available, just not automatic
|
||||
|
||||
The script stays runnable by maintainers and contributors. The pattern is:
|
||||
|
||||
```bash
|
||||
LAST30DAYS_PYTHON=python3.13 \
|
||||
python3 skills/last30days/scripts/evaluate_search_quality.py \
|
||||
--baseline main --candidate HEAD
|
||||
```
|
||||
|
||||
Reviewers can request a manual eval run when a PR is in the retrieval/ranking/synthesis path and the risk warrants it. Contributors can run it locally before submitting if they want signal upfront.
|
||||
|
||||
### 2. Standard PR CI gates remain deterministic and contract-shaped
|
||||
|
||||
`pytest` (offline-safe), plugin-contract checks, version-consistency contracts, ruff/lint. Anything that returns the same answer twice for the same input. Quality-of-output assessment lives outside that loop.
|
||||
|
||||
### 3. The middle ground is `workflow_dispatch`, not auto-PR-gating
|
||||
|
||||
If maintainers want a GitHub-triggered eval that doesn't make every PR pay the live-API cost, the right shape is a manually-dispatched workflow (or a label-triggered one) — not a `pull_request:` workflow that runs unconditionally. That keeps the cost knob in human hands.
|
||||
|
||||
### 4. Revisit if the eval can ever be made offline-deterministic
|
||||
|
||||
The blocker is the live-API + non-determinism combination. If a future iteration of the script can compute meaningful Jaccard/retention metrics against static fixtures (no live API calls, no LLM judging), the decision flips and it becomes a candidate for default CI. The decision below tracks that condition; revisit when it's met.
|
||||
|
||||
## What this means in practice
|
||||
|
||||
- Don't merge PRs that wire `evaluate_search_quality.py` into the default `validate.yml` workflow.
|
||||
- Do merge PRs that add `workflow_dispatch` triggers or label-gated runs.
|
||||
- When reviewing a retrieval/ranking change, request a manual eval if the diff suggests it could regress quality — don't expect CI to catch it.
|
||||
|
||||
## Links
|
||||
|
||||
- `skills/last30days/scripts/evaluate_search_quality.py` — the evaluator script
|
||||
- `docs/search-quality-eval.md` — user-facing usage documentation
|
||||
- `.github/workflows/validate.yml` — the default CI workflow (deterministic gates only)
|
||||
|
||||
---
|
||||
|
||||
*Adapted from a draft ADR proposed by @hnshah in [#374](https://github.com/mvanhorn/last30days-skill/pull/374), restructured into the `docs/solutions/` convention. The original ADR text correctly identified the constraint; this version adds the "why workflow_dispatch is the middle ground" framing and the revisit-condition.*
|
||||
@@ -1,117 +0,0 @@
|
||||
---
|
||||
title: Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
|
||||
date: 2026-06-09
|
||||
category: docs/solutions/logic-errors
|
||||
module: lib/rerank
|
||||
problem_type: logic_error
|
||||
component: search_ranking
|
||||
severity: high
|
||||
symptoms:
|
||||
- on-entity, high-engagement items that name the brand but omit the trailing descriptor of a multi-word query are demoted in keyless/fallback rerank results
|
||||
- observed case is a 323-point HN thread about Stripe scoring 0 on a "Stripe payments" query
|
||||
- the entity-miss demotion lands twice (ENTITY_MISS_PENALTY on rerank_score plus a secondary final_score penalty), so a false miss guarantees burial regardless of engagement
|
||||
- reddit keyless comment-enrichment slot selection skips the same on-entity threads via an independently duplicated full-phrase check in _slot_priority
|
||||
root_cause: logic_error
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- reddit_keyless
|
||||
- comment_enrichment
|
||||
tags:
|
||||
- entity-grounding
|
||||
- rerank
|
||||
- keyless-fallback
|
||||
- multi-word-entity
|
||||
- substring-match
|
||||
- false-demotion
|
||||
- reddit-keyless
|
||||
- duplicated-logic
|
||||
---
|
||||
|
||||
# Keyless rerank entity grounding required full multi-word phrase, falsely demoting on-entity items
|
||||
|
||||
## Problem
|
||||
|
||||
The keyless/fallback rerank path's entity-grounding demotion required the FULL multi-word primary-entity phrase as a contiguous substring of the candidate's text (`primary_entity.lower() not in haystack`), so on-entity items that omitted a trailing search descriptor were falsely flagged as entity misses and buried by a deliberately decisive double penalty.
|
||||
|
||||
## Symptoms
|
||||
|
||||
- On a "Stripe payments" query, a 323-point HN thread titled "Stripe is friendly to 'friendly fraud'" was demoted to score 0 — purely because its text never contained the literal phrase "stripe payments" (the trailing word "payments" was missing).
|
||||
- The burial is guaranteed by design, not incidental: a flagged entity miss takes −25 `ENTITY_MISS_PENALTY` on `rerank_score` in `_fallback_tuple`, PLUS `ENTITY_MISS_FINAL_PENALTY` applied directly in `_final_score` (added 2026-04-19 after engagement + freshness drowned the diluted penalty). A false positive on the check means confirmed-good signal cannot recover.
|
||||
- The same over-strict check had been independently re-implemented in `reddit_keyless._slot_priority` (keyless Reddit comment-enrichment slot selection), so scarce comment slots were also steered away from head-token-only posts.
|
||||
|
||||
## What Didn't Work
|
||||
|
||||
- **Naively relaxing the check** — the full-phrase check existed for a real reason: on 2026-04-19 an off-topic video with zero brand mentions ranked #2 on a Hermes query (documented in the `ENTITY_MISS_FINAL_PENALTY` comment in `skills/last30days/scripts/lib/rerank.py`). Any fix had to keep that demotion firing.
|
||||
- **Word-boundary matching** — rejected; it re-introduces over-demotion on plurals/possessives/compounds ("stripes", "Stripe's").
|
||||
- **Graded penalty** (full-phrase = 0, head-only = half, none = full) — rejected; it half-punishes items that are 100% about the entity. Lexical coverage is not topical degree.
|
||||
- **Any-token grounding** — rejected; "payments" alone would ground completely generic posts.
|
||||
- **Distinctiveness gate for generic heads** — rejected as complexity to patch a failure mode that is already a safe no-op (see Why This Works).
|
||||
- **Trusting the docstring** — `reddit_keyless._slot_priority`'s docstring claimed to "mirror rerank's demotion signal," but its inline reimplementation (`entity in _post_text(post).lower()`) had silently drifted from being a mirror into being a second copy of the bug. It was found only by a code-reuse review, not by tests.
|
||||
|
||||
## Solution
|
||||
|
||||
Ground on the **head token** of the primary entity instead of the full phrase, via one shared helper used by both paths.
|
||||
|
||||
**Site 1 — new helper in `skills/last30days/scripts/lib/rerank.py`:**
|
||||
|
||||
```python
|
||||
def _entity_grounded(haystack: str, primary_entity: str) -> bool:
|
||||
tokens = primary_entity.lower().split()
|
||||
if not tokens:
|
||||
return True
|
||||
return tokens[0] in haystack
|
||||
```
|
||||
|
||||
`_fallback_tuple` switches from the inline phrase check to the helper:
|
||||
|
||||
```python
|
||||
# before
|
||||
if haystack.strip() and primary_entity.lower() not in haystack:
|
||||
# after
|
||||
if haystack.strip() and not _entity_grounded(haystack, primary_entity):
|
||||
```
|
||||
|
||||
**Site 2 — secondary penalty in `_final_score`: no code change needed.** It keys off the explanation string set by site 1, so it inherits the fix automatically:
|
||||
|
||||
```python
|
||||
if candidate.explanation and "entity-miss" in candidate.explanation:
|
||||
base = max(0.0, base - ENTITY_MISS_FINAL_PENALTY)
|
||||
```
|
||||
|
||||
**Site 3 — `skills/last30days/scripts/lib/reddit_keyless.py` `_slot_priority`:** replace the drifted reimplementation with a call to the shared helper:
|
||||
|
||||
```python
|
||||
# before
|
||||
return entity in _post_text(post).lower()
|
||||
# after
|
||||
return rerank._entity_grounded(_post_text(post).lower(), entity)
|
||||
```
|
||||
|
||||
Tests: `tests/test_rerank_v3.py` gained `test_fallback_grounds_on_head_token_not_full_phrase` (the Stripe regression) and `test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` (guards the 2026-04-19 behavior). `tests/test_reddit_keyless.py`'s two old-contract tests were rewritten as `test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match`.
|
||||
|
||||
## Why This Works
|
||||
|
||||
- **Root cause:** trailing tokens of a multi-word query ("payments" in "Stripe payments") are usually category descriptors the user/planner appended for search, not part of the entity name. Requiring the whole phrase conflates "doesn't repeat my search phrasing" with "isn't about my entity." The brand head token alone is sufficient grounding; items that never name the brand at all still miss the head token and stay demoted — so the original 2026-04-19 fix keeps firing.
|
||||
- **Asymmetry argument:** the demotion is engineered to be decisive (double penalty across `rerank_score` and `final_score`), so a false entity-miss is fatal-by-design, while a false grounding merely defers the item to normal relevance/freshness/quality ranking. When the punishment is capital, the conviction standard should be conservative.
|
||||
- **Substring (not word-boundary) is deliberate:** it catches plurals/possessives/compounds ("stripes", "Stripe's"). Degenerate short heads ("X", "Go", "C") make the check vacuously true, which merely **disables** the penalty — reverting to the pre-grounding baseline — rather than burying good items. Every failure mode of this rule degrades toward "no penalty," never toward "bury good signal."
|
||||
- **Accepted, bounded limitation:** head-collision with a different famous entity ("Hermes Agent" → a "Hermes Birkin" thread now escapes demotion). This is lexically unfixable — any token rule strong enough to kill the collision re-kills the Stripe case; the discriminator is semantic. The LLM rerank path (which receives the full phrase as prompt guidance and judges semantically) covers this when API keys exist; the keyless path accepts the bounded risk.
|
||||
|
||||
## Prevention
|
||||
|
||||
- **Shared helper as single source of truth:** when one module's behavior must "mirror" another's signal, it must *call* the same function, not re-implement the check. The `reddit_keyless._slot_priority` drift happened precisely because the mirror was a copy. The fix wires it to `rerank._entity_grounded`, and the docstring now states this explicitly: "keying on the same head token keeps the two paths from diverging."
|
||||
- **Docstrings record deliberate trade-offs:** `_entity_grounded`'s docstring documents WHY head-token (not phrase), why substring (not word-boundary), and the safe-failure direction. Future readers see the rejected alternatives were considered, not overlooked — and won't "tighten" the check into a regression.
|
||||
- **Both directions pinned by named tests:**
|
||||
- `tests/test_rerank_v3.py::test_fallback_grounds_on_head_token_not_full_phrase` — false-demotion regression (the Stripe HN thread must not be flagged).
|
||||
- `tests/test_rerank_v3.py::test_fallback_still_demotes_when_head_token_absent_on_multiword_topic` — the fix must not neuter the demotion (guards the 2026-04-19 off-topic-video incident).
|
||||
- `tests/test_reddit_keyless.py::test_slot_priority_grounds_on_head_token_not_full_phrase` and `test_intent_modifier_topic_prioritizes_head_token_match` — the mirrored path asserts the same contract.
|
||||
- **Audit tests when changing a contract:** tests that encode the old behavior as correct must be rewritten to the new contract, not worked around — the two old `test_reddit_keyless.py` tests would have silently re-blessed the bug.
|
||||
- **For decisive penalties, route through one flag:** the `_final_score` backstop keys off `"entity-miss" in candidate.explanation` rather than re-running the check — so there was exactly one site to fix and the second penalty inherited it for free. Prefer this signal-propagation pattern over duplicating predicate logic at each penalty site.
|
||||
|
||||
## Related Issues
|
||||
|
||||
- [PR #484](https://github.com/mvanhorn/last30days-skill/pull/484) — "fix(reddit): relevance-aware comment-enrichment slot selection in keyless path" — introduced the `_slot_priority` mirror this fix reroutes through the shared helper.
|
||||
- [PR #457](https://github.com/mvanhorn/last30days-skill/pull/457) — "fix(reddit): restore free path via keyless RSS + shreddit scrape" — established the keyless Reddit path.
|
||||
- [PR #488](https://github.com/mvanhorn/last30days-skill/pull/488) (open) — "fix(reddit): relevance floor + relevance-first ranking" — external PR touching the same ranking surface; coordinate before merging both.
|
||||
- [Issue #468](https://github.com/mvanhorn/last30days-skill/issues/468) (open) — relevance scoring over-pruning on-topic YouTube items; same symptom family in a different source.
|
||||
- [../architecture/search-quality-eval-manual-by-default-2026-05-10.md](../architecture/search-quality-eval-manual-by-default-2026-05-10.md) — how to validate ranking/grounding changes like this one (manual eval, not CI-gated).
|
||||
- [../workflow-issues/release-consistency-test-cascade-2026-05-16.md](../workflow-issues/release-consistency-test-cascade-2026-05-16.md) — sibling prevention pattern: lockstep artifacts drift unless mechanically unified.
|
||||
@@ -1,219 +0,0 @@
|
||||
---
|
||||
title: Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
date: 2026-05-16
|
||||
category: docs/solutions/workflow-issues
|
||||
module: ci-release-engineering
|
||||
problem_type: workflow_issue
|
||||
component: testing_framework
|
||||
severity: high
|
||||
applies_when:
|
||||
- a test asserts consistency between two release-time artifacts (e.g., SKILL.md version and a hardcoded pin in a shell script)
|
||||
- one artifact is updated as part of a version bump and the other requires a manual lockstep update
|
||||
- multiple long-lived PRs are open simultaneously against the same base branch
|
||||
symptoms:
|
||||
- every open PR's CI fails after a version bump even though the PRs are unrelated to versioning
|
||||
- the failing test references a stale hardcoded value that was not updated alongside the bumped version
|
||||
- PR authors must rebase and manually fix an artifact they did not touch
|
||||
root_cause: missing_workflow_step
|
||||
resolution_type: code_fix
|
||||
related_components:
|
||||
- development_workflow
|
||||
- documentation
|
||||
tags:
|
||||
- ci
|
||||
- release-engineering
|
||||
- consistency-test
|
||||
- version-pin
|
||||
- cascade-failure
|
||||
- test-design
|
||||
- workflow
|
||||
---
|
||||
|
||||
# Release-time consistency tests cause cascade CI failures across all open PRs
|
||||
|
||||
## Context
|
||||
|
||||
A `tests/test_version_consistency.py::test_sync_cache_path_uses_skill_version` test was added to enforce that the version string embedded in `skills/last30days/scripts/sync.sh` (a hardcoded plugin-cache path segment) matched the version frontmatter in `skills/last30days/SKILL.md`. The intention was sound: the cache path had to stay in lockstep with the skill version or the sync would silently pull stale files.
|
||||
|
||||
The test worked as designed until a release shipped. At that point it turned into a cascade-failure machine:
|
||||
|
||||
1. A release PR bumps `SKILL.md` version (e.g., 3.2.0 → 3.2.1) **and** bumps the `sync.sh` pin. That PR's CI is green.
|
||||
2. The release PR merges to `main`.
|
||||
3. Every PR that was open at merge time was branched from pre-release `main`. Those PRs have `SKILL.md` 3.2.1 (inherited via merge-base with `main`) but their branch never touched `sync.sh`.
|
||||
4. CI for those PRs runs the consistency test against the new `main` — `SKILL.md` says 3.2.1, `sync.sh` still says 3.2.0 — and fails.
|
||||
5. All open PRs are now red simultaneously, with a failure that has nothing to do with their changes.
|
||||
|
||||
This affected at least five PRs during the 2026-05-13 to 2026-05-15 window: PR #400 (caught during rebase, required a manual pin bump), PRs #390 and #392 (OpenClaw `SCRAPECREATORS_API_KEY` fix, both stalled for the same stale-pin reason), and at least two others. A follow-up hotfix PR (#397 — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`) was required just to unblock the queue.
|
||||
|
||||
The permanent fix was PR #405: delete `sync.sh` entirely (the install workflow made it redundant) and drop `test_sync_cache_path_uses_skill_version`. Once both were gone, no version-consistency cascade was possible.
|
||||
|
||||
## Guidance
|
||||
|
||||
### 1. Don't write consistency tests that read two files and assert one matches a substring derived from the other
|
||||
|
||||
This pattern looks safe but is not:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # reads SKILL.md
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # asserts sync.sh contains that string
|
||||
)
|
||||
```
|
||||
|
||||
It encodes the assumption that both files are always updated together, in the same commit, on the same branch. That assumption breaks the moment two files have independent lifecycle owners — a versioned manifest and a deployment script are archetypal examples.
|
||||
|
||||
### 2. If the values genuinely need to stay in sync, derive one from the other at runtime
|
||||
|
||||
Remove the hardcoded pin from `sync.sh` and compute it:
|
||||
|
||||
```bash
|
||||
# sync.sh — derive version from SKILL.md at runtime, no pin to maintain
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "$(dirname "$0")/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
CACHE_PATH="last30days-skill/last30days/${SKILL_VERSION}"
|
||||
```
|
||||
|
||||
Now there is only one source of truth (`SKILL.md`). The test that asserted they matched becomes vacuous and should be deleted. If `SKILL.md` is wrong, the sync itself will fail loudly — which is better feedback than a CI gate on a different PR.
|
||||
|
||||
### 3. If two values must stay independent for legitimate reasons, update them together and make the test self-skip if either source is missing
|
||||
|
||||
If separate versioning is genuinely required (e.g., SKILL.md versions for harness consumers, sync.sh versions a private artifact store with its own cadence), update both in the same PR — never staggered — and write the test to self-skip rather than error when either file is absent:
|
||||
|
||||
```python
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_sh = SKILL_ROOT / "scripts" / "sync.sh"
|
||||
if not sync_sh.exists():
|
||||
self.skipTest("sync.sh not present; skipping pin consistency check")
|
||||
sync_text = sync_sh.read_text(encoding="utf-8")
|
||||
version = _skill_version()
|
||||
self.assertIn(
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text,
|
||||
)
|
||||
```
|
||||
|
||||
Self-skipping means deleting the file is a non-event in CI — no cascading red, no hotfix PR to the queue.
|
||||
|
||||
### 4. Run consistency tests against the merge-base diff, not main
|
||||
|
||||
If you keep a two-file consistency test, scope it so it only fails when the PR itself modifies one of the two files but not the other. A GitHub Actions step can do this:
|
||||
|
||||
```yaml
|
||||
- name: Check sync.sh version pin consistency
|
||||
run: |
|
||||
BASE=$(git merge-base HEAD origin/main)
|
||||
SKILL_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'SKILL\.md' || true)
|
||||
SYNC_CHANGED=$(git diff --name-only "$BASE" HEAD | grep -c 'sync\.sh' || true)
|
||||
if [ "$SKILL_CHANGED" -gt 0 ] && [ "$SYNC_CHANGED" -eq 0 ]; then
|
||||
echo "SKILL.md version bumped but sync.sh pin was not updated"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
This only fires when your PR touched `SKILL.md` and left `sync.sh` alone — never because a release merged to `main` after you branched.
|
||||
|
||||
### 5. Ask whether you actually need this test
|
||||
|
||||
If the values are wrong, downstream tooling will fail loudly: the sync will fetch the wrong artifact, installs will break, or the harness will reject the version. A test that exists only to catch a human-bookkeeping error at release time adds cascade-fail risk without offering a meaningfully earlier signal. Weigh that cost before adding any two-file consistency gate.
|
||||
|
||||
## Why This Matters
|
||||
|
||||
The damage from a stale-pin consistency test is asymmetric. It:
|
||||
|
||||
- Fails on every open PR simultaneously the moment a release lands on `main` — not just the PR that forgot to update the pin.
|
||||
- Produces a failure message that points at a line in a test file with no obvious relationship to the PR's actual changes.
|
||||
- Requires either a hotfix PR (touching a file the failing PRs have no business touching) or a manual rebase of every affected branch.
|
||||
- Blocks work that has already been reviewed and approved.
|
||||
|
||||
In this repo the effect was measurable: at least five PRs stalled across a two-day window, one hotfix PR was shipped just to unblock the queue, and multiple authors spent time debugging a failure completely unrelated to their changes.
|
||||
|
||||
The broader principle is that tests which gate on *bookkeeping consistency between files* impose their maintenance cost on every contributor, every time, even when those contributors did nothing wrong. That cost compounds with team size and release cadence.
|
||||
|
||||
## When to Apply
|
||||
|
||||
Apply this guidance whenever you find yourself:
|
||||
|
||||
- Writing a test that reads two files and asserts that a string in one matches a value derived from the other.
|
||||
- Adding a CI step labeled "consistency check," "sync check," or "pin check" where the check compares a hardcoded value against a computed one from a separate file.
|
||||
- Working in a repo where a versioned manifest (e.g., `SKILL.md`, `package.json`, `pyproject.toml`) and a deployment artifact (e.g., a shell script, a Dockerfile, a Helm values file) are both maintained by hand.
|
||||
- Reviewing a PR that touches only one of two "paired" files and fails a consistency test for the other.
|
||||
|
||||
It does *not* apply to tests that read a single source of truth and validate its internal structure (e.g., asserting that `SKILL.md`'s frontmatter version is double-quoted, or that `package.json`'s `version` field is a valid semver string). Those tests have one file and one assertion; they cannot cascade across branches.
|
||||
|
||||
## Examples
|
||||
|
||||
### Before — the pattern that caused the cascade
|
||||
|
||||
Original `tests/test_version_consistency.py` (deleted in commit `9fb19ea`):
|
||||
|
||||
```python
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_ROOT = ROOT / "skills" / "last30days"
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
match = re.search(r'^version:\s*"([^"]+)"\s*$', text, re.MULTILINE)
|
||||
if not match:
|
||||
raise AssertionError("SKILL.md version frontmatter not found")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
class TestVersionConsistency(unittest.TestCase):
|
||||
def test_sync_cache_path_uses_skill_version(self) -> None:
|
||||
sync_text = (SKILL_ROOT / "scripts" / "sync.sh").read_text(encoding="utf-8")
|
||||
version = _skill_version() # source 1: SKILL.md frontmatter
|
||||
self.assertIn( # assertion: sync.sh must contain
|
||||
f'last30days-skill/last30days/{version}"',
|
||||
sync_text, # source 2: hardcoded string in sync.sh
|
||||
)
|
||||
```
|
||||
|
||||
`sync.sh` contained a line like:
|
||||
|
||||
```bash
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/3.2.0"
|
||||
```
|
||||
|
||||
When SKILL.md bumped to `3.2.1` in a release PR, `sync.sh` was updated in the same PR and CI stayed green. But every PR branched before that release still had `sync.sh` at `3.2.0`. Their CI failed immediately, with an assertion error pointing at the test, not at the release PR.
|
||||
|
||||
### After — what we did: delete both
|
||||
|
||||
PR #405 deleted `sync.sh` (the install workflow replaced it) and dropped `test_sync_cache_path_uses_skill_version` in the same change. No consistency gate, no pin to maintain, no cascade possible.
|
||||
|
||||
### After — what we could have done instead: derive at runtime
|
||||
|
||||
If `sync.sh` had still been needed, the right fix would have been to remove the hardcoded version from the script and derive it from `SKILL.md`:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# sync.sh — no hardcoded version; reads SKILL.md as single source of truth
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SKILL_VERSION=$(grep -m1 '^version:' "${SCRIPT_DIR}/../SKILL.md" \
|
||||
| sed 's/version:[[:space:]]*"\([^"]*\)"/\1/')
|
||||
|
||||
if [ -z "$SKILL_VERSION" ]; then
|
||||
echo "error: could not parse version from SKILL.md" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PLUGIN_CACHE="$HOME/.cache/last30days-skill/last30days/${SKILL_VERSION}"
|
||||
# ... rest of sync logic
|
||||
```
|
||||
|
||||
With this in place, `test_sync_cache_path_uses_skill_version` has no reason to exist — there is nothing to assert. Delete it. If the version parsing breaks, `sync.sh` itself exits non-zero with a clear message.
|
||||
|
||||
## Related
|
||||
|
||||
- **PR #397** (merged) — `fix(sync): bump cache target to 3.2.1 to match SKILL.md`. The hotfix that unblocked the cascade temporarily by bumping the pin.
|
||||
- **PR #400** (merged) — caught the same cascade during rebase; had to bump the pin to clear CI.
|
||||
- **PR #390** (closed) and **PR #392** (rebased + merged) — OpenClaw `SCRAPECREATORS_API_KEY` fix; both blocked by the cascade until rebased onto post-#405 main.
|
||||
- **PR #405** (merged) — the permanent fix: deleted `sync.sh` + `test_sync_cache_path_uses_skill_version` together.
|
||||
- **PR #412** (merged) — adjacent work that consolidated SKILL.md version parsing into `lib/skill_meta.py`, reducing future drift risk by giving the version field one canonical reader.
|
||||
@@ -0,0 +1,388 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -0,0 +1,310 @@
|
||||
# V1 vs V2 Comparison Analysis
|
||||
|
||||
**Date:** 2026-02-06
|
||||
**Queries tested:** 4 (1 head-to-head, 3 V1-only)
|
||||
**Scope:** Quick smoke test, not full 17-query matrix
|
||||
|
||||
---
|
||||
|
||||
## Part 1: Head-to-Head -- "kanye west" (NEWS Query)
|
||||
|
||||
### Dimension-by-Dimension Scoring
|
||||
|
||||
#### 1. Query Parsing Display
|
||||
|
||||
Does it show the `🔍 **{TOPIC}** · {QUERY_TYPE}` line before running tools?
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 1 | No parsing display at all. Output starts with "## What I learned:" -- jumps straight into synthesis. No acknowledgment of topic or query type before research. |
|
||||
| V2 | 1 | No parsing display either. Output starts with "Here's what I found:" then "## What I learned:" -- same problem as V1. |
|
||||
|
||||
**Analysis:** Neither version actually rendered the query parsing display. V2 SKILL.md explicitly requires `🔍 **kanye west** · News` before any tools run, but the agent did not produce it. This is a V2 instruction that failed to land. Both score 1/5.
|
||||
|
||||
Possible cause: The parsing display is supposed to appear *before* tools are called -- it may have been shown during execution but not captured in the final output text. If so, both outputs represent only the post-research synthesis, not the full session. Regardless, based on what is in the output files, neither shows it.
|
||||
|
||||
---
|
||||
|
||||
#### 2. Source Coverage (Reddit/X/Web counts)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | `Reddit: 0 relevant threads` / `X: 30 posts │ ~10 likes` / `Web: 20+ pages`. Two of three sources returned results. Reddit was zero. |
|
||||
| V2 | 3 | `Reddit: 0 threads (no results this cycle)` / `X: 29 posts │ 33 likes │ 14 reposts` / `Web: 30+ pages`. Same pattern: two of three returned results. |
|
||||
|
||||
**Analysis:** Nearly identical coverage. Both got zero Reddit results (likely a script/API issue for this topic, not a SKILL.md problem). V2 has slightly more precise X metrics (33 likes, 14 reposts vs. V1's vague "~10 likes"). V2 has more web pages (30+ vs 20+). Both miss the 10+ Reddit threshold for a score of 4+.
|
||||
|
||||
---
|
||||
|
||||
#### 3. Citation Quality (sparse vs every-sentence)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 2 | No inline citations at all. The body text makes claims ("full-page Wall Street Journal apology," "Hellwatt Festival in Italy") but never attributes them to a specific source. The stats box lists "Washington Post, Billboard, AllHipHop" but the body has zero `per @handle` or `per Rolling Stone` attributions. |
|
||||
| V2 | 5 | Every bold section ends with a sparse, clean citation. Examples: `"per Rolling Stone"`, `"per The Washington Post"`, `"per Billboard"`, `"per AllHipHop"`, `"per The News International"`. One citation per topic, never chained. Exactly what V2 SKILL.md specifies. |
|
||||
|
||||
**Analysis:** This is the single biggest quality gap between V1 and V2. V1's output reads like a Wikipedia summary -- informative but ungrounded. V2 reads like a researched briefing where every claim has a named source. V2 nails the "sparse citation" rule from its SKILL.md: `"cite 1 source per pattern, short format: 'per @handle' or 'per r/sub'"`.
|
||||
|
||||
V1 quote (no citation): `"He'll headline the new Hellwatt Festival in Italy (July 4-18, 2026)."`
|
||||
V2 quote (cited): `"Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard."`
|
||||
|
||||
---
|
||||
|
||||
#### 4. Summary Structure (bold topic headers, organized sections)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | Has a coherent narrative structure with a paragraph of synthesis, then a `**KEY THEMES:**` numbered list. But the opening is a single dense paragraph, not broken into scannable sections with bold headers. |
|
||||
| V2 | 5 | Each storyline gets its own bold header: `**BULLY Album — March 20, 2026 via Gamma**`, `**Public Apology for Antisemitism**`, `**Hellwatt Festival in Italy**`, `**Health Concerns**`, `**Grammys Ban**`, `**Kim & Lewis Hamilton Buzz**`. Each is a standalone scannable unit with 1-3 sentences. |
|
||||
|
||||
**Analysis:** V2 follows the SKILL.md template exactly: `**{Topic 1}** — [1-2 sentences, per source]`. V1 uses a blob + list approach which is readable but less scannable. V2 is notably better for a user who wants to skim and find the story they care about.
|
||||
|
||||
V1 structure: 1 dense paragraph -> 5-item `KEY THEMES` list
|
||||
V2 structure: 6 bold topic cards, each self-contained -> no KEY THEMES list (but doesn't need one because the structure itself is the organization)
|
||||
|
||||
---
|
||||
|
||||
#### 5. Stats Box Format (emoji tree vs plain text)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 4 | Uses `├─` tree format with emoji: `├─ 🟠 Reddit: 0 relevant threads` / `├─ 🔵 X: 30 posts` / `├─ 🌐 Web: 20+ pages` / `└─ Top voices:`. Minor deviation: says "0 relevant threads (filtered out noise)" instead of the V1 SKILL.md template "0 threads (no results this cycle)". Also omits the `🗣️` emoji on the Top voices line. |
|
||||
| V2 | 5 | Perfect match to V2 SKILL.md template: `├─ 🟠 Reddit: 0 threads (no results this cycle)` / `├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)` / `├─ 🌐 Web: 30+ pages │ rollingstone.com, ...` / `└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex`. Includes `(via xAI)` notation, `🗣️` emoji, @handles with engagement counts. |
|
||||
|
||||
**Analysis:** V2 is tighter and matches its template exactly. V1 is close but has minor deviations (custom "filtered out noise" text, missing `🗣️` emoji, no @handles or engagement counts on Top voices). V2's inclusion of actual @handles with like counts (`@honest30bgfan_ (33 likes)`) adds credibility.
|
||||
|
||||
---
|
||||
|
||||
#### 6. Research Grounding (actual research vs generic knowledge)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 4 | Clearly grounded: mentions specific details like "Wall Street Journal apology (Jan 26, 2026)," "four-month-long manic episode," "frontal-lobe brain injury," "North West collaborated on 'Piercings on My Hand,'" "Monumental Plaza de Toros." These are specific enough to be from research, not pre-training. Minor generic leakage: the "KEY THEMES" list uses editorial framing ("Accountability arc," "Mental health transparency") that feels more like analysis than research extraction. |
|
||||
| V2 | 5 | Every fact is specific and attributed: "12th studio album," "13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign," "earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded," "103,000-capacity RCF Arena." The AI-deepfaked vocals detail is a standout -- it is clearly from research, not something a model would know from pre-training. The Kim/Lewis Hamilton item (`"X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton"`) is explicitly sourced from X data, not general knowledge. |
|
||||
|
||||
**Analysis:** Both are well-grounded, but V2 has more "could only come from research" details. The deepfaked vocals story, the exact venue capacity, and the explicit X chatter observation are details that prove the synthesis is from the research output, not hallucinated.
|
||||
|
||||
---
|
||||
|
||||
#### 7. Prompt Quality (invitation to share vision, not dumping prompts)
|
||||
|
||||
| Version | Score | Evidence |
|
||||
|---------|-------|----------|
|
||||
| V1 | 3 | Ends with: `"Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in."` This is a follow-up invitation, but it is NOT the SKILL.md-specified invitation. It is topic-specific and conversational, which is nice, but it does not ask the user to "share your vision for what you want to create." It misses the prompt-generation angle entirely. |
|
||||
| V2 | 5 | Ends with exactly: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice."` This matches the V2 SKILL.md template verbatim. It positions the skill correctly: not a news summarizer but a research-to-prompt pipeline. |
|
||||
|
||||
**Analysis:** V1's closing is friendly but off-brand. It treats the skill as a research tool, not a research-to-prompt tool. V2 correctly frames the next step as "tell me what to create and I'll write the prompt." This is a meaningful difference -- V1 would leave a user thinking they just got a summary, while V2 primes them to get a usable output.
|
||||
|
||||
---
|
||||
|
||||
### Head-to-Head Scorecard
|
||||
|
||||
| Dimension | V1 | V2 | Winner |
|
||||
|-----------|----|----|--------|
|
||||
| 1. Query Parsing Display | 1 | 1 | Tie (both failed) |
|
||||
| 2. Source Coverage | 3 | 3 | Tie |
|
||||
| 3. Citation Quality | 2 | 5 | **V2 (+3)** |
|
||||
| 4. Summary Structure | 3 | 5 | **V2 (+2)** |
|
||||
| 5. Stats Box Format | 4 | 5 | **V2 (+1)** |
|
||||
| 6. Research Grounding | 4 | 5 | **V2 (+1)** |
|
||||
| 7. Prompt Quality (invitation) | 3 | 5 | **V2 (+2)** |
|
||||
| **TOTAL** | **20/35** | **29/35** | **V2 wins by 9 points** |
|
||||
|
||||
**V2 is clearly better.** The biggest gaps are citation quality (+3) and summary structure (+2). V2's output reads like a professional research briefing; V1's reads like a decent but unstructured summary.
|
||||
|
||||
---
|
||||
|
||||
## Part 2: V1-Only Outputs Analysis
|
||||
|
||||
### Output 1: "open claw" (GENERAL query)
|
||||
|
||||
**What V1 does well:**
|
||||
- Strong research grounding. Mentions exact numbers: "145,000+ GitHub stars," "20,000+ forks," "700+ skills," "341 malicious skills." These are clearly from research.
|
||||
- The KEY PATTERNS section is excellent: 5 well-organized patterns with community quotes (`"I give it sudo and let it configure everything"` vs `"prompt injection is terrifying when you give the bot access to your actual bank account"`).
|
||||
- Good synthesis of the security vs. enthusiasm tension -- captures the community split accurately.
|
||||
- Stats box uses the emoji tree format correctly with `├──` (though note: uses double-dash `──` instead of single `─`, minor inconsistency).
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display (`🔍 **open claw** · General`).
|
||||
- No inline citations in the body text. The 5 KEY PATTERNS have no `per @handle` or `per r/sub` attribution. Which Reddit thread said "I give it sudo"? Which X post raised the security concern? We do not know.
|
||||
- The stats box says `├── 🟠 Reddit: 25 threads │ ~750+ upvotes` -- the tilde and plus are imprecise. V2 SKILL.md wants exact parsed numbers.
|
||||
- Top voices line lists subreddits and handles but no engagement counts: `@grok, @Starlink` -- are these the highest-engagement handles? No like counts shown.
|
||||
- No bold topic headers in the body -- it is a single paragraph followed by a numbered list, not the `**{Topic}** — sentence, per source` format V2 requires.
|
||||
|
||||
**V1 Score (estimated):** 22/35
|
||||
|
||||
---
|
||||
|
||||
### Output 2: "nano banana pro prompting" (PROMPTING query)
|
||||
|
||||
**What V1 does well:**
|
||||
- Correctly identifies two prompting styles (JSON structured vs. natural language "Creative Director") and explains when each works best. This is excellent PROMPTING-type synthesis.
|
||||
- KEY PATTERNS are specific and actionable: "85mm lens at f/1.8," "three-point lighting with key at 45 degrees," "text rendering works -- keep text under 3 words for best results (75% success rate)." These are concrete tips a user can apply immediately.
|
||||
- Research grounding is strong: cites specific upvote counts ("149-259 upvotes"), subreddit names (`r/nanobanana2pro`), and the Google AI blog.
|
||||
- The invitation correctly targets Nano Banana Pro: `"Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro."`
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display.
|
||||
- Stats box uses plain text dashes: `- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments` instead of the tree format `├─ 🟠 Reddit:`. Uses `|` pipe instead of `│` box-drawing character. V2 SKILL.md explicitly says: "NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji."
|
||||
- No inline body citations. KEY PATTERNS mention Reddit upvote ranges but no specific `per @handle` attributions.
|
||||
- Missing `✅ All agents reported back!` header -- just says "All agents reported back!" without the checkmark.
|
||||
- Body structure is paragraph + numbered list, not bold topic headers.
|
||||
|
||||
**V1 Score (estimated):** 23/35 (slightly higher than open claw due to better actionability)
|
||||
|
||||
---
|
||||
|
||||
### Output 3: "how to best setup clawdbot" (HOW-TO query)
|
||||
|
||||
**What V1 does well:**
|
||||
- This is the best V1 output of the batch. It goes beyond synthesis and actually delivers a **Quick-Start guide** with numbered steps, a **Security Hardening** checklist, and a **Budget Option** -- all grounded in research.
|
||||
- Excellent research grounding: `"per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0"` -- this is an actual citation with an @handle!
|
||||
- Specific, actionable recommendations: exact commands (`curl -fsSL https://clawd.bot/install.sh | bash`), specific model recommendations (Claude Opus 4.5 for best results, GLM 4.7 Flash for local), specific channel advice (Telegram first, WhatsApp QR code fails).
|
||||
- Stats box is correct emoji tree format with engagement counts: `@aashatwt (452 likes), @recap_david (329 likes)`.
|
||||
- Captures the naming confusion accurately: "Clawdbot -> Moltbot -> OpenClaw."
|
||||
|
||||
**What V1 is missing (per V2 SKILL.md features):**
|
||||
- No query parsing display.
|
||||
- Body text has no inline citations except the Budget Option section. The 5 KEY PATTERNS have no `per @handle` attribution.
|
||||
- Bold topic headers are used only in the Quick-Start and Security sections, not in the KEY PATTERNS or intro.
|
||||
- The output delivers the "answer" directly (setup guide) rather than waiting for the user's vision and offering to write a prompt. For a HOW-TO query this might be the right call, but it skips the SKILL.md flow of "show research -> invite vision -> write prompt."
|
||||
|
||||
**V1 Score (estimated):** 26/35 (best of the V1 outputs)
|
||||
|
||||
---
|
||||
|
||||
### Patterns Across All V1 Outputs
|
||||
|
||||
**Consistent strengths:**
|
||||
1. Research grounding is solid across all three. V1 does not hallucinate -- the facts are clearly from the research output, not pre-training.
|
||||
2. KEY PATTERNS lists are consistently useful and actionable.
|
||||
3. Stats boxes are present in all outputs (though formatting varies).
|
||||
4. The invitation/closing line is present in all outputs.
|
||||
|
||||
**Consistent weaknesses:**
|
||||
1. **No query parsing display** in any output (0 for 4, including Kanye West).
|
||||
2. **No inline citations** in the body text (except one @handle in the clawdbot output). The research feels real but is unattributed.
|
||||
3. **Stats box formatting is inconsistent.** Open claw uses `├──` (double dash), nano banana pro uses `- 🟠` (plain dash + pipe), clawdbot uses `├─` (correct). Three different formats in three outputs.
|
||||
4. **Body structure defaults to paragraph + numbered list** instead of bold topic headers. Only clawdbot partially uses bold headers (in the guide section, not the research section).
|
||||
5. **No `(via Bird/xAI)` notation** on X stats in any output.
|
||||
|
||||
---
|
||||
|
||||
## Part 3: SKILL.md Feature Diff
|
||||
|
||||
### Features in V2 but NOT V1
|
||||
|
||||
| Feature | V2 Lines | Impact |
|
||||
|---------|----------|--------|
|
||||
| **Query parsing display** (`🔍 **{TOPIC}** · {QUERY_TYPE}`) | 40-53 | HIGH -- confirms to user the skill understood their request before spending time on research. |
|
||||
| **Sparse citation rules** with BAD/GOOD examples | 186-193 | HIGH -- this is the #1 quality differentiator in the Kanye head-to-head. `"per @handle"` format, never chain multiple citations. |
|
||||
| **Bold topic headers** template (`**{Topic 1}** — [1-2 sentences, per source]`) | 195-208 | HIGH -- makes output scannable. |
|
||||
| **Strict stats template** with "NEVER use plain text dashes" instruction | 217-230 | MEDIUM -- prevents the formatting inconsistency seen across V1 outputs. |
|
||||
| **RECOMMENDATIONS source attribution** (each item MUST have Sources: line with @handles) | 178-182 | MEDIUM -- only affects RECOMMENDATIONS queries. |
|
||||
| **Reddit 0 results handling** (explicit instruction for what to write) | 229 | LOW -- edge case, but prevents ad-hoc text like V1's "filtered out noise." |
|
||||
| **Bird CLI / xAI notation** in stats | 223 | LOW -- cosmetic transparency about data source. |
|
||||
| **Step 2 phrasing: "DO WEBSEARCH WHILE SCRIPT RUNS"** | 71-73 | LOW -- execution optimization, no output impact. |
|
||||
|
||||
### Features in V1 but NOT V2
|
||||
|
||||
| Feature | V1 Lines | Impact | Should Restore? |
|
||||
|---------|----------|--------|-----------------|
|
||||
| **Use cases block** (4 examples in intro) | 12-17 | LOW | No |
|
||||
| **Setup Check section** (3 modes, bash script, "keys are OPTIONAL") | 50-78 | MEDIUM for new users | Yes, for public release |
|
||||
| **BAD/GOOD synthesis anti-pattern examples** | 172-191 | MEDIUM-HIGH | YES |
|
||||
| **Self-check instruction** ("Re-read your 'What I learned' section...") | 269 | MEDIUM | YES |
|
||||
| **Quality Checklist** (5-point checklist before delivering prompt) | 306-324 | HIGH | YES |
|
||||
| **Prompt format anti-pattern** ("Research says JSON but you write prose") | 302 | MEDIUM | YES |
|
||||
| **"IF USER ASKS FOR MORE OPTIONS"** section | 327-329 | LOW-MEDIUM | YES |
|
||||
| **Web-only mode stats template + promo** | 248-259 | MEDIUM for no-key users | For public release |
|
||||
| **TARGET_TOOL question template** (4 options) | 272-280 | LOW | No |
|
||||
| **Context Memory: explicit "don't re-search" instructions** | 342-358 | MEDIUM | YES |
|
||||
| **Output footer emoji + engagement counts** | 366-380 | LOW | YES |
|
||||
|
||||
### Features in BOTH (Shared)
|
||||
|
||||
| Feature | Notes |
|
||||
|---------|-------|
|
||||
| Parse User Intent (TOPIC, TARGET_TOOL, QUERY_TYPE) | Same 4 query types, same detection logic |
|
||||
| "Don't ask about tool before research" rule | Identical |
|
||||
| Research script execution command | Same `python3` command |
|
||||
| WebSearch queries by QUERY_TYPE | Same search strategies |
|
||||
| "Use user's exact terminology" instruction | V2 shorter but same intent |
|
||||
| Judge Agent synthesis logic | Same 5-step weighting process |
|
||||
| "Ground in actual research" instruction | Same core instruction, V1 has more examples |
|
||||
| RECOMMENDATIONS: extract specific names | Same logic |
|
||||
| Prompt format matching | Same instruction |
|
||||
| Wait for user's vision | Same |
|
||||
| Write ONE perfect prompt | Same structure |
|
||||
| Context Memory | V2 shorter version |
|
||||
| Output summary footer | Both have it, V1 has emoji |
|
||||
| Depth options (quick/default/deep) | Same |
|
||||
| "After each prompt: Stay in Expert Mode" | Same |
|
||||
|
||||
### Overall Assessment
|
||||
|
||||
**V2 is a clear upgrade in output formatting and citation quality.** The three features V2 adds (query parsing display, sparse citation rules, bold topic headers) directly address the three biggest weaknesses seen across all V1 outputs. The Kanye West head-to-head proves it: V2 scores 29/35 vs V1's 20/35.
|
||||
|
||||
**However, V2 dropped several quality guardrails from V1** that do not affect formatting but affect *correctness*: the self-check instruction, the anti-pattern examples, the quality checklist for prompts, and the "don't re-search" context memory rule. These are cheap to restore (under 25 lines total) and protect against subtle failure modes that may not show up in a 1-query test but will appear over dozens of uses.
|
||||
|
||||
---
|
||||
|
||||
## Part 4: Verdict
|
||||
|
||||
### Ship V2 or Not?
|
||||
|
||||
**Ship V2 -- but restore the guardrails first.**
|
||||
|
||||
V2 is unambiguously better on every formatting dimension. The citation quality improvement alone (V1: 2/5 -> V2: 5/5) makes it worth shipping. The bold topic headers and strict stats template fix the inconsistency problems visible across all V1 outputs.
|
||||
|
||||
But V2 dropped 6 guardrail features from V1 that cost almost nothing to include and protect against real failure modes. These should be restored before V2 goes public.
|
||||
|
||||
### Remaining Gaps
|
||||
|
||||
**Must fix before shipping (affects correctness):**
|
||||
|
||||
1. **Restore the quality checklist for prompts.** This is the test plan's #1 priority item. V1 had a 5-point checklist; V2 reduced it to one line. The checklist is what makes prompts feel polished -- it is the "that's a great prompt" mechanism. Add 8 lines.
|
||||
|
||||
2. **Restore BAD/GOOD anti-pattern examples.** V2 says "ground in actual research" but does not show what *bad* grounding looks like. V1's ClawdBot/Claude Code conflation example is exactly the kind of concrete negative example that prevents real failures. Add 5 lines.
|
||||
|
||||
3. **Restore self-check instruction.** One sentence: "Re-read your 'What I learned' section -- does it match what the research ACTUALLY says?" Zero cost, catches hallucination. Add 2 lines.
|
||||
|
||||
4. **Restore "don't re-search" context memory rule.** V2 only says "only do new research if user asks about a DIFFERENT topic." V1 explicitly bans re-searching and tells the agent to answer from existing research. Add 3 lines.
|
||||
|
||||
**Should fix (polish):**
|
||||
|
||||
5. Restore prompt format anti-pattern ("Research says JSON but you write prose"). Add 2 lines.
|
||||
6. Restore "IF USER ASKS FOR MORE OPTIONS" section. Add 2 lines.
|
||||
7. Add emoji + engagement counts back to the output summary footer. Edit 3 lines.
|
||||
|
||||
**Skip for now:**
|
||||
|
||||
8. Setup Check section -- add back for public release, not needed for execution.
|
||||
9. Web-only mode stats template -- lower priority, most testers have API keys.
|
||||
10. TARGET_TOOL question template -- agent handles this naturally.
|
||||
|
||||
### Query Parsing Display: Investigate
|
||||
|
||||
Both V1 and V2 scored 1/5 on query parsing display. V2 has the feature in its SKILL.md but the agent did not render it in the captured output. This could mean:
|
||||
- The display was shown during execution but not captured (likely -- it appears before tools run, and the output files may only contain post-research content).
|
||||
- The instruction is not strong enough and the agent skips it.
|
||||
|
||||
**Recommendation:** Verify in a live session whether the parsing display actually appears. If it does not, strengthen the instruction (e.g., "This line MUST be the first thing you output, before any tool calls").
|
||||
|
||||
### Total Effort
|
||||
|
||||
Restoring all 7 priority items: approximately 25 lines added to V2 SKILL.md. Under 15 minutes of work. The V2 formatting wins are substantial and proven; the V1 guardrails are small and proven. Combining both produces the best version.
|
||||
|
||||
### Final Score Summary
|
||||
|
||||
| | V1 (Kanye) | V2 (Kanye) | Delta |
|
||||
|--|-----------|-----------|-------|
|
||||
| Total | 20/35 | 29/35 | **V2 +9** |
|
||||
|
||||
| | V1 (Open Claw) | V1 (Nano Banana) | V1 (Clawdbot) | V1 Average |
|
||||
|--|---------------|-----------------|--------------|------------|
|
||||
| Estimated Total | 22/35 | 23/35 | 26/35 | **23.7/35** |
|
||||
|
||||
V2 at 29/35 beats every V1 output, including V1's best (clawdbot at 26/35).
|
||||
|
||||
**Decision: Ship V2 with guardrails restored.**
|
||||
@@ -0,0 +1,388 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: "[topic] for [tool]" or "[topic]"
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
Use cases:
|
||||
- **Prompting**: "photorealistic people in Nano Banana Pro", "Midjourney prompts", "ChatGPT image generation" → learn techniques, get copy-paste prompts
|
||||
- **Recommendations**: "best Claude Code skills", "top AI tools" → get a LIST of specific things people mention
|
||||
- **News**: "what's happening with OpenAI", "latest AI announcements" → current events and updates
|
||||
- **General**: any topic you're curious about → understand what the community is saying
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
---
|
||||
|
||||
## Setup Check
|
||||
|
||||
The skill works in three modes based on available API keys:
|
||||
|
||||
1. **Full Mode** (both keys): Reddit + X + WebSearch - best results with engagement metrics
|
||||
2. **Partial Mode** (one key): Reddit-only or X-only + WebSearch
|
||||
3. **Web-Only Mode** (no keys): WebSearch only - still useful, but no engagement metrics
|
||||
|
||||
**API keys are OPTIONAL.** The skill will work without them using WebSearch fallback.
|
||||
|
||||
### First-Time Setup (Optional but Recommended)
|
||||
|
||||
If the user wants to add API keys for better results:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.config/last30days
|
||||
cat > ~/.config/last30days/.env << 'ENVEOF'
|
||||
# last30days API Configuration
|
||||
# Both keys are optional - skill works with WebSearch fallback
|
||||
|
||||
# For Reddit research (uses OpenAI's web_search tool)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# For X/Twitter research (uses xAI's x_search tool)
|
||||
XAI_API_KEY=
|
||||
ENVEOF
|
||||
|
||||
chmod 600 ~/.config/last30days/.env
|
||||
echo "Config created at ~/.config/last30days/.env"
|
||||
echo "Edit to add your API keys for enhanced research."
|
||||
```
|
||||
|
||||
**DO NOT stop if no keys are configured.** Proceed with web-only mode.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**IMPORTANT: The script handles API key detection automatically.** Run it and check the output to determine mode.
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Show a promo banner if keys are missing (this is intentional marketing)
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
**Step 2: Check the output mode**
|
||||
|
||||
The script output will indicate the mode:
|
||||
- **"Mode: both"** or **"Mode: reddit-only"** or **"Mode: x-only"**: Script found results, WebSearch is supplementary
|
||||
- **"Mode: web-only"**: No API keys, Claude must do ALL research via WebSearch
|
||||
|
||||
**Step 3: Do WebSearch**
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- If user says "ChatGPT image prompting", search for "ChatGPT image prompting"
|
||||
- Do NOT add "DALL-E", "GPT-4o", or other terms you think are related
|
||||
- Your knowledge may be outdated - trust the user's terminology
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Step 3: Wait for background script to complete**
|
||||
Use TaskOutput to get the script results before proceeding to synthesis.
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords? THIS IS CRITICAL.
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
**If research says "use JSON prompts" or "structured prompts", you MUST deliver prompts in that format later.**
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**CRITICAL: Do NOT output any "Sources:" lists. The final display should be clean.**
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
1. [Specific name] - mentioned {n}x (r/sub, @handle, blog.com)
|
||||
2. [Specific name] - mentioned {n}x (sources)
|
||||
3. [Specific name] - mentioned {n}x (sources)
|
||||
4. [Specific name] - mentioned {n}x (sources)
|
||||
5. [Specific name] - mentioned {n}x (sources)
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
```
|
||||
What I learned:
|
||||
|
||||
[2-4 sentences synthesizing key insights FROM THE ACTUAL RESEARCH OUTPUT.]
|
||||
|
||||
KEY PATTERNS I'll use:
|
||||
1. [Pattern from research]
|
||||
2. [Pattern from research]
|
||||
3. [Pattern from research]
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
For **full/partial mode** (has API keys):
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {n} threads │ {sum} upvotes │ {sum} comments
|
||||
├─ 🔵 X: {n} posts │ {sum} likes │ {sum} reposts
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top voices: r/{sub1}, r/{sub2} │ @{handle1}, @{handle2} │ {web_author} on {site}
|
||||
```
|
||||
|
||||
For **web-only mode** (no API keys):
|
||||
```
|
||||
---
|
||||
✅ Research complete!
|
||||
├─ 🌐 Web: {n} pages │ {domains}
|
||||
└─ Top sources: {author1} on {site1}, {author2} on {site2}
|
||||
|
||||
💡 Want engagement metrics? Add API keys to ~/.config/last30days/.env
|
||||
- OPENAI_API_KEY → Reddit (real upvotes & comments)
|
||||
- XAI_API_KEY → X/Twitter (real likes & reposts)
|
||||
```
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
**Use real numbers from the research output.** The patterns should be actual insights from the research, not generic advice.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If the research was about ClawdBot (a self-hosted AI agent), your summary should be about ClawdBot, not Claude Code. If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**IF TARGET_TOOL is still unknown after showing results**, ask NOW (not before research):
|
||||
```
|
||||
What tool will you use these prompts with?
|
||||
|
||||
Options:
|
||||
1. [Most relevant tool based on research - e.g., if research mentioned Figma/Sketch, offer those]
|
||||
2. Nano Banana Pro (image generation)
|
||||
3. ChatGPT / Claude (text/code)
|
||||
4. Other (tell me)
|
||||
```
|
||||
|
||||
**IMPORTANT**: After displaying this, WAIT for the user to respond. Don't dump generic prompts.
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
When they respond with their vision (e.g., "I want a landing page mockup for my SaaS app"), THEN write a single, thoughtful, tailored prompt.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT:**
|
||||
|
||||
- Research says "JSON prompts" → Write the prompt AS JSON
|
||||
- Research says "structured parameters" → Use structured key: value format
|
||||
- Research says "natural language" → Use conversational prose
|
||||
- Research says "keyword lists" → Use comma-separated keywords
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS - if research said JSON, this is JSON. If research said natural language, this is prose. Match what works.]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
### Quality Checklist:
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
- **If they ask a question** - answer it from your research findings
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
For **full/partial mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
|
||||
For **web-only mode**:
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} web pages from {domains}
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
💡 Unlock Reddit & X data: Add API keys to ~/.config/last30days/.env
|
||||
```
|
||||
@@ -0,0 +1,25 @@
|
||||
## What I learned:
|
||||
|
||||
The last 30 days have been one of Kanye West's most significant periods in years. The biggest story is his **full-page Wall Street Journal apology** (Jan 26, 2026) titled "To Those I've Hurt" — a comprehensive open letter addressing his antisemitic remarks and erratic behavior. He attributed his actions to a "four-month-long manic episode" and revealed that a car accident 25 years ago caused a frontal-lobe brain injury that wasn't properly diagnosed until 2023, leading to his bipolar type-1 diagnosis. Multiple sources describe wife **Bianca Censori** as instrumental in getting him to seek treatment during a manic depressive episode.
|
||||
|
||||
On the music side, he's announced **two upcoming albums**: *BULLY* (now scheduled for March 20, 2026) and *IN A PERFECT WORLD*. Daughter **North West** collaborated with him on a track called "Piercings on My Hand." He performed a massive show in Mexico City (Jan 30) at the Monumental Plaza de Toros, and he'll headline the new **Hellwatt Festival** in Italy (July 4-18, 2026).
|
||||
|
||||
The **2026 Grammys** remain contentious — he's reportedly not invited after last year's red carpet drama. Health concerns also surfaced after a January LA outing where he appeared "noticeably bloated and moving with sluggish energy."
|
||||
|
||||
**KEY THEMES:**
|
||||
1. **Accountability arc** — The WSJ apology is being widely discussed as either genuine growth or calculated PR
|
||||
2. **Mental health transparency** — Brain injury + bipolar diagnosis framing his past behavior
|
||||
3. **Bianca Censori's role** — Increasingly seen as a stabilizing force (despite continued provocative public appearances)
|
||||
4. **Music comeback** — Two albums announced, active touring in Latin America and Europe
|
||||
5. **Grammy exile** — Still persona non grata at major award shows
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 0 relevant threads (filtered out noise)
|
||||
├─ 🔵 X: 30 posts │ ~10 likes │ casual fan chatter + Bianca Censori discourse
|
||||
├─ 🌐 Web: 20+ pages │ washingtonpost.com, billboard.com, allhiphop.com, euronews.com, rap-up.com
|
||||
└─ Top voices: Washington Post, Billboard, AllHipHop, Euronews, The Mirror
|
||||
|
||||
---
|
||||
|
||||
Want to dive deeper into any of these threads — the apology, the new albums, the Grammys situation, or Bianca Censori? Just tell me what angle you're interested in.
|
||||
@@ -0,0 +1,24 @@
|
||||
## What I learned:
|
||||
|
||||
OpenClaw (formerly ClawdBot, formerly Moltbot) is an open-source autonomous AI agent created by Peter Steinberger that went massively viral in late January 2026 — hitting 145,000+ GitHub stars and 20,000+ forks. It runs locally on your machine, connects to messaging platforms (Telegram, WhatsApp, Discord, Signal, etc.), and uses LLMs like Claude, GPT, or Gemini to actually perform tasks on your computer — file management, scheduling, web browsing, purchases, and more. Think "JARVIS for everyone."
|
||||
|
||||
The community is simultaneously excited and alarmed. The "ClawHub" skill marketplace has 700+ skills but a VirusTotal/Koi Security audit found **341 malicious skills** doing data exfiltration and crypto theft. A high-severity RCE vulnerability was also disclosed. The #1 most downloaded skill on ClawHub was confirmed malware.
|
||||
|
||||
**KEY PATTERNS from the research:**
|
||||
|
||||
1. **Cost is the top complaint** — API usage burns money fast; many users recommend Gemini CLI (free 1,000 req/day) over Claude/GPT for OpenClaw to control spend
|
||||
2. **Security is a dumpster fire** — Supply chain attacks via ClawHub skills, prompt injection risks when browsing the web, RCE vulnerabilities, and orgs are banning it at the EDR level
|
||||
3. **Setup is nontrivial** — Even after install, most users get stuck on skills/tools/automation config; the `openclaw onboard` wizard is the recommended path
|
||||
4. **The skill ecosystem is powerful but dangerous** — 700+ skills available, but no code signing, no real moderation, and active malware campaigns
|
||||
5. **Community is split** — Enthusiasts love the autonomy ("I give it sudo and let it configure everything"), while security folks are terrified ("prompt injection is terrifying when you give the bot access to your actual bank account")
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├── 🟠 Reddit: 25 threads │ ~750+ upvotes │ ~600+ comments
|
||||
├── 🔵 X: 27 posts │ 1,750+ likes │ 325+ reposts
|
||||
├── 🌐 Web: 20+ pages │ Wikipedia, Bloomberg, The Register, Hacker News, Nature, VirusTotal, DigitalOcean, Codecademy, Tom's Hardware
|
||||
└── Top voices: r/openclaw, r/cybersecurity, r/clawdbot, r/ArtificialInteligence │ @grok, @Starlink │ Nature, Bloomberg, ACM
|
||||
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly.
|
||||
@@ -0,0 +1,27 @@
|
||||
Now I have a comprehensive picture. Let me synthesize everything.
|
||||
|
||||
**What I learned:**
|
||||
|
||||
Nano Banana Pro is fundamentally different from older image models — it's a "thinking" model that understands intent, physics, and composition. The community has converged on two distinct prompting styles, and **both work well** depending on the use case:
|
||||
|
||||
1. **JSON structured prompts** — Dominant on Reddit (r/nanobanana2pro) and X. Power users share elaborate JSON objects with nested fields for subject, appearance, lighting, composition, and style. These produce the most consistent, controlled results for portraits, editorials, and product shots.
|
||||
|
||||
2. **Natural language "Creative Director" prompts** — Recommended by Google's official guide. Full sentences describing the scene as if briefing a photographer. Better for cinematic shots, complex edits, and iterative refinement.
|
||||
|
||||
**KEY PATTERNS I'll use:**
|
||||
1. **JSON format for maximum control** — The top Reddit posts (149-259 upvotes) all use structured JSON with nested objects: `subject > appearance > hair/skin/expression`, `lighting > key/fill/rim`, `camera > lens/angle/depth_of_field`
|
||||
2. **Be a Creative Director, not a tag spammer** — Replace "4k, realistic, beautiful" with specific technical directions: "85mm lens at f/1.8", "three-point lighting with key at 45°", "shallow depth of field with bokeh"
|
||||
3. **Provide context for WHY** — Adding purpose ("for a luxury fashion editorial", "for a high-end gourmet cookbook") helps the model infer appropriate styling, plating, lighting
|
||||
4. **Edit, don't regenerate** — When 80% is right, ask for the specific change conversationally
|
||||
5. **Text rendering works** — Keep text under 3 words for best results (75% success rate), specify font style and placement
|
||||
|
||||
---
|
||||
All agents reported back!
|
||||
- 🟠 Reddit: 5 threads | 638 upvotes | 66 comments
|
||||
- 🔵 X: 29 posts | 312 likes | 117 reposts
|
||||
- 🌐 Web: 10+ pages | blog.google, dev.to, eweek.com, imagine.art, github.com, leonardo.ai, medium.com
|
||||
- Top voices: r/nanobanana2pro | @Xmira_belle, @doctorwasif, @chatgptpaglu | Google AI blog, DEV Community
|
||||
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into Nano Banana Pro.
|
||||
@@ -0,0 +1,48 @@
|
||||
**What I learned:**
|
||||
|
||||
Clawdbot (now officially renamed **OpenClaw**) is a self-hosted, open-source personal AI assistant that connects to messaging apps (Telegram, WhatsApp, Discord, Slack, Signal, iMessage) and uses LLMs to act as an autonomous agent on your behalf. It's roughly one week old as a public project and the community is moving fast — the name has already changed twice (Clawdbot → Moltbot → OpenClaw).
|
||||
|
||||
**KEY PATTERNS from the research:**
|
||||
|
||||
1. **Security is the #1 concern** — Multiple Reddit threads and guides warn that many setups are wide open to the internet with zero auth. Run it on dedicated/isolated hardware (VM, Mac Mini, NAS), never your main machine. Block dangerous commands via allowlist. Never add it to group chats.
|
||||
|
||||
2. **Use the onboarding wizard, not Docker** — Docker was reported as unreliable (shims, bridges). The single-line install script (`curl -fsSL https://clawd.bot/install.sh | bash`) followed by `openclaw onboard` is the smoothest path. Docker is a "no-go" per multiple users.
|
||||
|
||||
3. **Model choice matters a lot** — Claude Opus 4.5 is recommended for best results (trained to resist prompt injection, ~99% resistance). For budget setups, Claude Haiku works at $0. Local models (Ollama/LM Studio) work but cause hallucination and token issues — GLM 4.7 Flash (30B) was recommended as the best local option.
|
||||
|
||||
4. **Telegram is the easiest channel to start with** — Most guides and users start with a Telegram bot token. WhatsApp QR code pairing fails frequently. Get Telegram working first, then expand.
|
||||
|
||||
5. **Sandbox and isolate** — Set `agents.defaults.sandbox.mode` to `"non-main"` for Docker sandboxing. Use a Linux VM or UTM on Mac to segregate traffic. Tailscale for secure remote access.
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 19 threads │ 415+ upvotes │ 290+ comments
|
||||
├─ 🔵 X: 29 posts │ 815+ likes │ 65+ reposts
|
||||
├─ 🌐 Web: 10+ pages │ Medium, DataCamp, DEV, GitHub, QNAP, docs.openclaw.ai
|
||||
└─ Top voices: r/clawdbot, r/LocalLLM, r/AskVibecoders │ @aashatwt (452 likes), @recap_david (329 likes) │ Lukas Niessen on Medium
|
||||
|
||||
---
|
||||
|
||||
**Here's the consensus best setup path based on what the community actually recommends:**
|
||||
|
||||
### Quick-Start (30 min)
|
||||
1. **Get dedicated hardware** — Mac Mini, old laptop, NAS (QNAP has an official guide), or a VPS (DigitalOcean gist available). Don't use your daily driver.
|
||||
2. **Install via script**: `curl -fsSL https://clawd.bot/install.sh | bash`
|
||||
3. **Run the wizard**: `openclaw onboard` — it walks through gateway, workspace, channels, and skills
|
||||
4. **Connect Telegram first** — create a bot via @BotFather, paste the token
|
||||
5. **Set your API key** — Anthropic API key for Claude, or connect Ollama for local models
|
||||
6. **Customize SOUL.md** — this controls your bot's personality and behavior
|
||||
|
||||
### Security Hardening (do this immediately)
|
||||
- Bind gateway to localhost only, use Tailscale for remote access
|
||||
- Set sandbox mode to `"non-main"`
|
||||
- Create an explicit command allowlist (block `rm -rf`, `sudo`, `chmod`)
|
||||
- Rotate API keys every 90 days
|
||||
- Never add the bot to group chats
|
||||
|
||||
### Budget Option ($0)
|
||||
Per @shynxbt: Use a free AWS VPS + Claude Haiku model + Telegram bot = fully functional for $0.
|
||||
|
||||
---
|
||||
|
||||
Want me to help you with a specific part of the setup, or do you have a particular use case in mind (home automation, CRM, coding assistant, etc.)?
|
||||
@@ -0,0 +1,332 @@
|
||||
---
|
||||
name: last30days
|
||||
description: Research a topic from the last 30 days on Reddit + X + Web, become an expert, and write copy-paste-ready prompts for the user's target tool.
|
||||
argument-hint: '"[topic] for [tool]" or "[topic]"'
|
||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||
---
|
||||
|
||||
# last30days: Research Any Topic from the Last 30 Days
|
||||
|
||||
Research ANY topic across Reddit, X, and the web. Surface what people are actually discussing, recommending, and debating right now.
|
||||
|
||||
## CRITICAL: Parse User Intent
|
||||
|
||||
Before doing anything, parse the user's input for:
|
||||
|
||||
1. **TOPIC**: What they want to learn about (e.g., "web app mockups", "Claude Code skills", "image generation")
|
||||
2. **TARGET TOOL** (if specified): Where they'll use the prompts (e.g., "Nano Banana Pro", "ChatGPT", "Midjourney")
|
||||
3. **QUERY TYPE**: What kind of research they want:
|
||||
- **PROMPTING** - "X prompts", "prompting for X", "X best practices" → User wants to learn techniques and get copy-paste prompts
|
||||
- **RECOMMENDATIONS** - "best X", "top X", "what X should I use", "recommended X" → User wants a LIST of specific things
|
||||
- **NEWS** - "what's happening with X", "X news", "latest on X" → User wants current events/updates
|
||||
- **GENERAL** - anything else → User wants broad understanding of the topic
|
||||
|
||||
Common patterns:
|
||||
- `[topic] for [tool]` → "web mockups for Nano Banana Pro" → TOOL IS SPECIFIED
|
||||
- `[topic] prompts for [tool]` → "UI design prompts for Midjourney" → TOOL IS SPECIFIED
|
||||
- Just `[topic]` → "iOS design mockups" → TOOL NOT SPECIFIED, that's OK
|
||||
- "best [topic]" or "top [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
- "what are the best [topic]" → QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**IMPORTANT: Do NOT ask about target tool before research.**
|
||||
- If tool is specified in the query, use it
|
||||
- If tool is NOT specified, run research first, then ask AFTER showing results
|
||||
|
||||
**Store these variables:**
|
||||
- `TOPIC = [extracted topic]`
|
||||
- `TARGET_TOOL = [extracted tool, or "unknown" if not specified]`
|
||||
- `QUERY_TYPE = [RECOMMENDATIONS | NEWS | HOW-TO | GENERAL]`
|
||||
|
||||
**DISPLAY your parsing to the user.** Before running any tools, output a single line:
|
||||
|
||||
🔍 **{TOPIC}** · {QUERY_TYPE}
|
||||
Searching Reddit, X, and the web for {natural language description of what you'll look for}...
|
||||
|
||||
Example outputs:
|
||||
- 🔍 **kanye west** · News — Searching Reddit, X, and the web for the latest kanye west news and discussions...
|
||||
- 🔍 **best MCP servers** · Recommendations — Searching Reddit, X, and the web for the most recommended MCP servers...
|
||||
- 🔍 **nano banana pro prompting** · Prompting — Searching Reddit, X, and the web for nano banana pro prompting techniques and tips...
|
||||
- 🔍 **open claw** · General — Searching Reddit, X, and the web for what people are saying about open claw...
|
||||
|
||||
If TARGET_TOOL is known, mention it: "...for nano banana pro prompting techniques to use in ChatGPT..."
|
||||
|
||||
This text MUST appear before you call any tools. It confirms to the user that you understood their request.
|
||||
|
||||
---
|
||||
|
||||
## Research Execution
|
||||
|
||||
**Step 1: Run the research script**
|
||||
```bash
|
||||
python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1
|
||||
```
|
||||
|
||||
The script will automatically:
|
||||
- Detect available API keys
|
||||
- Run Reddit/X searches if keys exist
|
||||
- Signal if WebSearch is needed
|
||||
|
||||
---
|
||||
|
||||
## STEP 2: DO WEBSEARCH WHILE SCRIPT RUNS
|
||||
|
||||
The script auto-detects sources (Bird CLI, API keys, etc). While waiting for it, do WebSearch.
|
||||
|
||||
For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode).
|
||||
|
||||
Choose search queries based on QUERY_TYPE:
|
||||
|
||||
**If RECOMMENDATIONS** ("best X", "top X", "what X should I use"):
|
||||
- Search for: `best {TOPIC} recommendations`
|
||||
- Search for: `{TOPIC} list examples`
|
||||
- Search for: `most popular {TOPIC}`
|
||||
- Goal: Find SPECIFIC NAMES of things, not generic advice
|
||||
|
||||
**If NEWS** ("what's happening with X", "X news"):
|
||||
- Search for: `{TOPIC} news 2026`
|
||||
- Search for: `{TOPIC} announcement update`
|
||||
- Goal: Find current events and recent developments
|
||||
|
||||
**If PROMPTING** ("X prompts", "prompting for X"):
|
||||
- Search for: `{TOPIC} prompts examples 2026`
|
||||
- Search for: `{TOPIC} techniques tips`
|
||||
- Goal: Find prompting techniques and examples to create copy-paste prompts
|
||||
|
||||
**If GENERAL** (default):
|
||||
- Search for: `{TOPIC} 2026`
|
||||
- Search for: `{TOPIC} discussion`
|
||||
- Goal: Find what people are actually saying
|
||||
|
||||
For ALL query types:
|
||||
- **USE THE USER'S EXACT TERMINOLOGY** - don't substitute or add tech names based on your knowledge
|
||||
- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
|
||||
- INCLUDE: blogs, tutorials, docs, news, GitHub repos
|
||||
- **DO NOT output "Sources:" list** - this is noise, we'll show stats at the end
|
||||
|
||||
**Depth options** (passed through from user's command):
|
||||
- `--quick` → Faster, fewer sources (8-12 each)
|
||||
- (default) → Balanced (20-30 each)
|
||||
- `--deep` → Comprehensive (50-70 Reddit, 40-60 X)
|
||||
|
||||
---
|
||||
|
||||
## Judge Agent: Synthesize All Sources
|
||||
|
||||
**After all searches complete, internally synthesize (don't display stats yet):**
|
||||
|
||||
The Judge Agent must:
|
||||
1. Weight Reddit/X sources HIGHER (they have engagement signals: upvotes, likes)
|
||||
2. Weight WebSearch sources LOWER (no engagement data)
|
||||
3. Identify patterns that appear across ALL three sources (strongest signals)
|
||||
4. Note any contradictions between sources
|
||||
5. Extract the top 3-5 actionable insights
|
||||
|
||||
**Do NOT display stats here - they come at the end, right before the invitation.**
|
||||
|
||||
---
|
||||
|
||||
## FIRST: Internalize the Research
|
||||
|
||||
**CRITICAL: Ground your synthesis in the ACTUAL research content, not your pre-existing knowledge.**
|
||||
|
||||
Read the research output carefully. Pay attention to:
|
||||
- **Exact product/tool names** mentioned (e.g., if research mentions "ClawdBot" or "@clawdbot", that's a DIFFERENT product than "Claude Code" - don't conflate them)
|
||||
- **Specific quotes and insights** from the sources - use THESE, not generic knowledge
|
||||
- **What the sources actually say**, not what you assume the topic is about
|
||||
|
||||
**ANTI-PATTERN TO AVOID**: If user asks about "clawdbot skills" and research returns ClawdBot content (self-hosted AI agent), do NOT synthesize this as "Claude Code skills" just because both involve "skills". Read what the research actually says.
|
||||
|
||||
### If QUERY_TYPE = RECOMMENDATIONS
|
||||
|
||||
**CRITICAL: Extract SPECIFIC NAMES, not generic patterns.**
|
||||
|
||||
When user asks "best X" or "top X", they want a LIST of specific things:
|
||||
- Scan research for specific product names, tool names, project names, skill names, etc.
|
||||
- Count how many times each is mentioned
|
||||
- Note which sources recommend each (Reddit thread, X post, blog)
|
||||
- List them by popularity/mention count
|
||||
|
||||
**BAD synthesis for "best Claude Code skills":**
|
||||
> "Skills are powerful. Keep them under 500 lines. Use progressive disclosure."
|
||||
|
||||
**GOOD synthesis for "best Claude Code skills":**
|
||||
> "Most mentioned skills: /commit (5 mentions), remotion skill (4x), git-worktree (3x), /pr (3x). The Remotion announcement got 16K likes on X."
|
||||
|
||||
### For all QUERY_TYPEs
|
||||
|
||||
Identify from the ACTUAL RESEARCH OUTPUT:
|
||||
- **PROMPT FORMAT** - Does research recommend JSON, structured params, natural language, keywords?
|
||||
- The top 3-5 patterns/techniques that appeared across multiple sources
|
||||
- Specific keywords, structures, or approaches mentioned BY THE SOURCES
|
||||
- Common pitfalls mentioned BY THE SOURCES
|
||||
|
||||
---
|
||||
|
||||
## THEN: Show Summary + Invite Vision
|
||||
|
||||
**Display in this EXACT sequence:**
|
||||
|
||||
**FIRST - What I learned (based on QUERY_TYPE):**
|
||||
|
||||
**If RECOMMENDATIONS** - Show specific things mentioned with sources:
|
||||
```
|
||||
🏆 Most mentioned:
|
||||
|
||||
[Tool Name] - {n}x mentions
|
||||
Use Case: [what it does]
|
||||
Sources: @handle1, @handle2, r/sub, blog.com
|
||||
|
||||
[Tool Name] - {n}x mentions
|
||||
Use Case: [what it does]
|
||||
Sources: @handle3, r/sub2, Complex
|
||||
|
||||
Notable mentions: [other specific things with 1-2 mentions]
|
||||
```
|
||||
|
||||
**CRITICAL for RECOMMENDATIONS:**
|
||||
- Each item MUST have a "Sources:" line with actual @handles from X posts (e.g., @LONGLIVE47, @ByDobson)
|
||||
- Include subreddit names (r/hiphopheads) and web sources (Complex, Variety)
|
||||
- Parse @handles from research output and include the highest-engagement ones
|
||||
- Format naturally - tables work well for wide terminals, stacked cards for narrow
|
||||
|
||||
**If PROMPTING/NEWS/GENERAL** - Show synthesis and patterns:
|
||||
|
||||
CITATION RULE: Cite sources sparingly to prove research is real.
|
||||
- In the "What I learned" intro: cite 1-2 top sources total, not every sentence
|
||||
- In KEY PATTERNS: cite 1 source per pattern, short format: "per @handle" or "per r/sub"
|
||||
- Do NOT include engagement metrics in citations (likes, upvotes) - save those for stats box
|
||||
- Do NOT chain multiple citations: "per @x, @y, @z" is too much. Pick the strongest one.
|
||||
|
||||
**BAD:** "His album is set for March 20 (per @cocoabutterbf; Rolling Stone; HotNewHipHop; Complex)."
|
||||
**GOOD:** "His album BULLY is set for March 20 via Gamma, per Rolling Stone."
|
||||
|
||||
```
|
||||
What I learned:
|
||||
|
||||
**{Topic 1}** — [1-2 sentences about this storyline, per source]
|
||||
|
||||
**{Topic 2}** — [1-2 sentences, per source]
|
||||
|
||||
**{Topic 3}** — [1-2 sentences, per source]
|
||||
|
||||
KEY PATTERNS from the research:
|
||||
1. [Pattern] — per @handle
|
||||
2. [Pattern] — per r/sub
|
||||
3. [Pattern] — per source
|
||||
```
|
||||
|
||||
**THEN - Stats (right before invitation):**
|
||||
|
||||
**CRITICAL: Calculate actual totals from the research output.**
|
||||
- Count posts/threads from each section
|
||||
- Sum engagement: parse `[Xlikes, Yrt]` from each X post, `[Xpts, Ycmt]` from Reddit
|
||||
- Identify top voices: highest-engagement @handles from X, most active subreddits
|
||||
|
||||
**Copy this EXACTLY, replacing only the {placeholders}:**
|
||||
|
||||
```
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments
|
||||
├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts (via Bird/xAI)
|
||||
├─ 🌐 Web: {N} pages │ {domain1}, {domain2}, {domain3}
|
||||
└─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2}
|
||||
---
|
||||
```
|
||||
|
||||
If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)"
|
||||
NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji.
|
||||
|
||||
**SELF-CHECK before displaying**: Re-read your "What I learned" section. Does it match what the research ACTUALLY says? If you catch yourself projecting your own knowledge instead of the research, rewrite it.
|
||||
|
||||
**LAST - Invitation:**
|
||||
```
|
||||
---
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into {TARGET_TOOL}.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## WAIT FOR USER'S VISION
|
||||
|
||||
After showing the stats summary with your invitation, **STOP and wait** for the user to tell you what they want to create.
|
||||
|
||||
---
|
||||
|
||||
## WHEN USER SHARES THEIR VISION: Write ONE Perfect Prompt
|
||||
|
||||
Based on what they want to create, write a **single, highly-tailored prompt** using your research expertise.
|
||||
|
||||
### CRITICAL: Match the FORMAT the research recommends
|
||||
|
||||
**If research says to use a specific prompt FORMAT, YOU MUST USE THAT FORMAT.**
|
||||
|
||||
**ANTI-PATTERN**: Research says "use JSON prompts with device specs" but you write plain prose. This defeats the entire purpose of the research.
|
||||
|
||||
### Quality Checklist (run before delivering):
|
||||
- [ ] **FORMAT MATCHES RESEARCH** - If research said JSON/structured/etc, prompt IS that format
|
||||
- [ ] Directly addresses what the user said they want to create
|
||||
- [ ] Uses specific patterns/keywords discovered in research
|
||||
- [ ] Ready to paste with zero edits (or minimal [PLACEHOLDERS] clearly marked)
|
||||
- [ ] Appropriate length and style for TARGET_TOOL
|
||||
|
||||
### Output Format:
|
||||
|
||||
```
|
||||
Here's your prompt for {TARGET_TOOL}:
|
||||
|
||||
---
|
||||
|
||||
[The actual prompt IN THE FORMAT THE RESEARCH RECOMMENDS]
|
||||
|
||||
---
|
||||
|
||||
This uses [brief 1-line explanation of what research insight you applied].
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## IF USER ASKS FOR MORE OPTIONS
|
||||
|
||||
Only if they ask for alternatives or more prompts, provide 2-3 variations. Don't dump a prompt pack unless requested.
|
||||
|
||||
---
|
||||
|
||||
## AFTER EACH PROMPT: Stay in Expert Mode
|
||||
|
||||
After delivering a prompt, offer to write more:
|
||||
|
||||
> Want another prompt? Just tell me what you're creating next.
|
||||
|
||||
---
|
||||
|
||||
## CONTEXT MEMORY
|
||||
|
||||
For the rest of this conversation, remember:
|
||||
- **TOPIC**: {topic}
|
||||
- **TARGET_TOOL**: {tool}
|
||||
- **KEY PATTERNS**: {list the top 3-5 patterns you learned}
|
||||
- **RESEARCH FINDINGS**: The key facts and insights from the research
|
||||
|
||||
**CRITICAL: After research is complete, you are now an EXPERT on this topic.**
|
||||
|
||||
When the user asks follow-up questions:
|
||||
- **DO NOT run new WebSearches** - you already have the research
|
||||
- **Answer from what you learned** - cite the Reddit threads, X posts, and web sources
|
||||
- **If they ask for a prompt** - write one using your expertise
|
||||
|
||||
Only do new research if the user explicitly asks about a DIFFERENT topic.
|
||||
|
||||
---
|
||||
|
||||
## Output Summary Footer (After Each Prompt)
|
||||
|
||||
After delivering a prompt, end with:
|
||||
|
||||
```
|
||||
---
|
||||
📚 Expert in: {TOPIC} for {TARGET_TOOL}
|
||||
📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} web pages
|
||||
|
||||
Want another prompt? Just tell me what you're creating next.
|
||||
```
|
||||
@@ -0,0 +1,25 @@
|
||||
Here's what I found:
|
||||
|
||||
## What I learned:
|
||||
|
||||
**BULLY Album — March 20, 2026 via Gamma** — After years of delays (first announced September 2024), Kanye's 12th studio album finally has a firm release date. The 13-track project features Peso Pluma, Playboi Carti, and Ty Dolla Sign. Sonically it recalls *808s & Heartbreak* and *MBDTF* — Ye mostly sings rather than raps. Notably, earlier leak versions used AI-deepfaked vocals, which have reportedly been re-recorded with his real voice for the official release, per Rolling Stone.
|
||||
|
||||
**Public Apology for Antisemitism** — On January 26, Ye took out a full-page Wall Street Journal ad titled "To Those I've Hurt," publicly apologizing for his antisemitic remarks. He disclosed a previously undiagnosed brain injury from his 2002 car accident and attributed his behavior to a four-month manic episode fueled by bipolar disorder, psychosis, and paranoia. "I lost touch with reality," he wrote. The apology also extended to the Black community, per The Washington Post.
|
||||
|
||||
**Hellwatt Festival in Italy** — Ye is headlining a brand-new festival at the 103,000-capacity RCF Arena in Italy over three weekends from July 4-18, 2026 — his first-ever live concert in Italy, per Billboard.
|
||||
|
||||
**Health Concerns** — A rare January 2026 outing in LA with Bianca Censori reignited concern about Ye's physical appearance. Insiders point to medication side effects, frequent travel, and inconsistent routines, per AllHipHop.
|
||||
|
||||
**Grammys Ban** — Ye is reportedly not welcome at the 2026 Grammy Awards after clashing with organizers last year over his invitation terms, per The News International.
|
||||
|
||||
**Kim & Lewis Hamilton Buzz** — X chatter is heavily focused on Kim Kardashian's relationship with Lewis Hamilton, with users contrasting her new relationship against her marriage to Ye.
|
||||
|
||||
---
|
||||
✅ All agents reported back!
|
||||
├─ 🟠 Reddit: 0 threads (no results this cycle)
|
||||
├─ 🔵 X: 29 posts │ 33 likes │ 14 reposts (via xAI)
|
||||
├─ 🌐 Web: 30+ pages │ rollingstone.com, washingtonpost.com, complex.com, billboard.com, npr.org
|
||||
└─ 🗣️ Top voices: @honest30bgfan_ (33 likes), @HipHopCrave_ │ Rolling Stone, Washington Post, Complex
|
||||
---
|
||||
|
||||
Share your vision for what you want to create and I'll write a thoughtful prompt you can copy-paste directly into your tool of choice.
|
||||
@@ -1,8 +0,0 @@
|
||||
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/community-more-posts/top/?name=technology&t=week on 2026-05-29; trimmed to 5 post cards (start-tag attrs only). -->
|
||||
<div id="feed">
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tq0zk7/the_netherlands_just_blocked_a_us_company_from/" content-href="https://www.techspot.com/news/112552-netherlands-blocked-us-company-buying-app-dutch-citizens.html" view-context="SubredditFeed" comment-count="1743" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="0" award-count="23" award-id="award_obsessed_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/Obsessed_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-28T11:37:01.506000+0000" domain="techspot.com" id="t3_1tq0zk7" post-title="The Netherlands just blocked a US company from buying the app Dutch citizens use for everything" post-language="en" post-type="link" score="52692" upvote-ratio="0.9606269354736776" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1toe7m2/erin_brockovich_launches_map_of_over_4200_data/" content-href="https://www.newsweek.com/erin-brockovich-asks-americans-for-help-as-she-launches-data-center-map-11989813" view-context="SubredditFeed" comment-count="673" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="2" award-count="6" award-id="award_this_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/this_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T17:39:43.272000+0000" domain="newsweek.com" id="t3_1toe7m2" post-title="Erin Brockovich launches map of over 4,200 data centres in the US, appeals for local communities to report environmental impact and other costs" post-language="en" post-type="link" score="33567" upvote-ratio="0.973297166968053" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_fj9vsvfd" author="marketrent" icon="https://www.redditstatic.com/avatars/defaults/v2/avatar_default_1.png" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tollgz/majority_of_americans_support_ban_on_surveillance/" content-href="https://gizmodo.com/majority-of-americans-support-ban-on-surveillance-pricing-and-electronic-shelf-labels-2000762717" view-context="SubredditFeed" comment-count="1043" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="3" award-count="7" award-id="award_free_bravo" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/bravo_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-26T21:55:07.322000+0000" domain="gizmodo.com" id="t3_1tollgz" post-title="Majority of Americans Support Ban on Surveillance Pricing and Electronic Shelf Labels" post-language="en" post-type="link" score="29791" upvote-ratio="0.9815063671850003" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_98wao505" author="Plastic_Ninja_9014" icon="https://preview.redd.it/snoovatar/avatars/69af2b53-b0a1-4ab6-b119-d90f21c423fe-headshot.png?width=64&height=64&crop=smart&auto=webp&s=f3661eb511798004968f8b115a689dcee30f1428" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tp5qz2/tech_ceos_are_apparently_suffering_from_ai/" content-href="https://techcrunch.com/2026/05/27/tech-ceos-are-apparently-suffering-from-ai-psychosis/" view-context="SubredditFeed" comment-count="1653" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="4" award-count="6" award-id="award_free_regret_2" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/regret_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-27T13:33:49.280000+0000" domain="techcrunch.com" id="t3_1tp5qz2" post-title="Tech CEOs are apparently suffering from AI psychosis" post-language="en" post-type="link" score="26419" upvote-ratio="0.9605741880002646" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_cc0n0rs5" author="AdSpecialist6598" icon="https://styles.redditmedia.com/t5_4heieb/styles/profileIcon_snoob7abf9c5-a18e-4228-a419-5179810e11df-headshot-f.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=94f6b9715ca039332ed1714f3abe0842cef23b81" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
<shreddit-post data-ks-item class="block relative cursor-pointer group bg-neutral-background focus-within:bg-neutral-background-hover hover:bg-neutral-background-hover xs:rounded-4 px-md py-2xs my-2xs nd:visible nd:pb-[var(--rem36)]" permalink="/r/technology/comments/1tn5g7s/pope_leo_issues_ai_encyclical_warning_that_opaque/" content-href="https://variety.com/2026/biz/global/pope-leo-ai-encyclical-algorithms-threaten-dehumanisation-1236758186/" view-context="SubredditFeed" comment-count="608" is-slim-card view-type="cardView" pdp-target="_self" feedIndex="6" award-count="7" award-id="award_hooray_3" award-icon-url="https://i.redd.it/snoovatar/snoo_assets/marketing/FTUE_40.png" moderation-verdict="" is-embeddable is-desktop-viewport is-awardable is-link-post created-timestamp="2026-05-25T10:45:04.093000+0000" domain="variety.com" id="t3_1tn5g7s" post-title="Pope Leo Issues AI Encyclical Warning That ‘Opaque Algorithms’ Controlled by a ‘Few’ Companies Can Bring ‘New Forms of Dehumanisation’" post-language="en" post-type="link" score="25835" upvote-ratio="0.9760626539506095" subreddit-id="t5_2qh16" subreddit-prefixed-name="r/technology" author-id="t2_1i1zizibn9" author="yourfavchoom" icon="https://styles.redditmedia.com/t5_dgdrt8/styles/profileIcon_k9x929ihm8rg1.png?width=64&height=64&frame=1&auto=webp&crop=64%3A64%2Csmart&s=2e8a5042cccc4555167f98d28bc0de4e13fd3ca5" data-expected-lcp subreddit-name="technology"></shreddit-post>
|
||||
</div>
|
||||
@@ -1,7 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- FIXTURE: captured live from reddit.com/r/Rakuten/top.rss on 2026-05-29; trimmed to 5 entries. Atom shape identical to search.rss. --><feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/"><category term="Rakuten" label="r/Rakuten"/><updated>2026-05-29T14:14:32+00:00</updated><icon>https://www.redditstatic.com/icon.png/</icon><id>/r/Rakuten/top.rss?t=month</id><link rel="self" href="https://www.reddit.com/r/Rakuten/top.rss?t=month" type="application/atom+xml" /><link rel="alternate" href="https://www.reddit.com/r/Rakuten/top?t=month" type="text/html" /><subtitle>This is an unofficial subreddit for Rakuten Rewards, the cash back website. We are not affiliated with, endorsed by, or sponsored by Rakuten or any of its subsidiaries.</subtitle><title>top scoring links : Rakuten</title><entry><author><name>/u/InternetUser52</name><uri>https://www.reddit.com/user/InternetUser52</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>I&#39;m rich!!</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/InternetUser52"> /u/InternetUser52 </a> <br/> <span><a href="https://i.redd.it/q8fgmxs29c2h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/">[comments]</a></span></content><id>t3_1tiv013</id><link href="https://www.reddit.com/r/Rakuten/comments/1tiv013/lets_goo_002/" /><updated>2026-05-20T18:48:31+00:00</updated><published>2026-05-20T18:48:31+00:00</published><title>LETS GOO! $0.02!!!</title></entry>
|
||||
<entry><author><name>/u/Immediate-Duck-6351</name><uri>https://www.reddit.com/user/Immediate-Duck-6351</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>I don’t travel and I’m buying a house in a few weeks so cash back is amazing 🙌 hoping to keep the pace in the next quarter so I can buy new kitchen appliances lol. </p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/Immediate-Duck-6351"> /u/Immediate-Duck-6351 </a> <br/> <span><a href="https://i.redd.it/d2a4s0ipvb1h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/">[comments]</a></span></content><id>t3_1te1fp8</id><link href="https://www.reddit.com/r/Rakuten/comments/1te1fp8/so_excited/" /><updated>2026-05-15T16:29:28+00:00</updated><published>2026-05-15T16:29:28+00:00</published><title>So excited 🥳</title></entry>
|
||||
<entry><author><name>/u/gnibgnib</name><uri>https://www.reddit.com/user/gnibgnib</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>128k for the May transfer</p> <p>41k pending for August </p> <p>Got another 9k at Asics not showing but overall pretty happy with Rakuten</p> <p>P2 was able to secure 85k for May transfer</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/gnibgnib"> /u/gnibgnib </a> <br/> <span><a href="https://www.reddit.com/gallery/1tb8674">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/">[comments]</a></span></content><id>t3_1tb8674</id><link href="https://www.reddit.com/r/Rakuten/comments/1tb8674/had_a_great_run_so_far_this_year_thanks_to_this/" /><updated>2026-05-12T17:17:19+00:00</updated><published>2026-05-12T17:17:19+00:00</published><title>Had a great run so far this year thanks to this sub!</title></entry>
|
||||
<entry><author><name>/u/TravelVet93</name><uri>https://www.reddit.com/user/TravelVet93</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html">&#32; submitted by &#32; <a href="https://www.reddit.com/user/TravelVet93"> /u/TravelVet93 </a> <br/> <span><a href="https://i.redd.it/x6b9whvupb1h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/">[comments]</a></span></content><id>t3_1te0hom</id><link href="https://www.reddit.com/r/Rakuten/comments/1te0hom/my_best_payout_so_far/" /><updated>2026-05-15T15:56:40+00:00</updated><published>2026-05-15T15:56:40+00:00</published><title>My best payout so far</title></entry>
|
||||
<entry><author><name>/u/Beautiful-Piece-4252</name><uri>https://www.reddit.com/user/Beautiful-Piece-4252</uri></author><category term="Rakuten" label="r/Rakuten"/><content type="html"><!-- SC_OFF --><div class="md"><p>The amount of $$ available in sign up bonuses is amazing. It&#39;s kind of a part time job ensuring Rakuten captures everything, but my August and November payout should be sizeable. I&#39;m new to this and it always seemed like a lot of work for little reward. I know it&#39;s not sustainable, but wow!</p> </div><!-- SC_ON --> &#32; submitted by &#32; <a href="https://www.reddit.com/user/Beautiful-Piece-4252"> /u/Beautiful-Piece-4252 </a> <br/> <span><a href="https://i.redd.it/1vqvajsci42h1.jpeg">[link]</a></span> &#32; <span><a href="https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/">[comments]</a></span></content><id>t3_1thsnm1</id><link href="https://www.reddit.com/r/Rakuten/comments/1thsnm1/how_can_this_be_real/" /><updated>2026-05-19T16:46:17+00:00</updated><published>2026-05-19T16:46:17+00:00</published><title>How can this be real?</title></entry>
|
||||
</feed>
|
||||
@@ -1,29 +0,0 @@
|
||||
<!-- FIXTURE: captured live from reddit.com/svc/shreddit/comments/r/Rakuten/t3_1taeiw0 on 2026-05-29;
|
||||
trimmed to 6 real comment elements (real attrs + real bodies) + 2 synthetic edge cases. -->
|
||||
<shreddit-comment-tree-stats total-comments="14"></shreddit-comment-tree-stats>
|
||||
<shreddit-comment-tree id="comment-tree" post-id="t3_1taeiw0">
|
||||
<shreddit-comment created="2026-05-11T20:16:57.590000+0000" author="Obvious_Painting_881" thingId="t1_ol8tp8n" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8tp8n/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_ol8tp8n-comment-rtjson-content" slot="comment"><div id="t1_ol8tp8n-post-rtjson-content" dir="auto"><p dir="auto">Where do you find $750? The highest available package for Total was $284.99 when I did the lifelock promotion. I did get the full 284.99 from Rakuten.</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-12T12:26:14.973000+0000" author="Stormtrooper149" thingId="t1_olcy1iv" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/olcy1iv/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_olcy1iv-comment-rtjson-content" slot="comment"><div id="t1_olcy1iv-post-rtjson-content" dir="auto"><p dir="auto">It went to pending ($712.49)</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-19T01:43:48.026000+0000" author="heythereyou01" thingId="t1_omlbiqg" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/omlbiqg/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_omlbiqg-comment-rtjson-content" slot="comment"><div id="t1_omlbiqg-post-rtjson-content" dir="auto"><p dir="auto">Hey I PM’d. can I get the screenshot ?</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-11T20:21:16.398000+0000" author="Stormtrooper149" thingId="t1_ol8undb" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8undb/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_ol8undb-comment-rtjson-content" slot="comment"><div id="t1_ol8undb-post-rtjson-content" dir="auto"><p dir="auto">Family plan</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-11T20:28:33.803000+0000" author="Obvious_Painting_881" thingId="t1_ol8w8w6" depth="2" permalink="/r/Rakuten/comments/1taeiw0/comment/ol8w8w6/" score="1" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_ol8w8w6-comment-rtjson-content" slot="comment"><div id="t1_ol8w8w6-post-rtjson-content" dir="auto"><p dir="auto">Price seems to change every time I go to the page but I see only 249.99-369.99 for Total/Advanced. No where near your $750. Just saying the Total plan for 299.99 worked for me and I got 284.99 which is 95%.</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-12T02:33:48.200000+0000" author="jwegener" thingId="t1_olaqzjk" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/olaqzjk/" score="2" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_olaqzjk-comment-rtjson-content" slot="comment"><div id="t1_olaqzjk-post-rtjson-content" dir="auto"><p dir="auto">I did that one. Let’s pray</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-13T10:00:00.000000+0000" author="[deleted]" thingId="t1_synthdel" depth="0" permalink="/r/Rakuten/comments/1taeiw0/comment/synthdel/" score="5" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_synthdel-comment-rtjson-content" slot="comment"><div id="t1_synthdel-post-rtjson-content" dir="auto"><p dir="auto">[removed]</p></div></div>
|
||||
</shreddit-comment>
|
||||
<shreddit-comment created="2026-05-13T11:00:00.000000+0000" author="NegScoreUser" thingId="t1_synthneg" depth="1" permalink="/r/Rakuten/comments/1taeiw0/comment/synthneg/" score="-7" postId="t3_1taeiw0" content-type="text">
|
||||
<div id="t1_synthneg-comment-rtjson-content" slot="comment"><div id="t1_synthneg-post-rtjson-content" dir="auto"><p dir="auto">A downvoted but real reply with negative score for edge-case coverage.</p></div></div>
|
||||
</shreddit-comment>
|
||||
</shreddit-comment-tree>
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "last30days-skill",
|
||||
"version": "3.3.2",
|
||||
"version": "3.0.5",
|
||||
"description": "Research a topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||
"settings": [
|
||||
{
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"triggerOnUpdates": true,
|
||||
"statusCheck": true
|
||||
}
|
||||
+2
-1
@@ -6,7 +6,8 @@
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash \"${CLAUDE_PLUGIN_ROOT:-${extensionPath:-.}}/hooks/scripts/check-config.sh\""
|
||||
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-config.sh",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -33,13 +33,8 @@ load_env_vars() {
|
||||
[[ -z "$key" ]] && continue
|
||||
key=$(echo "$key" | xargs)
|
||||
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||
# Strip inline comments (# preceded by whitespace) to prevent
|
||||
# command substitution in backtick-containing comments
|
||||
value="${value%%[[:space:]]#*}"
|
||||
if [[ -n "$key" && -n "$value" ]]; then
|
||||
# printf -v writes via assignment semantics (global from inside a
|
||||
# function), works on macOS's /bin/bash 3.2 — `declare -g` is 4.2+.
|
||||
printf -v "ENV_${key}" '%s' "$value"
|
||||
eval "ENV_${key}=\"${value}\""
|
||||
fi
|
||||
done < "$file"
|
||||
fi
|
||||
@@ -63,53 +58,14 @@ fi
|
||||
# Check SETUP_COMPLETE (from file or env)
|
||||
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||
|
||||
# Compute last-run summary line (if last-run.json exists)
|
||||
if [[ "${LAST30DAYS_CONFIG_DIR+x}" == "x" ]]; then
|
||||
if [[ -n "$LAST30DAYS_CONFIG_DIR" ]]; then
|
||||
LAST_RUN_FILE="$LAST30DAYS_CONFIG_DIR/last-run.json"
|
||||
else
|
||||
LAST_RUN_FILE=""
|
||||
fi
|
||||
else
|
||||
LAST_RUN_FILE="$HOME/.config/last30days/last-run.json"
|
||||
fi
|
||||
LAST_RUN_LINE=""
|
||||
if [[ -n "$LAST_RUN_FILE" && -f "$LAST_RUN_FILE" ]] && command -v python3 &>/dev/null; then
|
||||
LAST_RUN_LINE=$(LAST_RUN_FILE="$LAST_RUN_FILE" python3 - <<'PY' 2>/dev/null || true
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
|
||||
path = os.environ["LAST_RUN_FILE"]
|
||||
try:
|
||||
with open(path) as fh:
|
||||
d = json.load(fh)
|
||||
topic = (d.get("topic") or "?")[:60]
|
||||
ts = d.get("timestamp", "")
|
||||
dt = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
delta = (datetime.datetime.now(datetime.timezone.utc) - dt).total_seconds()
|
||||
if delta < 60: ago = f"{int(delta)}s ago"
|
||||
elif delta < 3600: ago = f"{int(delta//60)}m ago"
|
||||
elif delta < 86400: ago = f"{int(delta//3600)}h ago"
|
||||
else: ago = f"{int(delta//86400)}d ago"
|
||||
total = d.get("total", 0)
|
||||
print(f" Last run: \"{topic}\" · {ago} · {total} results")
|
||||
except Exception:
|
||||
pass
|
||||
PY
|
||||
)
|
||||
fi
|
||||
|
||||
# If setup has never been run, show welcome message for new users
|
||||
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||
cat <<'EOF'
|
||||
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||
Research any topic across Reddit, HN, X, YouTube, Polymarket (last 30 days).
|
||||
|
||||
Reddit, Hacker News, and Polymarket work out of the box.
|
||||
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||
EOF
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -141,33 +97,16 @@ if [[ -n "$HAS_BSKY" ]]; then
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||
fi
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
|
||||
# Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer
|
||||
# whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing.
|
||||
SC_ADD=3
|
||||
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
|
||||
EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \
|
||||
| sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//')
|
||||
if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then
|
||||
SC_ADD=$((SC_ADD - 1))
|
||||
fi
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
|
||||
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||
fi
|
||||
|
||||
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||
# Fully configured — compact ready message
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
else
|
||||
# Setup done but missing ScrapeCreators — recommend it
|
||||
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||
echo " Research any topic across social + market + web sources (last 30 days)."
|
||||
[[ -n "$LAST_RUN_LINE" ]] && echo "$LAST_RUN_LINE"
|
||||
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||
echo " 100 free credits, no credit card — scrapecreators.com"
|
||||
echo " 10,000 free API calls, no credit card — scrapecreators.com"
|
||||
echo " last30days has no affiliation with any API provider."
|
||||
fi
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.4 MiB |
+9
-6
@@ -1,14 +1,16 @@
|
||||
[project]
|
||||
name = "last30days-skill"
|
||||
version = "3.3.2"
|
||||
version = "3.0.0"
|
||||
description = "Multi-source last-30-days research skill"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
dependencies = [
|
||||
"requests>=2.32,<3",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.0.3,<10",
|
||||
"pytest>=9,<10",
|
||||
"pytest-cov>=7,<8",
|
||||
]
|
||||
|
||||
@@ -22,9 +24,9 @@ addopts = [
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["skills/last30days/scripts", "tests"]
|
||||
source = ["scripts", "tests"]
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
@@ -32,6 +34,7 @@ omit = [
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
omit = [
|
||||
"skills/last30days/scripts/lib/vendor/*",
|
||||
"scripts/lib/vendor/*",
|
||||
"dist/*",
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
The AI world reinvents itself every month. This skill keeps you current.
|
||||
|
||||
`/last30days` researches your topic across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and 5+ more sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations.
|
||||
|
||||
## v3 is the intelligent search release
|
||||
|
||||
v3 is a ground-up engine rewrite by [@j-sperling](https://github.com/j-sperling). The old engine searched keywords. The new engine understands your topic first, then searches the right people and communities.
|
||||
|
||||
Type "OpenClaw" and v3 resolves @steipete, r/openclaw, r/ClaudeCode, and the right YouTube channels and TikTok hashtags before a single API call fires. Type "Peter Steinberger" and it resolves his X handle and GitHub profile, switches to person mode, and shows what he shipped this month at 85% merge rate across 22 PRs. None of that was on Google.
|
||||
|
||||
## Headline features
|
||||
|
||||
### Intelligent pre-research
|
||||
|
||||
The killer feature. A new Python pre-research brain resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching. Bidirectional: person to company, product to founder, name to GitHub profile. The right subreddits, the right handles, the right hashtags, all resolved before a single API call.
|
||||
|
||||
### Best Takes
|
||||
|
||||
A second LLM judge scores every result for humor, wit, and virality alongside relevance. Every brief now ends with a Best Takes section surfacing the cleverest one-liners and most viral quotes. The Reddit and X people are funny, and the old engine buried their best stuff.
|
||||
|
||||
### Cross-source cluster merging
|
||||
|
||||
When the same story hits Reddit, X, and YouTube, v3 merges them into one cluster instead of three duplicates. Entity-based overlap detection catches matches even when the titles use different words.
|
||||
|
||||
### Single-pass comparisons
|
||||
|
||||
"X vs Y" used to run three serial passes (12+ minutes). v3 runs one pass with entity-aware subqueries for both sides at once. Same depth, 3 minutes.
|
||||
|
||||
### GitHub person-mode and project-mode
|
||||
|
||||
When the topic is a person, the engine switches from keyword search to author-scoped queries. PR velocity, top repos by stars, release notes for what shipped this month, woven into the narrative alongside X posts and Reddit threads.
|
||||
|
||||
When the topic is a project, it pulls live star counts, READMEs, releases, and top issues from the GitHub API. No stale blog posts.
|
||||
|
||||
### ELI5 mode
|
||||
|
||||
Say "eli5 on" after any research run. The synthesis rewrites in plain language. No jargon. Same data, same sources, same citations, just clearer. Say "eli5 off" to go back.
|
||||
|
||||
### 13+ sources
|
||||
|
||||
v3 adds Threads, Pinterest, Perplexity, Bluesky, and Parallel AI grounding to the existing Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, GitHub, and Web lineup. Perplexity Deep Research (`--deep-research`) gives you 50+ citation reports for serious investigation.
|
||||
|
||||
### Per-author cap and entity disambiguation
|
||||
|
||||
Max 3 items per author prevents single-voice dominance. Synthesis trusts resolved handles over fuzzy keyword matches.
|
||||
|
||||
## Install
|
||||
|
||||
Claude Code:
|
||||
|
||||
```
|
||||
/plugin marketplace add mvanhorn/last30days-skill
|
||||
```
|
||||
|
||||
OpenClaw:
|
||||
|
||||
```
|
||||
clawhub install last30days-official
|
||||
```
|
||||
|
||||
OpenAI Codex CLI: run `codex` from a checkout of this repo and v3's skill at `.agents/skills/last30days/SKILL.md` will be discovered automatically. Or copy `SKILL.md` to `~/.agents/skills/last30days/SKILL.md` for a global install.
|
||||
|
||||
Zero config. Reddit, Hacker News, Polymarket, and GitHub work immediately. Run it once and the setup wizard unlocks X, YouTube, TikTok, and more in 30 seconds.
|
||||
|
||||
## v3 Community
|
||||
|
||||
v3 was shaped by community contributors whose PRs and issues inspired core features. Their code wasn't merged directly (v3 was a ground-up rewrite), but their ideas drove what shipped.
|
||||
|
||||
Thanks to @uppinote20, @zerone0x, @thinkun, @thomasmktong, @fanispoulinakisai-boop, @pejmanjohn, @zl190, and @hnshah. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the full list.
|
||||
|
||||
Contributors who shaped the release itself:
|
||||
|
||||
- @Jah-yee (#153) surfaced the need for a real Codex CLI integration, which shipped in #219
|
||||
- @Cody-Coyote (#204) reported the marketplace validation bug that needed fixing before v3 could ship cleanly
|
||||
- @dannyshmueli pushed for v3 and Codex family support publicly on X
|
||||
|
||||
Full Added / Changed / Fixed detail lives in [CHANGELOG.md](CHANGELOG.md) under `[3.0.0]`.
|
||||
|
||||
## Earlier contributors
|
||||
|
||||
From the v1 and v2 lineage:
|
||||
|
||||
- [@galligan](https://github.com/galligan) for marketplace plugin inspiration
|
||||
- [@hutchins](https://x.com/hutchins) for pushing the YouTube feature
|
||||
|
||||
30 days of research. 30 seconds of work. Thirteen sources. Zero stale prompts.
|
||||
@@ -1,14 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh - package this repo as a claude.ai-upload-ready .skill file
|
||||
# Usage: bash skills/last30days/scripts/build-skill.sh (run from repo root)
|
||||
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||
#
|
||||
# Produces dist/last30days.skill, a zip with a single top-level `last30days/`
|
||||
# directory containing SKILL.md and the scripts/ runtime from skills/last30days.
|
||||
# See
|
||||
# directory containing SKILL.md and the scripts/ runtime. See
|
||||
# docs/plans/2026-04-14-001-fix-skill-upload-200-file-limit-plan.md.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
@@ -18,7 +17,14 @@ fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/last30days.skill"
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD:skills/last30days
|
||||
git archive --format=zip --prefix=last30days/ --output="$OUT" HEAD
|
||||
|
||||
# claude.ai's .skill bundle only needs the root SKILL.md + scripts/ runtime.
|
||||
# Claude Code needs skills/ and .claude-plugin/ in the git archive
|
||||
# (that's why they're NOT in .gitattributes export-ignore), but the .skill
|
||||
# bundle must strip them to keep a single canonical SKILL.md and stay under
|
||||
# the 200-file cap.
|
||||
zip -d "$OUT" "last30days/skills/*" "last30days/.claude-plugin/*" > /dev/null 2>&1 || true
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# A/B test runner: public release vs private beta
|
||||
# Usage: bash skills/last30days/scripts/compare.sh "Kanye West"
|
||||
# Usage: bash scripts/compare.sh "Kanye West"
|
||||
#
|
||||
# Runs /last30days (public release) and /last30days-beta (private beta)
|
||||
# sequentially with a 30s gap, saves raw results with distinct suffixes,
|
||||
@@ -9,14 +9,13 @@
|
||||
set -e
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: bash skills/last30days/scripts/compare.sh <topic>"
|
||||
echo " Example: bash skills/last30days/scripts/compare.sh Kevin Rose"
|
||||
echo "Usage: bash scripts/compare.sh <topic>"
|
||||
echo " Example: bash scripts/compare.sh Kevin Rose"
|
||||
exit 1
|
||||
fi
|
||||
TOPIC="$*"
|
||||
SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//')
|
||||
LAST30DAYS_MEMORY_DIR="${LAST30DAYS_MEMORY_DIR:-$HOME/Documents/Last30Days}"
|
||||
DIR="$LAST30DAYS_MEMORY_DIR"
|
||||
DIR="$HOME/Documents/Last30Days"
|
||||
DATE=$(date +%Y-%m-%d)
|
||||
|
||||
echo "=============================================="
|
||||
+3
-8
@@ -20,11 +20,9 @@ sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from lib import env as envlib
|
||||
from lib import schema
|
||||
from lib.providers import GEMINI_FLASH_LITE
|
||||
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json"
|
||||
|
||||
|
||||
@@ -44,7 +42,7 @@ def _load_default_topics() -> list[tuple[str, str]]:
|
||||
|
||||
DEFAULT_TOPICS = _load_default_topics()
|
||||
DEFAULT_SEARCH = ""
|
||||
DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE
|
||||
DEFAULT_JUDGE_MODEL = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||
|
||||
|
||||
@@ -309,10 +307,7 @@ def create_eval_env() -> dict[str, str]:
|
||||
|
||||
|
||||
def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]:
|
||||
engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py"
|
||||
if not engine.exists():
|
||||
engine = repo_dir / "scripts" / "last30days.py"
|
||||
cmd = [sys.executable, str(engine), topic, "--emit=json"]
|
||||
cmd = [sys.executable, "scripts/last30days.py", topic, "--emit=json"]
|
||||
if search:
|
||||
cmd.extend(["--search", search])
|
||||
if quick:
|
||||
@@ -0,0 +1,433 @@
|
||||
#!/usr/bin/env python3
|
||||
# ruff: noqa: E402
|
||||
"""last30days v3.0.0 CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
MIN_PYTHON = (3, 12)
|
||||
|
||||
|
||||
def ensure_supported_python(version_info: tuple[int, int, int] | object | None = None) -> None:
|
||||
if version_info is None:
|
||||
version_info = sys.version_info
|
||||
major, minor, micro = tuple(version_info[:3])
|
||||
if (major, minor) >= MIN_PYTHON:
|
||||
return
|
||||
sys.stderr.write(
|
||||
"last30days v3 requires Python 3.12+.\n"
|
||||
f"Detected Python {major}.{minor}.{micro}.\n"
|
||||
"Install and use python3.12 or python3.13, then rerun this command.\n"
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
ensure_supported_python()
|
||||
|
||||
if os.name == "nt":
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(stream, "reconfigure"):
|
||||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from lib import env, pipeline, render, schema, ui
|
||||
|
||||
_child_pids: set[int] = set()
|
||||
_child_pids_lock = threading.Lock()
|
||||
|
||||
|
||||
def register_child_pid(pid: int) -> None:
|
||||
with _child_pids_lock:
|
||||
_child_pids.add(pid)
|
||||
|
||||
|
||||
def unregister_child_pid(pid: int) -> None:
|
||||
with _child_pids_lock:
|
||||
_child_pids.discard(pid)
|
||||
|
||||
|
||||
def _cleanup_children() -> None:
|
||||
with _child_pids_lock:
|
||||
pids = list(_child_pids)
|
||||
for pid in pids:
|
||||
try:
|
||||
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
continue
|
||||
|
||||
|
||||
atexit.register(_cleanup_children)
|
||||
|
||||
|
||||
def parse_search_flag(raw: str) -> list[str]:
|
||||
sources = []
|
||||
for source in raw.split(","):
|
||||
source = source.strip().lower()
|
||||
if not source:
|
||||
continue
|
||||
normalized = pipeline.SEARCH_ALIAS.get(source, source)
|
||||
if normalized not in pipeline.MOCK_AVAILABLE_SOURCES:
|
||||
raise SystemExit(f"Unknown search source: {source}")
|
||||
if normalized not in sources:
|
||||
sources.append(normalized)
|
||||
if not sources:
|
||||
raise SystemExit("--search requires at least one source.")
|
||||
return sources
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
|
||||
return slug or "last30days"
|
||||
|
||||
|
||||
def save_output(report: schema.Report, emit: str, save_dir: str, suffix: str = "") -> Path:
|
||||
from datetime import datetime
|
||||
path = Path(save_dir).expanduser().resolve()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
slug = slugify(report.topic)
|
||||
extension = "json" if emit == "json" else "md"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
out_path = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
if out_path.exists():
|
||||
out_path = path / f"{slug}-raw{suffix_part}-{datetime.now().strftime('%Y-%m-%d')}.{extension}"
|
||||
# Always save the FULL dump to disk (all items, all sources, transcripts).
|
||||
# Claude sees compact clusters via --emit=compact on stdout.
|
||||
# The saved file is the complete debug artifact.
|
||||
if emit == "json":
|
||||
content = emit_output(report, emit)
|
||||
else:
|
||||
content = render.render_full(report)
|
||||
out_path.write_text(content, encoding="utf-8")
|
||||
return out_path
|
||||
|
||||
|
||||
def emit_output(report: schema.Report, emit: str, fun_level: str = "medium", save_path: str | None = None) -> str:
|
||||
if emit == "json":
|
||||
return json.dumps(schema.to_dict(report), indent=2, sort_keys=True)
|
||||
if emit in {"compact", "md"}:
|
||||
return render.render_compact(report, fun_level=fun_level, save_path=save_path)
|
||||
if emit == "context":
|
||||
return render.render_context(report)
|
||||
raise SystemExit(f"Unsupported emit mode: {emit}")
|
||||
|
||||
|
||||
def compute_save_path_display(save_dir: str, topic: str, suffix: str, emit: str) -> str:
|
||||
"""Compute the user-friendly save path string that will be shown in the footer.
|
||||
|
||||
Uses ~ for the home directory so the footer reads "~/Documents/Last30Days/slug-raw.md"
|
||||
instead of an absolute machine-local path.
|
||||
"""
|
||||
from pathlib import Path as _Path
|
||||
path = _Path(save_dir).expanduser().resolve()
|
||||
slug = slugify(topic)
|
||||
extension = "json" if emit == "json" else "md"
|
||||
suffix_part = f"-{suffix}" if suffix else ""
|
||||
raw = path / f"{slug}-raw{suffix_part}.{extension}"
|
||||
try:
|
||||
home = _Path.home().resolve()
|
||||
relative = raw.relative_to(home)
|
||||
return f"~/{relative}"
|
||||
except ValueError:
|
||||
return str(raw)
|
||||
|
||||
|
||||
def persist_report(report: schema.Report) -> dict[str, int]:
|
||||
import store
|
||||
|
||||
store.init_db()
|
||||
topic_row = store.add_topic(report.topic)
|
||||
topic_id = topic_row["id"]
|
||||
source_mode = ",".join(sorted(report.items_by_source)) or "v3"
|
||||
run_id = store.record_run(topic_id, source_mode=source_mode, status="running")
|
||||
try:
|
||||
findings = store.findings_from_report(report)
|
||||
counts = store.store_findings(run_id, topic_id, findings)
|
||||
store.update_run(
|
||||
run_id,
|
||||
status="completed",
|
||||
findings_new=counts["new"],
|
||||
findings_updated=counts["updated"],
|
||||
)
|
||||
return counts
|
||||
except Exception as exc:
|
||||
store.update_run(run_id, status="failed", error_message=str(exc)[:500])
|
||||
raise
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Research a topic across live social, market, and grounded web sources.")
|
||||
parser.add_argument("topic", nargs="*", help="Research topic")
|
||||
parser.add_argument("--emit", default="compact", choices=["compact", "json", "context", "md"])
|
||||
parser.add_argument("--search", help="Comma-separated source list")
|
||||
parser.add_argument("--quick", action="store_true", help="Lower-latency retrieval profile")
|
||||
parser.add_argument("--deep", action="store_true", help="Higher-recall retrieval profile")
|
||||
parser.add_argument("--debug", action="store_true", help="Enable HTTP debug logging")
|
||||
parser.add_argument("--mock", action="store_true", help="Use mock retrieval fixtures")
|
||||
parser.add_argument("--diagnose", action="store_true", help="Print provider and source availability")
|
||||
parser.add_argument("--save-dir", help="Optional directory for saving the rendered output")
|
||||
parser.add_argument("--store", action="store_true", help="Persist ranked findings to the SQLite research store")
|
||||
parser.add_argument("--x-handle", help="X handle for targeted supplemental search")
|
||||
parser.add_argument("--x-related", help="Comma-separated related X handles (searched with lower weight)")
|
||||
parser.add_argument("--web-backend", default="auto",
|
||||
choices=["auto", "brave", "exa", "serper", "parallel", "none"],
|
||||
help="Web search backend (default: auto, tries Brave then Exa then Serper then Parallel)")
|
||||
parser.add_argument("--deep-research", action="store_true",
|
||||
help="Use Perplexity Deep Research (~$0.90/query) for in-depth analysis. Requires OPENROUTER_API_KEY.")
|
||||
parser.add_argument("--plan", help="JSON query plan (skips internal LLM planner). Can be a JSON string or a file path.")
|
||||
parser.add_argument("--save-suffix", help="Suffix for saved output filename (e.g., 'gemini' → kanye-west-raw-gemini.md)")
|
||||
parser.add_argument("--subreddits", help="Comma-separated subreddit names to search (e.g., SaaS,Entrepreneur)")
|
||||
parser.add_argument("--tiktok-hashtags", help="Comma-separated TikTok hashtags without # (e.g., tella,screenrecording)")
|
||||
parser.add_argument("--tiktok-creators", help="Comma-separated TikTok creator handles (e.g., TellaHQ,taborplace)")
|
||||
parser.add_argument("--ig-creators", help="Comma-separated Instagram creator handles (e.g., tella.tv,laborstories)")
|
||||
parser.add_argument(
|
||||
"--days",
|
||||
"--lookback-days",
|
||||
dest="lookback_days",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Number of days to look back for research (default: 30, watchlist uses 90)",
|
||||
)
|
||||
parser.add_argument("--auto-resolve", action="store_true",
|
||||
help="Use web search to discover subreddits/handles before planning (for platforms without WebSearch)")
|
||||
parser.add_argument("--github-user", help="GitHub username for person-mode search (e.g., steipete)")
|
||||
parser.add_argument("--github-repo", help="Comma-separated owner/repo for project-mode search (e.g., openclaw/openclaw,paperclipai/paperclip)")
|
||||
return parser
|
||||
|
||||
|
||||
def _missing_sources_for_promo(diag: dict[str, object]) -> str | None:
|
||||
available = set(diag.get("available_sources") or [])
|
||||
missing = []
|
||||
if "reddit" not in available:
|
||||
missing.append("reddit")
|
||||
if "x" not in available:
|
||||
missing.append("x")
|
||||
if "grounding" not in available:
|
||||
missing.append("web")
|
||||
if not missing:
|
||||
return None
|
||||
if "reddit" in missing and "x" in missing:
|
||||
return "both"
|
||||
return missing[0]
|
||||
|
||||
|
||||
def _show_runtime_ui(report: schema.Report, progress: ui.ProgressDisplay, diag: dict[str, object]) -> None:
|
||||
counts = {source: len(items) for source, items in report.items_by_source.items()}
|
||||
display_sources = list(
|
||||
dict.fromkeys(
|
||||
[
|
||||
*report.query_plan.source_weights.keys(),
|
||||
*report.items_by_source.keys(),
|
||||
*report.errors_by_source.keys(),
|
||||
]
|
||||
)
|
||||
)
|
||||
progress.end_processing()
|
||||
progress.show_complete(
|
||||
source_counts=counts,
|
||||
display_sources=display_sources,
|
||||
)
|
||||
promo = _missing_sources_for_promo(diag)
|
||||
if promo:
|
||||
progress.show_promo(promo, diag=diag)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
# Use parse_known_args so setup sub-flags (--device-auth, --github,
|
||||
# --openclaw) pass through without argparse hard-exiting.
|
||||
args, extra_argv = parser.parse_known_args()
|
||||
if args.debug:
|
||||
os.environ["LAST30DAYS_DEBUG"] = "1"
|
||||
|
||||
config = env.get_config()
|
||||
|
||||
# Handle setup subcommand
|
||||
topic = " ".join(args.topic).strip()
|
||||
if topic.lower() == "setup":
|
||||
from lib import setup_wizard
|
||||
if "--openclaw" in extra_argv:
|
||||
results = setup_wizard.run_openclaw_setup(config)
|
||||
print(json.dumps(results))
|
||||
return 0
|
||||
if "--github" in extra_argv:
|
||||
results = setup_wizard.run_github_auth()
|
||||
print(json.dumps(results))
|
||||
return 0
|
||||
if "--device-auth" in extra_argv:
|
||||
results = setup_wizard.run_full_device_auth()
|
||||
print(json.dumps(results))
|
||||
return 0
|
||||
sys.stderr.write("Running auto-setup...\n")
|
||||
results = setup_wizard.run_auto_setup(config)
|
||||
from_browser = "auto"
|
||||
if results.get("cookies_found"):
|
||||
first_browser = next(iter(results["cookies_found"].values()))
|
||||
from_browser = first_browser
|
||||
setup_wizard.write_setup_config(env.CONFIG_FILE, from_browser=from_browser)
|
||||
results["env_written"] = True
|
||||
sys.stderr.write(setup_wizard.get_setup_status_text(results) + "\n")
|
||||
return 0
|
||||
|
||||
requested_sources = parse_search_flag(args.search) if args.search else None
|
||||
diag = pipeline.diagnose(config, requested_sources)
|
||||
|
||||
if args.diagnose:
|
||||
print(json.dumps(diag, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
if not topic:
|
||||
parser.print_usage(sys.stderr)
|
||||
return 2
|
||||
|
||||
if not os.environ.get("LAST30DAYS_SKIP_PREFLIGHT"):
|
||||
from lib import preflight
|
||||
refuse_msg = preflight.check_class_1_trap(topic)
|
||||
if refuse_msg:
|
||||
sys.stderr.write(refuse_msg)
|
||||
return 2
|
||||
|
||||
progress = ui.ProgressDisplay(topic, show_banner=True)
|
||||
progress.start_processing()
|
||||
|
||||
depth = "deep" if args.deep else "quick" if args.quick else "default"
|
||||
try:
|
||||
x_related = [h.strip() for h in args.x_related.split(",") if h.strip()] if args.x_related else None
|
||||
subreddits = [s.strip().lstrip("r/") for s in args.subreddits.split(",") if s.strip()] if args.subreddits else None
|
||||
tiktok_hashtags = [h.strip().lstrip("#") for h in args.tiktok_hashtags.split(",") if h.strip()] if args.tiktok_hashtags else None
|
||||
tiktok_creators = [c.strip().lstrip("@") for c in args.tiktok_creators.split(",") if c.strip()] if args.tiktok_creators else None
|
||||
ig_creators = [c.strip().lstrip("@") for c in args.ig_creators.split(",") if c.strip()] if args.ig_creators else None
|
||||
# Parse external plan if provided via --plan flag
|
||||
external_plan = None
|
||||
if args.plan:
|
||||
import json as _json
|
||||
plan_str = args.plan
|
||||
if os.path.isfile(plan_str):
|
||||
plan_str = open(plan_str).read()
|
||||
try:
|
||||
external_plan = _json.loads(plan_str)
|
||||
except _json.JSONDecodeError as exc:
|
||||
sys.stderr.write(f"[Planner] Invalid --plan JSON: {exc}\n")
|
||||
|
||||
# Auto-resolve: use web search to discover subreddits/handles before planning.
|
||||
# This is the engine-side equivalent of SKILL.md Steps 0.55/0.75 for platforms
|
||||
# without WebSearch (OpenClaw, Codex, raw CLI).
|
||||
if args.auto_resolve and not external_plan:
|
||||
from lib import resolve
|
||||
resolution = resolve.auto_resolve(topic, config)
|
||||
if resolution.get("subreddits") and not subreddits:
|
||||
subreddits = resolution["subreddits"]
|
||||
sys.stderr.write(f"[AutoResolve] Subreddits: {', '.join(subreddits)}\n")
|
||||
if resolution.get("x_handle") and not args.x_handle:
|
||||
args.x_handle = resolution["x_handle"]
|
||||
sys.stderr.write(f"[AutoResolve] X handle: @{args.x_handle}\n")
|
||||
if resolution.get("github_user") and not args.github_user:
|
||||
args.github_user = resolution["github_user"]
|
||||
sys.stderr.write(f"[AutoResolve] GitHub user: @{args.github_user}\n")
|
||||
if resolution.get("github_repos") and not args.github_repo:
|
||||
args.github_repo = ",".join(resolution["github_repos"])
|
||||
sys.stderr.write(f"[AutoResolve] GitHub repos: {args.github_repo}\n")
|
||||
if resolution.get("context"):
|
||||
# Inject context into external_plan metadata for the planner to use
|
||||
if not external_plan:
|
||||
external_plan = None # planner will use its own, but with context
|
||||
# Store context for the planner prompt injection
|
||||
config["_auto_resolve_context"] = resolution["context"]
|
||||
sys.stderr.write(f"[AutoResolve] Context: {resolution['context'][:80]}...\n")
|
||||
|
||||
github_user = args.github_user.lstrip("@").lower() if args.github_user else None
|
||||
github_repos = [r.strip() for r in args.github_repo.split(",") if r.strip() and "/" in r.strip()] if args.github_repo else None
|
||||
|
||||
# --deep-research: auto-enable perplexity source and set deep flag
|
||||
if args.deep_research:
|
||||
if not config.get("OPENROUTER_API_KEY"):
|
||||
print("Error: --deep-research requires OPENROUTER_API_KEY", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
config["_deep_research"] = True
|
||||
# Auto-enable perplexity in INCLUDE_SOURCES
|
||||
include = config.get("INCLUDE_SOURCES") or ""
|
||||
if "perplexity" not in include.lower():
|
||||
config["INCLUDE_SOURCES"] = f"{include},perplexity" if include else "perplexity"
|
||||
|
||||
report = pipeline.run(
|
||||
topic=topic,
|
||||
config=config,
|
||||
depth=depth,
|
||||
requested_sources=requested_sources,
|
||||
mock=args.mock,
|
||||
x_handle=args.x_handle,
|
||||
x_related=x_related,
|
||||
web_backend=args.web_backend,
|
||||
external_plan=external_plan,
|
||||
subreddits=subreddits,
|
||||
tiktok_hashtags=tiktok_hashtags,
|
||||
tiktok_creators=tiktok_creators,
|
||||
ig_creators=ig_creators,
|
||||
lookback_days=args.lookback_days,
|
||||
github_user=github_user,
|
||||
github_repos=github_repos,
|
||||
)
|
||||
except Exception as exc:
|
||||
progress.end_processing()
|
||||
progress.show_error(str(exc))
|
||||
raise
|
||||
_show_runtime_ui(report, progress, diag)
|
||||
if args.store:
|
||||
counts = persist_report(report)
|
||||
sys.stderr.write(
|
||||
f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
|
||||
# Show quality nudge if applicable
|
||||
try:
|
||||
from lib import quality_nudge
|
||||
quality = quality_nudge.compute_quality_score(config, {})
|
||||
if quality.get("nudge_text"):
|
||||
sys.stderr.write(f"\n{quality['nudge_text']}\n")
|
||||
sys.stderr.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
fun_level = config.get("FUN_LEVEL", "medium").lower()
|
||||
footer_save_path = None
|
||||
if args.save_dir:
|
||||
footer_save_path = compute_save_path_display(
|
||||
args.save_dir, report.topic, args.save_suffix or "", args.emit
|
||||
)
|
||||
|
||||
# Signal to render_compact whether pre-research flags were supplied.
|
||||
# Used to emit a Pre-Research Status warning when the model skipped
|
||||
# Step 0.5 / 0.55 and invoked the engine bare on an eligible topic.
|
||||
pre_research_flags_present = bool(
|
||||
args.x_handle
|
||||
or args.github_user
|
||||
or args.subreddits
|
||||
or args.plan
|
||||
or args.auto_resolve
|
||||
or args.tiktok_creators
|
||||
or args.ig_creators
|
||||
)
|
||||
report.artifacts["pre_research_flags_present"] = pre_research_flags_present
|
||||
|
||||
rendered = emit_output(report, args.emit, fun_level=fun_level, save_path=footer_save_path)
|
||||
if args.save_dir:
|
||||
save_path = save_output(report, args.emit, args.save_dir, suffix=args.save_suffix or "")
|
||||
sys.stderr.write(f"[last30days] Saved output to {save_path}\n")
|
||||
sys.stderr.flush()
|
||||
print(rendered)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -7,22 +7,18 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log, subproc
|
||||
from . import http, log
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
# How many times to retry the bird-search subprocess when stdout is non-JSON
|
||||
# (typically an HTML anti-bot interstitial from Twitter's edge).
|
||||
MAX_JSON_DECODE_RETRIES = 2
|
||||
JSON_DECODE_RETRY_DELAY = 5.0 # seconds between retry attempts
|
||||
|
||||
|
||||
def _first_of(*values):
|
||||
"""Return first value that is not None."""
|
||||
@@ -154,14 +150,16 @@ def get_bird_status() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"""Invoke the vendored bird-search.mjs subprocess once.
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
Returns (result, error_dict). If error_dict is non-None, treat it as the
|
||||
final result and do not retry — those errors are terminal (timeout,
|
||||
spawn failure). If error_dict is None, the subprocess ran to completion
|
||||
and `result` is the SubprocResult; the caller decides whether to retry
|
||||
based on the result.stdout content.
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
cmd = [
|
||||
"node", str(_BIRD_SEARCH_MJS),
|
||||
@@ -170,109 +168,62 @@ def _invoke_bird_subprocess(query: str, count: int, timeout: int):
|
||||
"--json",
|
||||
]
|
||||
|
||||
pid_holder: list[int] = []
|
||||
# Use process groups for clean cleanup on timeout/kill
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
|
||||
def _register(pid: int) -> None:
|
||||
pid_holder.append(pid)
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
# Register for cleanup tracking (if available)
|
||||
try:
|
||||
from last30days import register_child_pid
|
||||
register_child_pid(pid)
|
||||
from last30days import register_child_pid, unregister_child_pid
|
||||
register_child_pid(proc.pid)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(
|
||||
cmd,
|
||||
timeout=timeout,
|
||||
env=_subprocess_env(),
|
||||
on_pid=_register,
|
||||
)
|
||||
except subproc.SubprocTimeout:
|
||||
return None, {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
except Exception as e:
|
||||
return None, {"error": str(e), "items": []}
|
||||
finally:
|
||||
if pid_holder:
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
# Kill the entire process group
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||
finally:
|
||||
try:
|
||||
from last30days import unregister_child_pid
|
||||
unregister_child_pid(pid_holder[0])
|
||||
unregister_child_pid(proc.pid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result, None
|
||||
|
||||
|
||||
def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
"""Run a search using the vendored bird-search.mjs module.
|
||||
|
||||
Retries the subprocess on JSON-decode failure (typically a Twitter
|
||||
anti-bot HTML interstitial in stdout) up to MAX_JSON_DECODE_RETRIES
|
||||
times with JSON_DECODE_RETRY_DELAY seconds between attempts. Terminal
|
||||
errors (subprocess timeout, non-zero return code) are returned
|
||||
immediately without retry.
|
||||
|
||||
Args:
|
||||
query: Full search query string (including since: filter)
|
||||
count: Number of results to request
|
||||
timeout: Timeout in seconds (per attempt)
|
||||
|
||||
Returns:
|
||||
Raw Bird JSON response or error dict.
|
||||
"""
|
||||
last_decode_error: Optional[str] = None
|
||||
|
||||
for attempt in range(MAX_JSON_DECODE_RETRIES):
|
||||
result, terminal_error = _invoke_bird_subprocess(query, count, timeout)
|
||||
if terminal_error is not None:
|
||||
return terminal_error
|
||||
|
||||
if result.returncode != 0:
|
||||
error = result.stderr.strip() or "Bird search failed"
|
||||
if proc.returncode != 0:
|
||||
error = stderr.strip() if stderr else "Bird search failed"
|
||||
return {"error": error, "items": []}
|
||||
|
||||
output = result.stdout.strip()
|
||||
output = stdout.strip() if stdout else ""
|
||||
if not output:
|
||||
return {"items": []}
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError as e:
|
||||
# Twitter's edge sometimes serves an HTML anti-bot interstitial
|
||||
# in place of JSON. Tag the failure shape so it's distinguishable
|
||||
# from "no results" in logs, then retry the subprocess.
|
||||
looks_html = output.lstrip().lower().startswith(("<!doctype", "<html", "<"))
|
||||
attempt_num = attempt + 1
|
||||
log_msg = (
|
||||
f"Bird search returned non-JSON stdout "
|
||||
f"(looks_html={looks_html}, attempt {attempt_num}/{MAX_JSON_DECODE_RETRIES}, "
|
||||
f"first 80 chars: {output[:80]!r})"
|
||||
)
|
||||
last_decode_error = str(e)
|
||||
if attempt_num < MAX_JSON_DECODE_RETRIES:
|
||||
log.source_log(
|
||||
"X/bird",
|
||||
f"{log_msg}; retrying in {JSON_DECODE_RETRY_DELAY:.0f}s",
|
||||
)
|
||||
time.sleep(JSON_DECODE_RETRY_DELAY)
|
||||
continue
|
||||
log.source_log("X/bird", log_msg)
|
||||
return {
|
||||
"error": (
|
||||
f"Invalid JSON response after {MAX_JSON_DECODE_RETRIES} attempts "
|
||||
f"(likely Twitter anti-bot interstitial): {e}"
|
||||
),
|
||||
"items": [],
|
||||
}
|
||||
|
||||
parsed = json.loads(output)
|
||||
if isinstance(parsed, list):
|
||||
return {"items": parsed}
|
||||
return parsed
|
||||
|
||||
# Defensive fallthrough — loop should always return above.
|
||||
return {
|
||||
"error": f"Bird search exhausted retries: {last_decode_error}",
|
||||
"items": [],
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "items": []}
|
||||
|
||||
|
||||
def search_x(
|
||||
@@ -379,29 +330,47 @@ def search_handles(
|
||||
"--json",
|
||||
]
|
||||
|
||||
try:
|
||||
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
|
||||
except subproc.SubprocTimeout:
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
except OSError as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
if result.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
||||
return []
|
||||
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return []
|
||||
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
preexec_fn=preexec,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
return []
|
||||
|
||||
if proc.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||
return []
|
||||
|
||||
output = (stdout or "").strip()
|
||||
if not output:
|
||||
return []
|
||||
|
||||
response = json.loads(output)
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
return []
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
"""Bluesky search via AT Protocol (requires app password).
|
||||
|
||||
Uses bsky.social for auth and api.bsky.app for post search (the canonical
|
||||
authenticated AppView). The previous default `public.api.bsky.app` is the
|
||||
unauthenticated public mirror, which BunnyCDN now blocks for searchPosts
|
||||
regardless of auth header (verified 2026-05-04). Override the search host
|
||||
via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again.
|
||||
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are
|
||||
19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords.
|
||||
The createSession endpoint accepts main-account passwords too, but they're
|
||||
bad hygiene (no scope, can't revoke individually).
|
||||
Uses bsky.social for auth and public.api.bsky.app for post search.
|
||||
Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
@@ -23,64 +14,7 @@ from typing import Any, Dict, List, Optional
|
||||
from . import http, log
|
||||
|
||||
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
|
||||
_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app"
|
||||
|
||||
|
||||
def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""Resolve the Bluesky search URL with BSKY_SEARCH_HOST override.
|
||||
|
||||
Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or
|
||||
.env file. The project's env.py loads .env into config but not into
|
||||
os.environ, so check both — same hybrid pattern as last30days.py for
|
||||
LAST30DAYS_STORE.
|
||||
|
||||
Hardens user-supplied host values against three common mis-configurations:
|
||||
whitespace (e.g. " api.bsky.app "), embedded path components (e.g.
|
||||
"api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and
|
||||
embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these
|
||||
we log a warning and fall back to the default rather than building an
|
||||
invalid URL with an opaque downstream error.
|
||||
"""
|
||||
config = config or {}
|
||||
raw = (
|
||||
os.environ.get("BSKY_SEARCH_HOST")
|
||||
or config.get("BSKY_SEARCH_HOST")
|
||||
or _DEFAULT_BSKY_SEARCH_HOST
|
||||
)
|
||||
host = raw.strip().rstrip("/")
|
||||
# Strip embedded scheme so users who paste full URLs do not break the f-string.
|
||||
for prefix in ("https://", "http://"):
|
||||
if host.lower().startswith(prefix):
|
||||
host = host[len(prefix):]
|
||||
break
|
||||
if not host or "/" in host or " " in host:
|
||||
# Embedded path or whitespace remains — don't trust it. Default + log.
|
||||
if raw != _DEFAULT_BSKY_SEARCH_HOST:
|
||||
_log(
|
||||
f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; "
|
||||
f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}"
|
||||
)
|
||||
host = _DEFAULT_BSKY_SEARCH_HOST
|
||||
return f"https://{host}/xrpc/app.bsky.feed.searchPosts"
|
||||
|
||||
|
||||
# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric
|
||||
# with three hyphens at fixed positions).
|
||||
_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$")
|
||||
|
||||
|
||||
def _validate_app_password_format(value) -> bool:
|
||||
"""Return True if value matches Bluesky's 19-char app-password format.
|
||||
|
||||
False for non-strings (None, int, list) so callers passing config dict
|
||||
values directly don't crash. Detect-but-not-gate: the createSession
|
||||
endpoint also accepts main-account passwords, so failing this check is
|
||||
a hygiene smell, not a hard error.
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return bool(_APP_PASSWORD_RE.fullmatch(value))
|
||||
|
||||
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
|
||||
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 15,
|
||||
@@ -210,20 +144,6 @@ def search_bluesky(
|
||||
if not handle or not app_password:
|
||||
return {"posts": [], "error": "Bluesky credentials not configured"}
|
||||
|
||||
# One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password
|
||||
# form. createSession accepts main-account passwords too — but main
|
||||
# passwords have no scope (full account access), can't be revoked
|
||||
# individually, and rotating them breaks every service that holds them.
|
||||
# We warn but do not gate, matching the project's detect-don't-block
|
||||
# philosophy elsewhere.
|
||||
if not _validate_app_password_format(app_password):
|
||||
_log(
|
||||
"BSKY_APP_PASSWORD does not look like an app password "
|
||||
"(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main "
|
||||
"account password — those work but are bad hygiene. Generate "
|
||||
"an app password at https://bsky.app/settings/app-passwords"
|
||||
)
|
||||
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
@@ -235,7 +155,7 @@ def search_bluesky(
|
||||
"limit": str(min(count, 100)),
|
||||
"sort": "top",
|
||||
}
|
||||
url = f"{_resolve_search_url(config)}?{urlencode(params)}"
|
||||
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
|
||||
|
||||
def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
token = _create_session(handle, app_password)
|
||||
@@ -1,12 +1,9 @@
|
||||
"""Chrome and Brave cookie extraction for macOS.
|
||||
"""Chrome cookie extraction for macOS.
|
||||
|
||||
Extracts cookies from Chromium-based browser SQLite databases using only
|
||||
stdlib modules and the system openssl CLI (ships with macOS). Zero pip
|
||||
dependencies.
|
||||
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
|
||||
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
|
||||
|
||||
Chromium on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
Chrome and Brave share the same algorithm; only the DB path and Keychain
|
||||
service name differ.
|
||||
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||
"""
|
||||
|
||||
@@ -21,11 +18,10 @@ from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cookie DB locations on macOS
|
||||
# Chrome cookie DB location on macOS
|
||||
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||
BRAVE_BASE_DIR = Path.home() / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser"
|
||||
|
||||
# Chromium v10 encryption constants (shared by Chrome and Brave)
|
||||
# Chrome v10 encryption constants
|
||||
CHROME_SALT = b"saltysalt"
|
||||
CHROME_PBKDF2_ITERATIONS = 1003
|
||||
CHROME_KEY_LENGTH = 16
|
||||
@@ -33,8 +29,8 @@ CHROME_KEY_LENGTH = 16
|
||||
CHROME_IV_HEX = "20" * 16
|
||||
|
||||
|
||||
def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||
"""Retrieve the encryption passphrase for a Chromium-based browser from macOS Keychain.
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
|
||||
|
||||
Calls `security find-generic-password` which may trigger a system dialog
|
||||
on first access.
|
||||
@@ -43,34 +39,30 @@ def _get_chromium_encryption_key(service_name: str) -> Optional[bytes]:
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["security", "find-generic-password", "-w", "-s", service_name],
|
||||
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.info("%s Keychain access denied or browser not installed: %s", service_name, result.stderr.strip())
|
||||
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
|
||||
return None
|
||||
passphrase = result.stdout.strip()
|
||||
if not passphrase:
|
||||
logger.info("%s Keychain returned empty passphrase", service_name)
|
||||
logger.info("Chrome Keychain returned empty passphrase")
|
||||
return None
|
||||
return passphrase.encode("utf-8")
|
||||
except FileNotFoundError:
|
||||
logger.info("'security' command not found — not on macOS?")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.info("%s Keychain access timed out", service_name)
|
||||
logger.info("Chrome Keychain access timed out")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Failed to get %s encryption key: %s", service_name, e)
|
||||
logger.info("Failed to get Chrome encryption key: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||
return _get_chromium_encryption_key("Chrome Safe Storage")
|
||||
|
||||
|
||||
def _derive_aes_key(passphrase: bytes) -> bytes:
|
||||
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
||||
return hashlib.pbkdf2_hmac(
|
||||
@@ -173,42 +165,36 @@ def _get_db_version(cursor: sqlite3.Cursor) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_chromium_cookies_macos(
|
||||
db_path: Path,
|
||||
keychain_service: str,
|
||||
domain: str,
|
||||
cookie_names: list[str],
|
||||
) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from any Chromium-based browser on macOS.
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS.
|
||||
|
||||
Copies the locked Cookies database to a temp file, reads specified cookies,
|
||||
and decrypts v10-encrypted values using the Keychain-stored key.
|
||||
|
||||
Args:
|
||||
db_path: Path to the browser's Cookies SQLite file.
|
||||
keychain_service: macOS Keychain service name (e.g. "Chrome Safe Storage").
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
|
||||
cookie_names: List of cookie names to extract
|
||||
|
||||
Returns:
|
||||
Dict mapping cookie name to decrypted value, or None on failure.
|
||||
Only includes cookies that were successfully found and decrypted.
|
||||
"""
|
||||
if not db_path.exists():
|
||||
logger.info("%s cookies database not found at %s", keychain_service, db_path)
|
||||
if not CHROME_COOKIES_DB.exists():
|
||||
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
|
||||
return None
|
||||
|
||||
passphrase = _get_chromium_encryption_key(keychain_service)
|
||||
# Get encryption key from Keychain
|
||||
passphrase = _get_chrome_encryption_key()
|
||||
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||
|
||||
# Copy DB to temp file (browser locks the original while running)
|
||||
# Copy DB to temp file (Chrome locks the original)
|
||||
tmp_fd = None
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||
shutil.copy2(str(db_path), tmp_path)
|
||||
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
|
||||
except Exception as e:
|
||||
logger.info("Failed to copy %s cookies database: %s", keychain_service, e)
|
||||
logger.info("Failed to copy Chrome cookies database: %s", e)
|
||||
if tmp_path:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
@@ -225,22 +211,26 @@ def _extract_chromium_cookies_macos(
|
||||
cursor = conn.cursor()
|
||||
|
||||
db_version = _get_db_version(cursor)
|
||||
logger.debug("%s cookie DB version: %d", keychain_service, db_version)
|
||||
logger.debug("Chrome cookie DB version: %d", db_version)
|
||||
|
||||
# Build query with placeholders for cookie names
|
||||
placeholders = ",".join("?" for _ in cookie_names)
|
||||
query = (
|
||||
f"SELECT name, value, encrypted_value FROM cookies "
|
||||
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
||||
)
|
||||
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
|
||||
params = [f"%{domain}"] + list(cookie_names)
|
||||
cursor.execute(query, params)
|
||||
|
||||
results: dict[str, str] = {}
|
||||
for name, value, encrypted_value in cursor.fetchall():
|
||||
# Prefer unencrypted value if present
|
||||
if value:
|
||||
results[name] = value
|
||||
continue
|
||||
|
||||
# Handle encrypted value
|
||||
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||
if aes_key is None:
|
||||
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||
@@ -251,72 +241,25 @@ def _extract_chromium_cookies_macos(
|
||||
else:
|
||||
logger.debug("Failed to decrypt cookie %s", name)
|
||||
elif encrypted_value:
|
||||
# Unknown encryption version
|
||||
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
||||
|
||||
conn.close()
|
||||
|
||||
if not results:
|
||||
logger.info("No matching cookies found in %s for domain %s", keychain_service, domain)
|
||||
logger.info("No matching cookies found in Chrome for domain %s", domain)
|
||||
return None
|
||||
|
||||
return results
|
||||
|
||||
except sqlite3.Error as e:
|
||||
logger.info("Failed to read %s cookies database: %s", keychain_service, e)
|
||||
logger.info("Failed to read Chrome cookies database: %s", e)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info("Unexpected error reading %s cookies: %s", keychain_service, e)
|
||||
logger.info("Unexpected error reading Chrome cookies: %s", e)
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Chrome on macOS."""
|
||||
return _extract_chromium_cookies_macos(
|
||||
CHROME_COOKIES_DB, "Chrome Safe Storage", domain, cookie_names
|
||||
)
|
||||
|
||||
|
||||
def _find_brave_cookies_db() -> Optional[Path]:
|
||||
"""Find Brave's Cookies database on macOS.
|
||||
|
||||
Tries the Default profile first, then scans numbered Profile directories
|
||||
by most-recently-modified. Brave creates extra profiles as "Profile 1",
|
||||
"Profile 2", etc. alongside Default; the most recently used one is the
|
||||
likeliest to hold current cookies. Lexicographic sort would visit
|
||||
"Profile 10" before "Profile 2", which can return the wrong profile.
|
||||
"""
|
||||
default = BRAVE_BASE_DIR / "Default" / "Cookies"
|
||||
if default.exists():
|
||||
return default
|
||||
|
||||
try:
|
||||
candidates = [
|
||||
child for child in BRAVE_BASE_DIR.iterdir()
|
||||
if child.is_dir() and child.name.startswith("Profile ")
|
||||
]
|
||||
for child in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
||||
candidate = child / "Cookies"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_brave_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||
"""Extract cookies from Brave on macOS.
|
||||
|
||||
Brave uses the same v10 AES-128-CBC encryption as Chrome; only the DB
|
||||
path and Keychain service name differ.
|
||||
"""
|
||||
db_path = _find_brave_cookies_db()
|
||||
if db_path is None:
|
||||
logger.info("Brave cookies database not found under %s", BRAVE_BASE_DIR)
|
||||
return None
|
||||
return _extract_chromium_cookies_macos(db_path, "Brave Safe Storage", domain, cookie_names)
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Browser cookie extraction for last30days.
|
||||
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Brave, Safari)
|
||||
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
|
||||
to enable zero-config authentication for services like X/Twitter.
|
||||
|
||||
Only uses Python stdlib — no external dependencies.
|
||||
@@ -255,29 +255,6 @@ def extract_chrome_cookies(
|
||||
return None
|
||||
|
||||
|
||||
def extract_brave_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Extract cookies from Brave for the given domain and cookie names.
|
||||
|
||||
macOS only — Brave uses the same v10 AES-128-CBC encryption as Chrome,
|
||||
with a different DB path and Keychain service name ("Brave Safe Storage").
|
||||
Tries the Default profile first, then scans numbered Profile directories.
|
||||
|
||||
Returns:
|
||||
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||
"""
|
||||
if platform.system() != "Darwin":
|
||||
logger.debug("Brave cookie extraction only supported on macOS")
|
||||
return None
|
||||
try:
|
||||
from .chrome_cookies import extract_brave_cookies_macos
|
||||
return extract_brave_cookies_macos(domain, cookie_names)
|
||||
except Exception as exc:
|
||||
logger.debug("Brave cookie extraction failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def extract_safari_cookies(
|
||||
domain: str, cookie_names: List[str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
@@ -305,9 +282,9 @@ def extract_cookies(
|
||||
"""Extract cookies from the specified browser.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
'auto' tries browsers in platform-appropriate order:
|
||||
- macOS: Chrome -> Brave -> Firefox -> Safari
|
||||
- macOS: Chrome -> Firefox -> Safari
|
||||
- Linux: Firefox only
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
@@ -356,7 +333,7 @@ def extract_cookies_with_source(
|
||||
so callers can track the source.
|
||||
|
||||
Args:
|
||||
browser: One of 'firefox', 'chrome', 'brave', 'safari', or 'auto'.
|
||||
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||
domain: The cookie domain to match (e.g. ".x.com").
|
||||
cookie_names: List of cookie names to extract.
|
||||
|
||||
@@ -367,7 +344,6 @@ def extract_cookies_with_source(
|
||||
extractors = {
|
||||
"firefox": extract_firefox_cookies,
|
||||
"chrome": extract_chrome_cookies,
|
||||
"brave": extract_brave_cookies,
|
||||
"safari": extract_safari_cookies,
|
||||
}
|
||||
|
||||
@@ -384,7 +360,7 @@ def extract_cookies_with_source(
|
||||
# Auto mode: try browsers in platform-appropriate order
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
order = ["chrome", "brave", "firefox", "safari"]
|
||||
order = ["chrome", "firefox", "safari"]
|
||||
elif system == "Linux":
|
||||
order = ["firefox"]
|
||||
else:
|
||||
@@ -39,14 +39,11 @@ def normalize_text(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
|
||||
if len(norm) < n:
|
||||
return {norm} if norm else set()
|
||||
return {norm[index:index + n] for index in range(len(norm) - n + 1)}
|
||||
|
||||
|
||||
def get_ngrams(text: str, n: int = 3) -> set[str]:
|
||||
return _ngrams_of_normalized(normalize_text(text), n)
|
||||
text = normalize_text(text)
|
||||
if len(text) < n:
|
||||
return {text} if text else set()
|
||||
return {text[index:index + n] for index in range(len(text) - n + 1)}
|
||||
|
||||
|
||||
def jaccard_similarity(left: set[str], right: set[str]) -> float:
|
||||
@@ -93,7 +90,7 @@ class _PreparedText:
|
||||
|
||||
def __init__(self, raw: str) -> None:
|
||||
norm = normalize_text(raw)
|
||||
self.ngrams = _ngrams_of_normalized(norm)
|
||||
self.ngrams = get_ngrams(norm) if norm else set()
|
||||
self.tokens = _tokenize(norm)
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ def _extract_subreddits(reddit_items: List[Dict[str, Any]]) -> List[str]:
|
||||
|
||||
for item in reddit_items:
|
||||
# Primary subreddit
|
||||
sub = item.get("subreddit", "").strip().removeprefix("r/")
|
||||
sub = item.get("subreddit", "").strip().lstrip("r/")
|
||||
if sub:
|
||||
sub_counts[sub] += 1
|
||||
|
||||
@@ -29,23 +29,6 @@ else:
|
||||
|
||||
CODEX_AUTH_FILE = Path(os.environ.get("CODEX_AUTH_FILE", str(Path.home() / ".codex" / "auth.json")))
|
||||
|
||||
# macOS Keychain integration: items stored with this service prefix are picked
|
||||
# up automatically on Darwin as the lowest-priority credential source.
|
||||
# Example: `security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."`.
|
||||
KEYCHAIN_SERVICE_PREFIX = "last30days-"
|
||||
|
||||
# Single source of truth for which credentials the Keychain loader looks up.
|
||||
# The setup-keychain.sh helper mirrors this list and is held in sync via
|
||||
# tests/test_env_keychain.py::test_keychain_keys_match_setup_script.
|
||||
KEYCHAIN_KEYS = (
|
||||
"OPENAI_API_KEY", "XAI_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY",
|
||||
"GOOGLE_GENAI_API_KEY", "SCRAPECREATORS_API_KEY", "APIFY_API_TOKEN",
|
||||
"AUTH_TOKEN", "CT0", "BSKY_HANDLE", "BSKY_APP_PASSWORD",
|
||||
"TRUTHSOCIAL_TOKEN", "BRAVE_API_KEY", "EXA_API_KEY", "SERPER_API_KEY",
|
||||
"OPENROUTER_API_KEY", "PARALLEL_API_KEY", "XQUIK_API_KEY",
|
||||
"XIAOHONGSHU_API_BASE",
|
||||
)
|
||||
|
||||
AuthSource = Literal["api_key", "codex", "none"]
|
||||
AuthStatus = Literal["ok", "missing", "expired", "missing_account_id"]
|
||||
|
||||
@@ -70,10 +53,6 @@ class OpenAIAuth:
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file has overly permissive permissions."""
|
||||
if os.name == "nt":
|
||||
# Windows reports synthesized POSIX mode bits that do not reflect NTFS ACLs.
|
||||
return
|
||||
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
# Check if group or other can read (bits 0o044)
|
||||
@@ -112,46 +91,6 @@ def load_env_file(path: Path) -> dict[str, str]:
|
||||
return env
|
||||
|
||||
|
||||
def _load_keychain(keys: list[str]) -> dict[str, str]:
|
||||
"""Load credentials from macOS Keychain (no-op on other platforms).
|
||||
|
||||
Each key is looked up as a generic password with service name
|
||||
``f"{KEYCHAIN_SERVICE_PREFIX}{key}"`` for the current user. Missing items
|
||||
and lookup failures are silent — Keychain is the lowest-priority source
|
||||
and is meant to be additive over `.env` files and process environment.
|
||||
"""
|
||||
import platform
|
||||
if platform.system() != "Darwin":
|
||||
return {}
|
||||
|
||||
import shutil
|
||||
security = shutil.which("security")
|
||||
if not security:
|
||||
return {}
|
||||
|
||||
import subprocess
|
||||
import pwd
|
||||
# USER can be unset under sudo, in Docker without --env USER, or in some CI
|
||||
# runners; fall back to the OS user record so lookups still match items
|
||||
# stored by setup-keychain.sh (which uses $USER).
|
||||
user = os.environ.get("USER") or pwd.getpwuid(os.getuid()).pw_name
|
||||
env: dict[str, str] = {}
|
||||
for key in keys:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[security, "find-generic-password",
|
||||
"-a", user,
|
||||
"-s", f"{KEYCHAIN_SERVICE_PREFIX}{key}",
|
||||
"-w"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
continue
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
env[key] = result.stdout.strip()
|
||||
return env
|
||||
|
||||
|
||||
def _decode_jwt_payload(token: str) -> dict[str, Any] | None:
|
||||
"""Decode JWT payload without verification."""
|
||||
try:
|
||||
@@ -275,7 +214,6 @@ def get_config() -> dict[str, Any]:
|
||||
1. Environment variables (os.environ)
|
||||
2. .claude/last30days.env (per-project config)
|
||||
3. ~/.config/last30days/.env (global config)
|
||||
4. macOS Keychain items prefixed ``last30days-`` (Darwin only)
|
||||
"""
|
||||
# Load from global config file
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||
@@ -284,14 +222,9 @@ def get_config() -> dict[str, Any]:
|
||||
project_env_path = _find_project_env()
|
||||
project_env = load_env_file(project_env_path) if project_env_path else {}
|
||||
|
||||
# Merge file sources: project > global
|
||||
# Merge: project overrides global
|
||||
merged_env = {**file_env, **project_env}
|
||||
|
||||
# Keychain is the lowest-priority source (Darwin only; no-op elsewhere).
|
||||
# Loaded before openai_auth so OPENAI_API_KEY can come from Keychain too.
|
||||
keychain_env = _load_keychain(list(KEYCHAIN_KEYS))
|
||||
merged_env = {**keychain_env, **merged_env}
|
||||
|
||||
openai_auth = get_openai_auth(merged_env)
|
||||
|
||||
# Build config: Codex/OpenAI auth + process.env > project .env > global .env
|
||||
@@ -314,7 +247,6 @@ def get_config() -> dict[str, Any]:
|
||||
('LAST30DAYS_RERANK_MODEL', None),
|
||||
('LAST30DAYS_X_MODEL', None),
|
||||
('LAST30DAYS_X_BACKEND', None),
|
||||
('LAST30DAYS_STORE', None),
|
||||
('OPENAI_MODEL_PIN', None),
|
||||
('XAI_MODEL_PIN', None),
|
||||
('SCRAPECREATORS_API_KEY', None),
|
||||
@@ -323,7 +255,6 @@ def get_config() -> dict[str, Any]:
|
||||
('CT0', None),
|
||||
('BSKY_HANDLE', None),
|
||||
('BSKY_APP_PASSWORD', None),
|
||||
('BSKY_SEARCH_HOST', None),
|
||||
('TRUTHSOCIAL_TOKEN', None),
|
||||
('BRAVE_API_KEY', None),
|
||||
('EXA_API_KEY', None),
|
||||
@@ -334,41 +265,16 @@ def get_config() -> dict[str, Any]:
|
||||
('FROM_BROWSER', None),
|
||||
('SETUP_COMPLETE', None),
|
||||
('INCLUDE_SOURCES', ''),
|
||||
('EXCLUDE_SOURCES', ''),
|
||||
('LAST30DAYS_YOUTUBE_SSH_HOST', None),
|
||||
('LAST30DAYS_TRANSCRIPT_TIMEOUT', None),
|
||||
]
|
||||
|
||||
for key, default in keys:
|
||||
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
||||
|
||||
# Backward-compat: ScrapeCreators' own examples and tutorials use the
|
||||
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and
|
||||
# CREATORS). Accept that form too so users who follow the vendor's docs
|
||||
# don't silently end up with has_scrapecreators=False. Canonical name
|
||||
# wins when both are set.
|
||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||
legacy = os.environ.get('SCRAPE_CREATORS_API_KEY') or merged_env.get('SCRAPE_CREATORS_API_KEY')
|
||||
if legacy:
|
||||
config['SCRAPECREATORS_API_KEY'] = legacy
|
||||
|
||||
# Multi-key rotation: comma-separated SCRAPECREATORS_API_KEY round-robins
|
||||
# via random.choice per run. Originally added in #268, accidentally dropped
|
||||
# in v3.0.6, restored here.
|
||||
sc_key_raw = config.get('SCRAPECREATORS_API_KEY') or ''
|
||||
if ',' in sc_key_raw:
|
||||
import random
|
||||
sc_keys = [k.strip() for k in sc_key_raw.split(',') if k.strip()]
|
||||
config['SCRAPECREATORS_API_KEY'] = random.choice(sc_keys) if sc_keys else ''
|
||||
|
||||
# Track which config source was used (highest-priority file source wins
|
||||
# the label; keychain is only reported when nothing else is configured).
|
||||
# Track which config source was used
|
||||
if project_env_path:
|
||||
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
||||
elif CONFIG_FILE and CONFIG_FILE.exists():
|
||||
config['_CONFIG_SOURCE'] = f'global:{CONFIG_FILE}'
|
||||
elif keychain_env:
|
||||
config['_CONFIG_SOURCE'] = 'keychain'
|
||||
else:
|
||||
config['_CONFIG_SOURCE'] = 'env_only'
|
||||
|
||||
@@ -450,10 +356,6 @@ def get_x_source_with_method(config: dict[str, Any]) -> tuple[str | None, str]:
|
||||
if config.get("AUTH_TOKEN") and config.get("CT0"):
|
||||
method = config.get("_AUTH_TOKEN_SOURCE", "env")
|
||||
return "bird", method
|
||||
# Fall back to xurl CLI (official X API v2, OAuth2, free developer app)
|
||||
from . import xurl_x
|
||||
if xurl_x.is_available():
|
||||
return "xurl", "oauth2"
|
||||
return None, "none"
|
||||
|
||||
|
||||
@@ -466,6 +368,14 @@ def config_exists() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def is_reddit_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Reddit search is available.
|
||||
|
||||
v3 uses ScrapeCreators only.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
|
||||
|
||||
def get_reddit_source(config: dict[str, Any]) -> str | None:
|
||||
"""Determine which Reddit backend to use.
|
||||
|
||||
@@ -491,7 +401,6 @@ def get_x_source(config: dict[str, Any]) -> str | None:
|
||||
Returns:
|
||||
'bird' if Bird is installed and explicit cookies are configured,
|
||||
'xai' if XAI_API_KEY is configured,
|
||||
'xurl' if xurl CLI is installed and authenticated,
|
||||
None if no X source available.
|
||||
"""
|
||||
# Import here to avoid circular dependency
|
||||
@@ -512,11 +421,6 @@ def get_x_source(config: dict[str, Any]) -> str | None:
|
||||
if has_bird_creds and bird_x.is_bird_installed():
|
||||
return 'bird'
|
||||
|
||||
# Fall back to xurl CLI (official X API v2, OAuth2, free developer app)
|
||||
from . import xurl_x
|
||||
if xurl_x.is_available():
|
||||
return 'xurl'
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@@ -611,12 +515,12 @@ def _parse_include_sources(config: dict[str, Any]) -> set[str]:
|
||||
def is_threads_available(config: dict[str, Any]) -> bool:
|
||||
"""Check if Threads source is available.
|
||||
|
||||
Returns True when SCRAPECREATORS_API_KEY is set. Threads runs alongside
|
||||
TikTok and Instagram as part of the SC family — same key, same per-call
|
||||
cost shape, so the same default-on rule applies. Suppress via
|
||||
EXCLUDE_SOURCES=threads.
|
||||
Requires SCRAPECREATORS_API_KEY AND 'threads' in INCLUDE_SOURCES.
|
||||
Threads is an opt-in source - it is not activated by default.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
if not config.get('SCRAPECREATORS_API_KEY'):
|
||||
return False
|
||||
return 'threads' in _parse_include_sources(config)
|
||||
|
||||
|
||||
def is_instagram_available(config: dict[str, Any]) -> bool:
|
||||
@@ -698,18 +602,14 @@ def get_x_source_status(config: dict[str, Any]) -> dict[str, Any]:
|
||||
elif xai_available:
|
||||
source = 'xai'
|
||||
else:
|
||||
# Fall back to xurl CLI
|
||||
from . import xurl_x as _xurl_check
|
||||
source = 'xurl' if _xurl_check.is_available() else None
|
||||
source = None
|
||||
|
||||
from . import xurl_x as _xurl_x
|
||||
return {
|
||||
"source": source,
|
||||
"bird_installed": bird_status["installed"],
|
||||
"bird_authenticated": bird_status["authenticated"],
|
||||
"bird_username": bird_status["username"],
|
||||
"xai_available": xai_available,
|
||||
"xurl_available": _xurl_x.is_available(),
|
||||
"can_install_bird": bird_status["can_install"],
|
||||
}
|
||||
|
||||
@@ -116,8 +116,6 @@ def weighted_rrf(
|
||||
"""Fuse ranked lists into a single candidate pool."""
|
||||
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
|
||||
candidates: dict[str, schema.Candidate] = {}
|
||||
# Track (source, item_id) pairs already attached to each candidate for O(1) dedup.
|
||||
seen_source_items: dict[str, set[tuple[str, str]]] = {}
|
||||
|
||||
for (label, source), items in streams.items():
|
||||
subquery = subqueries[label]
|
||||
@@ -156,7 +154,6 @@ def weighted_rrf(
|
||||
]
|
||||
},
|
||||
)
|
||||
seen_source_items[key] = {(item.source, item.item_id)}
|
||||
continue
|
||||
|
||||
candidate = candidates[key]
|
||||
@@ -182,9 +179,7 @@ def weighted_rrf(
|
||||
candidate.subquery_labels.append(label)
|
||||
if item.source not in candidate.sources:
|
||||
candidate.sources.append(item.source)
|
||||
source_item_key = (item.source, item.item_id)
|
||||
if source_item_key not in seen_source_items[key]:
|
||||
seen_source_items[key].add(source_item_key)
|
||||
if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
|
||||
candidate.source_items.append(item)
|
||||
candidate.metadata.setdefault("provenance", []).append(
|
||||
{
|
||||
@@ -62,17 +62,6 @@ def _resolve_token(token: Optional[str] = None) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def resolve_token(token: Optional[str] = None) -> Optional[str]:
|
||||
"""Public alias for ``_resolve_token``.
|
||||
|
||||
The pipeline calls this once before ``search_github`` and
|
||||
``enrich_with_comments`` so the ``gh auth token`` subprocess fallback
|
||||
only fires once per query when ``GITHUB_TOKEN`` is unset, instead of
|
||||
twice (once per call site).
|
||||
"""
|
||||
return _resolve_token(token)
|
||||
|
||||
|
||||
def _fetch_json(
|
||||
url: str,
|
||||
token: Optional[str] = None,
|
||||
@@ -153,14 +142,8 @@ def search_github(
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
token: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Search GitHub Issues and PRs (HTTP fetch only).
|
||||
|
||||
Returns a raw envelope shaped like every other adapter's ``search_X``:
|
||||
``{"items": [raw GitHub API items], "context": {core, from_date,
|
||||
to_date, count}}``. Normalization, date filtering, and sorting move
|
||||
to ``parse_github_response``; comment enrichment moves to
|
||||
``enrich_with_comments``.
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search GitHub Issues and PRs.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
@@ -170,23 +153,15 @@ def search_github(
|
||||
token: Optional GitHub token (falls back to env/gh CLI)
|
||||
|
||||
Returns:
|
||||
Dict envelope. Empty ``items`` list on any failure.
|
||||
List of normalized item dicts. Empty list on any failure.
|
||||
"""
|
||||
count = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
|
||||
core = extract_core_subject(topic)
|
||||
resolved_token = _resolve_token(token)
|
||||
if not resolved_token:
|
||||
_log("No GitHub token available (set GITHUB_TOKEN or install gh CLI)")
|
||||
return {
|
||||
"items": [],
|
||||
"error": "no token",
|
||||
"context": {
|
||||
"core": core,
|
||||
"from_date": from_date,
|
||||
"to_date": to_date,
|
||||
"count": count,
|
||||
},
|
||||
}
|
||||
return []
|
||||
|
||||
count = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
|
||||
core = extract_core_subject(topic)
|
||||
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
|
||||
# Build search query with date filter
|
||||
@@ -201,41 +176,12 @@ def search_github(
|
||||
|
||||
data = _fetch_json(url, token=resolved_token, timeout=30)
|
||||
if not data:
|
||||
return {"items": [], "context": {"core": core, "from_date": from_date,
|
||||
"to_date": to_date, "count": count}}
|
||||
return []
|
||||
|
||||
raw_items = data.get("items", [])
|
||||
_log(f"Found {len(raw_items)} issues/PRs")
|
||||
|
||||
return {
|
||||
"items": raw_items,
|
||||
"context": {
|
||||
"core": core,
|
||||
"from_date": from_date,
|
||||
"to_date": to_date,
|
||||
"count": count,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def parse_github_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Normalize a ``search_github`` envelope into the skill's item shape.
|
||||
|
||||
Pure function: no I/O, no token, no enrichment. Applies the date
|
||||
filter using the search context and sorts by relevance.
|
||||
"""
|
||||
if not isinstance(response, dict):
|
||||
return []
|
||||
raw_items = response.get("items") or []
|
||||
if not isinstance(raw_items, list):
|
||||
return []
|
||||
context = response.get("context") or {}
|
||||
core = context.get("core") or ""
|
||||
from_date = context.get("from_date") or ""
|
||||
to_date = context.get("to_date") or ""
|
||||
count = context.get("count") or DEPTH_LIMITS["default"]
|
||||
|
||||
items: List[Dict[str, Any]] = []
|
||||
items = []
|
||||
for i, item in enumerate(raw_items[:count]):
|
||||
html_url = item.get("html_url", "")
|
||||
repo = _parse_repo_from_url(html_url)
|
||||
@@ -278,34 +224,20 @@ def parse_github_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
},
|
||||
})
|
||||
|
||||
# Enrich top items with comments
|
||||
items = _enrich_top_items(items, depth, resolved_token)
|
||||
|
||||
# Date filter
|
||||
if from_date and to_date:
|
||||
items = [
|
||||
item for item in items
|
||||
if item.get("date") is None or (from_date <= item["date"] <= to_date)
|
||||
]
|
||||
filtered = []
|
||||
for item in items:
|
||||
d = item.get("date")
|
||||
if d is None or (from_date <= d <= to_date):
|
||||
filtered.append(item)
|
||||
|
||||
items.sort(key=lambda x: x.get("relevance", 0), reverse=True)
|
||||
return items
|
||||
# Sort by relevance
|
||||
filtered.sort(key=lambda x: x.get("relevance", 0), reverse=True)
|
||||
|
||||
|
||||
def enrich_with_comments(
|
||||
items: List[Dict[str, Any]],
|
||||
depth: str = "default",
|
||||
token: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Fetch top comments for top-K items by reactions and attach to metadata.
|
||||
|
||||
Mutates and returns ``items``. Resolves ``token`` via env/gh CLI when
|
||||
not supplied, matching ``search_github``'s fallback chain.
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
resolved_token = _resolve_token(token)
|
||||
if not resolved_token:
|
||||
_log("No GitHub token available for comment enrichment")
|
||||
return items
|
||||
return _enrich_top_items(items, depth, resolved_token)
|
||||
return filtered
|
||||
|
||||
|
||||
def _enrich_top_items(
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
@@ -140,10 +139,7 @@ def parallel_search(
|
||||
data = http.request(
|
||||
"POST", "https://api.parallel.ai/v1/search",
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
json_data={
|
||||
"search_queries": [query],
|
||||
"advanced_settings": {"max_results": count},
|
||||
},
|
||||
json_data={"query": query, "max_results": count},
|
||||
timeout=15,
|
||||
)
|
||||
items = []
|
||||
@@ -153,7 +149,7 @@ def parallel_search(
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
raw_date = r.get("publish_date") or ""
|
||||
raw_date = r.get("published_date") or ""
|
||||
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
|
||||
if not _in_date_range(pub_date, date_range):
|
||||
continue
|
||||
@@ -162,7 +158,7 @@ def parallel_search(
|
||||
"title": r.get("title", ""),
|
||||
"url": url,
|
||||
"source_domain": _domain(url),
|
||||
"snippet": ((r.get("excerpts") or [""])[0] or "")[:500],
|
||||
"snippet": r.get("snippet", ""),
|
||||
"date": pub_date,
|
||||
"relevance": 0.8,
|
||||
"why_relevant": "Parallel AI web search",
|
||||
@@ -209,90 +205,29 @@ def web_search(
|
||||
backend = "parallel"
|
||||
else:
|
||||
return [], {}
|
||||
items: list[dict] = []
|
||||
artifact: dict = {}
|
||||
if backend == "brave":
|
||||
key = config.get("BRAVE_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
|
||||
items, artifact = brave_search(query, date_range, key)
|
||||
elif backend == "exa":
|
||||
return brave_search(query, date_range, key)
|
||||
if backend == "exa":
|
||||
key = config.get("EXA_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
|
||||
items, artifact = exa_search(query, date_range, key)
|
||||
elif backend == "serper":
|
||||
return exa_search(query, date_range, key)
|
||||
if backend == "serper":
|
||||
key = config.get("SERPER_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
|
||||
items, artifact = serper_search(query, date_range, key)
|
||||
elif backend == "parallel":
|
||||
return serper_search(query, date_range, key)
|
||||
if backend == "parallel":
|
||||
key = config.get("PARALLEL_API_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
|
||||
items, artifact = parallel_search(query, date_range, key)
|
||||
elif backend != "none":
|
||||
return parallel_search(query, date_range, key)
|
||||
if backend != "none":
|
||||
raise ValueError(f"Unsupported web backend: {backend!r}")
|
||||
else:
|
||||
return [], {}
|
||||
if items and not _reddit_excluded(config):
|
||||
items = _enrich_reddit_items(items)
|
||||
return items, artifact
|
||||
|
||||
|
||||
def _reddit_excluded(config: dict) -> bool:
|
||||
"""Return True when EXCLUDE_SOURCES contains 'reddit'.
|
||||
|
||||
Respects the same suppression knob the pipeline uses for source gating,
|
||||
so a user who set EXCLUDE_SOURCES=reddit doesn't get Reddit content
|
||||
smuggled back in via web-search URLs.
|
||||
"""
|
||||
raw = (config.get("EXCLUDE_SOURCES") or "").split(",")
|
||||
return any(s.strip().lower() == "reddit" for s in raw)
|
||||
|
||||
|
||||
def _enrich_reddit_items(items: list[dict]) -> list[dict]:
|
||||
"""Enrich web search results that are Reddit URLs with thread body and comments.
|
||||
|
||||
Claude Code's WebFetch blocks reddit.com, so the model can't retrieve
|
||||
Reddit content from web search results. This fetches it via the public
|
||||
JSON API (reddit.com/.../.json) which bypasses that restriction.
|
||||
|
||||
Callers should gate this with EXCLUDE_SOURCES=reddit handling (see
|
||||
`_reddit_excluded`) so a user who explicitly excluded Reddit doesn't
|
||||
get Reddit content via web-search URLs.
|
||||
"""
|
||||
from . import reddit_enrich
|
||||
from .reddit_enrich import RedditRateLimitError
|
||||
|
||||
for item in items:
|
||||
url = item.get("url", "")
|
||||
if "reddit.com" not in url or "/comments/" not in url:
|
||||
continue
|
||||
try:
|
||||
thread_data = reddit_enrich.fetch_thread_data(url, timeout=8)
|
||||
if not thread_data:
|
||||
continue
|
||||
parsed = reddit_enrich.parse_thread_data(thread_data)
|
||||
# selftext lives under parsed["submission"], not at the top level
|
||||
selftext = (parsed.get("submission") or {}).get("selftext", "")
|
||||
if selftext:
|
||||
item["snippet"] = selftext[:2000]
|
||||
comments = parsed.get("comments", [])
|
||||
top = reddit_enrich.get_top_comments(comments)
|
||||
if top:
|
||||
item["top_comments"] = [
|
||||
{"score": c.get("score", 0), "excerpt": (c.get("body") or "")[:200]}
|
||||
for c in top[:5]
|
||||
]
|
||||
item["enriched_via"] = "reddit_json_api"
|
||||
except RedditRateLimitError as exc:
|
||||
# Stop iterating to avoid flooding more 429s
|
||||
sys.stderr.write(f"[Web] Reddit rate-limited, halting enrichment: {exc}\n")
|
||||
break
|
||||
except Exception as exc:
|
||||
sys.stderr.write(f"[Web] Reddit enrichment failed for {url}: {exc}\n")
|
||||
return items
|
||||
return [], {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -88,26 +88,17 @@ def search_hackernews(
|
||||
|
||||
# Use extracted core subject instead of raw topic for cleaner Algolia matching
|
||||
core = extract_core_subject(topic)
|
||||
# Hyphens and commas tokenize awkwardly in Algolia; flatten them so themed
|
||||
# queries like "ts-bun-node" or "claude, personal agents" become plain words.
|
||||
core_flat = _flatten_query_for_algolia(core)
|
||||
_log(f"Searching for '{core_flat}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
|
||||
|
||||
# Use relevance-sorted search with minimum engagement filter.
|
||||
# NOTE: restrictSearchableAttributes=title omitted intentionally — it would
|
||||
# miss Ask HN/Show HN threads where the topic appears in the body.
|
||||
params = {
|
||||
"query": core_flat,
|
||||
"query": core,
|
||||
"tags": "story",
|
||||
"numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
|
||||
"hitsPerPage": str(count),
|
||||
}
|
||||
# Algolia defaults to AND across query tokens, so a 4-5 word theme query
|
||||
# matches no stories. Mark all-but-the-first token as optional so Algolia
|
||||
# ranks by how many tokens match instead of requiring every one.
|
||||
tokens = core_flat.split()
|
||||
if len(tokens) > 1:
|
||||
params["optionalWords"] = " ".join(tokens[1:])
|
||||
|
||||
from urllib.parse import urlencode
|
||||
url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
|
||||
@@ -126,56 +117,28 @@ def search_hackernews(
|
||||
return response
|
||||
|
||||
|
||||
_WORD_BOUNDARY_RE_CACHE: Dict[str, "re.Pattern[str]"] = {}
|
||||
|
||||
|
||||
def _flatten_query_for_algolia(text: str) -> str:
|
||||
"""Normalise query for Algolia + post-filter comparison.
|
||||
|
||||
Multi-keyword theme queries frequently contain commas (delimiters) or
|
||||
hyphens (compound terms like ``ts-bun-node``); both tokenize awkwardly.
|
||||
Flatten them to spaces and collapse runs of whitespace so the search
|
||||
parameter and the post-filter operate on the same shape.
|
||||
"""
|
||||
return " ".join(text.replace(",", " ").replace("-", " ").split())
|
||||
|
||||
|
||||
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
|
||||
"""Check if any query token appears as a whole word in the title.
|
||||
"""Check if the query term appears in the title content, not just an HN prefix or author.
|
||||
|
||||
Returns True when the query is empty (no filter), or when at least one
|
||||
query token matches as a whole word in the title after stripping
|
||||
"Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes.
|
||||
|
||||
We previously required *every* token to appear (all-words), which killed
|
||||
every Algolia hit on multi-keyword themes like "claude, personal agents,
|
||||
agentic infra" because real HN titles never contain all five tokens
|
||||
verbatim. Relaxing to any-word matches Algolia's `optionalWords` behaviour
|
||||
in `search_hackernews`. Token-overlap relevance scoring at parse time
|
||||
demotes hits where only one weak token matched, so the loosened gate
|
||||
won't surface noise to the top of the ranking.
|
||||
|
||||
Word-boundary matching (rather than naive substring) prevents short
|
||||
tokens like ``ai`` or ``ts`` from matching unrelated words like
|
||||
``email`` or ``artists``.
|
||||
Returns True if the query (or any multi-word token) appears in the title
|
||||
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
|
||||
and ignoring the author name. Returns True when query is empty (no filter).
|
||||
"""
|
||||
if not query:
|
||||
return True
|
||||
stripped = _HN_PREFIXES.sub("", title).strip()
|
||||
# Also check that the match isn't solely in the author's username
|
||||
check_text = stripped.lower()
|
||||
# Normalise the query the same way search_hackernews does so post-filter
|
||||
# tokens line up with what Algolia actually saw.
|
||||
query_words = [w for w in _flatten_query_for_algolia(query.lower()).split() if w]
|
||||
if not query_words:
|
||||
return True
|
||||
query_lower = query.lower()
|
||||
# Check each word of the query independently; all must appear somewhere
|
||||
# in the stripped title (not just the prefix).
|
||||
query_words = query_lower.split()
|
||||
for word in query_words:
|
||||
pattern = _WORD_BOUNDARY_RE_CACHE.get(word)
|
||||
if pattern is None:
|
||||
pattern = re.compile(rf"\b{re.escape(word)}\b")
|
||||
_WORD_BOUNDARY_RE_CACHE[word] = pattern
|
||||
if pattern.search(check_text):
|
||||
return True
|
||||
return False
|
||||
if word in check_text:
|
||||
continue
|
||||
# Word not found in stripped title — reject
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -23,19 +22,9 @@ def log(msg: str):
|
||||
MAX_RETRIES = 5
|
||||
MAX_429_RETRIES = 2
|
||||
RETRY_DELAY = 2.0
|
||||
# DNS resolution failures (gaierror) are transient — typically resolved by a
|
||||
# brief backoff and retry. Use a dedicated minimum attempt count + exponential
|
||||
# delays (1s, 2s, 4s) so callers that pass a small `retries` value still get a
|
||||
# meaningful chance to recover from a transient resolution failure.
|
||||
MIN_DNS_RETRIES = 3
|
||||
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
||||
|
||||
|
||||
def _is_dns_failure(err: urllib.error.URLError) -> bool:
|
||||
"""Return True if a URLError was caused by DNS resolution (gaierror)."""
|
||||
return isinstance(getattr(err, "reason", None), socket.gaierror)
|
||||
|
||||
|
||||
class HTTPError(Exception):
|
||||
"""HTTP request error with status code."""
|
||||
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
||||
@@ -96,13 +85,7 @@ def request(
|
||||
|
||||
last_error = None
|
||||
rate_limit_count = 0
|
||||
# DNS failures get a dedicated minimum attempt count + exponential backoff.
|
||||
# `effective_retries` is the actual loop bound; we expand it on the first
|
||||
# gaierror if the caller passed a smaller `retries` value than MIN_DNS_RETRIES.
|
||||
effective_retries = retries
|
||||
dns_attempts = 0
|
||||
attempt = 0
|
||||
while attempt < effective_retries:
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
body = response.read().decode('utf-8')
|
||||
@@ -132,8 +115,6 @@ def request(
|
||||
if rate_limit_count >= max_429_retries:
|
||||
raise last_error
|
||||
|
||||
# HTTP errors respect the caller's original `retries`; only DNS
|
||||
# failures get the widened `effective_retries` budget.
|
||||
if attempt < retries - 1:
|
||||
if e.code == 429:
|
||||
# Respect Retry-After header, fall back to exponential backoff
|
||||
@@ -149,43 +130,11 @@ def request(
|
||||
else:
|
||||
delay = RETRY_DELAY * (2 ** attempt)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure may have widened `effective_retries`, but that
|
||||
# widening is DNS-only — don't grant extra HTTP attempts.
|
||||
break
|
||||
except urllib.error.URLError as e:
|
||||
log(f"URL Error: {e.reason}")
|
||||
last_error = HTTPError(f"URL Error: {e.reason}")
|
||||
if _is_dns_failure(e):
|
||||
# DNS resolution failures are transient; expand the retry budget
|
||||
# to MIN_DNS_RETRIES if the caller passed fewer, and use
|
||||
# exponential backoff (1s, 2s, 4s, ...) instead of the linear
|
||||
# default. Counts DNS attempts separately so other URLError
|
||||
# causes don't bypass the regular retry budget.
|
||||
dns_attempts += 1
|
||||
if effective_retries < MIN_DNS_RETRIES:
|
||||
log(
|
||||
f"DNS resolution failed; expanding retry budget from "
|
||||
f"{effective_retries} to {MIN_DNS_RETRIES}"
|
||||
)
|
||||
effective_retries = MIN_DNS_RETRIES
|
||||
if attempt < effective_retries - 1:
|
||||
delay = 2 ** (dns_attempts - 1) # 1s, 2s, 4s, 8s, ...
|
||||
log(
|
||||
f"DNS resolution failure (attempt {dns_attempts}); "
|
||||
f"retrying in {delay:.1f}s"
|
||||
)
|
||||
time.sleep(delay)
|
||||
elif attempt < retries - 1:
|
||||
# Non-DNS URLError (e.g. ConnectionRefused) respects the
|
||||
# caller's original retry budget, not the DNS-widened bound.
|
||||
if attempt < retries - 1:
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Caller's original retry budget exhausted; an earlier DNS
|
||||
# failure widening `effective_retries` does not carry over
|
||||
# to non-DNS error paths.
|
||||
break
|
||||
except json.JSONDecodeError as e:
|
||||
log(f"JSON decode error: {e}")
|
||||
last_error = HTTPError(f"Invalid JSON response: {e}")
|
||||
@@ -195,13 +144,7 @@ def request(
|
||||
log(f"Connection error: {type(e).__name__}: {e}")
|
||||
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
||||
if attempt < retries - 1:
|
||||
# Socket errors respect the caller's original retry budget.
|
||||
time.sleep(RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
# Original budget exhausted; DNS widening doesn't apply here.
|
||||
break
|
||||
|
||||
attempt += 1
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
@@ -223,53 +166,6 @@ def post_raw(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, st
|
||||
return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs)
|
||||
|
||||
|
||||
BROWSER_USER_AGENT = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
def get_text(
|
||||
url: str,
|
||||
timeout: int = DEFAULT_TIMEOUT,
|
||||
retries: int = 2,
|
||||
accept: str = "*/*",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Optional[str]:
|
||||
"""Fetch a URL and return decoded text, or None on any failure.
|
||||
|
||||
Keyless helper for Reddit RSS and shreddit HTML endpoints — the free path
|
||||
that replaced the now-403 ``.json`` endpoints. Sends a browser User-Agent
|
||||
and never raises: returns None on HTTP error, network failure, or timeout
|
||||
so tiered callers can fall through to the next source.
|
||||
|
||||
Args:
|
||||
url: Request URL
|
||||
timeout: HTTP timeout per attempt in seconds
|
||||
retries: Number of retries on failure (kept low — these tiers fail fast)
|
||||
accept: Accept header value (e.g. "application/atom+xml", "text/html")
|
||||
headers: Optional extra headers merged over the defaults
|
||||
|
||||
Returns:
|
||||
Decoded response body as text, or None on failure.
|
||||
"""
|
||||
merged = {
|
||||
"User-Agent": BROWSER_USER_AGENT,
|
||||
"Accept": accept,
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
if headers:
|
||||
merged.update(headers)
|
||||
try:
|
||||
return request(
|
||||
"GET", url, headers=merged, timeout=timeout, retries=retries, raw=True
|
||||
)
|
||||
except HTTPError as e:
|
||||
log(f"get_text failed ({e}): {url}")
|
||||
return None
|
||||
|
||||
|
||||
def scrapecreators_headers(token: str) -> Dict[str, str]:
|
||||
"""Build ScrapeCreators request headers (x-api-key + JSON content type)."""
|
||||
return {
|
||||
@@ -7,14 +7,17 @@ Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
|
||||
API docs: https://scrapecreators.com/docs
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import dates, http, log
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||
|
||||
@@ -28,42 +31,7 @@ DEPTH_CONFIG = {
|
||||
# Max words to keep from each caption
|
||||
CAPTION_MAX_WORDS = 500
|
||||
|
||||
# Default transcript fetch timeout (seconds). SC's
|
||||
# /v2/instagram/media/transcript regularly takes >15s on real workloads,
|
||||
# so the default is generous; override via LAST30DAYS_TRANSCRIPT_TIMEOUT.
|
||||
DEFAULT_TRANSCRIPT_TIMEOUT = 30
|
||||
|
||||
|
||||
def _resolve_transcript_timeout(
|
||||
timeout: Optional[float] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> float:
|
||||
"""Resolve the IG transcript-fetch timeout.
|
||||
|
||||
Priority (highest wins):
|
||||
1. Explicit ``timeout`` kwarg
|
||||
2. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in os.environ
|
||||
3. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in caller-supplied config dict
|
||||
4. ``DEFAULT_TRANSCRIPT_TIMEOUT`` (30s)
|
||||
|
||||
Mirrors the ``os.environ.get(X) or config.get(X)`` pattern used for
|
||||
LAST30DAYS_STORE in last30days.py so the env var works whether it's
|
||||
shell-exported or set in ~/.config/last30days/.env.
|
||||
"""
|
||||
if timeout is not None:
|
||||
try:
|
||||
return float(timeout)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
raw = os.environ.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||
if not raw and config:
|
||||
raw = config.get("LAST30DAYS_TRANSCRIPT_TIMEOUT")
|
||||
if raw:
|
||||
try:
|
||||
return float(raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return float(DEFAULT_TRANSCRIPT_TIMEOUT)
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
@@ -81,17 +49,6 @@ def _extract_core_subject(topic: str) -> str:
|
||||
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
|
||||
|
||||
|
||||
def _to_hashtag_form(query: str) -> str:
|
||||
"""Collapse a multi-word query to hashtag form (no spaces, lowercase).
|
||||
|
||||
SC's /v2/instagram/reels/search wraps Google Search and is documented
|
||||
to be flaky on multi-token queries. Single-token queries map to a
|
||||
hashtag page lookup which is the stable path. Used as a 500-retry
|
||||
fallback before the request bubbles up as a silent failure.
|
||||
"""
|
||||
return ''.join(query.split()).lower()
|
||||
|
||||
|
||||
def _infer_query_intent(topic: str) -> str:
|
||||
"""Tiny local intent classifier for Instagram query expansion."""
|
||||
text = topic.lower().strip()
|
||||
@@ -279,17 +236,30 @@ def _user_reels(
|
||||
"""
|
||||
_log(f"User reels: @{handle}")
|
||||
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
|
||||
try:
|
||||
data = http.get(
|
||||
reels_url,
|
||||
params={"handle": handle},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"User reels error for @{handle}: {e}")
|
||||
return []
|
||||
if not _requests:
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"handle": handle})
|
||||
url = f"{reels_url}?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"User reels error (urllib) for @{handle}: {e}")
|
||||
return []
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
reels_url,
|
||||
params={"handle": handle},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"User reels error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
|
||||
_log(f" -> {len(raw_items)} reels from @{handle}")
|
||||
@@ -323,37 +293,31 @@ def search_instagram(
|
||||
|
||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except http.HTTPError as e:
|
||||
# SC's v2 reels search wraps Google Search and 500s frequently on
|
||||
# multi-token queries. Single tokens hit the stable hashtag-page
|
||||
# path. Retry once with hashtag form before bubbling up.
|
||||
if getattr(e, "status_code", None) == 500 and ' ' in core_topic:
|
||||
_log(f"IG search 500 on '{core_topic}', retrying with hashtag form")
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": _to_hashtag_form(core_topic)},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as retry_e:
|
||||
_log(f"IG search retry failed: {retry_e}")
|
||||
return {"items": [], "error": f"{type(retry_e).__name__}: {retry_e}"}
|
||||
else:
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"query": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
||||
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
||||
@@ -385,8 +349,6 @@ def fetch_captions(
|
||||
video_items: List[Dict[str, Any]],
|
||||
token: str,
|
||||
depth: str = "default",
|
||||
timeout: Optional[float] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
||||
|
||||
@@ -398,21 +360,14 @@ def fetch_captions(
|
||||
video_items: Items from search_instagram()
|
||||
token: ScrapeCreators API key
|
||||
depth: Depth level for caption limit
|
||||
timeout: Optional per-request transcript timeout in seconds. When
|
||||
None, resolves from LAST30DAYS_TRANSCRIPT_TIMEOUT (env or
|
||||
config), defaulting to DEFAULT_TRANSCRIPT_TIMEOUT (30s).
|
||||
config: Optional config dict (from env.get_config()) used as a
|
||||
fallback source for LAST30DAYS_TRANSCRIPT_TIMEOUT when the
|
||||
value is not exported in os.environ.
|
||||
|
||||
Returns:
|
||||
Dict mapping video_id -> caption text (truncated to 500 words)
|
||||
"""
|
||||
depth_cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = depth_cfg["max_captions"]
|
||||
transcript_timeout = _resolve_transcript_timeout(timeout, config)
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = config["max_captions"]
|
||||
|
||||
if not video_items or not token:
|
||||
if not video_items or not token or not _requests:
|
||||
return {}
|
||||
|
||||
top_items = video_items[:max_captions]
|
||||
@@ -437,24 +392,26 @@ def fetch_captions(
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
data = http.get(
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||
params={"url": url},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=transcript_timeout,
|
||||
retries=1,
|
||||
timeout=15,
|
||||
)
|
||||
transcripts = data.get("transcripts") or []
|
||||
if transcripts and isinstance(transcripts, list):
|
||||
transcript_text = " ".join(
|
||||
t.get("text", "") for t in transcripts
|
||||
if isinstance(t, dict) and t.get("text")
|
||||
)
|
||||
if transcript_text:
|
||||
words = transcript_text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = transcript_text
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
transcripts = data.get("transcripts") or []
|
||||
if transcripts and isinstance(transcripts, list):
|
||||
# Combine all transcript segments
|
||||
transcript_text = " ".join(
|
||||
t.get("text", "") for t in transcripts
|
||||
if isinstance(t, dict) and t.get("text")
|
||||
)
|
||||
if transcript_text:
|
||||
words = transcript_text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = transcript_text
|
||||
except Exception as e:
|
||||
_log(f"Transcript fetch failed for {vid}: {e}")
|
||||
|
||||
@@ -49,7 +49,6 @@ def normalize_source_items(
|
||||
"xquik": _normalize_x,
|
||||
"pinterest": _normalize_pinterest,
|
||||
"polymarket": _normalize_polymarket,
|
||||
"digg": _normalize_digg,
|
||||
"grounding": _normalize_grounding,
|
||||
"xiaohongshu": _normalize_grounding,
|
||||
"github": _normalize_github,
|
||||
@@ -111,19 +110,6 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
|
||||
return default
|
||||
|
||||
|
||||
def _join_comment_excerpts(
|
||||
top_comments: list[Any],
|
||||
key: str,
|
||||
limit: int = 3,
|
||||
) -> str:
|
||||
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
|
||||
return " ".join(
|
||||
str(comment.get(key) or "").strip()
|
||||
for comment in top_comments[:limit]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
|
||||
|
||||
def _domain_from_url(url: str) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
@@ -183,7 +169,11 @@ def _normalize_reddit(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
body = "\n".join(
|
||||
part
|
||||
for part in [
|
||||
@@ -251,11 +241,6 @@ def _normalize_youtube(
|
||||
metadata: dict[str, Any] = {}
|
||||
if highlights:
|
||||
metadata["transcript_highlights"] = highlights
|
||||
if item.get("captions_disabled"):
|
||||
# Surfaced for quality_nudge: uploader disabled captions, so this
|
||||
# video should be subtracted from the degraded-transcript-ratio
|
||||
# denominator (it was never going to produce a transcript).
|
||||
metadata["captions_disabled"] = True
|
||||
metadata["top_comments"] = _remap_comments(
|
||||
item.get("top_comments") or [],
|
||||
score_keys=("score", "likes"),
|
||||
@@ -353,7 +338,11 @@ def _normalize_hackernews(
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
top_comments = item.get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "text")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("text") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
title = str(item.get("title") or "").strip()
|
||||
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
||||
return _source_item(
|
||||
@@ -405,53 +394,6 @@ def _normalize_microblog(
|
||||
)
|
||||
|
||||
|
||||
def _normalize_digg(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
index: int,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> schema.SourceItem:
|
||||
"""Normalizer for Digg AI 1000 clusters.
|
||||
|
||||
Each cluster is one item. The TLDR carries the most useful body for
|
||||
rerank and synthesis. Top-ranked X posts attached at search time are
|
||||
passed through under metadata['posts'] so render can emit them as
|
||||
inline 'via Digg' quotes.
|
||||
"""
|
||||
title = str(item.get("title") or "").strip()
|
||||
tldr = str(item.get("tldr") or "").strip()
|
||||
body = "\n\n".join(part for part in [title, tldr] if part)
|
||||
posts = item.get("posts") or []
|
||||
if not isinstance(posts, list):
|
||||
posts = []
|
||||
cluster_url_id = str(item.get("id") or f"DG{index + 1}")
|
||||
return _source_item(
|
||||
item_id=cluster_url_id,
|
||||
source=source,
|
||||
title=title or f"Digg cluster {index + 1}",
|
||||
body=body,
|
||||
url=str(item.get("url") or f"https://di.gg/ai/{cluster_url_id}"),
|
||||
author="",
|
||||
container="Digg",
|
||||
published_at=item.get("date"),
|
||||
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
|
||||
engagement=item.get("engagement") or {},
|
||||
relevance_hint=item.get("relevance", 0.5),
|
||||
why_relevant=str(item.get("why_relevant") or ""),
|
||||
snippet=tldr[:400],
|
||||
metadata={
|
||||
"clusterUrlId": cluster_url_id,
|
||||
"tldr": tldr,
|
||||
"rank": (item.get("engagement") or {}).get("rank"),
|
||||
"uniqueAuthors": (item.get("engagement") or {}).get("uniqueAuthors"),
|
||||
"postCount": (item.get("engagement") or {}).get("postCount"),
|
||||
"firstPostAge": item.get("first_post_age"),
|
||||
"posts": posts,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalize_polymarket(
|
||||
source: str,
|
||||
item: dict[str, Any],
|
||||
@@ -499,7 +441,11 @@ def _normalize_github(
|
||||
title = str(item.get("title") or "").strip()
|
||||
snippet_text = str(item.get("snippet") or "").strip()
|
||||
top_comments = item.get("metadata", {}).get("top_comments") or []
|
||||
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||
comment_text = " ".join(
|
||||
str(comment.get("excerpt") or "").strip()
|
||||
for comment in top_comments[:3]
|
||||
if isinstance(comment, dict)
|
||||
)
|
||||
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
||||
metadata = item.get("metadata") or {}
|
||||
return _source_item(
|
||||
@@ -11,6 +11,11 @@ import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import dates, http, log
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
|
||||
@@ -135,17 +140,31 @@ def search_pinterest(
|
||||
|
||||
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"keyword": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/search?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Extract items from response - try common SC response shapes
|
||||
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
|
||||
@@ -15,7 +15,6 @@ from . import (
|
||||
bluesky,
|
||||
dates,
|
||||
dedupe,
|
||||
digg,
|
||||
entity_extract,
|
||||
env,
|
||||
github,
|
||||
@@ -31,7 +30,6 @@ from . import (
|
||||
query,
|
||||
reddit,
|
||||
reddit_public,
|
||||
relevance,
|
||||
rerank,
|
||||
schema,
|
||||
signals,
|
||||
@@ -42,7 +40,6 @@ from . import (
|
||||
xai_x,
|
||||
xiaohongshu_api,
|
||||
xquik,
|
||||
xurl_x,
|
||||
youtube_yt,
|
||||
)
|
||||
from .cluster import cluster_candidates
|
||||
@@ -79,10 +76,7 @@ MOCK_AVAILABLE_SOURCES = [
|
||||
"xiaohongshu",
|
||||
"github",
|
||||
"perplexity",
|
||||
"threads",
|
||||
"pinterest",
|
||||
"xquik",
|
||||
"digg",
|
||||
]
|
||||
|
||||
|
||||
@@ -110,8 +104,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.extend(["hackernews", "polymarket"])
|
||||
if config.get("GITHUB_TOKEN") or which("gh"):
|
||||
available.append("github")
|
||||
if which("digg-pp-cli"):
|
||||
available.append("digg")
|
||||
if env.is_bluesky_available(config):
|
||||
available.append("bluesky")
|
||||
if env.is_truthsocial_available(config):
|
||||
@@ -120,9 +112,7 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("grounding")
|
||||
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
|
||||
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
|
||||
if config.get("OPENROUTER_API_KEY") and (
|
||||
"perplexity" in include_sources or (requested_sources and "perplexity" in requested_sources)
|
||||
):
|
||||
if config.get("OPENROUTER_API_KEY") and "perplexity" in include_sources:
|
||||
available.append("perplexity")
|
||||
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
|
||||
available.append("xiaohongshu")
|
||||
@@ -132,9 +122,6 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("pinterest")
|
||||
if env.is_xquik_available(config):
|
||||
available.append("xquik")
|
||||
exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()}
|
||||
if exclude:
|
||||
available = [s for s in available if s not in exclude]
|
||||
return available
|
||||
|
||||
|
||||
@@ -190,7 +177,6 @@ def run(
|
||||
lookback_days: int = 30,
|
||||
github_user: str | None = None,
|
||||
github_repos: list[str] | None = None,
|
||||
internal_subrun: bool = False,
|
||||
) -> schema.Report:
|
||||
settings = DEPTH_SETTINGS[depth]
|
||||
requested_sources = normalize_requested_sources(requested_sources)
|
||||
@@ -207,7 +193,7 @@ def run(
|
||||
available = [source for source in available if source in requested_sources]
|
||||
if web_backend == "none":
|
||||
available = [s for s in available if s != "grounding"]
|
||||
elif web_backend in ("brave", "exa", "serper", "parallel") and "grounding" not in available:
|
||||
elif web_backend in ("brave", "exa", "serper") and "grounding" not in available:
|
||||
available.append("grounding")
|
||||
if not available:
|
||||
raise RuntimeError("No sources are available for this run.")
|
||||
@@ -228,7 +214,6 @@ def run(
|
||||
provider=None if mock else reasoning_provider,
|
||||
model=None if mock else runtime.planner_model,
|
||||
context=config.get("_auto_resolve_context", ""),
|
||||
internal_subrun=internal_subrun,
|
||||
)
|
||||
# Source labelling: the fallback path annotates notes with "fallback-plan"
|
||||
# or "deterministic-comparison-plan"; anything else came from the LLM.
|
||||
@@ -455,7 +440,7 @@ def run(
|
||||
if bundle.items_by_source.get(source):
|
||||
del bundle.errors_by_source[source]
|
||||
|
||||
items_by_source = _finalize_items_by_source(bundle.items_by_source, topic=topic, config=config)
|
||||
items_by_source = _finalize_items_by_source(bundle.items_by_source, topic=topic)
|
||||
candidates = weighted_rrf(bundle.items_by_source_and_query, plan, pool_limit=settings["pool_limit"])
|
||||
ranked_candidates = rerank.rerank_candidates(
|
||||
topic=topic,
|
||||
@@ -512,19 +497,17 @@ def _normalize_score_dedupe(
|
||||
source, raw_items, from_date, to_date,
|
||||
freshness_mode=freshness_mode,
|
||||
)
|
||||
prepared_query = relevance.PreparedQuery(ranking_query)
|
||||
normalized = signals.annotate_stream(normalized, prepared_query, freshness_mode)
|
||||
normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
|
||||
normalized = signals.prune_low_relevance(normalized)
|
||||
normalized = dedupe.dedupe_items(normalized)
|
||||
for item in normalized:
|
||||
item.snippet = snippet.extract_best_snippet(item, prepared_query)
|
||||
item.snippet = snippet.extract_best_snippet(item, ranking_query)
|
||||
return normalized
|
||||
|
||||
|
||||
def _finalize_items_by_source(
|
||||
items_by_source_raw: dict[str, list[schema.SourceItem]],
|
||||
topic: str = "",
|
||||
config: dict | None = None,
|
||||
) -> dict[str, list[schema.SourceItem]]:
|
||||
finalized = {}
|
||||
for source, items in items_by_source_raw.items():
|
||||
@@ -537,17 +520,6 @@ def _finalize_items_by_source(
|
||||
# (e.g., WTI crude oil, Elon tweet counts) before footer emission.
|
||||
if source == "polymarket" and topic:
|
||||
items = polymarket.filter_items_against_topic(topic, items)
|
||||
# --polymarket-keywords (via config): additional keyword filter
|
||||
# for ambiguous single-token topics (e.g., "Warriors" → nba,gsw).
|
||||
keywords = config.get("_polymarket_keywords") if isinstance(config, dict) else None
|
||||
if keywords:
|
||||
items = polymarket.filter_items_against_keywords(items, keywords)
|
||||
if source == "digg" and items:
|
||||
# Pull top-ranked X posts only for the survivors that will appear
|
||||
# in the brief. Spending the enrichment budget here (rather than
|
||||
# at retrieval time) keeps the inline 'via Digg' quotes
|
||||
# paired with the clusters dedupe actually kept.
|
||||
digg.enrich_source_items(items, top_k=3)
|
||||
finalized[source] = items
|
||||
return finalized
|
||||
|
||||
@@ -923,9 +895,6 @@ def _retrieve_stream(
|
||||
depth=depth,
|
||||
)
|
||||
return xai_x.parse_x_response(result), {}
|
||||
if backend == "xurl":
|
||||
result = xurl_x.search_x(subquery.search_query, depth=depth)
|
||||
return xurl_x.parse_x_response(result, topic=subquery.search_query), {}
|
||||
raise RuntimeError("No X backend is available.")
|
||||
if source == "youtube":
|
||||
# Use raw_topic so expand_youtube_queries() generates diverse variants
|
||||
@@ -983,13 +952,6 @@ def _retrieve_stream(
|
||||
if source == "hackernews":
|
||||
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
|
||||
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
|
||||
if source == "digg":
|
||||
result = digg.search_digg(subquery.search_query, from_date, to_date, depth=depth)
|
||||
items = digg.parse_digg_response(result, query=subquery.search_query)
|
||||
# Enrichment with attached X posts is deferred to
|
||||
# _finalize_items_by_source so it runs on the items that actually
|
||||
# survive dedupe rather than on top-K of the raw fanout.
|
||||
return items, {}
|
||||
if source == "bluesky":
|
||||
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
|
||||
return bluesky.parse_bluesky_response(result), {}
|
||||
@@ -1007,14 +969,8 @@ def _retrieve_stream(
|
||||
result = polymarket.search_polymarket(subquery.search_query, from_date, to_date, depth=depth)
|
||||
return polymarket.parse_polymarket_response(result, topic=subquery.search_query), {}
|
||||
if source == "github":
|
||||
# Resolve once at the pipeline boundary so search and enrich
|
||||
# share the result; otherwise each call would re-run the env
|
||||
# lookup and gh-CLI subprocess fallback (up to 5s timeout each).
|
||||
token = github.resolve_token(config.get("GITHUB_TOKEN"))
|
||||
response = github.search_github(subquery.search_query, from_date, to_date, depth=depth, token=token)
|
||||
items = github.parse_github_response(response)
|
||||
items = github.enrich_with_comments(items, depth=depth, token=token)
|
||||
return items, {}
|
||||
result = github.search_github(subquery.search_query, from_date, to_date, depth=depth, token=config.get("GITHUB_TOKEN"))
|
||||
return result, {}
|
||||
if source == "pinterest":
|
||||
result = pinterest.search_pinterest(
|
||||
subquery.search_query, from_date, to_date,
|
||||
@@ -1088,45 +1044,6 @@ def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[d
|
||||
"why_relevant": "Brave web search",
|
||||
}
|
||||
],
|
||||
"digg": [
|
||||
{
|
||||
"id": "mock1abc",
|
||||
"title": f"Digg cluster about {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock1abc",
|
||||
"tldr": f"Curated cluster summarizing recent {subquery.search_query} discussion across the AI 1000.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(3)[0],
|
||||
"engagement": {"postCount": 8, "uniqueAuthors": 5, "rank": 2, "rank_score": 49.0},
|
||||
"first_post_age": "3d",
|
||||
"posts": [
|
||||
{
|
||||
"username": "exampledev",
|
||||
"display_name": "Example Dev",
|
||||
"category": "Engineer",
|
||||
"rank": 142,
|
||||
"body": f"Quote from the AI 1000 about {subquery.search_query}.",
|
||||
"post_type": "tweet",
|
||||
"x_url": "https://x.com/exampledev/status/1",
|
||||
"posted_at": dates.get_date_range(3)[0],
|
||||
},
|
||||
],
|
||||
"relevance": 0.84,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
{
|
||||
"id": "mock2def",
|
||||
"title": f"Second Digg cluster on {subquery.search_query}",
|
||||
"url": "https://di.gg/ai/mock2def",
|
||||
"tldr": f"Another angle on {subquery.search_query}.",
|
||||
"author": "",
|
||||
"date": dates.get_date_range(8)[0],
|
||||
"engagement": {"postCount": 3, "uniqueAuthors": 2, "rank": 18, "rank_score": 33.0},
|
||||
"first_post_age": "8d",
|
||||
"posts": [],
|
||||
"relevance": 0.71,
|
||||
"why_relevant": "Mock Digg cluster",
|
||||
},
|
||||
],
|
||||
}
|
||||
if source == "grounding":
|
||||
return payloads.get(source, []), {
|
||||
@@ -19,14 +19,14 @@ ALLOWED_INTENTS = {
|
||||
}
|
||||
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
|
||||
QUICK_SOURCE_PRIORITY = {
|
||||
"factual": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||
"product": ["youtube", "reddit", "x", "xquik", "tiktok"],
|
||||
"concept": ["hackernews", "reddit", "x", "xquik", "youtube"],
|
||||
"opinion": ["reddit", "x", "xquik", "youtube", "hackernews"],
|
||||
"how_to": ["youtube", "reddit", "x", "xquik", "hackernews"],
|
||||
"comparison": ["reddit", "x", "xquik", "hackernews", "youtube"],
|
||||
"breaking_news": ["x", "xquik", "reddit", "hackernews", "youtube", "polymarket"],
|
||||
"prediction": ["polymarket", "x", "xquik", "hackernews", "reddit", "youtube"],
|
||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||
"product": ["youtube", "reddit", "x", "tiktok"],
|
||||
"concept": ["hackernews", "reddit", "x", "youtube"],
|
||||
"opinion": ["reddit", "x", "youtube", "hackernews"],
|
||||
"how_to": ["youtube", "reddit", "x", "hackernews"],
|
||||
"comparison": ["reddit", "x", "hackernews", "youtube"],
|
||||
"breaking_news": ["x", "reddit", "hackernews", "youtube", "polymarket"],
|
||||
"prediction": ["polymarket", "x", "hackernews", "reddit", "youtube"],
|
||||
}
|
||||
SOURCE_PRIORITY = {
|
||||
"factual": ["hackernews", "reddit", "x", "youtube"],
|
||||
@@ -60,7 +60,6 @@ INTENT_SOURCE_EXCLUSIONS: dict[str, set[str]] = {
|
||||
SOURCE_CAPABILITIES = {
|
||||
"reddit": {"discussion", "social"},
|
||||
"x": {"discussion", "social"},
|
||||
"xquik": {"discussion", "social"},
|
||||
"youtube": {"video", "video_longform", "discussion"},
|
||||
"tiktok": {"video", "video_shortform", "social"},
|
||||
"instagram": {"video", "video_shortform", "social"},
|
||||
@@ -68,7 +67,6 @@ SOURCE_CAPABILITIES = {
|
||||
"bluesky": {"discussion", "social"},
|
||||
"truthsocial": {"discussion", "social"},
|
||||
"polymarket": {"market"},
|
||||
"digg": {"discussion", "social", "link"},
|
||||
"xiaohongshu": {"video", "video_shortform", "social"},
|
||||
"github": {"discussion", "link"},
|
||||
"grounding": {"web", "reference", "link"},
|
||||
@@ -88,16 +86,9 @@ def plan_query(
|
||||
provider: providers.ReasoningClient | None,
|
||||
model: str | None,
|
||||
context: str = "",
|
||||
internal_subrun: bool = False,
|
||||
) -> schema.QueryPlan:
|
||||
"""Create a query plan. Comparison queries with extractable entities use a
|
||||
deterministic plan; other intents prefer the configured reasoning provider.
|
||||
|
||||
internal_subrun: when True, suppress the LAW 7 "No --plan passed" stderr
|
||||
warning. LAW 7 targets the hosting-reasoning-model path; competitor
|
||||
fan-out sub-runs are engine-internal and the warning is a false positive
|
||||
there. Default False preserves the warning on every user-facing invocation.
|
||||
"""
|
||||
deterministic plan; other intents prefer the configured reasoning provider."""
|
||||
if _should_force_deterministic_plan(topic):
|
||||
return _fallback_plan(
|
||||
topic,
|
||||
@@ -131,17 +122,16 @@ def plan_query(
|
||||
# planner credentials - NOT a prerequisite the caller needs. If you are
|
||||
# the hosting reasoning model, YOU are the provider. LAW 7 / 2026-04-19
|
||||
# Hermes Agent Use Cases failure mode.
|
||||
if not internal_subrun:
|
||||
import sys
|
||||
print(
|
||||
"[Planner] No --plan passed. If you are the reasoning model hosting "
|
||||
"this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime), "
|
||||
"YOU ARE the planner: generate a JSON query plan yourself and pass it "
|
||||
"via --plan. You do not need an API key or credentials; you ARE the "
|
||||
"LLM. The deterministic fallback below is the headless/cron path only. "
|
||||
"See LAW 7 in SKILL.md and Step 0.75 for the plan schema.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
import sys
|
||||
print(
|
||||
"[Planner] No --plan passed. If you are the reasoning model hosting "
|
||||
"this skill (Claude Code, Codex, Hermes, Gemini, or any agent runtime), "
|
||||
"YOU ARE the planner: generate a JSON query plan yourself and pass it "
|
||||
"via --plan. You do not need an API key or credentials; you ARE the "
|
||||
"LLM. The deterministic fallback below is the headless/cron path only. "
|
||||
"See LAW 7 in SKILL.md and Step 0.75 for the plan schema.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return _fallback_plan(topic, available_sources, requested_sources, depth)
|
||||
|
||||
|
||||
@@ -274,15 +264,7 @@ def _sanitize_plan(
|
||||
freshness_mode=freshness_mode,
|
||||
cluster_mode=cluster_mode,
|
||||
raw_topic=topic,
|
||||
subqueries=_normalize_subquery_weights(
|
||||
_trim_subqueries_for_depth(
|
||||
subqueries,
|
||||
intent,
|
||||
depth,
|
||||
eligible_sources,
|
||||
requested_sources=requested_sources,
|
||||
)
|
||||
),
|
||||
subqueries=_normalize_subquery_weights(_trim_subqueries_for_depth(subqueries, intent, depth, eligible_sources)),
|
||||
source_weights=source_weights,
|
||||
notes=[str(note).strip() for note in raw.get("notes") or [] if str(note).strip()],
|
||||
)
|
||||
@@ -315,7 +297,6 @@ def _trim_subqueries_for_depth(
|
||||
intent: str,
|
||||
depth: str,
|
||||
available_sources: list[str],
|
||||
requested_sources: list[str] | None = None,
|
||||
) -> list[schema.SubQuery]:
|
||||
# At non-quick depth, expand sources: use capability routing for intents
|
||||
# that define it, or all available sources otherwise. The LLM planner may
|
||||
@@ -345,15 +326,6 @@ def _trim_subqueries_for_depth(
|
||||
for subquery in subqueries:
|
||||
if depth in {"quick", "default"}:
|
||||
preferred_sources = ranked_sources[:limit]
|
||||
if requested_sources:
|
||||
requested = [
|
||||
source
|
||||
for source in requested_sources
|
||||
if source in available_sources and source in subquery.sources
|
||||
]
|
||||
for source in requested:
|
||||
if source not in preferred_sources:
|
||||
preferred_sources.append(source)
|
||||
else:
|
||||
preferred_sources = [source for source in ranked_sources if source in subquery.sources][:limit]
|
||||
if len(preferred_sources) < limit:
|
||||
@@ -446,13 +418,7 @@ def _fallback_plan(
|
||||
cluster_mode=_default_cluster_mode(intent),
|
||||
raw_topic=topic,
|
||||
subqueries=_normalize_subquery_weights(
|
||||
_trim_subqueries_for_depth(
|
||||
subqueries[:_max_subqueries(intent, topic)],
|
||||
intent,
|
||||
depth,
|
||||
list(source_weights),
|
||||
requested_sources=requested_sources,
|
||||
)
|
||||
_trim_subqueries_for_depth(subqueries[:_max_subqueries(intent, topic)], intent, depth, list(source_weights))
|
||||
),
|
||||
source_weights=_normalize_weights(source_weights),
|
||||
notes=[note],
|
||||
@@ -232,39 +232,6 @@ def filter_items_against_topic(topic: str, items: List[Any]) -> List[Any]:
|
||||
return filtered
|
||||
|
||||
|
||||
def filter_items_against_keywords(items: List[Any], keywords: List[str]) -> List[Any]:
|
||||
"""Keep only items whose title contains at least one keyword (case-insensitive).
|
||||
|
||||
Intended for disambiguating ambiguous single-token topics like 'Warriors'
|
||||
via --polymarket-keywords (e.g., 'nba,gsw,golden-state') to filter out
|
||||
Glasgow Warriors rugby, Honor of Kings Rogue Warriors markets that share
|
||||
the 'Warriors' token but are not the target entity.
|
||||
"""
|
||||
if not keywords:
|
||||
return items
|
||||
normalized_keywords = [kw.strip().lower() for kw in keywords if kw and kw.strip()]
|
||||
if not normalized_keywords:
|
||||
return items
|
||||
|
||||
filtered = []
|
||||
for item in items:
|
||||
title = getattr(item, "title", None)
|
||||
if title is None and isinstance(item, dict):
|
||||
title = item.get("title", "")
|
||||
title = (title or "").lower()
|
||||
if any(kw in title for kw in normalized_keywords):
|
||||
filtered.append(item)
|
||||
|
||||
dropped = len(items) - len(filtered)
|
||||
if dropped:
|
||||
_log(
|
||||
f"Keyword filter dropped {dropped} Polymarket items; "
|
||||
f"kept {len(filtered)} matching {normalized_keywords}"
|
||||
)
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
def _extract_domain_queries(topic: str, events: List[Dict]) -> List[str]:
|
||||
"""Extract domain-indicator search terms from first-pass event tags.
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
from . import env, http, schema
|
||||
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite"
|
||||
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite-preview"
|
||||
GEMINI_PRO = "gemini-3.1-pro-preview"
|
||||
OPENAI_DEFAULT = "gpt-5.4-nano"
|
||||
XAI_DEFAULT = "grok-4-1-fast"
|
||||
@@ -19,11 +19,7 @@ OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
|
||||
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
|
||||
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
|
||||
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||
# OpenRouter routes the Gemini Flash Lite tier as the -preview slug; that is the
|
||||
# stable form on that routing layer even though native Gemini's GEMINI_FLASH_LITE
|
||||
# constant is suffix-free. If GEMINI_FLASH_LITE moves to a non-preview stable ID,
|
||||
# double-check that OpenRouter's slug still maps to the same upstream model.
|
||||
OPENROUTER_DEFAULT = "google/gemini-3.1-flash-lite-preview"
|
||||
OPENROUTER_DEFAULT = "google/gemini-flash-2.0"
|
||||
|
||||
|
||||
class ReasoningClient:
|
||||
@@ -97,6 +93,13 @@ class GeminiClient(ReasoningClient):
|
||||
)
|
||||
return extract_gemini_text(payload)
|
||||
|
||||
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
|
||||
|
||||
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
|
||||
|
||||
|
||||
class OpenAIClient(ReasoningClient):
|
||||
name = "openai"
|
||||
|
||||
@@ -236,8 +239,8 @@ def _resolve_model_pins(config: dict[str, Any], depth: str, provider_name: str)
|
||||
rerank_model = config.get("LAST30DAYS_RERANK_MODEL") or default_rerank
|
||||
|
||||
if provider_name == "gemini":
|
||||
_require_gemini_31(planner_model, role="planner")
|
||||
_require_gemini_31(rerank_model, role="rerank")
|
||||
_require_gemini_31_preview(planner_model, role="planner")
|
||||
_require_gemini_31_preview(rerank_model, role="rerank")
|
||||
|
||||
return planner_model, rerank_model
|
||||
|
||||
@@ -348,11 +351,11 @@ def _resolve_x_backend(config: dict[str, Any]) -> str | None:
|
||||
return env.get_x_source(config)
|
||||
|
||||
|
||||
def _require_gemini_31(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-"):
|
||||
def _require_gemini_31_preview(model: str, *, role: str) -> None:
|
||||
if model.startswith("gemini-3.1-") and model.endswith("-preview"):
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"{role} must use a Gemini 3.1 model. Got: {model}"
|
||||
f"{role} must use a Gemini 3.1 preview model. Got: {model}"
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Post-research quality score and upgrade nudge.
|
||||
|
||||
Computes a quality score based on 5 core sources and builds
|
||||
a nudge message describing what the user missed and how to fix it.
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
|
||||
|
||||
# The 5 core sources
|
||||
CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit"]
|
||||
|
||||
# Labels for display
|
||||
SOURCE_LABELS = {
|
||||
"hn": "Hacker News",
|
||||
"polymarket": "Polymarket",
|
||||
"x": "X/Twitter",
|
||||
"youtube": "YouTube",
|
||||
"reddit": "Reddit",
|
||||
}
|
||||
|
||||
|
||||
def _is_x_active(config: dict, research_results: dict) -> bool:
|
||||
"""Check if X source is active (has credentials AND didn't error)."""
|
||||
has_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||
if not has_creds:
|
||||
return False
|
||||
# If X errored this run, it's configured but broken
|
||||
if research_results.get("x_error"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_youtube_active(config: dict, research_results: dict) -> bool:
|
||||
"""Check if YouTube source is active (yt-dlp installed)."""
|
||||
try:
|
||||
from . import youtube_yt
|
||||
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||
except Exception:
|
||||
has_ytdlp = False
|
||||
if not has_ytdlp:
|
||||
return False
|
||||
if research_results.get("youtube_error"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||
"""Compute research quality score based on 5 core sources.
|
||||
|
||||
Args:
|
||||
config: Configuration dict from env.get_config()
|
||||
research_results: Dict with keys like x_error, youtube_error,
|
||||
reddit_error reflecting what happened this run.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"score_pct": 40-100,
|
||||
"core_active": ["hn", "polymarket", ...],
|
||||
"core_missing": ["x", "youtube"],
|
||||
"core_errored": [], # configured but errored
|
||||
"nudge_text": "..." or None if 100%
|
||||
}
|
||||
"""
|
||||
core_active: List[str] = []
|
||||
core_missing: List[str] = []
|
||||
core_errored: List[str] = []
|
||||
|
||||
# HN, Polymarket, and Reddit are always active
|
||||
core_active.append("hn")
|
||||
core_active.append("polymarket")
|
||||
core_active.append("reddit")
|
||||
|
||||
# X
|
||||
has_x_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||
if _is_x_active(config, research_results):
|
||||
core_active.append("x")
|
||||
else:
|
||||
core_missing.append("x")
|
||||
if has_x_creds and research_results.get("x_error"):
|
||||
core_errored.append("x")
|
||||
|
||||
# YouTube
|
||||
yt_active = _is_youtube_active(config, research_results)
|
||||
if yt_active:
|
||||
core_active.append("youtube")
|
||||
else:
|
||||
core_missing.append("youtube")
|
||||
# Check if configured but errored (yt-dlp installed but failed this run)
|
||||
try:
|
||||
from . import youtube_yt
|
||||
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||
except Exception:
|
||||
has_ytdlp = False
|
||||
if has_ytdlp and research_results.get("youtube_error"):
|
||||
core_errored.append("youtube")
|
||||
|
||||
score_pct = int(len(core_active) / 5 * 100)
|
||||
|
||||
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||
active_sources = research_results.get("active_sources") or []
|
||||
nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None
|
||||
|
||||
return {
|
||||
"score_pct": score_pct,
|
||||
"core_active": core_active,
|
||||
"core_missing": core_missing,
|
||||
"core_errored": core_errored,
|
||||
"nudge_text": nudge_text,
|
||||
}
|
||||
|
||||
|
||||
def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str:
|
||||
"""Build human-readable nudge text describing what was missed.
|
||||
|
||||
Prioritizes free suggestions. Optionally mentions bonus sources
|
||||
(TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured.
|
||||
"""
|
||||
lines: List[str] = []
|
||||
|
||||
# Describe what was missed
|
||||
missed_parts: List[str] = []
|
||||
for src in core_missing:
|
||||
label = SOURCE_LABELS[src]
|
||||
if src in core_errored:
|
||||
missed_parts.append(f"{label} (errored this run)")
|
||||
else:
|
||||
missed_parts.append(label)
|
||||
|
||||
active_count = 5 - len(core_missing)
|
||||
lines.append(f"Research quality: {active_count}/5 core sources.")
|
||||
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
||||
lines.append("")
|
||||
|
||||
# Free suggestions
|
||||
free_suggestions: List[str] = []
|
||||
|
||||
if "x" in core_missing:
|
||||
if "x" in core_errored:
|
||||
free_suggestions.append(
|
||||
"X/Twitter errored - log into x.com in your browser, then re-run."
|
||||
)
|
||||
else:
|
||||
free_suggestions.append(
|
||||
"X/Twitter: real-time posts with likes and reposts - the fastest "
|
||||
"signal for breaking topics. Two options: log into x.com in your "
|
||||
"browser and re-run (cookies detected automatically), or add "
|
||||
"XAI_API_KEY to your .env (no browser access, get key at api.x.ai)."
|
||||
)
|
||||
|
||||
if "youtube" in core_missing:
|
||||
if "youtube" in core_errored:
|
||||
free_suggestions.append(
|
||||
"YouTube errored - update yt-dlp: brew upgrade yt-dlp"
|
||||
)
|
||||
else:
|
||||
free_suggestions.append(
|
||||
"YouTube: video transcripts with key moments - often the deepest "
|
||||
"explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
|
||||
)
|
||||
|
||||
# Mention bonus opt-in sources when SC key is present
|
||||
if has_sc:
|
||||
bonus_hints = []
|
||||
if "threads" not in (active_sources or []):
|
||||
bonus_hints.append("Threads")
|
||||
if "pinterest" not in (active_sources or []):
|
||||
bonus_hints.append("Pinterest")
|
||||
if bonus_hints:
|
||||
free_suggestions.append(
|
||||
f"Your SC key also powers {', '.join(bonus_hints)} and YouTube comments. "
|
||||
"Add them to INCLUDE_SOURCES in your .env to enable."
|
||||
)
|
||||
|
||||
if free_suggestions:
|
||||
lines.append("Free fixes:")
|
||||
for s in free_suggestions:
|
||||
lines.append(f" - {s}")
|
||||
lines.append("")
|
||||
|
||||
# Bonus sources mention (non-blocking)
|
||||
if not has_sc:
|
||||
lines.append(
|
||||
"Bonus: TikTok and Instagram are available with a free "
|
||||
"ScrapeCreators key at scrapecreators.com (no affiliation)."
|
||||
)
|
||||
else:
|
||||
lines.append("last30days has no affiliation with any API provider.")
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -334,7 +334,7 @@ def _global_search(
|
||||
)
|
||||
return data.get("posts", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
if e.status_code in (401, 403):
|
||||
raise
|
||||
_log(f"Global search error: {e}")
|
||||
return []
|
||||
@@ -376,11 +376,6 @@ def _subreddit_search(
|
||||
retries=2,
|
||||
)
|
||||
return data.get("posts", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
raise
|
||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
_log(f"Subreddit search error for r/{subreddit}: {e}")
|
||||
return []
|
||||
@@ -408,11 +403,6 @@ def fetch_post_comments(
|
||||
retries=2,
|
||||
)
|
||||
return data.get("comments", data.get("data", []))
|
||||
except http.HTTPError as e:
|
||||
if e.status_code in (401, 402, 403):
|
||||
raise
|
||||
_log(f"Comment fetch error: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
_log(f"Comment fetch error: {e}")
|
||||
return []
|
||||
@@ -1,16 +1,9 @@
|
||||
"""Reddit public ``.json`` search module (demoted to keyless Tier 0).
|
||||
"""Standalone Reddit public JSON search module.
|
||||
|
||||
Reddit's public ``.json`` endpoints now return HTTP 403 from most contexts
|
||||
(shreddit anti-bot), so this is no longer the primary free path. The keyless
|
||||
pipeline (see reddit_keyless.py) still calls ``search`` as a cheap one-shot
|
||||
Tier 0 attempt — a residential machine may occasionally get a 200 — before
|
||||
falling through to RSS discovery (reddit_rss.py) and shreddit comment
|
||||
enrichment (reddit_shreddit.py).
|
||||
Searches Reddit using the free public JSON endpoints (no API key required).
|
||||
Promoted from last-resort fallback to robust primary free path.
|
||||
|
||||
``search_reddit_public`` is retained as a compatibility shim that delegates to
|
||||
the keyless pipeline, so existing callers (pipeline.py) need no change.
|
||||
|
||||
Endpoints (Tier 0):
|
||||
Endpoints:
|
||||
- Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit}
|
||||
- Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month
|
||||
|
||||
@@ -18,21 +11,17 @@ Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
|
||||
network timeouts, and missing subreddits.
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
USER_AGENT = "last30days/3.0 (research tool)"
|
||||
|
||||
# Depth-aware limits for thread counts
|
||||
DEPTH_LIMITS = {
|
||||
@@ -41,6 +30,13 @@ DEPTH_LIMITS = {
|
||||
"deep": 50,
|
||||
}
|
||||
|
||||
# How many top posts to enrich with comments, by depth
|
||||
ENRICH_LIMITS = {
|
||||
"quick": 3,
|
||||
"default": 5,
|
||||
"deep": 8,
|
||||
}
|
||||
|
||||
MAX_RETRIES = 3
|
||||
BASE_BACKOFF = 2.0 # seconds
|
||||
|
||||
@@ -64,9 +60,6 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
|
||||
headers = {
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
|
||||
@@ -78,10 +71,7 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
|
||||
_log(f"Anti-bot HTML response (Content-Type: {content_type})")
|
||||
return None
|
||||
|
||||
raw = resp.read()
|
||||
if resp.headers.get("Content-Encoding", "").lower() == "gzip":
|
||||
raw = gzip.decompress(raw)
|
||||
body = raw.decode("utf-8")
|
||||
body = resp.read().decode("utf-8")
|
||||
return json.loads(body)
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
@@ -208,7 +198,7 @@ def search(
|
||||
encoded_query = _url_encode(query)
|
||||
|
||||
if subreddit:
|
||||
sub = subreddit.removeprefix("r/").strip()
|
||||
sub = subreddit.lstrip("r/").strip()
|
||||
url = (
|
||||
f"https://www.reddit.com/r/{sub}/search.json"
|
||||
f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
|
||||
@@ -236,6 +226,78 @@ def search(
|
||||
return unique[:limit]
|
||||
|
||||
|
||||
def _enrich_post(item: Dict[str, Any], timeout: int = 10) -> Dict[str, Any]:
|
||||
"""Enrich a single post with top comments. Never raises."""
|
||||
try:
|
||||
from . import reddit_enrich
|
||||
thread_data = reddit_enrich.fetch_thread_data(item["url"], timeout=timeout)
|
||||
if not thread_data:
|
||||
return item
|
||||
parsed = reddit_enrich.parse_thread_data(thread_data)
|
||||
comments = parsed.get("comments", [])
|
||||
top = reddit_enrich.get_top_comments(comments)
|
||||
item["top_comments"] = [
|
||||
{
|
||||
"score": c.get("score", 0),
|
||||
"excerpt": (c.get("body") or "")[:200],
|
||||
"author": c.get("author", ""),
|
||||
}
|
||||
for c in top[:10]
|
||||
]
|
||||
except Exception:
|
||||
# Never discard — keep post with empty metadata
|
||||
pass
|
||||
return item
|
||||
|
||||
|
||||
def _enrich_posts(posts: List[Dict[str, Any]], depth: str = "default") -> List[Dict[str, Any]]:
|
||||
"""Enrich top N posts with comment data using threads. Total budget 45s."""
|
||||
limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])
|
||||
to_enrich = posts[:limit]
|
||||
rest = posts[limit:]
|
||||
|
||||
if not to_enrich:
|
||||
return posts
|
||||
|
||||
enriched = []
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=min(limit, 4)) as executor:
|
||||
futures = {
|
||||
executor.submit(_enrich_post, post, 10): i
|
||||
for i, post in enumerate(to_enrich)
|
||||
}
|
||||
# Collect results with 45s total budget
|
||||
import concurrent.futures
|
||||
done, not_done = concurrent.futures.wait(futures, timeout=45)
|
||||
# Build result list preserving order
|
||||
result_map: Dict[int, Dict[str, Any]] = {}
|
||||
for future in done:
|
||||
idx = futures[future]
|
||||
try:
|
||||
result_map[idx] = future.result(timeout=0)
|
||||
except Exception:
|
||||
result_map[idx] = to_enrich[idx]
|
||||
# Any not-done futures: keep original post
|
||||
for future in not_done:
|
||||
idx = futures[future]
|
||||
result_map[idx] = to_enrich[idx]
|
||||
future.cancel()
|
||||
enriched = [result_map[i] for i in range(len(to_enrich))]
|
||||
except Exception:
|
||||
enriched = to_enrich
|
||||
|
||||
return enriched + rest
|
||||
|
||||
|
||||
def _search_subreddit(sub: str, topic: str, depth: str, timeout: int = 15) -> List[Dict[str, Any]]:
|
||||
"""Search a single subreddit. Never raises."""
|
||||
try:
|
||||
return search(topic, depth=depth, subreddit=sub, timeout=timeout)
|
||||
except Exception as e:
|
||||
_log(f"Subreddit search failed for r/{sub}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def search_reddit_public(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
@@ -243,17 +305,12 @@ def search_reddit_public(
|
||||
depth: str = "default",
|
||||
subreddits: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""High-level free Reddit search + enrichment (keyless).
|
||||
"""High-level Reddit public search matching the openai_reddit interface.
|
||||
|
||||
Thin compatibility shim over the tiered keyless pipeline: the legacy
|
||||
``.json`` search/enrichment endpoints now return HTTP 403, so this delegates
|
||||
to ``reddit_keyless.search_and_enrich`` (Tier 0 one-shot ``.json`` →
|
||||
Tier 1 RSS discovery → Tier 2 shreddit comment enrichment). The name and
|
||||
signature are preserved so ``pipeline.py`` and other callers need no change
|
||||
and the ScrapeCreators backup still engages when this returns empty.
|
||||
|
||||
The module-level ``search`` / ``_parse_posts`` helpers remain in use as the
|
||||
keyless pipeline's demoted Tier 0 ``.json`` attempt.
|
||||
When subreddits are provided (from agent planning), searches each targeted
|
||||
sub first, then does global search, and deduplicates across both. This
|
||||
mirrors the SC search_and_enrich() flow where pre-resolved subreddits get
|
||||
priority.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
@@ -264,9 +321,57 @@ def search_reddit_public(
|
||||
|
||||
Returns:
|
||||
List of normalized item dicts matching ScrapeCreators output format.
|
||||
Empty list on total failure (so SC backup can engage).
|
||||
"""
|
||||
from . import reddit_keyless
|
||||
return reddit_keyless.search_and_enrich(
|
||||
topic, from_date, to_date, depth=depth, subreddits=subreddits
|
||||
all_posts: List[Dict[str, Any]] = []
|
||||
|
||||
# Phase 1: Search targeted subreddits in parallel (if provided)
|
||||
if subreddits:
|
||||
_log(f"Searching {len(subreddits)} targeted subreddits: {subreddits}")
|
||||
workers = min(4, len(subreddits))
|
||||
with ThreadPoolExecutor(max_workers=workers) as executor:
|
||||
futures = {
|
||||
executor.submit(_search_subreddit, sub, topic, depth): sub
|
||||
for sub in subreddits
|
||||
}
|
||||
for future in futures:
|
||||
sub = futures[future]
|
||||
try:
|
||||
sub_posts = future.result(timeout=30)
|
||||
_log(f" -> {len(sub_posts)} results from r/{sub}")
|
||||
all_posts.extend(sub_posts)
|
||||
except (Exception, FuturesTimeoutError) as e:
|
||||
_log(f" -> r/{sub} failed: {e}")
|
||||
|
||||
# Phase 2: Global search
|
||||
global_posts = search(topic, depth=depth)
|
||||
all_posts.extend(global_posts)
|
||||
|
||||
# Deduplicate by URL (targeted results keep priority since they come first)
|
||||
seen_urls: set = set()
|
||||
results: List[Dict[str, Any]] = []
|
||||
for post in all_posts:
|
||||
if post["url"] not in seen_urls:
|
||||
seen_urls.add(post["url"])
|
||||
results.append(post)
|
||||
|
||||
# Date filter: keep posts in range or with unknown dates
|
||||
filtered = []
|
||||
for item in results:
|
||||
d = item.get("date")
|
||||
if d is None or (from_date <= d <= to_date):
|
||||
filtered.append(item)
|
||||
|
||||
# Sort by engagement (score desc)
|
||||
filtered.sort(
|
||||
key=lambda x: x.get("engagement", {}).get("score", 0),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
# Enrich top posts with comments
|
||||
filtered = _enrich_posts(filtered, depth=depth)
|
||||
|
||||
# Re-index IDs
|
||||
for i, item in enumerate(filtered):
|
||||
item["id"] = f"R{i + 1}"
|
||||
|
||||
return filtered
|
||||
@@ -71,29 +71,8 @@ def _normalize_phrase(text: str) -> str:
|
||||
return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split())
|
||||
|
||||
|
||||
class PreparedQuery:
|
||||
"""Precomputed query shape reused across items in a stream.
|
||||
|
||||
Built once per ranking_query; reused by token_overlap_relevance so the
|
||||
per-item normalize/score loops don't re-tokenize the same query N times.
|
||||
"""
|
||||
|
||||
__slots__ = ("raw", "q_tokens", "informative_q_tokens", "normalized_phrase")
|
||||
|
||||
def __init__(self, query: str) -> None:
|
||||
self.raw = query
|
||||
self.q_tokens = tokenize(query)
|
||||
informative = {t for t in self.q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
||||
self.informative_q_tokens = informative or self.q_tokens
|
||||
self.normalized_phrase = _normalize_phrase(query)
|
||||
|
||||
|
||||
def _as_prepared(query: "str | PreparedQuery") -> PreparedQuery:
|
||||
return query if isinstance(query, PreparedQuery) else PreparedQuery(query)
|
||||
|
||||
|
||||
def token_overlap_relevance(
|
||||
query: "str | PreparedQuery",
|
||||
query: str,
|
||||
text: str,
|
||||
hashtags: Optional[List[str]] = None,
|
||||
) -> float:
|
||||
@@ -116,8 +95,7 @@ def token_overlap_relevance(
|
||||
Returns:
|
||||
Float between 0.0 and 1.0 (0.5 for empty queries)
|
||||
"""
|
||||
prepared = _as_prepared(query)
|
||||
q_tokens = prepared.q_tokens
|
||||
q_tokens = tokenize(query)
|
||||
|
||||
# Combine text and hashtags for matching
|
||||
combined = text
|
||||
@@ -141,7 +119,9 @@ def token_overlap_relevance(
|
||||
if overlap == 0:
|
||||
return 0.0
|
||||
|
||||
informative_q_tokens = prepared.informative_q_tokens
|
||||
informative_q_tokens = {t for t in q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS}
|
||||
if not informative_q_tokens:
|
||||
informative_q_tokens = q_tokens
|
||||
|
||||
coverage = overlap / len(q_tokens)
|
||||
informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens)
|
||||
@@ -149,7 +129,7 @@ def token_overlap_relevance(
|
||||
precision = overlap / precision_denominator
|
||||
|
||||
phrase_bonus = 0.0
|
||||
normalized_query = prepared.normalized_phrase
|
||||
normalized_query = _normalize_phrase(query)
|
||||
normalized_text = _normalize_phrase(combined)
|
||||
if normalized_query and normalized_query in normalized_text:
|
||||
phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16
|
||||
@@ -8,40 +8,24 @@ from collections import Counter
|
||||
from datetime import date
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import dates, schema, skill_meta
|
||||
from . import dates, schema
|
||||
|
||||
|
||||
def _skill_version() -> str:
|
||||
"""Read plugin version from .claude-plugin/plugin.json, falling back to SKILL.md frontmatter.
|
||||
"""Read plugin version from .claude-plugin/plugin.json if available.
|
||||
|
||||
Per-harness skill install dirs (`~/.claude/skills`, `~/.codex/skills`, `~/.agents/skills`,
|
||||
Hermes, etc.) do not always carry `.claude-plugin/plugin.json` — that file ships with
|
||||
plugin-cache installs but not with per-harness skill installs. SKILL.md frontmatter is
|
||||
the fallback that keeps the badge from emitting v? on those installs. Returns "?" only
|
||||
if no usable version string is found from either source (missing files, corrupt JSON,
|
||||
or SKILL.md without a version line).
|
||||
|
||||
A corrupt manifest at one ancestor does not shadow a valid manifest at a deeper one
|
||||
(continue, not break). SKILL.md parsing accepts double-quoted, single-quoted, or
|
||||
unquoted YAML version scalars (delegated to skill_meta.read_skill_version).
|
||||
Tries nearest plugin.json by walking up from render.py's own location.
|
||||
Falls back to "?" if not found. This keeps the badge emission from
|
||||
crashing on non-plugin-cache installs (repo checkout, Gemini, Codex).
|
||||
"""
|
||||
here = pathlib.Path(__file__).resolve()
|
||||
for parent in here.parents:
|
||||
manifest = parent / ".claude-plugin" / "plugin.json"
|
||||
if manifest.is_file():
|
||||
for parent in [here.parent, *here.parents]:
|
||||
candidate = parent / ".claude-plugin" / "plugin.json"
|
||||
if candidate.is_file():
|
||||
try:
|
||||
version = json.loads(manifest.read_text()).get("version")
|
||||
return json.loads(candidate.read_text()).get("version", "?")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
if version:
|
||||
return version
|
||||
|
||||
# No usable manifest found at any ancestor — fall back to SKILL.md frontmatter.
|
||||
# First SKILL.md found in the walk is THIS skill's; never traverse past it.
|
||||
for parent in here.parents:
|
||||
skill_md = parent / "SKILL.md"
|
||||
if skill_md.is_file():
|
||||
return skill_meta.read_skill_version(skill_md) or "?"
|
||||
return "?"
|
||||
return "?"
|
||||
|
||||
|
||||
@@ -68,7 +52,6 @@ SOURCE_LABELS = {
|
||||
"xiaohongshu": "Xiaohongshu",
|
||||
"x": "X",
|
||||
"github": "GitHub",
|
||||
"digg": "Digg",
|
||||
"perplexity": "Perplexity",
|
||||
}
|
||||
|
||||
@@ -96,7 +79,7 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
f"# last30days v{_skill_version()}: {report.topic}",
|
||||
f"# last30days v3.0.0: {report.topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Date range: {report.range_from} to {report.range_to}",
|
||||
@@ -187,168 +170,6 @@ def render_compact(report: schema.Report, cluster_limit: int = 8, fun_level: str
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_for_html(
|
||||
report: schema.Report,
|
||||
synthesis_md: str | None = None,
|
||||
*,
|
||||
save_path: str | None = None,
|
||||
) -> str:
|
||||
"""Render markdown intended for shareable HTML conversion.
|
||||
|
||||
This output keeps the public badge, compact source/date metadata, an
|
||||
optional one-line data quality note, optional synthesized brief markdown,
|
||||
and the engine footer. It deliberately omits the debug file header,
|
||||
model-facing safety note, and evidence scratchpad emitted by
|
||||
render_compact().
|
||||
|
||||
When synthesis_md is None, the body is intentionally sparse: badge,
|
||||
metadata, optional data quality note, and engine footer only.
|
||||
"""
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
*_render_html_metadata(report),
|
||||
]
|
||||
if synthesis_md:
|
||||
lines.extend(["", synthesis_md.strip()])
|
||||
# Data quality warnings are NOT rendered into the HTML artifact. The HTML
|
||||
# is meant to be shared (Slack, email, Notion); recipients haven't asked
|
||||
# for technical commentary about how the run was produced. Generators see
|
||||
# the same warnings via collect_html_warnings() routed to stderr by the
|
||||
# CLI, so they can fix quality issues before sharing.
|
||||
_append_html_footer(lines, report, save_path)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_for_html_comparison(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
synthesis_md: str | None = None,
|
||||
*,
|
||||
save_path: str | None = None,
|
||||
) -> str:
|
||||
"""Render comparison markdown intended for shareable HTML conversion.
|
||||
|
||||
Same semantics as render_for_html(), but metadata and data quality notes
|
||||
are aggregated across the compared entities.
|
||||
"""
|
||||
if not entity_reports:
|
||||
raise ValueError("render_for_html_comparison requires at least one report")
|
||||
|
||||
entities = [label for label, _ in entity_reports]
|
||||
main_report = entity_reports[0][1]
|
||||
meta = (
|
||||
f"<!-- META: {main_report.range_from} to {main_report.range_to} "
|
||||
f"· comparing {len(entities)}: {', '.join(entities)} -->"
|
||||
)
|
||||
lines = [
|
||||
*_render_badge(),
|
||||
meta,
|
||||
]
|
||||
if synthesis_md:
|
||||
lines.extend(["", synthesis_md.strip()])
|
||||
# Comparison data quality notes also go to stderr, not into the artifact.
|
||||
_append_html_footer(lines, main_report, save_path)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def collect_html_warnings(report: schema.Report) -> list[str]:
|
||||
"""Collect data quality warnings for stderr output (NOT for the HTML artifact).
|
||||
|
||||
Returns a list of human-readable warning strings. Empty list if the run
|
||||
was clean. Used by the CLI to emit diagnostics to stderr after writing
|
||||
the HTML to stdout/file.
|
||||
"""
|
||||
notes: list[str] = []
|
||||
if _render_degraded_run_warning(report):
|
||||
notes.append("Run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
||||
elif _render_pre_research_warning(report):
|
||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
||||
freshness_warning = _assess_data_freshness(report)
|
||||
if freshness_warning:
|
||||
notes.append(freshness_warning)
|
||||
notes.extend(report.warnings)
|
||||
return _dedupe_notes(notes)
|
||||
|
||||
|
||||
def collect_html_warnings_comparison(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
) -> list[str]:
|
||||
"""Collect comparison-mode warnings, prefixed by entity label."""
|
||||
notes: list[str] = []
|
||||
for label, report in entity_reports:
|
||||
for w in collect_html_warnings(report):
|
||||
notes.append(f"{label}: {w}")
|
||||
return notes
|
||||
|
||||
|
||||
def _render_html_metadata(report: schema.Report) -> list[str]:
|
||||
"""Inline metadata as an HTML comment marker.
|
||||
|
||||
html_render.py post-processes ``<!-- META: ... -->`` markers into a
|
||||
``<div class="meta">`` after markdown conversion, so the metadata escapes
|
||||
the markdown converter's HTML-escaping pass cleanly. Same pattern as the
|
||||
PASS_THROUGH_FOOTER marker used for the engine tree.
|
||||
"""
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
if non_empty:
|
||||
sources = ", ".join(_source_label(s) for s in non_empty)
|
||||
else:
|
||||
sources = "no active sources"
|
||||
return [
|
||||
f"<!-- META: {report.range_from} to {report.range_to} · {sources} -->",
|
||||
]
|
||||
|
||||
|
||||
def _render_html_data_quality_note(report: schema.Report) -> str | None:
|
||||
notes: list[str] = []
|
||||
degraded_warning = _render_degraded_run_warning(report)
|
||||
if degraded_warning:
|
||||
notes.append("This run was missing pre-flight resolution. Re-run with `--plan` for richer results.")
|
||||
pre_research_warning = _render_pre_research_warning(report)
|
||||
if pre_research_warning and not degraded_warning:
|
||||
notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")
|
||||
freshness_warning = _assess_data_freshness(report)
|
||||
if freshness_warning:
|
||||
notes.append(freshness_warning)
|
||||
notes.extend(report.warnings)
|
||||
if not notes:
|
||||
return None
|
||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
||||
|
||||
|
||||
def _render_html_comparison_data_quality_note(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
) -> str | None:
|
||||
notes: list[str] = []
|
||||
for label, report in entity_reports:
|
||||
note = _render_html_data_quality_note(report)
|
||||
if note:
|
||||
clean = note.removeprefix("> **Data quality note:** ").strip()
|
||||
notes.append(f"{label}: {clean}")
|
||||
if not notes:
|
||||
return None
|
||||
return f"> **Data quality note:** {' '.join(_dedupe_notes(notes))}"
|
||||
|
||||
|
||||
def _dedupe_notes(notes: list[str]) -> list[str]:
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for note in notes:
|
||||
normalized = " ".join(str(note).split())
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
out.append(normalized)
|
||||
return out
|
||||
|
||||
|
||||
def _append_html_footer(lines: list[str], report: schema.Report, save_path: str | None) -> None:
|
||||
footer = _render_emoji_footer(report, save_path)
|
||||
lines.append("")
|
||||
lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")
|
||||
lines.extend(footer)
|
||||
lines.append("<!-- END PASS-THROUGH FOOTER -->")
|
||||
|
||||
|
||||
def _render_canonical_boundary() -> list[str]:
|
||||
"""Emit the explicit END-OF-CANONICAL-OUTPUT boundary.
|
||||
|
||||
@@ -533,8 +354,7 @@ def _render_comparison_scaffold(topic: str) -> list[str]:
|
||||
|
||||
Axes match the April 9 launch-video exemplar (9 axes suited to AI-tool
|
||||
comparisons). For non-AI-tool comparisons, the synthesizer writes N/A
|
||||
or topic-appropriate substitutes in irrelevant rows. The "What it is" row
|
||||
grounds in first-party positioning fetched during the run when available.
|
||||
or topic-appropriate substitutes in irrelevant rows.
|
||||
"""
|
||||
entities = _parse_comparison_entities(topic)
|
||||
if not entities:
|
||||
@@ -559,18 +379,10 @@ def _render_comparison_scaffold(topic: str) -> list[str]:
|
||||
]
|
||||
body = [f"| {axis} | " + " | ".join([" "] * len(entities)) + " |" for axis in axes]
|
||||
|
||||
fill_instructions = (
|
||||
"Fill each cell based on the research above. Keep cells short (5-15 words). "
|
||||
"Use ' - ' (hyphen with spaces) not em-dashes. Write N/A for axes that do not apply to this topic class. "
|
||||
"Ground the \"What it is\" row in first-party positioning fetched during this run's research when "
|
||||
"available - describe each entity as it pitches itself today, never from memory. "
|
||||
"This scaffold matches the April 9 launch-video exemplar shape."
|
||||
)
|
||||
|
||||
return [
|
||||
"## Head-to-Head",
|
||||
"",
|
||||
fill_instructions,
|
||||
"Fill each cell based on the research above. Keep cells short (5-15 words). Use ' - ' (hyphen with spaces) not em-dashes. Write N/A for axes that do not apply to this topic class. This scaffold matches the April 9 launch-video exemplar shape.",
|
||||
"",
|
||||
header,
|
||||
separator,
|
||||
@@ -580,226 +392,12 @@ def _render_comparison_scaffold(topic: str) -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def render_comparison_multi(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
*,
|
||||
cluster_limit: int = 4,
|
||||
fun_level: str = "medium",
|
||||
save_path: str | None = None,
|
||||
) -> str:
|
||||
"""Render N (entity, Report) pairs as a single comparison output.
|
||||
|
||||
Reuses _render_comparison_scaffold for the synthesis table and emits
|
||||
per-entity evidence sections inside one EVIDENCE FOR SYNTHESIS envelope.
|
||||
The single-Report render_compact path is unchanged.
|
||||
|
||||
Args:
|
||||
entity_reports: Ordered (label, Report) pairs. The first pair is the
|
||||
user's main topic; the remainder are discovered/explicit competitors.
|
||||
cluster_limit: Max clusters to surface per entity (kept lower than the
|
||||
single-entity default to keep N-way comparisons readable).
|
||||
fun_level: Same fun-level knob as render_compact, applied to each
|
||||
entity's best-takes block.
|
||||
save_path: Optional save-path display string for the footer.
|
||||
"""
|
||||
if not entity_reports:
|
||||
raise ValueError("render_comparison_multi requires at least one report")
|
||||
|
||||
entities = [label for label, _ in entity_reports]
|
||||
main_label, main_report = entity_reports[0]
|
||||
synthesized_topic = " vs ".join(entities)
|
||||
|
||||
lines: list[str] = [
|
||||
*_render_badge(),
|
||||
f"# last30days v{_skill_version()}: {synthesized_topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Comparison mode: {len(entities)} entities ({', '.join(entities)})",
|
||||
f"- Date range: {main_report.range_from} to {main_report.range_to}",
|
||||
"",
|
||||
]
|
||||
|
||||
aggregated_warnings: list[str] = []
|
||||
for label, report in entity_reports:
|
||||
aggregated_warnings.extend(f"[{label}] {w}" for w in report.warnings)
|
||||
if aggregated_warnings:
|
||||
lines.append("## Warnings")
|
||||
lines.extend(f"- {w}" for w in aggregated_warnings)
|
||||
lines.append("")
|
||||
|
||||
lines.append(
|
||||
"<!-- EVIDENCE FOR SYNTHESIS: read this, do not emit verbatim. Transform into "
|
||||
"`What I learned:` prose per LAW 2. Each entity has its own evidence subsection. -->"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
resolved_block = _render_resolved_entities_block(entity_reports)
|
||||
if resolved_block:
|
||||
lines.extend(resolved_block)
|
||||
lines.append("")
|
||||
|
||||
fun_params = _FUN_LEVELS.get(fun_level, _FUN_LEVELS["medium"])
|
||||
for label, report in entity_reports:
|
||||
lines.extend(_render_entity_evidence_block(
|
||||
label=label,
|
||||
report=report,
|
||||
cluster_limit=cluster_limit,
|
||||
fun_params=fun_params,
|
||||
))
|
||||
|
||||
lines.append("<!-- END EVIDENCE FOR SYNTHESIS -->")
|
||||
lines.append("")
|
||||
|
||||
# Reuse the existing comparison scaffold by feeding it the synthesized
|
||||
# topic. _parse_comparison_entities splits on " vs " so the scaffold
|
||||
# picks up all N entities automatically.
|
||||
scaffold = _render_comparison_scaffold(synthesized_topic)
|
||||
lines.extend(scaffold)
|
||||
|
||||
footer = _render_emoji_footer(main_report, save_path)
|
||||
if footer:
|
||||
lines.append("")
|
||||
lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")
|
||||
lines.extend(footer)
|
||||
lines.append("<!-- END PASS-THROUGH FOOTER -->")
|
||||
|
||||
lines.extend(_render_canonical_boundary())
|
||||
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def _render_resolved_entities_block(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
) -> list[str]:
|
||||
"""Emit a visible per-entity Step 0.55 resolution summary.
|
||||
|
||||
Reads `resolved` dicts from each Report's artifacts. Returns an empty
|
||||
list when no entity has a resolved payload (mock mode, no web backend,
|
||||
or artifacts not populated). Missing per-entity fields render as `-`.
|
||||
Context strings truncate at 120 chars.
|
||||
"""
|
||||
any_resolved = any(
|
||||
isinstance(report.artifacts.get("resolved"), dict)
|
||||
for _label, report in entity_reports
|
||||
)
|
||||
if not any_resolved:
|
||||
return []
|
||||
|
||||
out: list[str] = ["## Resolved Entities", ""]
|
||||
for label, report in entity_reports:
|
||||
resolved = report.artifacts.get("resolved") or {}
|
||||
x_handle = resolved.get("x_handle") or ""
|
||||
subs = resolved.get("subreddits") or []
|
||||
gh_user = resolved.get("github_user") or ""
|
||||
gh_repos = resolved.get("github_repos") or []
|
||||
context = resolved.get("context") or ""
|
||||
|
||||
x_display = f"@{x_handle}" if x_handle else "-"
|
||||
subs_display = (
|
||||
", ".join(f"r/{s}" for s in subs[:5]) + (
|
||||
f" (+{len(subs) - 5})" if len(subs) > 5 else ""
|
||||
)
|
||||
) if subs else "-"
|
||||
gh_display = f"@{gh_user}" if gh_user else "-"
|
||||
if gh_repos:
|
||||
gh_display += f" ({', '.join(gh_repos[:3])}" + (
|
||||
f" +{len(gh_repos) - 3}" if len(gh_repos) > 3 else ""
|
||||
) + ")"
|
||||
context_display = _truncate(context, 120) if context else "-"
|
||||
|
||||
out.append(
|
||||
f"- **{label}**: X {x_display} | Subs {subs_display} | "
|
||||
f"GitHub {gh_display} | Context: {context_display}"
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _render_entity_evidence_block(
|
||||
*,
|
||||
label: str,
|
||||
report: schema.Report,
|
||||
cluster_limit: int,
|
||||
fun_params: dict,
|
||||
) -> list[str]:
|
||||
"""Render one entity's clusters and best-takes inside the evidence envelope."""
|
||||
candidate_by_id = {c.candidate_id: c for c in report.ranked_candidates}
|
||||
out: list[str] = [f"## {label}", ""]
|
||||
|
||||
if not report.clusters:
|
||||
out.append("(no significant discussion this month)")
|
||||
out.append("")
|
||||
return out
|
||||
|
||||
out.append("### Ranked Evidence Clusters")
|
||||
out.append("")
|
||||
for index, cluster in enumerate(report.clusters[:cluster_limit], start=1):
|
||||
out.append(
|
||||
f"#### {index}. {cluster.title} "
|
||||
f"(score {cluster.score:.0f}, {len(cluster.candidate_ids)} item"
|
||||
f"{'s' if len(cluster.candidate_ids) != 1 else ''}, "
|
||||
f"sources: {', '.join(_source_label(s) for s in cluster.sources)})"
|
||||
)
|
||||
if cluster.uncertainty:
|
||||
out.append(f"- Uncertainty: {cluster.uncertainty}")
|
||||
for rep_index, candidate_id in enumerate(cluster.representative_ids, start=1):
|
||||
candidate = candidate_by_id.get(candidate_id)
|
||||
if not candidate:
|
||||
continue
|
||||
out.extend(_render_candidate(candidate, prefix=f"{rep_index}."))
|
||||
out.append("")
|
||||
|
||||
best_takes = _render_best_takes(
|
||||
report.ranked_candidates,
|
||||
limit=fun_params["limit"],
|
||||
threshold=fun_params["threshold"],
|
||||
)
|
||||
if best_takes:
|
||||
out.extend(best_takes)
|
||||
out.append("")
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def render_comparison_multi_context(
|
||||
entity_reports: list[tuple[str, schema.Report]],
|
||||
cluster_limit: int = 4,
|
||||
) -> str:
|
||||
"""Context-mode rendering for the multi-entity comparison."""
|
||||
if not entity_reports:
|
||||
raise ValueError("render_comparison_multi_context requires at least one report")
|
||||
|
||||
entities = [label for label, _ in entity_reports]
|
||||
lines = [
|
||||
f"Comparison: {' vs '.join(entities)}",
|
||||
f"Entities: {len(entities)}",
|
||||
_AI_SAFETY_NOTE,
|
||||
"",
|
||||
]
|
||||
resolved_block = _render_resolved_entities_block(entity_reports)
|
||||
if resolved_block:
|
||||
lines.extend(resolved_block)
|
||||
lines.append("")
|
||||
for label, report in entity_reports:
|
||||
lines.append(f"## {label}")
|
||||
lines.append(f"Intent: {report.query_plan.intent}")
|
||||
if not report.clusters:
|
||||
lines.append("- (no significant discussion this month)")
|
||||
else:
|
||||
for cluster in report.clusters[:cluster_limit]:
|
||||
lines.append(
|
||||
f"- {cluster.title} "
|
||||
f"[{', '.join(_source_label(s) for s in cluster.sources)}]"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_full(report: schema.Report) -> str:
|
||||
"""Full data dump: ALL clusters + ALL items by source. For saved files and debugging."""
|
||||
# Start with the same header as compact
|
||||
non_empty = [s for s, items in sorted(report.items_by_source.items()) if items]
|
||||
lines = [
|
||||
f"# last30days v{_skill_version()}: {report.topic}",
|
||||
f"# last30days v3.0.0: {report.topic}",
|
||||
"",
|
||||
*_assistant_safety_lines(),
|
||||
f"- Date range: {report.range_from} to {report.range_to}",
|
||||
@@ -812,17 +410,6 @@ def render_full(report: schema.Report) -> str:
|
||||
lines.extend(f"- {warning}" for warning in report.warnings)
|
||||
lines.append("")
|
||||
|
||||
# When this Report is a per-entity sub-run from vs-mode / --competitors,
|
||||
# include the single-row Resolved Entities block so the saved file is
|
||||
# self-describing. The artifact is populated by last30days.py's
|
||||
# _competitor_runner and _main_runner closures.
|
||||
resolved = report.artifacts.get("resolved")
|
||||
if isinstance(resolved, dict) and resolved.get("entity"):
|
||||
single_row = _render_resolved_entities_block([(resolved["entity"], report)])
|
||||
if single_row:
|
||||
lines.extend(single_row)
|
||||
lines.append("")
|
||||
|
||||
# ALL clusters (no limit)
|
||||
lines.append("## Ranked Evidence Clusters")
|
||||
lines.append("")
|
||||
@@ -851,7 +438,7 @@ def render_full(report: schema.Report) -> str:
|
||||
lines.append("## All Items by Source")
|
||||
lines.append("")
|
||||
source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest",
|
||||
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "digg", "perplexity"]
|
||||
"hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "perplexity"]
|
||||
for source in source_order:
|
||||
items = report.items_by_source.get(source, [])
|
||||
if not items:
|
||||
@@ -875,11 +462,7 @@ def render_full(report: schema.Report) -> str:
|
||||
for tc in top_comments[:3]:
|
||||
excerpt = tc.get("excerpt", tc.get("text", ""))[:200]
|
||||
tc_score = tc.get("score", "")
|
||||
attribution = _comment_attribution(item.source, tc.get("author"))
|
||||
lines.append(f" Top comment {attribution} ({tc_score} {vote_label}): {excerpt}")
|
||||
# Digg: inline X-post quotes attached to the cluster.
|
||||
for post in _digg_posts_for(item, limit=3):
|
||||
lines.append(f" > {_format_digg_quote(post)}")
|
||||
lines.append(f" Top comment ({tc_score} {vote_label}): {excerpt}")
|
||||
# Comment insights for Reddit
|
||||
insights = item.metadata.get("comment_insights", [])
|
||||
if insights:
|
||||
@@ -998,11 +581,7 @@ def _render_candidate(candidate: schema.Candidate, prefix: str) -> list[str]:
|
||||
excerpt = tc.get("excerpt") or tc.get("text") or ""
|
||||
score = tc.get("score", "")
|
||||
vote_label = _vote_label_for(primary.source) if primary else "upvotes"
|
||||
source = primary.source if primary else None
|
||||
attribution = _comment_attribution(source, tc.get("author"))
|
||||
lines.append(f" - {attribution} ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
|
||||
for post in _digg_posts_for(primary):
|
||||
lines.append(f" - {_format_digg_quote(post)}")
|
||||
lines.append(f" - Comment ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
|
||||
insight = _comment_insight(primary)
|
||||
if insight:
|
||||
lines.append(f" - Insight: {_truncate(insight, 220)}")
|
||||
@@ -1253,7 +832,6 @@ _FOOTER_SOURCES: list[tuple[str, str, str, str, list[tuple[str, str]]]] = [
|
||||
("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||
("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]),
|
||||
("github", "🐙", "GitHub", "item", [("reactions", "reactions"), ("comments", "comments")]),
|
||||
("digg", "⛏️", "Digg", "cluster", [("postCount", "posts"), ("uniqueAuthors", "authors")]),
|
||||
]
|
||||
|
||||
|
||||
@@ -1294,16 +872,15 @@ def _build_source_footer_lines(report: schema.Report) -> list[str]:
|
||||
if total > 0:
|
||||
total_str = f"{total:,}" if total >= 1000 else str(total)
|
||||
parts.append(f"{total_str} {word}")
|
||||
# YouTube: always append "M/N with transcripts" so a zero-transcript run
|
||||
# (typically caused by a stale yt-dlp binary) is visible at the conclusion
|
||||
# surface. Hiding zero converts a problem signal into an absence; the very
|
||||
# case that needs to be loud is the one previously omitted from the footer.
|
||||
# YouTube: append "N with transcripts" instead of a third likes-based column.
|
||||
# Transcripts are a more meaningful research-depth signal than likes.
|
||||
if source_key == "youtube":
|
||||
with_transcripts = sum(
|
||||
1 for it in items
|
||||
if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))
|
||||
)
|
||||
parts.append(f"{with_transcripts}/{len(items)} with transcripts")
|
||||
if with_transcripts > 0:
|
||||
parts.append(f"{with_transcripts} with transcripts")
|
||||
stats = " │ ".join(parts)
|
||||
out.append(_footer_line_for_source(emoji, label, len(items), item_word, stats))
|
||||
|
||||
@@ -1512,7 +1089,6 @@ ENGAGEMENT_DISPLAY: dict[str, list[tuple[str, str]]] = {
|
||||
"polymarket": [],
|
||||
"github": [("reactions", "react"), ("comments", "cmt")],
|
||||
"perplexity": [("citations", "cite")],
|
||||
"digg": [("postCount", "posts"), ("uniqueAuthors", "auth")],
|
||||
}
|
||||
|
||||
|
||||
@@ -1656,33 +1232,6 @@ def _vote_label_for(source: str) -> str:
|
||||
return _TOP_COMMENT_VOTE_LABEL.get(source, "votes")
|
||||
|
||||
|
||||
# Handle prefixes for commenter attribution. Reddit uses `u/`; everyone else
|
||||
# uses `@`. Missing source or unknown platform falls back to plain-text so
|
||||
# we never emit `u/` or `@` with no handle attached.
|
||||
_HANDLE_PREFIX: dict[str, str] = {
|
||||
"reddit": "u/",
|
||||
"tiktok": "@",
|
||||
"youtube": "@",
|
||||
"instagram": "@",
|
||||
"bluesky": "@",
|
||||
"x": "@",
|
||||
"threads": "@",
|
||||
}
|
||||
|
||||
|
||||
def _comment_attribution(source: str | None, author: str | None) -> str:
|
||||
"""Build the attribution prefix for a top comment line.
|
||||
|
||||
Returns a string like ``u/Cyrisaurus`` or ``@moosanoormahomed`` when an
|
||||
author is captured, or the legacy ``Comment`` marker when the author is
|
||||
missing, empty, deleted, or removed.
|
||||
"""
|
||||
if not author or author in ("[deleted]", "[removed]"):
|
||||
return "Comment"
|
||||
prefix = _HANDLE_PREFIX.get(source or "", "")
|
||||
return f"{prefix}{author}" if prefix else author
|
||||
|
||||
|
||||
def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score: int | None = None) -> list[dict]:
|
||||
"""Return up to `limit` top comments with score at or above the source's minimum.
|
||||
|
||||
@@ -1700,6 +1249,16 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score
|
||||
return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]
|
||||
|
||||
|
||||
def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None:
|
||||
if not item:
|
||||
return None
|
||||
comments = item.metadata.get("top_comments") or []
|
||||
if not comments or not isinstance(comments[0], dict):
|
||||
return None
|
||||
top = comments[0]
|
||||
return str(top.get("excerpt") or top.get("text") or "").strip() or None
|
||||
|
||||
|
||||
def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
||||
if not item:
|
||||
return None
|
||||
@@ -1709,39 +1268,6 @@ def _comment_insight(item: schema.SourceItem | None) -> str | None:
|
||||
return str(insights[0]).strip() or None
|
||||
|
||||
|
||||
def _digg_posts_for(item: schema.SourceItem | None, limit: int = 3) -> list[dict]:
|
||||
"""Return up to `limit` parsed Digg posts attached as enrichment to a cluster.
|
||||
|
||||
Returns an empty list for non-digg sources or clusters without enrichment.
|
||||
"""
|
||||
if not item or item.source != "digg":
|
||||
return []
|
||||
posts = item.metadata.get("posts") or []
|
||||
if not isinstance(posts, list):
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for entry in posts:
|
||||
if isinstance(entry, dict) and entry.get("body") and entry.get("username"):
|
||||
out.append(entry)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _format_digg_quote(post: dict, body_limit: int = 200) -> str:
|
||||
"""Format a Digg-attached X post as an inline 'via Digg' quote line."""
|
||||
handle = post.get("username") or ""
|
||||
x_url = post.get("x_url") or ""
|
||||
body = (post.get("body") or "").replace("\n", " ").strip()
|
||||
if len(body) > body_limit:
|
||||
body = body[: body_limit - 1].rstrip() + "…"
|
||||
if x_url and handle:
|
||||
return f"[@{handle}]({x_url}) via Digg: {body}"
|
||||
if handle:
|
||||
return f"@{handle} via Digg: {body}"
|
||||
return f"via Digg: {body}"
|
||||
|
||||
|
||||
def _transcript_highlights(item: schema.SourceItem | None) -> list[str]:
|
||||
if not item or item.source != "youtube":
|
||||
return []
|
||||
@@ -247,29 +247,6 @@ def _candidate_haystack(candidate: schema.Candidate) -> str:
|
||||
return " ".join(parts).lower()
|
||||
|
||||
|
||||
def _entity_grounded(haystack: str, primary_entity: str) -> bool:
|
||||
"""True if the candidate text plausibly mentions the primary entity.
|
||||
|
||||
Grounds on the HEAD token of the primary entity (the brand / proper-noun
|
||||
core), not the full multi-word phrase. Trailing tokens are usually category
|
||||
descriptors the user/planner appended for search ("Stripe payments"), not
|
||||
part of the entity, so requiring the whole phrase over-demotes on-entity
|
||||
items that omit the descriptor. Items that never name the brand at all still
|
||||
miss the head token and stay demoted.
|
||||
|
||||
Trade-off: a proper noun with a generic head ("New York Times" -> "new")
|
||||
under-demotes rather than over-demotes - the safe direction, since the
|
||||
observed harm was burying real high-engagement signal. Substring (not
|
||||
word-boundary) matching is likewise deliberate: it catches plurals and
|
||||
compounds ("stripes"), and vacuous matches from very short heads ("X",
|
||||
"Go") merely disable the penalty rather than burying good items.
|
||||
"""
|
||||
tokens = primary_entity.lower().split()
|
||||
if not tokens:
|
||||
return True
|
||||
return tokens[0] in haystack
|
||||
|
||||
|
||||
def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]:
|
||||
score = (
|
||||
(candidate.local_relevance * 100.0 * 0.7)
|
||||
@@ -277,15 +254,17 @@ def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") ->
|
||||
+ (candidate.source_quality * 100.0 * 0.1)
|
||||
)
|
||||
reason = "fallback-local-score"
|
||||
# Entity-grounding demotion: subtract ENTITY_MISS_PENALTY when the candidate
|
||||
# never mentions the primary entity's head token, across all text surfaces
|
||||
# Entity-grounding demotion: if the primary entity (topic minus intent
|
||||
# modifier) is not present anywhere in the candidate's text surfaces
|
||||
# (title, snippet, transcript, transcript highlights, top comments,
|
||||
# insights). Skip for candidates with NO text anywhere (e.g. image-only
|
||||
# TikToks) so thin-text sources aren't penalized unfairly. See
|
||||
# _entity_grounded for why grounding keys on the head token, not the phrase.
|
||||
# insights), subtract ENTITY_MISS_PENALTY. Skip for candidates with
|
||||
# NO text anywhere (e.g., image-only TikToks) to avoid penalizing
|
||||
# thin-text sources unfairly. 2026-04-19 Nate Herk "Managed Agents"
|
||||
# video ranked #2 on a Hermes query despite zero Hermes mentions
|
||||
# because the old haystack only checked title + snippet.
|
||||
if primary_entity:
|
||||
haystack = _candidate_haystack(candidate)
|
||||
if haystack.strip() and not _entity_grounded(haystack, primary_entity):
|
||||
if haystack.strip() and primary_entity.lower() not in haystack:
|
||||
score -= ENTITY_MISS_PENALTY
|
||||
reason = "fallback-local-score (entity-miss demotion)"
|
||||
return max(0.0, min(100.0, score)), reason
|
||||
@@ -11,64 +11,14 @@ import re
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from . import categories, dates, grounding
|
||||
|
||||
MAX_SUBS = 10
|
||||
from . import dates, grounding
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
print(f"[Resolve] {msg}", file=sys.stderr)
|
||||
|
||||
|
||||
def _merge_category_peers(topic: str, subreddits: list[str]) -> tuple[list[str], Optional[str]]:
|
||||
"""Extend the WebSearch-extracted subreddit list with category peers.
|
||||
|
||||
Classifies the topic, fetches the category's peer subs, dedupes
|
||||
case-insensitively against the existing list, and appends missing
|
||||
peers in priority order. Caps the final list at MAX_SUBS, preserving
|
||||
every WebSearch-returned sub (they are the freshest signal) and
|
||||
trimming from the peer-additions end.
|
||||
|
||||
Returns a tuple of (merged_subs, matched_category_id_or_None).
|
||||
Emits a [Resolve] Matched category log line only when peers were
|
||||
actually added (not when every peer was already in the WebSearch set).
|
||||
|
||||
Classification failures degrade to "no match" — the unwidened list
|
||||
is returned and a warning is logged.
|
||||
"""
|
||||
try:
|
||||
category = categories.detect_category(topic)
|
||||
except Exception as exc:
|
||||
_log(f"Category classification failed: {exc}")
|
||||
return list(subreddits)[:MAX_SUBS], None
|
||||
|
||||
if category is None:
|
||||
return list(subreddits)[:MAX_SUBS], None
|
||||
|
||||
peers = categories.peer_subs_for(category)
|
||||
if not peers:
|
||||
return list(subreddits)[:MAX_SUBS], category
|
||||
|
||||
existing_lower = {s.lower() for s in subreddits}
|
||||
merged = list(subreddits)
|
||||
added: list[str] = []
|
||||
for peer in peers:
|
||||
if len(merged) >= MAX_SUBS:
|
||||
break
|
||||
if peer.lower() in existing_lower:
|
||||
continue
|
||||
merged.append(peer)
|
||||
existing_lower.add(peer.lower())
|
||||
added.append(peer)
|
||||
|
||||
if added:
|
||||
_log(f"Matched category={category}, adding peers: {', '.join(added)}")
|
||||
|
||||
return merged, category
|
||||
|
||||
|
||||
def _has_backend(config: dict) -> bool:
|
||||
"""Check if any web search backend is available."""
|
||||
return bool(
|
||||
@@ -160,93 +110,6 @@ def _extract_github_repos(items: list[dict]) -> list[str]:
|
||||
return repos[:5] # cap at 5 repos
|
||||
|
||||
|
||||
_INTEGRATION_SUFFIX_KEYWORDS: dict[str, set[str]] = {
|
||||
"-action": {"action", "actions", "workflow", "workflows"},
|
||||
"-sdk": {"sdk", "client", "library"},
|
||||
"-plugin": {"plugin", "plugins", "extension", "extensions"},
|
||||
"-plugins": {"plugin", "plugins", "extension", "extensions"},
|
||||
"-docs": {"docs", "documentation"},
|
||||
"-examples": {"example", "examples", "sample", "samples"},
|
||||
"-template": {"template", "templates", "starter", "boilerplate"},
|
||||
}
|
||||
|
||||
|
||||
def _topic_tokens(topic: str) -> set[str]:
|
||||
return set(re.findall(r"[a-z0-9]+", (topic or "").lower()))
|
||||
|
||||
|
||||
def _topic_entity_slugs(topic: str) -> list[str]:
|
||||
entities = re.split(r"\b(?:vs|versus)\b", (topic or "").lower())
|
||||
slugs: list[str] = []
|
||||
for entity in entities:
|
||||
tokens = re.findall(r"[a-z0-9]+", entity)
|
||||
if tokens:
|
||||
slugs.append("-".join(tokens))
|
||||
return slugs
|
||||
|
||||
|
||||
def _repo_slug(repo: str) -> str:
|
||||
parts = repo.split("/", 1)
|
||||
if len(parts) != 2:
|
||||
return ""
|
||||
return parts[1].lower()
|
||||
|
||||
|
||||
def _canonicalize_integration_repo(topic: str, repo: str) -> str:
|
||||
"""Map integration repos back to canonical product repos when intent allows.
|
||||
|
||||
Example:
|
||||
anthropics/claude-code-action -> anthropics/claude-code
|
||||
unless topic explicitly asks for "action"/"workflow".
|
||||
"""
|
||||
parts = repo.split("/", 1)
|
||||
if len(parts) != 2:
|
||||
return repo
|
||||
owner, name = parts[0], parts[1]
|
||||
lower_name = name.lower()
|
||||
topic_words = _topic_tokens(topic)
|
||||
for suffix, intent_words in _INTEGRATION_SUFFIX_KEYWORDS.items():
|
||||
if not lower_name.endswith(suffix):
|
||||
continue
|
||||
if topic_words.intersection(intent_words):
|
||||
return repo
|
||||
base = name[: -len(suffix)]
|
||||
if base:
|
||||
return f"{owner}/{base}"
|
||||
return repo
|
||||
|
||||
|
||||
def canonicalize_github_repos(topic: str, repos: list[str], *, cap: int | None = 5) -> list[str]:
|
||||
"""Normalize/priority-sort GitHub repos for the current topic.
|
||||
|
||||
- Rewrites common integration suffixes to canonical product repos when
|
||||
topic intent does not mention those integrations.
|
||||
- Promotes exact topic slug matches (e.g., `claude-code`) over partials.
|
||||
"""
|
||||
canonicalized: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for repo in repos:
|
||||
candidate = _canonicalize_integration_repo(topic, repo.strip())
|
||||
if "/" not in candidate:
|
||||
continue
|
||||
key = candidate.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
canonicalized.append(candidate)
|
||||
|
||||
topic_slugs = set(_topic_entity_slugs(topic))
|
||||
if topic_slugs:
|
||||
exact = [r for r in canonicalized if _repo_slug(r) in topic_slugs]
|
||||
prefixed = [r for r in canonicalized if any(_repo_slug(r).startswith(f"{slug}-") for slug in topic_slugs) and r not in exact]
|
||||
rest = [r for r in canonicalized if r not in exact and r not in prefixed]
|
||||
canonicalized = exact + prefixed + rest
|
||||
|
||||
if cap is not None:
|
||||
return canonicalized[:cap]
|
||||
return canonicalized
|
||||
|
||||
|
||||
def _build_context_summary(items: list[dict]) -> str:
|
||||
"""Build a 1-2 sentence current events summary from news search results."""
|
||||
snippets: list[str] = []
|
||||
@@ -271,19 +134,10 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
||||
config: Dict with API keys (BRAVE_API_KEY, EXA_API_KEY, SERPER_API_KEY).
|
||||
|
||||
Returns:
|
||||
Dict with keys: subreddits, x_handle, github_user, github_repos,
|
||||
context, category, searches_run. Returns empty result if no web
|
||||
search backend is available.
|
||||
Dict with keys: subreddits, x_handle, context, searches_run.
|
||||
Returns empty result if no web search backend is available.
|
||||
"""
|
||||
empty = {
|
||||
"subreddits": [],
|
||||
"x_handle": "",
|
||||
"github_user": "",
|
||||
"github_repos": [],
|
||||
"context": "",
|
||||
"category": None,
|
||||
"searches_run": 0,
|
||||
}
|
||||
empty = {"subreddits": [], "x_handle": "", "context": "", "searches_run": 0}
|
||||
|
||||
if not _has_backend(config):
|
||||
_log("No web search backend available, skipping resolve")
|
||||
@@ -327,12 +181,10 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
||||
subreddits = _extract_subreddits(results.get("subreddit", []))
|
||||
x_handle = _extract_x_handle(results.get("x_handle", []))
|
||||
github_user = _extract_github_user(results.get("github", []))
|
||||
github_repos = canonicalize_github_repos(topic, _extract_github_repos(results.get("github", [])))
|
||||
github_repos = _extract_github_repos(results.get("github", []))
|
||||
context = _build_context_summary(results.get("news", []))
|
||||
|
||||
subreddits, category = _merge_category_peers(topic, subreddits)
|
||||
|
||||
_log(f"Resolved {len(subreddits)} subreddits, x_handle={x_handle!r}, github_user={github_user!r}, github_repos={github_repos!r}, context_len={len(context)}, category={category!r}")
|
||||
_log(f"Resolved {len(subreddits)} subreddits, x_handle={x_handle!r}, github_user={github_user!r}, github_repos={github_repos!r}, context_len={len(context)}")
|
||||
|
||||
return {
|
||||
"subreddits": subreddits,
|
||||
@@ -340,6 +192,5 @@ def auto_resolve(topic: str, config: dict) -> dict:
|
||||
"github_user": github_user,
|
||||
"github_repos": github_repos,
|
||||
"context": context,
|
||||
"category": category,
|
||||
"searches_run": searches_run,
|
||||
}
|
||||
@@ -107,18 +107,7 @@ def extract_safari_cookies_macos(
|
||||
if sys.platform != "darwin":
|
||||
return None
|
||||
|
||||
cookie_paths = [
|
||||
Path.home()
|
||||
/ "Library"
|
||||
/ "Containers"
|
||||
/ "com.apple.Safari"
|
||||
/ "Data"
|
||||
/ "Library"
|
||||
/ "Cookies"
|
||||
/ "Cookies.binarycookies",
|
||||
Path.home() / "Library" / "Cookies" / "Cookies.binarycookies",
|
||||
]
|
||||
cookie_path = next((path for path in cookie_paths if path.exists()), cookie_paths[0])
|
||||
cookie_path = Path.home() / "Library" / "Cookies" / "Cookies.binarycookies"
|
||||
|
||||
try:
|
||||
raw = cookie_path.read_bytes()
|
||||
@@ -335,9 +335,8 @@ def poll_device_auth(
|
||||
"""
|
||||
import sys
|
||||
|
||||
started_at = time.time()
|
||||
deadline = started_at + timeout
|
||||
last_reminder = started_at
|
||||
deadline = time.time() + timeout
|
||||
last_reminder = time.time()
|
||||
reminder_count = 0
|
||||
max_reminders = 4
|
||||
reminder_interval = 30 # seconds between reminders
|
||||
@@ -12,7 +12,6 @@ SOURCE_QUALITY = {
|
||||
"xiaohongshu": 0.7,
|
||||
"hackernews": 0.8,
|
||||
"youtube": 0.85,
|
||||
"digg": 0.85,
|
||||
"reddit": 0.6,
|
||||
"x": 0.68,
|
||||
"bluesky": 0.66,
|
||||
@@ -27,10 +26,7 @@ def source_quality(source: str) -> float:
|
||||
return SOURCE_QUALITY.get(source, 0.6)
|
||||
|
||||
|
||||
def local_relevance(
|
||||
item: schema.SourceItem,
|
||||
ranking_query: "str | relevance.PreparedQuery",
|
||||
) -> float:
|
||||
def local_relevance(item: schema.SourceItem, ranking_query: str) -> float:
|
||||
text = "\n".join(
|
||||
part
|
||||
for part in [item.title, item.body, item.snippet]
|
||||
@@ -96,7 +92,6 @@ ENGAGEMENT_WEIGHTS: dict[str, list[tuple[str, float]]] = {
|
||||
"bluesky": [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)],
|
||||
"truthsocial": [("likes", 0.45), ("reposts", 0.30), ("replies", 0.25)],
|
||||
"polymarket": [("volume", 0.60), ("liquidity", 0.40)],
|
||||
"digg": [("postCount", 0.40), ("uniqueAuthors", 0.30), ("rank_score", 0.30)],
|
||||
}
|
||||
|
||||
|
||||
@@ -180,14 +175,13 @@ def normalize(values: list[float | None]) -> list[int | None]:
|
||||
|
||||
def annotate_stream(
|
||||
items: list[schema.SourceItem],
|
||||
ranking_query: "str | relevance.PreparedQuery",
|
||||
ranking_query: str,
|
||||
freshness_mode: str,
|
||||
) -> list[schema.SourceItem]:
|
||||
"""Attach local scoring metadata and return items sorted by local_rank_score."""
|
||||
prepared_query = ranking_query if isinstance(ranking_query, relevance.PreparedQuery) else relevance.PreparedQuery(ranking_query)
|
||||
engagement_scores = normalize([engagement_raw(item) for item in items])
|
||||
for item, eng_score in zip(items, engagement_scores, strict=True):
|
||||
item.local_relevance = local_relevance(item, prepared_query)
|
||||
item.local_relevance = local_relevance(item, ranking_query)
|
||||
item.freshness = freshness(item, freshness_mode)
|
||||
item.engagement_score = eng_score
|
||||
item.source_quality = source_quality(item.source)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user