Compare commits

...

8 Commits

Author SHA1 Message Date
Matt Van Horn b76655c52e fix: neutral call-usage copy — works for free and paid tiers 2026-03-30 06:22:06 -07:00
Matt Van Horn 86deed2844 feat: INCLUDE_SOURCES config + TikTok/Instagram opt-in in NUX
- INCLUDE_SOURCES=tiktok,instagram in .env forces sources on for all
  query types, bypassing the tier system
- NUX shows opt-in modal after ScrapeCreators key is saved: "Also
  search TikTok and Instagram?" with honest call-usage warning
- Tier system preserved as default — override only when INCLUDE_SOURCES set

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-30 06:17:29 -07:00
Matt Van Horn bb79d54b2b Revert "fix: ScrapeCreators modal sells all 5 platforms, not just Reddit"
This reverts commit 2eb9cd6fba.
2026-03-29 18:17:25 -07:00
Matt Van Horn 2eb9cd6fba fix: ScrapeCreators modal sells all 5 platforms, not just Reddit 2026-03-29 18:16:58 -07:00
Matt Van Horn 775596ce21 feat: v2.9.6 — free-first NUX, cookie extraction, quality scoring
Setup wizard with consent-first cookie extraction (Chrome/Firefox/Safari),
yt-dlp auto-install, ScrapeCreators push, quality scoring (5 core sources),
status banner redesign, honest Reddit labeling, inline YouTube transcripts,
Exa free web search, Reddit public fallback, and post-research quality nudge.

Co-authored-by: Matt Van Horn <mvanhorn@MacBook-Pro.local>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-29 14:33:17 -07:00
Matt Van Horn 4d6224f79a feat(youtube): extract transcript highlights like Reddit comment gems
Add extract_transcript_highlights() that scores sentences by specificity
(numbers, proper nouns, topic relevance) and filters YouTube filler
(subscribe, welcome back, etc). Top 5 highlights shown as structured
bullets in compact output. Full transcript moved to collapsible <details>
block so the LLM reads highlights first, full text on demand.

SKILL.md updated to instruct the judge agent to quote highlights
directly in synthesis, same as Reddit top comments.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-23 17:49:38 -07:00
Matt Van Horn 499074b564 fix(youtube): pass full transcripts to LLM instead of truncating to 200 chars
TRANSCRIPT_MAX_WORDS raised from 500 to 5000 so the LLM gets the full
content of most videos (up to ~25 minutes). Removed the second 200-char
truncation in render.py that was reducing transcripts to a single sentence
before the judge agent ever saw them.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-23 17:28:01 -07:00
Matt Van Horn 6a5a0013c0 Merge pull request #70 from mvanhorn/fix/bluesky-cloudflare-error-messages
fix(bluesky): surface real error instead of misleading 'auth failed'
2026-03-15 22:56:00 -07:00
36 changed files with 6428 additions and 330 deletions
+1 -1
View File
@@ -10,7 +10,7 @@
{ {
"name": "last30days", "name": "last30days",
"description": "Research any topic from the last 30 days. Become an expert and write copy-paste-ready prompts.", "description": "Research any topic from the last 30 days. Become an expert and write copy-paste-ready prompts.",
"version": "2.9.5", "version": "2.9.6",
"author": { "author": {
"name": "Matt Van Horn", "name": "Matt Van Horn",
"url": "https://github.com/mvanhorn" "url": "https://github.com/mvanhorn"
+2 -12
View File
@@ -1,6 +1,6 @@
{ {
"name": "last30days", "name": "last30days",
"version": "2.9.5", "version": "2.9.6",
"description": "Research any topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.", "description": "Research any topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
"author": { "author": {
"name": "Matt Van Horn", "name": "Matt Van Horn",
@@ -10,17 +10,7 @@
"homepage": "https://github.com/mvanhorn/last30days-skill", "homepage": "https://github.com/mvanhorn/last30days-skill",
"repository": "https://github.com/mvanhorn/last30days-skill", "repository": "https://github.com/mvanhorn/last30days-skill",
"license": "MIT", "license": "MIT",
"keywords": [ "keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket"],
"research",
"reddit",
"twitter",
"youtube",
"tiktok",
"instagram",
"trends",
"prompts",
"polymarket"
],
"skills": ["./"], "skills": ["./"],
"hooks": {} "hooks": {}
} }
+94 -54
View File
@@ -12,12 +12,12 @@
clawhub install last30days-official clawhub install last30days-official
``` ```
**The AI world reinvents itself every month. This skill keeps you current.** /last30days researches your topic across Reddit, X, Bluesky, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations. Whether it's Seedance 2.0 access, paper.design prompts, or the latest Nano Banana Pro techniques, you'll know what people who are paying attention already know. **The AI world reinvents itself every month. This skill keeps you current.** /last30days researches your topic across Reddit, X, YouTube, and other sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations. Whether it's Seedance 2.0 access, paper.design prompts, or the latest Nano Banana Pro techniques, you'll know what people who are paying attention already know.
**New in v2.9.5 — Bluesky, Comparative Mode, and Config Improvements:** **New in v2.9.5 — Bluesky, Comparative Mode, and Config Improvements:**
- **Bluesky/AT Protocol** is now a social source. Opt-in via `BSKY_HANDLE` + `BSKY_APP_PASSWORD` (create at bsky.app/settings/app-passwords). Full pipeline: search, score, dedupe, render. - **Bluesky/AT Protocol** is now a social source. Opt-in via `BSKY_HANDLE` + `BSKY_APP_PASSWORD` (create at bsky.app/settings/app-passwords). Full pipeline: search, score, dedupe, render.
- **Comparative mode** - ask "X vs Y" (e.g., `/last30 cursor vs windsurf`) and get 3 parallel research passes with a side-by-side comparison: strengths, weaknesses, head-to-head table, and a data-driven verdict. - **Comparative mode** - ask "X vs Y" (e.g., `/last30 Claude Code vs Codex`) and get 3 parallel research passes with a side-by-side comparison: strengths, weaknesses, head-to-head table, and a data-driven verdict.
- **Per-project .env config** - drop a `.claude/last30days.env` in your project root for per-project API keys. - **Per-project .env config** - drop a `.claude/last30days.env` in your project root for per-project API keys.
- **SessionStart config check** - validates your config automatically when a Claude Code session starts. - **SessionStart config check** - validates your config automatically when a Claude Code session starts.
- **Expanded test coverage** - 455+ tests across all modules. - **Expanded test coverage** - 455+ tests across all modules.
@@ -63,44 +63,101 @@ gemini extensions install https://github.com/mvanhorn/last30days-skill.git
### Manual Install (Claude Code / Codex) ### Manual Install (Claude Code / Codex)
```bash ```bash
# Clone the repo
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
# Add your API keys (optional if signed in to Codex)
mkdir -p ~/.config/last30days
cat > ~/.config/last30days/.env << 'EOF'
SCRAPECREATORS_API_KEY=... # Reddit + TikTok + Instagram (one key, all three) - scrapecreators.com
OPENAI_API_KEY=sk-... # optional - legacy Reddit fallback if using `codex login`
AUTH_TOKEN=... # recommended for X search - copy once from x.com cookies
CT0=... # recommended for X search - copy once from x.com cookies
XAI_API_KEY=xai-... # optional - X fallback if you do not want cookie-based auth
BSKY_HANDLE=you.bsky.social # optional - Bluesky search (create app password below)
BSKY_APP_PASSWORD=xxxx-xxxx-xxxx # optional - bsky.app/settings/app-passwords
EOF
chmod 600 ~/.config/last30days/.env
``` ```
If you're signed in to Codex (`codex login`), the skill will use your Codex credentials for the OpenAI Responses API and you can omit `OPENAI_API_KEY`. If you're not signed in, run `codex login` first. That's it. Reddit, Hacker News, and Polymarket work immediately with zero configuration. Run `/last30days` to unlock more sources.
---
## Setup: Progressive Source Unlocking
Start using /last30days immediately. Add sources when you want better results.
### 1. Zero Config (3 sources) — Just install
Reddit (public JSON), Hacker News, and Polymarket work out of the box. No API keys, no configuration.
### 2. Run the setup wizard (5+ sources)
```
/last30days setup
```
The setup wizard automatically extracts X/Twitter login cookies from your browsers (Chrome, Firefox, Safari) and checks for yt-dlp. Takes about 30 seconds. Your cookies stay in memory and are never saved to disk.
### 3. Add Exa (FREE — semantic web search)
Register at [exa.ai](https://exa.ai) for 1,000 free searches/month, no credit card required.
```bash
# Add to ~/.config/last30days/.env
EXA_API_KEY=...
```
### 4. Add ScrapeCreators (RECOMMENDED — Reddit comments + TikTok + Instagram)
**This is the single most impactful upgrade.** Reddit comments are often the highest-value research content — top-voted replies with real insights. ScrapeCreators unlocks comment enrichment plus TikTok and Instagram. Register at [scrapecreators.com](https://scrapecreators.com) for 100 free API calls (no credit card required). After that, pay-as-you-go. last30days receives no money from any API provider — no referrals, no kickbacks.
```bash
# Add to ~/.config/last30days/.env
SCRAPECREATORS_API_KEY=...
```
### 5. Add Bluesky (FREE — app password)
Create an app password at [bsky.app/settings/app-passwords](https://bsky.app/settings/app-passwords).
```bash
# Add to ~/.config/last30days/.env
BSKY_HANDLE=you.bsky.social
BSKY_APP_PASSWORD=xxxx-xxxx-xxxx
```
### 6. Optional paid web search backends
```bash
# Add to ~/.config/last30days/.env
PARALLEL_API_KEY=... # Parallel AI (preferred — LLM-optimized results)
BRAVE_API_KEY=... # Brave Search (free tier: 2,000 queries/month)
OPENROUTER_API_KEY=... # OpenRouter/Perplexity Sonar Pro
```
---
### Do I need API keys?
| Source | Free Method | API Key | Do you need the API key? |
|--------|------------|---------|--------------------------|
| Reddit | Public JSON (always works) | ScrapeCreators | **Yes, strongly recommended.** Unlocks top comments — often the most valuable content. |
| X/Twitter | Browser cookies (auto-extracted) | xAI API key (`XAI_API_KEY`) | **No.** Cookies give identical quality. The setup wizard handles this. |
| YouTube | yt-dlp (`brew install yt-dlp`) | N/A | **No API key exists.** Install yt-dlp for search; transcripts work without it. |
| Hacker News | Always free | N/A | **No.** Always works, no config needed. |
| Polymarket | Always free | N/A | **No.** Always works, no config needed. |
| Web search | N/A | Exa (`EXA_API_KEY`) | **Optional.** 1,000 free searches/month at exa.ai. |
| Bluesky | Free app password | N/A | **Optional.** Free app password at bsky.app. |
| TikTok | N/A | ScrapeCreators | **Optional.** Included with ScrapeCreators key. |
| Instagram | N/A | ScrapeCreators | **Optional.** Included with ScrapeCreators key. |
| Truth Social | Browser cookies | N/A | **Optional.** Auto-extracted if logged in. |
*last30days receives no money from any API provider — no referrals, no kickbacks.*
---
### Config file locations
For project-specific overrides, create `.claude/last30days.env` in the repo root. It overrides the global `~/.config/last30days/.env`. For project-specific overrides, create `.claude/last30days.env` in the repo root. It overrides the global `~/.config/last30days/.env`.
### X Search Authentication
X search prefers explicit env auth. This keeps local runs headless and avoids browser-cookie and macOS Keychain prompts.
**Recommended setup:**
1. While logged into x.com once, open browser dev tools and copy the `auth_token` and `ct0` cookies for `x.com`.
2. Save them as `AUTH_TOKEN` and `CT0` in `~/.config/last30days/.env`, export them in your shell, or add them to `.claude/last30days.env` for a single project.
3. Re-run `/last30days`.
**xAI fallback:** If you do not want to provide `AUTH_TOKEN` and `CT0`, set `XAI_API_KEY` and the skill will use xAI's `x_search` backend instead.
**Verify it's working:**
```bash ```bash
node ~/.claude/skills/last30days/scripts/lib/vendor/bird-search/bird-search.mjs --whoami # Global config
mkdir -p ~/.config/last30days
chmod 600 ~/.config/last30days/.env
# Project-specific config (optional)
# .claude/last30days.env
``` ```
**Requirements:** Node.js 22+ (for the vendored Twitter GraphQL client). Check source availability: `python3 scripts/last30days.py --diagnose`
### Codex CLI ### Codex CLI
@@ -144,21 +201,6 @@ The open variant adds four modes on top of one-shot research:
Both variants use the same Python engine and scripts directory. The open variant adds command routing (`watch`, `briefing`, `history`) and references mode-specific instruction files. Both variants use the same Python engine and scripts directory. The open variant adds command routing (`watch`, `briefing`, `history`) and references mode-specific instruction files.
**Optional web search API keys** (add to `~/.config/last30days/.env`):
```bash
PARALLEL_API_KEY=... # Parallel AI (preferred - LLM-optimized results)
BRAVE_API_KEY=... # Brave Search (free tier: 2,000 queries/month)
OPENROUTER_API_KEY=... # OpenRouter/Perplexity Sonar Pro
```
**Optional Bluesky credentials** (add to `~/.config/last30days/.env`):
```bash
BSKY_HANDLE=you.bsky.social # Your Bluesky handle
BSKY_APP_PASSWORD=xxxx-xxxx-xxxx # Create at bsky.app/settings/app-passwords
```
Check source availability: `python3 scripts/last30days.py --diagnose`
## Usage ## Usage
``` ```
@@ -174,7 +216,7 @@ Examples:
## What It Does ## What It Does
1. **Researches** - Scans Reddit, X, Bluesky, Truth Social, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web for discussions from the last 30 days 1. **Researches** - Scans Reddit, X, YouTube, and other sources for discussions from the last 30 days
2. **Synthesizes** - Identifies patterns, best practices, and what actually works 2. **Synthesizes** - Identifies patterns, best practices, and what actually works
3. **Delivers** - Either writes copy-paste-ready prompts for your target tool, or gives you a curated expert-level answer 3. **Delivers** - Either writes copy-paste-ready prompts for your target tool, or gives you a curated expert-level answer
@@ -928,13 +970,11 @@ This example shows /last30days discovering **emerging developer workflows** - re
## Requirements ## Requirements
- **OpenAI auth** - For Reddit research (uses web search via Responses API). Use `OPENAI_API_KEY` or `codex login`. - **Python 3** - For the research engine
- **Node.js 22+** - For X search (bundled Twitter GraphQL client) - **Node.js 22+** - For X search (bundled Twitter GraphQL client)
- **Bundled X auth** - Set `AUTH_TOKEN` and `CT0` for popup-free local X search - **yt-dlp** (recommended) - For YouTube search. Install via `brew install yt-dlp` or `pip install yt-dlp`. Transcripts work without it.
- **Alternate X backend** - Set `XAI_API_KEY` if bundled X auth is not configured
- **yt-dlp** (optional) - For YouTube search + transcript extraction. Install via `brew install yt-dlp` or `pip install yt-dlp`. When present, automatically searches YouTube and extracts video transcripts as an additional source.
At least one auth path is required. Reddit needs OpenAI auth. X needs either `AUTH_TOKEN` plus `CT0` or `XAI_API_KEY`. YouTube search activates automatically when yt-dlp is in your PATH. No API keys are required to start. Reddit, Hacker News, and Polymarket work out of the box. Run `/last30days setup` to unlock X/Twitter via browser cookies and configure additional sources. See [Setup: Progressive Source Unlocking](#setup-progressive-source-unlocking) for the full progression.
## Troubleshooting ## Troubleshooting
@@ -986,7 +1026,7 @@ If your OpenAI org doesn't have access to a model (e.g., unverified for gpt-4.1)
### ScrapeCreators Reddit as default ### ScrapeCreators Reddit as default
Reddit now runs on [ScrapeCreators](https://scrapecreators.com) by default. One `SCRAPECREATORS_API_KEY` powers Reddit, TikTok, and Instagram — three sources, one key. No more `OPENAI_API_KEY` required for Reddit search. Reddit now runs on [ScrapeCreators](https://scrapecreators.com) by default. One `SCRAPECREATORS_API_KEY` powers Reddit, TikTok, and Instagram — three sources, one key. No more `OPENAI_API_KEY` required for Reddit search. 100 free API calls, no credit card required — just register at [scrapecreators.com](https://scrapecreators.com), then pay-as-you-go. last30days receives no money from any API provider — no referrals, no kickbacks.
```bash ```bash
echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env
@@ -1039,7 +1079,7 @@ Search "AI tools" and you get:
### TikTok + Instagram on ScrapeCreators ### TikTok + Instagram on ScrapeCreators
Both TikTok and Instagram are powered by [ScrapeCreators](https://scrapecreators.com) — one API key covers both sources. 100 free credits, then pay-as-you-go. Both TikTok and Instagram are powered by [ScrapeCreators](https://scrapecreators.com) — one API key covers both sources. Register at [scrapecreators.com](https://scrapecreators.com) for 100 free API calls (no credit card required). After that, pay-as-you-go. last30days receives no money from any API provider — no referrals, no kickbacks.
```bash ```bash
echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env
+147 -12
View File
@@ -1,7 +1,7 @@
--- ---
name: last30days name: last30days
version: "2.9.5" version: "2.9.6"
description: "Deep research engine covering the last 30 days across 10+ sources - Reddit, X/Twitter, YouTube, TikTok, Instagram, Hacker News, Polymarket, Bluesky, Truth Social, web. AI synthesizes findings into grounded, cited reports. The most comprehensive recency research skill on ClawHub." description: "Deep research engine covering the last 30 days across 10+ sources - Reddit, X/Twitter, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web. AI synthesizes findings into grounded, cited reports."
argument-hint: 'last30 AI video tools, last30 best project management tools' argument-hint: 'last30 AI video tools, last30 best project management tools'
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
homepage: https://github.com/mvanhorn/last30days-skill homepage: https://github.com/mvanhorn/last30days-skill
@@ -61,9 +61,139 @@ metadata:
# last30days v2.9.5: Research Any Topic from the Last 30 Days # last30days v2.9.5: Research Any Topic from the Last 30 Days
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). Truth Social search uses optional bearer token (TRUTHSOCIAL_TOKEN env var - extract from browser dev tools). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section. > **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
Research ANY topic across Reddit, X, Bluesky, Truth Social, YouTube, TikTok, Hacker News, Polymarket, and the web. Surface what people are actually discussing, recommending, betting on, and debating right now. Research ANY topic across Reddit, X, YouTube, and other sources. Surface what people are actually discussing, recommending, betting on, and debating right now.
## Step 0: First-Run Setup Wizard
**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even if the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST check for FIRST_RUN and present the wizard BEFORE running research. The topic "Mercer Island" is preserved — research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. The wizard takes 10 seconds and only runs once ever.
To detect first run: check if `~/.config/last30days/.env` exists. If it does NOT exist, this is a first run. **Do NOT run any Bash commands or show any command output to detect this — just check the file existence silently.** If the file exists and contains `SETUP_COMPLETE=true`, skip this section and proceed to Step 1.
**When first run is detected, you MUST follow these steps IN ORDER. Do NOT skip ahead to the topic picker or research. The sequence is: (1) welcome text → (2) setup modal → (3) run setup if chosen → (4) ScrapeCreators modal → (5) topic picker. You MUST start at step 1.**
**Step 1: Display the following welcome text ONCE as a normal message (not blockquoted). Then IMMEDIATELY call AskUserQuestion — do NOT repeat any of the welcome text inside the AskUserQuestion call.**
👋 Welcome to /last30days!
I research any topic across Reddit, X, YouTube, and other sources — synthesizing what people are actually saying right now.
To get the best results, I can:
🔍 **Scan your browser** for X/Twitter cookies (free X search — reads x.com only, never saved)
📺 **Install yt-dlp** for YouTube search + transcripts (free, open source, 190K+ GitHub stars)
**ScrapeCreators API key** unlocks Reddit comments + TikTok + Instagram (100 free to start — scrapecreators.com)
We recommend all 3 before your first run — it's what makes the magic. More community sources available later. We get no kickbacks from any of these.
**Then call AskUserQuestion with ONLY this question and these options — no additional text:**
Question: "How would you like to set up?"
Options:
- "Auto setup (~30 seconds) — scans browser for X cookies (free X search) and installs yt-dlp (free YouTube transcripts)"
- "Manual setup — show me what to configure"
- "Skip for now — Reddit (threads only), HN, Polymarket, Web"
**If the user picks 1 (Auto setup):**
Run the setup subcommand:
```bash
cd {SKILL_DIR} && python3 scripts/last30days.py setup
```
Show the user the results (what cookies were found, whether yt-dlp was installed).
**Then show the ScrapeCreators push (plain text, then modal):**
⭐ One more thing — Reddit comments are some of the best content on the internet. The top-voted replies often have sharper insights than the posts themselves. ScrapeCreators unlocks these (plus TikTok + Instagram) — 100 free to start, no credit card.
**Call AskUserQuestion:**
Question: "Want to add Reddit comments to your research?"
Options:
- "Open scrapecreators.com to get my free key" — run `open https://scrapecreators.com` via Bash to open in the user's browser. Then ask them to paste the API key they get. When they paste it, write SCRAPECREATORS_API_KEY={key} to ~/.config/last30days/.env
- "I have a key — let me paste it" — accept the key, write to .env
- "Skip for now — start researching" — proceed without ScrapeCreators
**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:**
Your ScrapeCreators key also powers TikTok and Instagram search. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.)
**Call AskUserQuestion:**
Question: "Also search TikTok and Instagram?"
Options:
- "Yes, search everything — TikTok + Instagram on every run" — append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "All set — Reddit comments, TikTok, and Instagram will run on every search."
- "Just Reddit comments for now" — don't write the flag. Confirm: "Got it — Reddit comments are on. TikTok and Instagram will kick in automatically for relevant topics like product reviews and trends. Add INCLUDE_SOURCES=tiktok,instagram to your .env anytime to force them on."
**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:**
**Call AskUserQuestion:**
Question: "What do you want to research first?"
Options:
- "Claude Code vs Codex" — tech comparison
- "Sam Altman" — person in the news
- "Warriors Basketball" — sports
- "AI Legal Prompting Techniques" — niche/professional
- "Type my own topic"
If user picks an example, run research with that topic. If they pick "Type my own", ask them what they want to research. If the user originally provided a topic with the command (e.g., `/last30days Mercer Island`), skip this modal and use their topic directly.
**END OF FIRST-RUN WIZARD. Everything above in Step 0 ONLY runs on first run. If SETUP_COMPLETE=true exists in .env, skip ALL of Step 0 — no welcome, no setup, no ScrapeCreators modal, no topic picker. Go directly to Step 1 (Parse User Intent). The topic picker is ONLY for first-time users who haven't run /last30days before.**
**If the user picks 2 (Manual setup):**
Show them this guide (present as plain text, not blockquoted):
**The magic of /last30days is Reddit comments + X posts together.** Here's how to unlock each source.
Add these to `~/.config/last30days/.env`:
**🔍 X/Twitter** (pick one — this is the most important):
- `FROM_BROWSER=auto` — easiest, free. Scans your browser cookies for x.com login.
- `AUTH_TOKEN=xxx` + `CT0=xxx` — paste your X cookies manually (x.com → F12 → Application → Cookies)
- `XAI_API_KEY=xxx` — use an xAI/Grok API key instead (api.x.ai)
**⭐ Reddit Comments + TikTok + Instagram + YouTube backup** (one key, 5 platforms):
- `SCRAPECREATORS_API_KEY=xxx` — 100 free to start at scrapecreators.com. This is the single most impactful key — Reddit comments are where the sharpest insights live.
**💡 Reddit discovery fallback** (optional):
- `OPENAI_API_KEY=xxx` — ChatGPT web search for Reddit threads. Only used if ScrapeCreators isn't configured — SC is better.
**📺 YouTube** (primary engine):
- Run `brew install yt-dlp` — free, open source. ScrapeCreators covers YouTube as backup if you skip this.
**Bonus sources** (add anytime):
- `EXA_API_KEY=xxx` — semantic web search, 1K free/month (exa.ai)
- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` — Bluesky (free app password)
- `BRAVE_API_KEY=xxx` — Brave web search
- `OPENROUTER_API_KEY=xxx` — Perplexity Sonar Pro search
Always add this last line: `SETUP_COMPLETE=true`
You can paste your API keys right here and I'll write the file for you. Some people say giving LLMs your keys is risky, others live in YOLO mode. Your call.
**If the user pastes keys**, write them to `~/.config/last30days/.env` (create the file and parent dirs if needed, append without overwriting existing keys, always include `SETUP_COMPLETE=true`). Then proceed with research.
**If the user says they'll do it themselves**, tell them: "Save the file, then run `/last30days <topic>` to research anything." Then proceed with research using whatever sources are currently available.
**If the user picks Skip:**
Proceed with research immediately using the user's original topic. Note: without setup, sources are limited to Reddit (threads only, no comments), HN, and Polymarket. X/Twitter and YouTube require setup.
---
## Do I Need API Keys?
When users ask about API keys, setup, or how to unlock more sources, reference this:
**You do NOT need API keys to use last30days.** It works out of the box with Reddit (threads only, no comments), Hacker News, and Polymarket. Browser cookies for X/Twitter are equivalent to an API key — just log into x.com in any browser and last30days will find your session automatically.
**Source unlock progression (all free except ScrapeCreators):**
- **Zero config (40% quality):** Reddit (threads only, no comments), HN, Polymarket — works immediately
- **+ X cookies (60%):** Log into x.com in any browser. last30days scans your cookies automatically. This is the same as having an API key — no signup required.
- **+ yt-dlp (80%):** `brew install yt-dlp` — open source, 190K+ GitHub stars. Enables YouTube search and transcripts.
- **+ ScrapeCreators (100%):** The only paid option worth mentioning. Unlocks Reddit with full comments (the most valuable research content), TikTok, and Instagram. 100 free API calls, no credit card — scrapecreators.com
**Key comparison:** X browser cookies = same access as an API key (free, no signup). ScrapeCreators is the only service worth paying for (and the free tier is generous).
last30days has no affiliation with any API provider — no referrals, no kickbacks.
---
## CRITICAL: Parse User Intent ## CRITICAL: Parse User Intent
@@ -100,7 +230,7 @@ Common patterns:
**DISPLAY your parsing to the user.** Before running any tools, output: **DISPLAY your parsing to the user.** Before running any tools, output:
``` ```
I'll research {TOPIC} across Reddit, X, Bluesky, Truth Social, TikTok, and the web to find what's been discussed in the last 30 days. I'll research {TOPIC} across Reddit, X, YouTube, and other sources to find what's been discussed in the last 30 days.
Parsed intent: Parsed intent:
- TOPIC = {TOPIC} - TOPIC = {TOPIC}
@@ -164,7 +294,7 @@ Agent mode report format:
``` ```
## Research Report: {TOPIC} ## Research Report: {TOPIC}
Generated: {date} | Sources: Reddit, X, Bluesky, Truth Social, YouTube, TikTok, HN, Polymarket, Web Generated: {date} | Sources: Reddit, X, Bluesky, YouTube, TikTok, HN, Polymarket, Web
### Key Findings ### Key Findings
[3-5 bullet points, highest-signal insights with citations] [3-5 bullet points, highest-signal insights with citations]
@@ -240,7 +370,7 @@ The script will automatically:
**Read the ENTIRE output.** It contains EIGHT data sections in this order: Reddit items, X items, YouTube items, TikTok items, Instagram Reels items, Hacker News items, Polymarket items, and WebSearch items. If you miss sections, you will produce incomplete stats. **Read the ENTIRE output.** It contains EIGHT data sections in this order: Reddit items, X items, YouTube items, TikTok items, Instagram Reels items, Hacker News items, Polymarket items, and WebSearch items. If you miss sections, you will produce incomplete stats.
**YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, and optional transcript snippet. Count them and include them in your synthesis and stats block. **YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, **transcript highlights** (pre-extracted quotable excerpts from the video), and an optional full transcript in a collapsible section. **Quote the highlights directly in your synthesis** - they are the YouTube equivalent of Reddit top comments. Attribute quotes to the channel name. Count them and include them in your synthesis and stats block.
**TikTok items in the output look like:** `**{TK_id}** (score:N) @{creator} [N views, N likes]` followed by a caption, URL, hashtags, and optional caption snippet. Count them and include them in your synthesis and stats block. **TikTok items in the output look like:** `**{TK_id}** (score:N) @{creator} [N views, N likes]` followed by a caption, URL, hashtags, and optional caption snippet. Count them and include them in your synthesis and stats block.
@@ -303,9 +433,10 @@ The Judge Agent must:
3. Weight TikTok sources HIGH (they have views, likes, and caption content — viral signal) 3. Weight TikTok sources HIGH (they have views, likes, and caption content — viral signal)
4. Weight WebSearch sources LOWER (no engagement data) 4. Weight WebSearch sources LOWER (no engagement data)
5. **For Reddit: Pay special attention to top comments** — they often contain the wittiest, most insightful, or funniest take. When a top comment has high upvotes (shown as `💬 Top comment (N upvotes)`), quote it directly in your synthesis. Reddit's value is in the comments. 5. **For Reddit: Pay special attention to top comments** — they often contain the wittiest, most insightful, or funniest take. When a top comment has high upvotes (shown as `💬 Top comment (N upvotes)`), quote it directly in your synthesis. Reddit's value is in the comments.
6. Identify patterns that appear across ALL sources (strongest signals) 6. **For YouTube: Quote transcript highlights directly in your synthesis.** These are pre-extracted key moments from the video - treat them like Reddit top comments. Attribute to the channel name and include the actual quote. YouTube's value is in what creators SAY, not just their view counts.
7. Note any contradictions between sources 7. Identify patterns that appear across ALL sources (strongest signals)
8. Extract the top 3-5 actionable insights 8. Note any contradictions between sources
9. Extract the top 3-5 actionable insights
7. **Cross-platform signals are the strongest evidence.** When items have `[also on: Reddit, HN]` or similar tags, it means the same story appears across multiple platforms. Lead with these cross-platform findings - they're the most important signals in the research. 7. **Cross-platform signals are the strongest evidence.** When items have `[also on: Reddit, HN]` or similar tags, it means the same story appears across multiple platforms. Lead with these cross-platform findings - they're the most important signals in the research.
@@ -498,6 +629,10 @@ KEY PATTERNS from the research:
3. [Pattern] — per @handle 3. [Pattern] — per @handle
``` ```
**THEN - Quality Nudge (if present in the output):**
If the research output contains a `**🔍 Research Coverage:**` block, render it verbatim right before the stats block. This tells the user which core sources are missing and how to unlock them. Do NOT render this block if it is absent from the output (100% coverage = no nudge).
**THEN - Stats (right before invitation):** **THEN - Stats (right before invitation):**
**CRITICAL: Calculate actual totals from the research output.** **CRITICAL: Calculate actual totals from the research output.**
@@ -725,7 +860,7 @@ Want another prompt? Just tell me what you're creating next.
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth) - Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth) - Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data) - Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (same SCRAPECREATORS_API_KEY as Reddit, PAYG after 100 free credits) - Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (same SCRAPECREATORS_API_KEY as Reddit, PAYG after 100 free API calls)
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search - Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
- Fetches public Reddit thread data from `reddit.com` for engagement metrics - Fetches public Reddit thread data from `reddit.com` for engagement metrics
- Stores research findings in local SQLite database (watchlist mode only) - Stores research findings in local SQLite database (watchlist mode only)
@@ -738,7 +873,7 @@ Want another prompt? Just tell me what you're creating next.
- Does not log, cache, or write API keys to output files - Does not log, cache, or write API keys to output files
- Does not send data to any endpoint not listed above - Does not send data to any endpoint not listed above
- Hacker News and Polymarket sources are always available (no API key, no binary dependency) - Hacker News and Polymarket sources are always available (no API key, no binary dependency)
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (same key covers both; 100 free credits, then PAYG) - TikTok and Instagram sources require SCRAPECREATORS_API_KEY (same key covers both; 100 free API calls, then PAYG)
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output - Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed) **Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
+78 -21
View File
@@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
set -euo pipefail set -euo pipefail
# Check if last30days has any configuration source available. # Check last30days configuration status and show appropriate welcome message.
# Priority: .claude/last30days.env > ~/.config/last30days/.env > env vars # Priority: .claude/last30days.env > ~/.config/last30days/.env > env vars
PROJECT_ENV=".claude/last30days.env" PROJECT_ENV=".claude/last30days.env"
@@ -19,33 +19,90 @@ check_perms() {
fi fi
} }
# Check per-project config # Load env file into variables for inspection (without exporting)
load_env_vars() {
local file="$1"
if [[ -f "$file" ]]; then
while IFS='=' read -r key value; do
# Skip comments, empty lines
[[ "$key" =~ ^[[:space:]]*# ]] && continue
[[ -z "$key" ]] && continue
key=$(echo "$key" | xargs)
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
if [[ -n "$key" && -n "$value" ]]; then
eval "ENV_${key}=\"${value}\""
fi
done < "$file"
fi
}
# Determine which config file is active
CONFIG_FILE=""
if [[ -f "$PROJECT_ENV" ]]; then if [[ -f "$PROJECT_ENV" ]]; then
CONFIG_FILE="$PROJECT_ENV"
check_perms "$PROJECT_ENV" check_perms "$PROJECT_ENV"
exit 0 elif [[ -f "$GLOBAL_ENV" ]]; then
fi CONFIG_FILE="$GLOBAL_ENV"
# Check global config
if [[ -f "$GLOBAL_ENV" ]]; then
check_perms "$GLOBAL_ENV" check_perms "$GLOBAL_ENV"
exit 0
fi fi
# Check if OPENAI_API_KEY is set in environment (minimum requirement) # Load config if found
if [[ -n "${OPENAI_API_KEY:-}" ]]; then if [[ -n "$CONFIG_FILE" ]]; then
exit 0 load_env_vars "$CONFIG_FILE"
fi fi
# Check if SCRAPECREATORS_API_KEY is set (also sufficient) # Check SETUP_COMPLETE (from file or env)
if [[ -n "${SCRAPECREATORS_API_KEY:-}" ]]; then SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
exit 0
fi
# No config found — inform user # If setup has never been run, show welcome message for new users
cat <<'EOF' if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
/last30days: No API keys configured. cat <<'EOF'
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
Create .claude/last30days.env with your API keys, or create Reddit, Hacker News, and Polymarket work out of the box.
~/.config/last30days/.env globally. At minimum, SCRAPECREATORS_API_KEY The setup wizard can unlock X/Twitter, YouTube, and more.
or OPENAI_API_KEY is required. See the README for setup instructions.
EOF EOF
exit 0
fi
# Setup done but check for ScrapeCreators
HAS_SCRAPECREATORS="${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}"
HAS_X="${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}"
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
HAS_YTDLP=""
if command -v yt-dlp &>/dev/null; then
HAS_YTDLP="yes"
fi
HAS_BSKY="${ENV_BSKY_HANDLE:-${BSKY_HANDLE:-}}"
HAS_EXA="${ENV_EXA_API_KEY:-${EXA_API_KEY:-}}"
# Count active sources
SOURCE_COUNT=2 # HN + Polymarket are always free
if [[ -n "$HAS_X" || -n "$HAS_XAI" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
# Reddit public JSON always works
SOURCE_COUNT=$((SOURCE_COUNT + 1))
if [[ -n "$HAS_YTDLP" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
if [[ -n "$HAS_EXA" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
if [[ -n "$HAS_BSKY" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
fi
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
# Fully configured — compact ready message
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
else
# Setup done but missing ScrapeCreators — recommend it
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
echo " 100 free API calls, no credit card — scrapecreators.com"
echo " last30days has no affiliation with any API provider."
fi
+136 -79
View File
@@ -149,17 +149,20 @@ from lib import (
normalize, normalize,
openai_reddit, openai_reddit,
reddit, reddit,
reddit_public,
reddit_enrich, reddit_enrich,
render, render,
schema, schema,
score, score,
scrapecreators_x, scrapecreators_x,
setup_wizard,
ui, ui,
tiktok, tiktok,
instagram, instagram,
websearch, websearch,
xai_x, xai_x,
youtube_yt, youtube_yt,
quality_nudge,
query_type as qt, query_type as qt,
) )
@@ -184,8 +187,10 @@ def _search_reddit(
) -> tuple: ) -> tuple:
"""Search Reddit (runs in thread). """Search Reddit (runs in thread).
Uses ScrapeCreators when SCRAPECREATORS_API_KEY is available (preferred). Hierarchy:
Falls back to OpenAI Responses API otherwise. 1. ScrapeCreators (if SCRAPECREATORS_API_KEY exists) — premium, best quality
2. Public Reddit JSON (always available) — free, good for thread discovery
3. OpenAI Responses API — legacy fallback for backwards compatibility
Returns: Returns:
Tuple of (reddit_items, raw_response, error, used_scrapecreators) Tuple of (reddit_items, raw_response, error, used_scrapecreators)
@@ -199,7 +204,7 @@ def _search_reddit(
if mock: if mock:
raw_response = load_fixture("openai_sample.json") raw_response = load_fixture("openai_sample.json")
elif sc_token: elif sc_token:
# === ScrapeCreators path (preferred) === # === Tier 1: ScrapeCreators path (preferred) ===
used_scrapecreators = True used_scrapecreators = True
try: try:
sys.stderr.write("[Reddit] Using ScrapeCreators API\n") sys.stderr.write("[Reddit] Using ScrapeCreators API\n")
@@ -216,61 +221,56 @@ def _search_reddit(
reddit_error = f"ScrapeCreators: {type(e).__name__}: {e}" reddit_error = f"ScrapeCreators: {type(e).__name__}: {e}"
sys.stderr.write(f"[Reddit] ScrapeCreators failed: {e}\n") sys.stderr.write(f"[Reddit] ScrapeCreators failed: {e}\n")
sys.stderr.flush() sys.stderr.flush()
# Fall through to OpenAI if we have that key
if not config.get("OPENAI_API_KEY"):
# No OpenAI either: try public Reddit fallback.
try:
reddit_items = openai_reddit.search_reddit_public(
topic, from_date, to_date, depth=depth,
)
raw_response = {"source": "reddit_public", "items": reddit_items}
return reddit_items, raw_response, None, False
except Exception as e2:
return [], {"error": str(e)}, reddit_error, used_scrapecreators
used_scrapecreators = False used_scrapecreators = False
sys.stderr.write("[Reddit] Falling back to OpenAI\n") # Fall through to Tier 2 (public JSON)
sys.stderr.flush()
# === OpenAI path (fallback) === # === Tier 2: Public Reddit JSON (free, always available) ===
if not mock: if not mock:
if config.get("OPENAI_API_KEY"): try:
try: sys.stderr.write("[Reddit] Trying public Reddit JSON\n")
raw_response = openai_reddit.search_reddit( sys.stderr.flush()
config["OPENAI_API_KEY"], reddit_items = reddit_public.search_reddit_public(
selected_models["openai"], topic, from_date, to_date, depth=depth,
topic, )
from_date, if reddit_items:
to_date,
depth=depth,
auth_source=config.get("OPENAI_AUTH_SOURCE", "api_key"),
account_id=config.get("OPENAI_CHATGPT_ACCOUNT_ID"),
)
except http.HTTPError as e:
raw_response = {"error": str(e)}
reddit_error = f"API error: {e}"
except Exception as e:
raw_response = {"error": str(e)}
reddit_error = f"{type(e).__name__}: {e}"
else:
# No OpenAI auth: direct Reddit public JSON fallback.
try:
reddit_items = openai_reddit.search_reddit_public(
topic, from_date, to_date, depth=depth,
)
raw_response = {"source": "reddit_public", "items": reddit_items} raw_response = {"source": "reddit_public", "items": reddit_items}
except http.HTTPError as e: sys.stderr.write(f"[Reddit] Public JSON returned {len(reddit_items)} results\n")
reddit_items = [] sys.stderr.flush()
raw_response = {"error": str(e), "source": "reddit_public"} return reddit_items, raw_response, None, False
reddit_error = f"Reddit public API error: {e}" # Empty results — fall through to Tier 3
except Exception as e: sys.stderr.write("[Reddit] Public JSON returned 0 results, trying OpenAI\n")
reddit_items = [] sys.stderr.flush()
raw_response = {"error": str(e), "source": "reddit_public"} except Exception as e:
reddit_error = f"Reddit public search error: {type(e).__name__}: {e}" sys.stderr.write(f"[Reddit] Public JSON failed: {e}\n")
sys.stderr.flush()
# Fall through to Tier 3
# Parse response # === Tier 3: OpenAI Responses API (legacy fallback) ===
if not mock and config.get("OPENAI_API_KEY"):
try:
sys.stderr.write("[Reddit] Falling back to OpenAI Responses API\n")
sys.stderr.flush()
raw_response = openai_reddit.search_reddit(
config["OPENAI_API_KEY"],
selected_models["openai"],
topic,
from_date,
to_date,
depth=depth,
auth_source=config.get("OPENAI_AUTH_SOURCE", "api_key"),
account_id=config.get("OPENAI_CHATGPT_ACCOUNT_ID"),
)
except http.HTTPError as e:
raw_response = {"error": str(e)}
reddit_error = f"API error: {e}"
except Exception as e:
raw_response = {"error": str(e)}
reddit_error = f"{type(e).__name__}: {e}"
# Parse response (OpenAI path)
reddit_items = openai_reddit.parse_reddit_response(raw_response or {}) reddit_items = openai_reddit.parse_reddit_response(raw_response or {})
# Quick retry with simpler query if few results # Quick retry with simpler query if few results (OpenAI path only)
if len(reddit_items) < 5 and not mock and not reddit_error and config.get("OPENAI_API_KEY"): if len(reddit_items) < 5 and not mock and not reddit_error and config.get("OPENAI_API_KEY"):
core = openai_reddit._extract_core_subject(topic) core = openai_reddit._extract_core_subject(topic)
if core.lower() != topic.lower(): if core.lower() != topic.lower():
@@ -292,7 +292,7 @@ def _search_reddit(
except Exception: except Exception:
pass pass
# Subreddit-targeted fallback if still < 3 results # Subreddit-targeted fallback if still < 3 results (OpenAI path only)
if len(reddit_items) < 3 and not mock and not reddit_error and config.get("OPENAI_API_KEY"): if len(reddit_items) < 3 and not mock and not reddit_error and config.get("OPENAI_API_KEY"):
sub_query = openai_reddit._build_subreddit_query(topic) sub_query = openai_reddit._build_subreddit_query(topic)
try: try:
@@ -617,7 +617,7 @@ def _search_web(
Tuple of (web_items, web_error) Tuple of (web_items, web_error)
web_items are raw dicts ready for websearch.normalize_websearch_items() web_items are raw dicts ready for websearch.normalize_websearch_items()
""" """
from lib import brave_search, parallel_search, openrouter_search from lib import brave_search, parallel_search, openrouter_search, exa_search
backend = env.get_web_search_source(config) backend = env.get_web_search_source(config)
if not backend: if not backend:
@@ -627,7 +627,11 @@ def _search_web(
raw_results = [] raw_results = []
try: try:
if backend == "parallel": if backend == "exa":
raw_results = exa_search.search_web(
topic, from_date, to_date, config["EXA_API_KEY"], depth=depth,
)
elif backend == "parallel":
raw_results = parallel_search.search_web( raw_results = parallel_search.search_web(
topic, from_date, to_date, config["PARALLEL_API_KEY"], depth=depth, topic, from_date, to_date, config["PARALLEL_API_KEY"], depth=depth,
) )
@@ -1537,12 +1541,29 @@ def main():
# Load config # Load config
config = env.get_config() config = env.get_config()
# Inject .env credentials into Bird module before auth check # Detect first run (no SETUP_COMPLETE in config)
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0')) first_run = setup_wizard.is_first_run(config)
# On first run, block Bird's Node.js sweet-cookie scanner from probing
# browser cookies before the user has given consent via the setup wizard.
# Explicit AUTH_TOKEN (from env var) is fine — only block browser scanning.
if first_run and config.get('_AUTH_TOKEN_SOURCE') != 'env':
os.environ['BIRD_DISABLE_BROWSER_COOKIES'] = '1'
# Inject .env credentials into Bird module before auth check.
# On first run (no SETUP_COMPLETE), only inject explicit env var credentials —
# skip if AUTH_TOKEN came from browser cookies (no consent yet).
if first_run:
auth_source = config.get('_AUTH_TOKEN_SOURCE')
if auth_source == 'env':
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0'))
else:
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0'))
# Auto-detect Bird (no prompts - just use it if available) # Auto-detect Bird (no prompts - just use it if available)
x_source_status = env.get_x_source_status(config) x_source_status = env.get_x_source_status(config)
x_source = x_source_status["source"] # 'bird', 'xai', or None x_source = x_source_status["source"] # 'bird', 'xai', or None
x_method = x_source_status.get("method") # 'env', 'browser-firefox', 'api', etc.
# Auto-detect yt-dlp for YouTube search # Auto-detect yt-dlp for YouTube search
has_ytdlp = env.is_ytdlp_available() has_ytdlp = env.is_ytdlp_available()
@@ -1570,6 +1591,7 @@ def main():
"reddit_public": True, "reddit_public": True,
"xai": bool(config.get("XAI_API_KEY")), "xai": bool(config.get("XAI_API_KEY")),
"x_source": x_source_status["source"], "x_source": x_source_status["source"],
"x_method": x_source_status.get("method"),
"bird_installed": x_source_status["bird_installed"], "bird_installed": x_source_status["bird_installed"],
"bird_authenticated": x_source_status["bird_authenticated"], "bird_authenticated": x_source_status["bird_authenticated"],
"bird_username": x_source_status.get("bird_username"), "bird_username": x_source_status.get("bird_username"),
@@ -1583,6 +1605,7 @@ def main():
"truthsocial": has_truthsocial, "truthsocial": has_truthsocial,
"polymarket": True, "polymarket": True,
"web_search_backend": web_source, "web_search_backend": web_source,
"exa": bool(config.get("EXA_API_KEY")),
"parallel_ai": bool(config.get("PARALLEL_API_KEY")), "parallel_ai": bool(config.get("PARALLEL_API_KEY")),
"brave": bool(config.get("BRAVE_API_KEY")), "brave": bool(config.get("BRAVE_API_KEY")),
"openrouter": bool(config.get("OPENROUTER_API_KEY")), "openrouter": bool(config.get("OPENROUTER_API_KEY")),
@@ -1590,6 +1613,19 @@ def main():
print(json.dumps(diag, indent=2)) print(json.dumps(diag, indent=2))
sys.exit(0) sys.exit(0)
# Handle 'setup' subcommand
if args.topic and args.topic.strip().lower() == "setup":
results = setup_wizard.run_auto_setup(config)
# Write config
env_path = env.CONFIG_FILE
if env_path:
written = setup_wizard.write_setup_config(env_path)
results["env_written"] = written
else:
results["env_written"] = False
print(setup_wizard.get_setup_status_text(results))
sys.exit(0)
# Validate topic (--diagnose doesn't need one) # Validate topic (--diagnose doesn't need one)
if not args.topic: if not args.topic:
print("Error: Please provide a topic to research.", file=sys.stderr) print("Error: Please provide a topic to research.", file=sys.stderr)
@@ -1599,40 +1635,30 @@ def main():
# Initialize progress display with topic # Initialize progress display with topic
progress = ui.ProgressDisplay(args.topic, show_banner=True) progress = ui.ProgressDisplay(args.topic, show_banner=True)
# Show diagnostic banner when sources are missing # Show status banner (free-first design — lead with what works)
web_source = env.get_web_search_source(config) web_source = env.get_web_search_source(config)
reddit_source = env.get_reddit_source(config)
diag = { diag = {
"openai": bool(config.get("OPENAI_API_KEY")), "setup_complete": bool(config.get("SETUP_COMPLETE")),
"reddit_public": True, "reddit_source": reddit_source, # 'scrapecreators', 'openai', or None
"xai": bool(config.get("XAI_API_KEY")),
"x_source": x_source_status["source"], "x_source": x_source_status["source"],
"bird_installed": x_source_status["bird_installed"], "x_method": x_source_status.get("method"),
"bird_authenticated": x_source_status["bird_authenticated"],
"bird_username": x_source_status.get("bird_username"),
"youtube": has_ytdlp, "youtube": has_ytdlp,
"tiktok": has_tiktok, "tiktok": has_tiktok,
"instagram": has_instagram, "instagram": has_instagram,
"xiaohongshu": has_xiaohongshu,
"hackernews": True, "hackernews": True,
"bluesky": True,
"truthsocial": has_truthsocial,
"polymarket": True, "polymarket": True,
"bluesky": has_bluesky,
"truthsocial": has_truthsocial,
"xiaohongshu": has_xiaohongshu,
"scrapecreators": bool(config.get("SCRAPECREATORS_API_KEY")),
"web_search_backend": "deferred to assistant" if args.no_native_web else web_source, "web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
} }
ui.show_diagnostic_banner(diag) ui.show_diagnostic_banner(diag)
# Check available sources (accounting for Bird auto-detection) # Check available sources (now accounts for Bird/cookie auth automatically)
available = env.get_available_sources(config) available = env.get_available_sources(config)
# Override available if Bird provides X
if x_source == 'bird':
if available == 'reddit':
available = 'both' # Now have both Reddit + X
elif available == 'reddit-web':
available = 'all' # Reddit + X + Web
elif available == 'web':
available = 'x-web' # X + Web
# Mock mode can work without keys # Mock mode can work without keys
if args.mock: if args.mock:
if args.sources == "auto": if args.sources == "auto":
@@ -1709,6 +1735,18 @@ def main():
search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type) search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type)
search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type) search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type)
search_run_xiaohongshu = has_xiaohongshu search_run_xiaohongshu = has_xiaohongshu
# INCLUDE_SOURCES override: force specific sources on regardless of tier
_include_sources = {s.strip().lower() for s in config.get('INCLUDE_SOURCES', '').split(',') if s.strip()}
if _include_sources:
if 'tiktok' in _include_sources and has_tiktok:
if not search_run_tiktok:
sys.stderr.write("[Config] INCLUDE_SOURCES override: forcing tiktok\n")
search_run_tiktok = True
if 'instagram' in _include_sources and has_instagram:
if not search_run_instagram:
sys.stderr.write("[Config] INCLUDE_SOURCES override: forcing instagram\n")
search_run_instagram = True
if args.search: if args.search:
search_sources = parse_search_flag(args.search) search_sources = parse_search_flag(args.search)
has_reddit = "reddit" in search_sources has_reddit = "reddit" in search_sources
@@ -1902,9 +1940,9 @@ def main():
elif has_ytdlp and not report.youtube: elif has_ytdlp and not report.youtube:
source_info["youtube_skip_reason"] = "0 results (query may be too specific)" source_info["youtube_skip_reason"] = "0 results (query may be too specific)"
if not has_tiktok: if not has_tiktok:
source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)" source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free API calls, no credit card)"
if not has_instagram: if not has_instagram:
source_info["instagram_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)" source_info["instagram_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free API calls, no credit card)"
if not has_xiaohongshu: if not has_xiaohongshu:
source_info["xiaohongshu_skip_reason"] = ( source_info["xiaohongshu_skip_reason"] = (
f"Xiaohongshu API unavailable or not logged in - start xiaohongshu-mcp and login " f"Xiaohongshu API unavailable or not logged in - start xiaohongshu-mcp and login "
@@ -1913,8 +1951,16 @@ def main():
if not web_source: if not web_source:
source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)" source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)"
# Compute quality score and upgrade nudge
research_results = {
"x_error": x_error,
"youtube_error": youtube_error,
"reddit_error": reddit_error,
}
quality = quality_nudge.compute_quality_score(config, research_results)
# Output result # Output result
output_result(report, args.emit, web_needed, args.topic, from_date, to_date, missing_keys, args.days, source_info) output_result(report, args.emit, web_needed, args.topic, from_date, to_date, missing_keys, args.days, source_info, first_run=first_run, quality=quality)
# Auto-save raw research to file if --save-dir is set # Auto-save raw research to file if --save-dir is set
if args.save_dir: if args.save_dir:
@@ -1926,6 +1972,8 @@ def main():
if save_path.exists(): if save_path.exists():
save_path = save_dir / f"{slug}-raw-{datetime.now().strftime('%Y-%m-%d')}.md" save_path = save_dir / f"{slug}-raw-{datetime.now().strftime('%Y-%m-%d')}.md"
content = render.render_compact(report, missing_keys=missing_keys) content = render.render_compact(report, missing_keys=missing_keys)
if quality and quality.get("nudge_text"):
content += "\n" + render.render_quality_nudge(quality)
content += "\n" + render.render_source_status(report, source_info) content += "\n" + render.render_source_status(report, source_info)
save_path.write_text(content, encoding="utf-8") save_path.write_text(content, encoding="utf-8")
print(f"📎 {save_path}", file=sys.stderr) print(f"📎 {save_path}", file=sys.stderr)
@@ -2043,10 +2091,19 @@ def output_result(
missing_keys: str = "none", missing_keys: str = "none",
days: int = 30, days: int = 30,
source_info: dict = None, source_info: dict = None,
first_run: bool = False,
quality: dict = None,
): ):
"""Output the result based on emit mode.""" """Output the result based on emit mode."""
if emit_mode == "compact": if emit_mode == "compact":
# Emit first-run flag before research output so SKILL.md can detect it
if first_run:
print("FIRST_RUN: true")
print("")
print(render.render_compact(report, missing_keys=missing_keys)) print(render.render_compact(report, missing_keys=missing_keys))
# Quality nudge (right before source status/stats block)
if quality and quality.get("nudge_text"):
print(render.render_quality_nudge(quality))
# Append source status footer # Append source status footer
print(render.render_source_status(report, source_info)) print(render.render_source_status(report, source_info))
elif emit_mode == "json": elif emit_mode == "json":
+265
View File
@@ -0,0 +1,265 @@
"""Chrome cookie extraction for macOS.
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
This is NOT affected by Windows App-Bound Encryption (v20).
"""
import hashlib
import logging
import shutil
import sqlite3
import subprocess
import tempfile
from pathlib import Path
from typing import Optional
logger = logging.getLogger(__name__)
# Chrome cookie DB location on macOS
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
# Chrome v10 encryption constants
CHROME_SALT = b"saltysalt"
CHROME_PBKDF2_ITERATIONS = 1003
CHROME_KEY_LENGTH = 16
# IV is 16 space characters (0x20)
CHROME_IV_HEX = "20" * 16
def _get_chrome_encryption_key() -> Optional[bytes]:
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
Calls `security find-generic-password` which may trigger a system dialog
on first access.
Returns the raw passphrase bytes, or None on failure.
"""
try:
result = subprocess.run(
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
capture_output=True,
text=True,
timeout=10,
)
if result.returncode != 0:
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
return None
passphrase = result.stdout.strip()
if not passphrase:
logger.info("Chrome Keychain returned empty passphrase")
return None
return passphrase.encode("utf-8")
except FileNotFoundError:
logger.info("'security' command not found — not on macOS?")
return None
except subprocess.TimeoutExpired:
logger.info("Chrome Keychain access timed out")
return None
except Exception as e:
logger.info("Failed to get Chrome encryption key: %s", e)
return None
def _derive_aes_key(passphrase: bytes) -> bytes:
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
return hashlib.pbkdf2_hmac(
"sha1",
passphrase,
CHROME_SALT,
CHROME_PBKDF2_ITERATIONS,
dklen=CHROME_KEY_LENGTH,
)
def _decrypt_v10_value(encrypted_value: bytes, aes_key: bytes, db_version: int) -> Optional[str]:
"""Decrypt a Chrome v10-encrypted cookie value.
Uses system openssl CLI for AES-128-CBC decryption (zero pip deps).
For Chrome 130+ (db_version >= 24), strips 32-byte SHA-256 prefix after decryption.
Returns decrypted string or None on failure.
"""
# Strip the 'v10' prefix
ciphertext = encrypted_value[3:]
if not ciphertext:
return None
hex_key = aes_key.hex()
try:
result = subprocess.run(
[
"openssl", "enc", "-aes-128-cbc", "-d",
"-K", hex_key,
"-iv", CHROME_IV_HEX,
"-nopad",
],
input=ciphertext,
capture_output=True,
timeout=5,
)
if result.returncode != 0:
logger.debug("openssl decryption failed: %s", result.stderr.decode(errors="replace").strip())
return None
decrypted = result.stdout
if not decrypted:
return None
# Remove PKCS7 padding
decrypted = _remove_pkcs7_padding(decrypted)
if decrypted is None:
return None
# Chrome 130+ (db version >= 24): strip 32-byte SHA-256 prefix
if db_version >= 24 and len(decrypted) > 32:
decrypted = decrypted[32:]
return decrypted.decode("utf-8", errors="replace")
except FileNotFoundError:
logger.info("openssl not found — cannot decrypt Chrome cookies")
return None
except subprocess.TimeoutExpired:
logger.info("openssl decryption timed out")
return None
except Exception as e:
logger.debug("Chrome cookie decryption error: %s", e)
return None
def _remove_pkcs7_padding(data: bytes) -> Optional[bytes]:
"""Remove PKCS7 padding from decrypted data.
The last byte indicates the number of padding bytes added.
All padding bytes must have the same value.
Returns unpadded data or None if padding is invalid.
"""
if not data:
return None
pad_len = data[-1]
if pad_len < 1 or pad_len > 16:
return None
# Verify all padding bytes match
if data[-pad_len:] != bytes([pad_len]) * pad_len:
return None
return data[:-pad_len]
def _get_db_version(cursor: sqlite3.Cursor) -> int:
"""Get Chrome cookie database version from the meta table.
Returns 0 if meta table doesn't exist or version can't be read.
"""
try:
cursor.execute("SELECT value FROM meta WHERE key = 'version'")
row = cursor.fetchone()
if row:
return int(row[0])
except Exception:
pass
return 0
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
"""Extract cookies from Chrome on macOS.
Copies the locked Cookies database to a temp file, reads specified cookies,
and decrypts v10-encrypted values using the Keychain-stored key.
Args:
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
cookie_names: List of cookie names to extract
Returns:
Dict mapping cookie name to decrypted value, or None on failure.
Only includes cookies that were successfully found and decrypted.
"""
if not CHROME_COOKIES_DB.exists():
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
return None
# Get encryption key from Keychain
passphrase = _get_chrome_encryption_key()
aes_key = _derive_aes_key(passphrase) if passphrase else None
# Copy DB to temp file (Chrome locks the original)
tmp_fd = None
tmp_path = None
try:
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
except Exception as e:
logger.info("Failed to copy Chrome cookies database: %s", e)
if tmp_path:
try:
Path(tmp_path).unlink(missing_ok=True)
except Exception:
pass
return None
finally:
if tmp_fd is not None:
import os
os.close(tmp_fd)
try:
conn = sqlite3.connect(tmp_path)
cursor = conn.cursor()
db_version = _get_db_version(cursor)
logger.debug("Chrome cookie DB version: %d", db_version)
# Build query with placeholders for cookie names
placeholders = ",".join("?" for _ in cookie_names)
query = (
f"SELECT name, value, encrypted_value FROM cookies "
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
)
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
params = [f"%{domain}"] + list(cookie_names)
cursor.execute(query, params)
results: dict[str, str] = {}
for name, value, encrypted_value in cursor.fetchall():
# Prefer unencrypted value if present
if value:
results[name] = value
continue
# Handle encrypted value
if encrypted_value and encrypted_value[:3] == b"v10":
if aes_key is None:
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
continue
decrypted = _decrypt_v10_value(encrypted_value, aes_key, db_version)
if decrypted:
results[name] = decrypted
else:
logger.debug("Failed to decrypt cookie %s", name)
elif encrypted_value:
# Unknown encryption version
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
conn.close()
if not results:
logger.info("No matching cookies found in Chrome for domain %s", domain)
return None
return results
except sqlite3.Error as e:
logger.info("Failed to read Chrome cookies database: %s", e)
return None
except Exception as e:
logger.info("Unexpected error reading Chrome cookies: %s", e)
return None
finally:
try:
Path(tmp_path).unlink(missing_ok=True)
except Exception:
pass
+295
View File
@@ -0,0 +1,295 @@
"""Browser cookie extraction for last30days.
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
to enable zero-config authentication for services like X/Twitter.
Only uses Python stdlib — no external dependencies.
"""
import configparser
import logging
import platform
import shutil
import sqlite3
import tempfile
from pathlib import Path
from typing import Dict, List, Optional
logger = logging.getLogger(__name__)
def _get_firefox_profiles_dir() -> Optional[Path]:
"""Return the Firefox profiles directory for the current platform, or None."""
system = platform.system()
if system == "Darwin":
path = Path.home() / "Library" / "Application Support" / "Firefox"
elif system == "Linux":
path = Path.home() / ".mozilla" / "firefox"
else:
# Windows: %APPDATA%\Mozilla\Firefox — best-effort
appdata = Path.home() / "AppData" / "Roaming" / "Mozilla" / "Firefox"
path = appdata
return path if path.is_dir() else None
def _find_default_profile(profiles_dir: Path) -> Optional[Path]:
"""Parse profiles.ini to find the default profile directory.
Looks for a section with Default=1. Falls back to the first profile
directory found on disk if profiles.ini is missing or malformed.
"""
ini_path = profiles_dir / "profiles.ini"
if ini_path.is_file():
try:
config = configparser.ConfigParser()
config.read(str(ini_path), encoding="utf-8")
# First pass: look for Default=1
for section in config.sections():
if config.has_option(section, "Default") and config.get(section, "Default") == "1":
return _resolve_profile_path(profiles_dir, config, section)
# Second pass: first Install* section with Default key (Firefox >= 67 format)
for section in config.sections():
if section.startswith("Install") and config.has_option(section, "Default"):
raw = config.get(section, "Default")
candidate = profiles_dir / raw
if candidate.is_dir():
return candidate
# Third pass: first Profile section that exists on disk
for section in config.sections():
if section.startswith("Profile"):
resolved = _resolve_profile_path(profiles_dir, config, section)
if resolved and resolved.is_dir():
return resolved
except (configparser.Error, OSError) as exc:
logger.debug("Failed to parse profiles.ini: %s", exc)
# Fallback: scan directory for anything that looks like a profile
return _fallback_find_profile(profiles_dir)
def _resolve_profile_path(
profiles_dir: Path, config: configparser.ConfigParser, section: str
) -> Optional[Path]:
"""Resolve a profile path from a ConfigParser section."""
if not config.has_option(section, "Path"):
return None
raw_path = config.get(section, "Path")
is_relative = config.has_option(section, "IsRelative") and config.get(section, "IsRelative") == "1"
if is_relative:
candidate = profiles_dir / raw_path
else:
candidate = Path(raw_path)
return candidate if candidate.is_dir() else None
def _fallback_find_profile(profiles_dir: Path) -> Optional[Path]:
"""Find the first directory that contains cookies.sqlite."""
try:
for child in sorted(profiles_dir.iterdir()):
if child.is_dir() and (child / "cookies.sqlite").is_file():
return child
except OSError:
pass
return None
def _query_cookies_db(
db_path: Path, domain: str, cookie_names: List[str]
) -> Optional[Dict[str, str]]:
"""Copy the cookies database to a temp file and query it.
Firefox locks cookies.sqlite while running, so we copy first.
Returns {name: value} dict or None if no matching cookies found.
"""
if not db_path.is_file():
return None
tmp_fd = None
tmp_path = None
try:
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
shutil.copy2(str(db_path), tmp_path)
conn = sqlite3.connect(tmp_path)
try:
# Build parameterized query — SQLite doesn't support array params,
# so we build the IN clause with individual placeholders.
placeholders = ",".join("?" for _ in cookie_names)
query = (
f"SELECT name, value FROM moz_cookies "
f"WHERE host LIKE ? AND name IN ({placeholders})"
)
# domain pattern: match .x.com, x.com, etc.
domain_pattern = f"%{domain}"
params = [domain_pattern] + list(cookie_names)
cursor = conn.execute(query, params)
rows = cursor.fetchall()
finally:
conn.close()
if not rows:
return None
return {name: value for name, value in rows}
except (sqlite3.Error, OSError) as exc:
logger.debug("Failed to query cookies database %s: %s", db_path, exc)
return None
finally:
if tmp_path:
try:
Path(tmp_path).unlink(missing_ok=True)
except OSError:
pass
if tmp_fd is not None:
try:
import os
os.close(tmp_fd)
except OSError:
pass
def extract_firefox_cookies(
domain: str, cookie_names: List[str]
) -> Optional[Dict[str, str]]:
"""Extract cookies from Firefox for the given domain and cookie names.
Finds the default Firefox profile, copies cookies.sqlite to a temp file
(to avoid lock conflicts), and queries for the requested cookies.
Args:
domain: The cookie domain to match (e.g. ".x.com"). Matched with LIKE %domain.
cookie_names: List of cookie names to extract (e.g. ["auth_token", "ct0"]).
Returns:
Dict of {cookie_name: cookie_value} or None if extraction fails.
"""
profiles_dir = _get_firefox_profiles_dir()
if profiles_dir is None:
logger.debug("Firefox profiles directory not found")
return None
profile_path = _find_default_profile(profiles_dir)
if profile_path is None:
logger.debug("No Firefox profile found in %s", profiles_dir)
return None
db_path = profile_path / "cookies.sqlite"
return _query_cookies_db(db_path, domain, cookie_names)
def extract_chrome_cookies(
domain: str, cookie_names: List[str]
) -> Optional[Dict[str, str]]:
"""Extract cookies from Chrome for the given domain and cookie names.
macOS only — uses Keychain + system openssl for AES-128-CBC decryption.
Linux/Windows not supported (Chrome uses platform-specific encryption).
Returns:
Dict of {cookie_name: cookie_value} or None if extraction fails.
"""
if platform.system() != "Darwin":
logger.debug("Chrome cookie extraction only supported on macOS")
return None
try:
from .chrome_cookies import extract_chrome_cookies_macos
return extract_chrome_cookies_macos(domain, cookie_names)
except Exception as exc:
logger.debug("Chrome cookie extraction failed: %s", exc)
return None
def extract_safari_cookies(
domain: str, cookie_names: List[str]
) -> Optional[Dict[str, str]]:
"""Extract cookies from Safari for the given domain and cookie names.
macOS only — parses the unencrypted binary cookie file.
Returns:
Dict of {cookie_name: cookie_value} or None if extraction fails.
"""
if platform.system() != "Darwin":
logger.debug("Safari cookie extraction only supported on macOS")
return None
try:
from .safari_cookies import extract_safari_cookies_macos
return extract_safari_cookies_macos(domain, cookie_names)
except Exception as exc:
logger.debug("Safari cookie extraction failed: %s", exc)
return None
def extract_cookies(
browser: str, domain: str, cookie_names: list[str]
) -> Optional[dict[str, str]]:
"""Extract cookies from the specified browser.
Args:
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
'auto' tries browsers in platform-appropriate order:
- macOS: Chrome -> Firefox -> Safari
- Linux: Firefox only
domain: The cookie domain to match (e.g. ".x.com").
cookie_names: List of cookie names to extract.
Returns:
Dict of {cookie_name: cookie_value} or None if extraction fails.
"""
result = extract_cookies_with_source(browser, domain, cookie_names)
if result is None:
return None
cookies, _browser_name = result
return cookies
def extract_cookies_with_source(
browser: str, domain: str, cookie_names: list[str]
) -> Optional[tuple[dict[str, str], str]]:
"""Extract cookies and report which browser they came from.
Same as extract_cookies() but returns a (cookies, browser_name) tuple
so callers can track the source.
Args:
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
domain: The cookie domain to match (e.g. ".x.com").
cookie_names: List of cookie names to extract.
Returns:
Tuple of ({cookie_name: cookie_value}, browser_name) or None.
"""
extractors = {
"firefox": extract_firefox_cookies,
"chrome": extract_chrome_cookies,
"safari": extract_safari_cookies,
}
if browser != "auto":
extractor = extractors.get(browser)
if extractor is None:
logger.warning("Unknown browser: %s", browser)
return None
result = extractor(domain, cookie_names)
return (result, browser) if result is not None else None
# Auto mode: try browsers in platform-appropriate order
system = platform.system()
if system == "Darwin":
order = ["chrome", "firefox", "safari"]
elif system == "Linux":
order = ["firefox"]
else:
order = ["firefox"]
for name in order:
result = extractors[name](domain, cookie_names)
if result is not None:
return (result, name)
return None
+230 -25
View File
@@ -2,11 +2,38 @@
import base64 import base64
import json import json
import logging
import os import os
import time import time
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Optional, Dict, Any, Literal from typing import Optional, Dict, Any, List, Literal
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Cookie domain registry: maps source names to browser cookie extraction params.
# Each entry: (domain, cookie_names, config_key_mapping)
# config_key_mapping: {cookie_name: config_key} so we know which config key
# each extracted cookie should populate.
# ---------------------------------------------------------------------------
COOKIE_DOMAINS: Dict[str, Dict[str, Any]] = {
"x": {
"domain": ".x.com",
"cookies": ["auth_token", "ct0"],
"mapping": {
"auth_token": "AUTH_TOKEN",
"ct0": "CT0",
},
},
"truthsocial": {
"domain": ".truthsocial.com",
"cookies": ["_session_id"],
"mapping": {
"_session_id": "TRUTHSOCIAL_TOKEN",
},
},
}
# Allow override via environment variable for testing # Allow override via environment variable for testing
# Set LAST30DAYS_CONFIG_DIR="" for clean/no-config mode # Set LAST30DAYS_CONFIG_DIR="" for clean/no-config mode
@@ -212,6 +239,87 @@ def _find_project_env() -> Optional[Path]:
return None return None
def extract_browser_credentials(config: Dict[str, Any]) -> Dict[str, str]:
"""Extract credentials from browser cookies for sources that need them.
Checks the FROM_BROWSER config key to decide whether/how to extract:
- 'auto': try browsers in platform order
- 'firefox', 'chrome', 'safari': try only that browser
- 'off': skip extraction entirely
If SETUP_COMPLETE is not set AND FROM_BROWSER is not explicitly set,
defaults to 'off' (wizard hasn't run yet — no extraction without consent).
If SETUP_COMPLETE is set and FROM_BROWSER is not set, defaults to 'auto'.
Explicit env var/config values always take priority over extracted cookies.
Returns:
Dict of {config_key: value} for credentials discovered from cookies.
"""
setup_complete = config.get("SETUP_COMPLETE")
from_browser = config.get("FROM_BROWSER")
# Determine effective browser setting
if from_browser is None:
if setup_complete:
from_browser = "auto"
else:
from_browser = "off"
from_browser = from_browser.lower().strip() if isinstance(from_browser, str) else "off"
if from_browser == "off":
return {}
# Lazy import to avoid loading cookie_extract at module level
try:
from . import cookie_extract
except Exception:
logger.debug("cookie_extract module not available")
return {}
credentials: Dict[str, str] = {}
for source_name, spec in COOKIE_DOMAINS.items():
domain = spec["domain"]
cookie_names: List[str] = spec["cookies"]
mapping: Dict[str, str] = spec["mapping"]
# Skip if ALL mapped config keys already have values
all_present = all(config.get(config_key) for config_key in mapping.values())
if all_present:
logger.debug(
"Skipping cookie extraction for %s: credentials already set",
source_name,
)
continue
try:
result = cookie_extract.extract_cookies_with_source(from_browser, domain, cookie_names)
except Exception as exc:
logger.debug(
"Cookie extraction failed for %s: %s", source_name, exc
)
continue
if result is None:
continue
cookies, browser_name = result
filled_any = False
for cookie_name, config_key in mapping.items():
# Only fill in keys not already present
if not config.get(config_key) and cookie_name in cookies:
credentials[config_key] = cookies[cookie_name]
filled_any = True
# Track which browser provided the credentials for this source
if filled_any:
credentials[f"__{source_name.upper()}_BROWSER"] = browser_name
return credentials
def get_config() -> Dict[str, Any]: def get_config() -> Dict[str, Any]:
"""Load configuration from multiple sources. """Load configuration from multiple sources.
@@ -219,6 +327,7 @@ def get_config() -> Dict[str, Any]:
1. Environment variables (os.environ) 1. Environment variables (os.environ)
2. .claude/last30days.env (per-project config) 2. .claude/last30days.env (per-project config)
3. ~/.config/last30days/.env (global config) 3. ~/.config/last30days/.env (global config)
4. Browser cookies (only fills in missing keys)
""" """
# Load from global config file # Load from global config file
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {} file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
@@ -249,6 +358,7 @@ def get_config() -> Dict[str, Any]:
('OPENROUTER_API_KEY', None), ('OPENROUTER_API_KEY', None),
('PARALLEL_API_KEY', None), ('PARALLEL_API_KEY', None),
('BRAVE_API_KEY', None), ('BRAVE_API_KEY', None),
('EXA_API_KEY', None),
('XIAOHONGSHU_API_BASE', None), ('XIAOHONGSHU_API_BASE', None),
('GEMINI_MODEL', None), ('GEMINI_MODEL', None),
('OPENAI_MODEL_POLICY', 'auto'), ('OPENAI_MODEL_POLICY', 'auto'),
@@ -262,11 +372,32 @@ def get_config() -> Dict[str, Any]:
('BSKY_HANDLE', None), ('BSKY_HANDLE', None),
('BSKY_APP_PASSWORD', None), ('BSKY_APP_PASSWORD', None),
('TRUTHSOCIAL_TOKEN', None), ('TRUTHSOCIAL_TOKEN', None),
('FROM_BROWSER', None),
('SETUP_COMPLETE', None),
('INCLUDE_SOURCES', None),
] ]
for key, default in keys: for key, default in keys:
config[key] = os.environ.get(key) or merged_env.get(key, default) config[key] = os.environ.get(key) or merged_env.get(key, default)
# Inject browser cookies for any credentials not already set
browser_creds = extract_browser_credentials(config)
for key, value in browser_creds.items():
if not config.get(key):
config[key] = value
# Track AUTH_TOKEN source for status reporting
if config.get('AUTH_TOKEN'):
if os.environ.get('AUTH_TOKEN') or merged_env.get('AUTH_TOKEN'):
config['_AUTH_TOKEN_SOURCE'] = 'env'
elif browser_creds.get('AUTH_TOKEN'):
browser_name = browser_creds.get('__X_BROWSER', 'unknown')
config['_AUTH_TOKEN_SOURCE'] = f'browser-{browser_name}'
else:
config['_AUTH_TOKEN_SOURCE'] = 'env' # fallback
else:
config['_AUTH_TOKEN_SOURCE'] = None
# Track which config source was used # Track which config source was used
if project_env_path: if project_env_path:
config['_CONFIG_SOURCE'] = f'project:{project_env_path}' config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
@@ -314,14 +445,19 @@ def get_reddit_source(config: Dict[str, Any]) -> Optional[str]:
def get_available_sources(config: Dict[str, Any]) -> str: def get_available_sources(config: Dict[str, Any]) -> str:
"""Determine which sources are available. """Determine which sources are available.
X is available if ANY auth method works: AUTH_TOKEN/CT0 (env or cookies),
XAI_API_KEY, or Bird installed+authenticated.
Reddit is always available (public JSON fallback).
HN and Polymarket are always available.
YouTube available if yt-dlp installed.
Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none' Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
""" """
# Reddit is available via public JSON fallback even without OpenAI auth.
has_reddit = True has_reddit = True
has_xai = bool(config.get('XAI_API_KEY')) has_x = get_x_source(config) is not None
has_web = has_web_search_keys(config) has_web = has_web_search_keys(config)
if has_reddit and has_xai: if has_reddit and has_x:
return 'all' if has_web else 'both' return 'all' if has_web else 'both'
elif has_reddit: elif has_reddit:
return 'reddit-web' if has_web else 'reddit' return 'reddit-web' if has_web else 'reddit'
@@ -330,16 +466,18 @@ def get_available_sources(config: Dict[str, Any]) -> str:
def has_web_search_keys(config: Dict[str, Any]) -> bool: def has_web_search_keys(config: Dict[str, Any]) -> bool:
"""Check if any web search API keys are configured.""" """Check if any web search API keys are configured."""
return bool(config.get('OPENROUTER_API_KEY') or config.get('PARALLEL_API_KEY') or config.get('BRAVE_API_KEY')) return bool(config.get('EXA_API_KEY') or config.get('OPENROUTER_API_KEY') or config.get('PARALLEL_API_KEY') or config.get('BRAVE_API_KEY'))
def get_web_search_source(config: Dict[str, Any]) -> Optional[str]: def get_web_search_source(config: Dict[str, Any]) -> Optional[str]:
"""Determine the best available web search backend. """Determine the best available web search backend.
Priority: Parallel AI > Brave > OpenRouter/Sonar Pro Priority: Exa (free) > Parallel AI > Brave > OpenRouter/Sonar Pro
Returns: 'parallel', 'brave', 'openrouter', or None Returns: 'exa', 'parallel', 'brave', 'openrouter', or None
""" """
if config.get('EXA_API_KEY'):
return 'exa'
if config.get('PARALLEL_API_KEY'): if config.get('PARALLEL_API_KEY'):
return 'parallel' return 'parallel'
if config.get('BRAVE_API_KEY'): if config.get('BRAVE_API_KEY'):
@@ -441,9 +579,13 @@ def validate_sources(requested: str, available: str, include_web: bool = False)
def get_x_source(config: Dict[str, Any]) -> Optional[str]: def get_x_source(config: Dict[str, Any]) -> Optional[str]:
"""Determine the best available X/Twitter source. """Determine the best available X/Twitter source.
Priority: Bird (free) → xAI (paid API) Priority chain:
1. AUTH_TOKEN/CT0 from env var or .env file → Bird with method "env"
2. AUTH_TOKEN/CT0 from browser cookie extraction → Bird with method "browser-{browser}"
3. XAI_API_KEY → xAI with method "api"
4. None
Keep X selection limited to documented, verified search backends. Use get_x_source_with_method() to also get the method string.
Args: Args:
config: Configuration dict from get_config() config: Configuration dict from get_config()
@@ -453,20 +595,58 @@ def get_x_source(config: Dict[str, Any]) -> Optional[str]:
'xai' if XAI_API_KEY is configured, 'xai' if XAI_API_KEY is configured,
None if no X source available. None if no X source available.
""" """
# Import here to avoid circular dependency source, _method = get_x_source_with_method(config)
return source
def get_x_source_with_method(config: Dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
"""Determine the best available X/Twitter source and auth method.
Priority chain:
1. AUTH_TOKEN/CT0 (env var or .env) → Bird with method "env"
2. AUTH_TOKEN/CT0 (browser cookies) → Bird with method "browser-{browser}"
3. XAI_API_KEY → xAI with method "api"
4. None
Args:
config: Configuration dict from get_config()
Returns:
Tuple of (source, method) where source is 'bird', 'xai', or None
and method is 'env', 'browser-chrome', 'browser-firefox', 'browser-safari', 'api', or None.
"""
from . import bird_x from . import bird_x
# Check Bird first (free option) setup_complete = config.get('SETUP_COMPLETE')
# Check Bird first (free option — uses AUTH_TOKEN/CT0 from any source)
if bird_x.is_bird_installed(): if bird_x.is_bird_installed():
username = bird_x.is_bird_authenticated() auth_source = config.get('_AUTH_TOKEN_SOURCE')
if username:
return 'bird' # If SETUP_COMPLETE is not set, only allow explicit env var credentials.
# Do NOT call is_bird_authenticated() for browser-cookie probing —
# that requires user consent via the setup wizard.
if not setup_complete:
# Explicit AUTH_TOKEN from env var / .env file is always allowed
if auth_source == 'env' and config.get('AUTH_TOKEN'):
username = bird_x.is_bird_authenticated()
if username:
return 'bird', 'env'
else:
# SETUP_COMPLETE is set — normal flow, probe cookies if needed
username = bird_x.is_bird_authenticated()
if username:
if auth_source and auth_source.startswith('browser-'):
method = auth_source # e.g. "browser-firefox"
else:
method = 'env'
return 'bird', method
# Fall back to xAI if key exists # Fall back to xAI if key exists
if config.get('XAI_API_KEY'): if config.get('XAI_API_KEY'):
return 'xai' return 'xai', 'api'
return None return None, None
def is_ytdlp_available() -> bool: def is_ytdlp_available() -> bool:
@@ -576,24 +756,49 @@ def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
"""Get detailed X source status for UI decisions. """Get detailed X source status for UI decisions.
Returns: Returns:
Dict with keys: source, bird_installed, bird_authenticated, Dict with keys: source, method, bird_installed, bird_authenticated,
bird_username, xai_available, can_install_bird bird_username, xai_available, can_install_bird
The ``method`` field indicates HOW the active source is authenticated:
- "env" — AUTH_TOKEN came from an env var or .env file
- "browser-chrome", "browser-firefox", "browser-safari" — from cookie extraction
- "api" — using xAI API key
- None — no X source available
""" """
from . import bird_x from . import bird_x
bird_status = bird_x.get_bird_status() setup_complete = config.get('SETUP_COMPLETE')
xai_available = bool(config.get('XAI_API_KEY')) xai_available = bool(config.get('XAI_API_KEY'))
# Determine active source if not setup_complete:
if bird_status["authenticated"]: # Before consent: do NOT call get_bird_status() which probes cookies.
source = 'bird' # Only check if Bird is installed (no cookie probing) and use the
elif xai_available: # gated get_x_source_with_method() which blocks cookie detection.
source = 'xai' bird_installed = bird_x.is_bird_installed()
else: source, method = get_x_source_with_method(config)
source = None
# Bird "authenticated" only if get_x_source_with_method found explicit creds
bird_authenticated = (source == 'bird')
return {
"source": source,
"method": method,
"bird_installed": bird_installed,
"bird_authenticated": bird_authenticated,
"bird_username": None if not bird_authenticated else "env AUTH_TOKEN",
"xai_available": xai_available,
"can_install_bird": True,
}
# SETUP_COMPLETE is set — normal flow
bird_status = bird_x.get_bird_status()
# Use the unified resolution function for source + method
source, method = get_x_source_with_method(config)
return { return {
"source": source, "source": source,
"method": method,
"bird_installed": bird_status["installed"], "bird_installed": bird_status["installed"],
"bird_authenticated": bird_status["authenticated"], "bird_authenticated": bird_status["authenticated"],
"bird_username": bird_status["username"], "bird_username": bird_status["username"],
+176
View File
@@ -0,0 +1,176 @@
"""Exa AI web search for last30days skill.
Uses the Exa Search API as a free web search backend.
Free tier: 1,000 searches/month, semantic search, no credit card required.
API docs: https://docs.exa.ai/reference/search
"""
import sys
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from . import http
ENDPOINT = "https://api.exa.ai/search"
# Domains to exclude (handled by Reddit/X search)
EXCLUDED_DOMAINS = {
"reddit.com", "www.reddit.com", "old.reddit.com",
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
}
def search_web(
topic: str,
from_date: str,
to_date: str,
api_key: str,
depth: str = "default",
) -> List[Dict[str, Any]]:
"""Search the web via Exa AI Search API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
api_key: Exa API key
depth: 'quick', 'default', or 'deep'
Returns:
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
"""
num_results = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
max_chars = {"quick": 1000, "default": 2000, "deep": 3000}.get(depth, 2000)
payload = {
"query": f"{topic} (from {from_date} to {to_date})",
"type": "auto",
"numResults": num_results,
"contents": {"text": {"maxCharacters": max_chars}},
}
# Add date filtering if dates are provided
if from_date:
payload["startPublishedDate"] = f"{from_date}T00:00:00.000Z"
if to_date:
payload["endPublishedDate"] = f"{to_date}T23:59:59.999Z"
sys.stderr.write(f"[Web] Searching Exa for: {topic}\n")
sys.stderr.flush()
try:
response = http.post(
ENDPOINT,
json_data=payload,
headers={
"x-api-key": api_key,
},
timeout=20,
retries=2,
)
except http.HTTPError as e:
status = e.status_code
if status == 401:
sys.stderr.write("[Web] Exa: invalid API key (401)\n")
sys.stderr.flush()
return []
if status == 429:
sys.stderr.write("[Web] Exa: rate limited (429)\n")
sys.stderr.flush()
return []
sys.stderr.write(f"[Web] Exa: HTTP error {status}: {e}\n")
sys.stderr.flush()
return []
except Exception as e:
sys.stderr.write(f"[Web] Exa: request failed: {e}\n")
sys.stderr.flush()
return []
return _normalize_results(response)
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Convert Exa API response to websearch item schema.
Exa results have: title, url, text, publishedDate, score, author.
"""
items = []
results = response.get("results", [])
if not isinstance(results, list):
return items
for i, result in enumerate(results):
if not isinstance(result, dict):
continue
url = result.get("url", "")
if not url:
continue
# Skip excluded domains
try:
domain = urlparse(url).netloc.lower()
if domain in EXCLUDED_DOMAINS:
continue
if domain.startswith("www."):
domain = domain[4:]
except Exception:
domain = ""
title = str(result.get("title", "")).strip()
# Exa returns page content in "text" field
snippet = str(result.get("text", "")).strip()
if not title and not snippet:
continue
# Parse publishedDate (ISO format from Exa: "2026-03-15T00:00:00.000Z")
date = _parse_exa_date(result.get("publishedDate"))
date_confidence = "med" if date else "low"
# Exa provides a relevance score
relevance = result.get("score", 0.6)
try:
relevance = min(1.0, max(0.0, float(relevance)))
except (TypeError, ValueError):
relevance = 0.6
items.append({
"id": f"W{i+1}",
"title": title[:200],
"url": url,
"source_domain": domain,
"snippet": snippet[:500],
"date": date,
"date_confidence": date_confidence,
"relevance": relevance,
"why_relevant": "",
})
sys.stderr.write(f"[Web] Exa: {len(items)} results\n")
sys.stderr.flush()
return items
def _parse_exa_date(published_date: Optional[str]) -> Optional[str]:
"""Parse Exa's publishedDate to YYYY-MM-DD.
Exa returns ISO format like "2026-03-15T00:00:00.000Z".
"""
if not published_date:
return None
try:
# Extract YYYY-MM-DD from ISO datetime
if "T" in published_date:
return published_date.split("T")[0]
# Already YYYY-MM-DD
if len(published_date) >= 10:
return published_date[:10]
except Exception:
pass
return None
+1 -1
View File
@@ -3,7 +3,7 @@
Uses ScrapeCreators REST API to search Instagram Reels by keyword, extract Uses ScrapeCreators REST API to search Instagram Reels by keyword, extract
engagement metrics (views, likes, comments), and fetch video transcripts. engagement metrics (views, likes, comments), and fetch video transcripts.
Requires SCRAPECREATORS_API_KEY in config. 100 free credits, then PAYG. Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
API docs: https://scrapecreators.com/docs API docs: https://scrapecreators.com/docs
""" """
+1
View File
@@ -193,6 +193,7 @@ def normalize_youtube_items(
date_confidence="high", date_confidence="high",
engagement=engagement, engagement=engagement,
transcript_snippet=item.get("transcript_snippet", ""), transcript_snippet=item.get("transcript_snippet", ""),
transcript_highlights=item.get("transcript_highlights", []),
relevance=item.get("relevance", 0.7), relevance=item.get("relevance", 0.7),
why_relevant=item.get("why_relevant", ""), why_relevant=item.get("why_relevant", ""),
)) ))
+201
View File
@@ -0,0 +1,201 @@
"""Post-research quality score and upgrade nudge.
Computes a quality score based on 5 core sources and builds
a nudge message describing what the user missed and how to fix it.
"""
from typing import Any, Dict, List, Optional
# The 5 core sources
CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit_comments"]
# Labels for display
SOURCE_LABELS = {
"hn": "Hacker News",
"polymarket": "Polymarket",
"x": "X/Twitter",
"youtube": "YouTube",
"reddit_comments": "Reddit with comments",
}
def _is_x_active(config: dict, research_results: dict) -> bool:
"""Check if X source is active (has credentials AND didn't error)."""
has_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
if not has_creds:
return False
# If X errored this run, it's configured but broken
if research_results.get("x_error"):
return False
return True
def _is_youtube_active(config: dict, research_results: dict) -> bool:
"""Check if YouTube source is active (yt-dlp installed)."""
try:
from . import youtube_yt
has_ytdlp = youtube_yt.is_ytdlp_installed()
except Exception:
has_ytdlp = False
if not has_ytdlp:
return False
if research_results.get("youtube_error"):
return False
return True
def _is_reddit_comments_active(config: dict, research_results: dict) -> bool:
"""Check if Reddit with comments is active (ScrapeCreators)."""
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
if not has_sc:
return False
if research_results.get("reddit_error"):
return False
return True
def compute_quality_score(config: dict, research_results: dict) -> dict:
"""Compute research quality score based on 5 core sources.
Args:
config: Configuration dict from env.get_config()
research_results: Dict with keys like x_error, youtube_error,
reddit_error reflecting what happened this run.
Returns:
{
"score_pct": 40-100,
"core_active": ["hn", "polymarket", ...],
"core_missing": ["x", "youtube", "reddit_comments"],
"core_errored": ["reddit_comments"], # configured but errored
"nudge_text": "..." or None if 100%
}
"""
core_active: List[str] = []
core_missing: List[str] = []
core_errored: List[str] = []
# HN and Polymarket are always active
core_active.append("hn")
core_active.append("polymarket")
# X
has_x_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
if _is_x_active(config, research_results):
core_active.append("x")
else:
core_missing.append("x")
if has_x_creds and research_results.get("x_error"):
core_errored.append("x")
# YouTube
try:
from . import youtube_yt
has_ytdlp = youtube_yt.is_ytdlp_installed()
except Exception:
has_ytdlp = False
if _is_youtube_active(config, research_results):
core_active.append("youtube")
else:
core_missing.append("youtube")
if has_ytdlp and research_results.get("youtube_error"):
core_errored.append("youtube")
# Reddit with comments (ScrapeCreators)
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
if _is_reddit_comments_active(config, research_results):
core_active.append("reddit_comments")
else:
core_missing.append("reddit_comments")
if has_sc and research_results.get("reddit_error"):
core_errored.append("reddit_comments")
score_pct = int(len(core_active) / 5 * 100)
nudge_text = _build_nudge_text(core_missing, core_errored) if core_missing else None
return {
"score_pct": score_pct,
"core_active": core_active,
"core_missing": core_missing,
"core_errored": core_errored,
"nudge_text": nudge_text,
}
def _build_nudge_text(core_missing: List[str], core_errored: List[str]) -> str:
"""Build human-readable nudge text describing what was missed.
Prioritizes free suggestions before paid ones.
"""
lines: List[str] = []
# Describe what was missed
missed_parts: List[str] = []
for src in core_missing:
label = SOURCE_LABELS[src]
if src in core_errored:
missed_parts.append(f"{label} (errored this run)")
else:
missed_parts.append(label)
active_count = 5 - len(core_missing)
lines.append(f"Research quality: {active_count}/5 core sources.")
lines.append(f"Missing: {', '.join(missed_parts)}.")
lines.append("")
# Free suggestions first
free_suggestions: List[str] = []
paid_suggestions: List[str] = []
if "x" in core_missing:
if "x" in core_errored:
free_suggestions.append(
"X errored — try refreshing your browser cookies "
"(log into x.com, then re-run)."
)
else:
free_suggestions.append(
"X/Twitter: scan browser cookies automatically — "
"just log into x.com in any browser and re-run."
)
if "youtube" in core_missing:
if "youtube" in core_errored:
free_suggestions.append(
"YouTube errored — check that yt-dlp is up to date: "
"brew upgrade yt-dlp"
)
else:
free_suggestions.append(
"YouTube: install yt-dlp — brew install yt-dlp"
)
if "reddit_comments" in core_missing:
if "reddit_comments" in core_errored:
paid_suggestions.append(
"Reddit comments errored — check your ScrapeCreators API key "
"at scrapecreators.com."
)
else:
paid_suggestions.append(
"Reddit with comments: add SCRAPECREATORS_API_KEY — "
"100 free API calls, no credit card — scrapecreators.com"
)
if free_suggestions:
lines.append("Free fixes:")
for s in free_suggestions:
lines.append(f" - {s}")
lines.append("")
if paid_suggestions:
lines.append("Paid options:")
for s in paid_suggestions:
lines.append(f" - {s}")
lines.append("")
lines.append("last30days has no affiliation with any API provider.")
return "\n".join(lines)
+259
View File
@@ -0,0 +1,259 @@
"""Standalone Reddit public JSON search module.
Searches Reddit using the free public JSON endpoints (no API key required).
Promoted from last-resort fallback to robust primary free path.
Endpoints:
- Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit}
- Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month
Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
network timeouts, and missing subreddits.
"""
import json
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Any, Dict, List, Optional
USER_AGENT = "last30days/3.0 (research tool)"
# Depth-aware limits for thread counts
DEPTH_LIMITS = {
"quick": 10,
"default": 25,
"deep": 50,
}
MAX_RETRIES = 3
BASE_BACKOFF = 2.0 # seconds
def _log(msg: str):
"""Log to stderr."""
sys.stderr.write(f"[RedditPublic] {msg}\n")
sys.stderr.flush()
def _url_encode(text: str) -> str:
"""URL-encode a query string."""
return urllib.parse.quote_plus(text)
def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
"""Fetch JSON from a URL with retry on 429 and error handling.
Returns parsed JSON dict, or None on unrecoverable failure.
"""
headers = {
"User-Agent": USER_AGENT,
"Accept": "application/json",
}
req = urllib.request.Request(url, headers=headers)
for attempt in range(MAX_RETRIES):
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
content_type = resp.headers.get("Content-Type", "")
if "json" not in content_type and "text/html" in content_type:
_log(f"Anti-bot HTML response (Content-Type: {content_type})")
return None
body = resp.read().decode("utf-8")
return json.loads(body)
except urllib.error.HTTPError as e:
if e.code == 429:
delay = BASE_BACKOFF * (2 ** attempt)
retry_after = None
if hasattr(e, "headers"):
retry_after = e.headers.get("Retry-After")
if retry_after:
try:
delay = float(retry_after)
except ValueError:
pass
_log(f"429 rate limited, retry {attempt + 1}/{MAX_RETRIES} after {delay:.1f}s")
if attempt < MAX_RETRIES - 1:
time.sleep(delay)
continue
# Last attempt exhausted
_log("429 retries exhausted")
return None
elif e.code == 404:
_log(f"404 not found: {url}")
return None
elif e.code == 403:
_log(f"403 forbidden: {url}")
return None
else:
_log(f"HTTP {e.code}: {e.reason}")
return None
except (urllib.error.URLError, OSError, TimeoutError) as e:
_log(f"Network error: {e}")
return None
except json.JSONDecodeError as e:
_log(f"JSON decode error: {e}")
return None
return None
def _parse_posts(data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Parse Reddit listing JSON into normalized post dicts."""
if not data:
return []
children = data.get("data", {}).get("children", [])
posts = []
for child in children:
if child.get("kind") != "t3":
continue
post = child.get("data", {})
permalink = str(post.get("permalink", "")).strip()
if not permalink or "/comments/" not in permalink:
continue
score = int(post.get("score", 0) or 0)
num_comments = int(post.get("num_comments", 0) or 0)
selftext = str(post.get("selftext", ""))
author = str(post.get("author", "[deleted]"))
created_utc = post.get("created_utc")
# Parse date
date_str = None
if created_utc:
try:
from datetime import datetime, timezone
dt = datetime.fromtimestamp(float(created_utc), tz=timezone.utc)
date_str = dt.strftime("%Y-%m-%d")
except (ValueError, TypeError, OSError):
pass
posts.append({
"id": "", # Will be assigned after dedup
"title": str(post.get("title", "")).strip(),
"url": f"https://www.reddit.com{permalink}",
"score": score,
"num_comments": num_comments,
"subreddit": str(post.get("subreddit", "")).strip(),
"created_utc": float(created_utc) if created_utc else None,
"author": author if author not in ("[deleted]", "[removed]") else "[deleted]",
"selftext": selftext[:500] if selftext else "",
# Normalized fields matching ScrapeCreators output
"date": date_str,
"engagement": {
"score": score,
"num_comments": num_comments,
"upvote_ratio": post.get("upvote_ratio"),
},
"relevance": _compute_relevance(score, num_comments),
"why_relevant": "Reddit public search",
})
return posts
def _compute_relevance(score: int, num_comments: int) -> float:
"""Estimate relevance from engagement signals."""
score_component = min(1.0, max(0.0, score / 500.0))
comments_component = min(1.0, max(0.0, num_comments / 200.0))
return round((score_component * 0.6) + (comments_component * 0.4), 3)
def search(
query: str,
depth: str = "default",
subreddit: Optional[str] = None,
timeout: int = 15,
) -> List[Dict[str, Any]]:
"""Search Reddit via the public JSON endpoint.
Args:
query: Search query string
depth: 'quick', 'default', or 'deep' — controls result limit
subreddit: Optional subreddit name (without r/) for scoped search
timeout: HTTP timeout in seconds
Returns:
List of normalized post dicts. Empty list on any failure.
"""
limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
encoded_query = _url_encode(query)
if subreddit:
sub = subreddit.lstrip("r/").strip()
url = (
f"https://www.reddit.com/r/{sub}/search.json"
f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
)
else:
url = (
f"https://www.reddit.com/search.json"
f"?q={encoded_query}&sort=relevance&t=month&limit={limit}&raw_json=1"
)
data = _fetch_json(url, timeout=timeout)
posts = _parse_posts(data)
# Dedupe by URL and assign IDs
seen_urls = set()
unique = []
for post in posts:
if post["url"] not in seen_urls:
seen_urls.add(post["url"])
unique.append(post)
for i, post in enumerate(unique):
post["id"] = f"R{i + 1}"
return unique[:limit]
def search_reddit_public(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
) -> List[Dict[str, Any]]:
"""High-level Reddit public search matching the openai_reddit interface.
Runs global search, deduplicates, filters by date range, and sorts
by engagement. Compatible as a drop-in replacement in the fallback chain.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
Returns:
List of normalized item dicts matching ScrapeCreators output format.
"""
results = search(topic, depth=depth)
# Date filter: keep posts in range or with unknown dates
filtered = []
for item in results:
d = item.get("date")
if d is None or (from_date <= d <= to_date):
filtered.append(item)
# Sort by engagement (score desc)
filtered.sort(
key=lambda x: x.get("engagement", {}).get("score", 0),
reverse=True,
)
# Re-index IDs
for i, item in enumerate(filtered):
item["id"] = f"R{i + 1}"
return filtered
+32 -6
View File
@@ -115,7 +115,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
lines.append("") lines.append("")
lines.append("---") lines.append("---")
lines.append("**⚡ Want better results?** Add API keys to unlock Reddit, TikTok, Instagram & X data:") lines.append("**⚡ Want better results?** Add API keys to unlock Reddit, TikTok, Instagram & X data:")
lines.append("- `SCRAPECREATORS_API_KEY` → Reddit + TikTok + Instagram (one key, all three!) — real upvotes, comments, views") lines.append("- `SCRAPECREATORS_API_KEY` → Reddit + TikTok + Instagram (one key, all three!) — 100 free calls, no CC — scrapecreators.com (no affiliation)")
lines.append("- `XAI_API_KEY` → X posts with real likes & reposts") lines.append("- `XAI_API_KEY` → X posts with real likes & reposts")
lines.append("- `OPENAI_API_KEY` (legacy) → Reddit threads (slower, higher cost)") lines.append("- `OPENAI_API_KEY` (legacy) → Reddit threads (slower, higher cost)")
lines.append("- Edit `~/.config/last30days/.env` to add keys") lines.append("- Edit `~/.config/last30days/.env` to add keys")
@@ -143,7 +143,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*") lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*")
lines.append("") lines.append("")
elif report.mode == "x-only" and missing_keys in ("reddit", "none"): elif report.mode == "x-only" and missing_keys in ("reddit", "none"):
lines.append("*💡 Tip: Add `SCRAPECREATORS_API_KEY` for Reddit + TikTok + Instagram data (one key, all three) and better triangulation.*") lines.append("*💡 Tip: Add `SCRAPECREATORS_API_KEY` for Reddit + TikTok + Instagram data (one key, all three) — 100 free calls, no CC — scrapecreators.com (no affiliation)*")
lines.append("") lines.append("")
# Reddit items # Reddit items
@@ -257,11 +257,15 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}{_xref_tag(item)}") lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}{_xref_tag(item)}")
lines.append(f" {item.title}") lines.append(f" {item.title}")
lines.append(f" {item.url}") lines.append(f" {item.url}")
if item.transcript_highlights:
lines.append(" Highlights:")
for hl in item.transcript_highlights[:5]:
lines.append(f' - "{hl}"')
if item.transcript_snippet: if item.transcript_snippet:
snippet = item.transcript_snippet[:200] word_count = len(item.transcript_snippet.split())
if len(item.transcript_snippet) > 200: lines.append(f" <details><summary>Full transcript ({word_count} words)</summary>")
snippet += "..." lines.append(f" {item.transcript_snippet}")
lines.append(f" Transcript: {snippet}") lines.append(" </details>")
lines.append(f" *{item.why_relevant}*") lines.append(f" *{item.why_relevant}*")
lines.append("") lines.append("")
@@ -521,6 +525,28 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
return "\n".join(lines) return "\n".join(lines)
def render_quality_nudge(quality: dict) -> str:
"""Render the quality score nudge block.
Args:
quality: Dict from quality_nudge.compute_quality_score()
Returns:
Markdown string with quality nudge, or empty string if no nudge.
"""
nudge_text = quality.get("nudge_text")
if not nudge_text:
return ""
lines = []
lines.append("---")
lines.append(f"**🔍 Research Coverage: {quality['score_pct']}%**")
lines.append("")
lines.append(nudge_text)
lines.append("")
return "\n".join(lines)
def render_source_status(report: schema.Report, source_info: dict = None) -> str: def render_source_status(report: schema.Report, source_info: dict = None) -> str:
"""Render source status footer showing what was used/skipped and why. """Render source status footer showing what was used/skipped and why.
+182
View File
@@ -0,0 +1,182 @@
"""
Safari binary cookie extractor for macOS.
Parses ~/Library/Cookies/Cookies.binarycookies (unencrypted binary format)
using only stdlib. Zero pip dependencies.
Reference: github.com/mdegrazia/Safari-Binary-Cookie-Parser
"""
from __future__ import annotations
import io
import struct
import sys
from pathlib import Path
# Mac epoch: 2001-01-01 00:00:00 UTC (not used for filtering, but documented)
_MAC_EPOCH_OFFSET = 978307200 # seconds between Unix epoch and Mac epoch
_MAGIC = b"cook"
def _read_null_terminated(data: bytes, offset: int) -> str:
"""Read a null-terminated string from data starting at offset."""
end = data.find(b"\x00", offset)
if end == -1:
end = len(data)
return data[offset:end].decode("utf-8", errors="replace")
def _parse_cookie_record(data: bytes) -> dict | None:
"""Parse a single cookie record. Returns dict with url, name, value, path or None."""
if len(data) < 44:
return None
try:
(size,) = struct.unpack("<I", data[0:4])
# flags at offset 4 (4 bytes, little-endian) — not needed for extraction
(url_offset,) = struct.unpack("<I", data[16:20])
(name_offset,) = struct.unpack("<I", data[20:24])
(path_offset,) = struct.unpack("<I", data[24:28])
(value_offset,) = struct.unpack("<I", data[28:32])
# expiry at offset 40 (8-byte double, little-endian) — not needed for filtering
# creation at offset 48 (8-byte double, little-endian) — not needed
url = _read_null_terminated(data, url_offset)
name = _read_null_terminated(data, name_offset)
path = _read_null_terminated(data, path_offset)
value = _read_null_terminated(data, value_offset)
return {"url": url, "name": name, "value": value, "path": path}
except (struct.error, IndexError, UnicodeDecodeError):
return None
def _parse_page(page_data: bytes) -> list[dict]:
"""Parse a single page of cookies. Returns list of cookie dicts."""
cookies = []
if len(page_data) < 8:
return cookies
# Page header: 4 bytes (always 00 00 01 00), then 4-byte LE cookie count
try:
(num_cookies,) = struct.unpack("<I", page_data[4:8])
except struct.error:
return cookies
# Sanity check
if num_cookies > 10000:
return cookies
# Cookie offsets: array of 4-byte LE uint32 starting at offset 8
offsets_end = 8 + num_cookies * 4
if offsets_end > len(page_data):
return cookies
for i in range(num_cookies):
off_start = 8 + i * 4
try:
(cookie_offset,) = struct.unpack("<I", page_data[off_start : off_start + 4])
except struct.error:
continue
if cookie_offset >= len(page_data):
continue
cookie_data = page_data[cookie_offset:]
record = _parse_cookie_record(cookie_data)
if record:
cookies.append(record)
return cookies
def extract_safari_cookies_macos(
domain: str, cookie_names: list[str]
) -> dict[str, str] | None:
"""
Extract cookies from Safari on macOS.
Args:
domain: Domain to match (substring match, e.g. "x.com")
cookie_names: List of cookie names to extract (e.g. ["auth_token", "ct0"])
Returns:
Dict mapping cookie name to value for found cookies, or None on failure.
"""
if sys.platform != "darwin":
return None
cookie_path = Path.home() / "Library" / "Cookies" / "Cookies.binarycookies"
try:
raw = cookie_path.read_bytes()
except FileNotFoundError:
return None
except PermissionError:
print(
"[safari] Permission denied reading Cookies.binarycookies. "
"Enable Full Disk Access for Terminal in System Settings > "
"Privacy & Security > Full Disk Access.",
file=sys.stderr,
)
return None
except OSError:
return None
return _parse_binary_cookies(raw, domain, cookie_names)
def _parse_binary_cookies(
raw: bytes, domain: str, cookie_names: list[str]
) -> dict[str, str] | None:
"""Parse raw binary cookie data. Separated for testability."""
if len(raw) < 8:
return None
# Validate magic
if raw[:4] != _MAGIC:
return None
try:
(num_pages,) = struct.unpack(">I", raw[4:8])
except struct.error:
return None
if num_pages > 100000:
return None
# Read page sizes (big-endian uint32 array)
page_sizes_end = 8 + num_pages * 4
if page_sizes_end > len(raw):
return None
page_sizes = []
for i in range(num_pages):
off = 8 + i * 4
try:
(ps,) = struct.unpack(">I", raw[off : off + 4])
page_sizes.append(ps)
except struct.error:
return None
# Parse each page
names_set = set(cookie_names)
result: dict[str, str] = {}
offset = page_sizes_end
for ps in page_sizes:
if offset + ps > len(raw):
break
page_data = raw[offset : offset + ps]
cookies = _parse_page(page_data)
for c in cookies:
# Substring match on domain (handles leading dots like ".x.com")
if domain in c["url"] and c["name"] in names_set:
result[c["name"]] = c["value"]
offset += ps
if not result:
return None
return result
+3
View File
@@ -210,6 +210,7 @@ class YouTubeItem:
date_confidence: str = "high" # YouTube dates are always reliable date_confidence: str = "high" # YouTube dates are always reliable
engagement: Optional[Engagement] = None engagement: Optional[Engagement] = None
transcript_snippet: str = "" transcript_snippet: str = ""
transcript_highlights: List[str] = field(default_factory=list)
relevance: float = 0.7 relevance: float = 0.7
why_relevant: str = "" why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores) subs: SubScores = field(default_factory=SubScores)
@@ -226,6 +227,7 @@ class YouTubeItem:
'date_confidence': self.date_confidence, 'date_confidence': self.date_confidence,
'engagement': self.engagement.to_dict() if self.engagement else None, 'engagement': self.engagement.to_dict() if self.engagement else None,
'transcript_snippet': self.transcript_snippet, 'transcript_snippet': self.transcript_snippet,
'transcript_highlights': self.transcript_highlights,
'relevance': self.relevance, 'relevance': self.relevance,
'why_relevant': self.why_relevant, 'why_relevant': self.why_relevant,
'subs': self.subs.to_dict(), 'subs': self.subs.to_dict(),
@@ -655,6 +657,7 @@ class Report:
date_confidence=y.get('date_confidence', 'high'), date_confidence=y.get('date_confidence', 'high'),
engagement=eng, engagement=eng,
transcript_snippet=y.get('transcript_snippet', ''), transcript_snippet=y.get('transcript_snippet', ''),
transcript_highlights=y.get('transcript_highlights', []),
relevance=y.get('relevance', 0.7), relevance=y.get('relevance', 0.7),
why_relevant=y.get('why_relevant', ''), why_relevant=y.get('why_relevant', ''),
subs=subs, subs=subs,
+186
View File
@@ -0,0 +1,186 @@
"""First-run setup wizard for last30days.
Detects first run, performs auto-setup (cookie extraction + yt-dlp check),
and writes configuration. The actual wizard UI is SKILL.md-driven (the LLM
presents it), but this module provides the detection and setup actions.
"""
import logging
import shutil
import subprocess
from pathlib import Path
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
def is_first_run(config: Dict[str, Any]) -> bool:
"""Return True if the setup wizard has not been completed.
Checks for SETUP_COMPLETE in the config dict. If it's not set
(None or empty string), the user hasn't gone through setup yet.
"""
return not config.get("SETUP_COMPLETE")
def run_auto_setup(config: Dict[str, Any]) -> Dict[str, Any]:
"""Perform the auto-setup actions.
- Runs cookie extraction in auto mode for all registered domains
- Checks if yt-dlp is installed
Returns:
Dict with keys:
cookies_found: {source_name: browser_name} for each source where cookies were found
ytdlp_installed: bool
env_written: bool (always False here caller writes config separately)
"""
from . import cookie_extract
from .env import COOKIE_DOMAINS
cookies_found: Dict[str, str] = {}
for source_name, spec in COOKIE_DOMAINS.items():
domain = spec["domain"]
cookie_names = spec["cookies"]
try:
result = cookie_extract.extract_cookies_with_source("auto", domain, cookie_names)
except Exception as exc:
logger.debug("Cookie extraction failed for %s: %s", source_name, exc)
continue
if result is not None:
_cookies, browser_name = result
cookies_found[source_name] = browser_name
# Check yt-dlp availability and install via Homebrew if missing
ytdlp_action: str
if shutil.which("yt-dlp") is not None:
ytdlp_installed = True
ytdlp_action = "already_installed"
elif shutil.which("brew") is not None:
brew_stderr = ""
try:
proc = subprocess.run(
["brew", "install", "yt-dlp"],
capture_output=True, text=True, timeout=120,
)
if proc.returncode == 0:
ytdlp_installed = True
ytdlp_action = "installed"
else:
ytdlp_installed = False
ytdlp_action = "install_failed"
brew_stderr = proc.stderr
logger.warning("brew install yt-dlp failed: %s", proc.stderr)
except Exception as exc:
ytdlp_installed = False
ytdlp_action = "install_failed"
brew_stderr = str(exc)
logger.warning("brew install yt-dlp exception: %s", exc)
else:
ytdlp_installed = False
ytdlp_action = "no_homebrew"
results: Dict[str, Any] = {
"cookies_found": cookies_found,
"ytdlp_installed": ytdlp_installed,
"ytdlp_action": ytdlp_action,
"env_written": False,
}
if ytdlp_action == "install_failed":
results["ytdlp_stderr"] = brew_stderr
return results
def write_setup_config(env_path: Path, from_browser: str = "auto") -> bool:
"""Write SETUP_COMPLETE and FROM_BROWSER to the .env file.
Creates the file and parent directories if needed.
Appends to existing file without overwriting existing keys.
Args:
env_path: Path to the .env file (e.g. ~/.config/last30days/.env)
from_browser: Browser extraction mode to write (default: "auto")
Returns:
True if config was written successfully, False on error.
"""
try:
env_path = Path(env_path)
env_path.parent.mkdir(parents=True, exist_ok=True)
# Read existing content to avoid overwriting keys
existing_keys: set = set()
existing_content = ""
if env_path.exists():
existing_content = env_path.read_text(encoding="utf-8")
for line in existing_content.splitlines():
stripped = line.strip()
if stripped and not stripped.startswith("#") and "=" in stripped:
key = stripped.split("=", 1)[0].strip()
existing_keys.add(key)
lines_to_add = []
if "SETUP_COMPLETE" not in existing_keys:
lines_to_add.append("SETUP_COMPLETE=true")
if "FROM_BROWSER" not in existing_keys:
lines_to_add.append(f"FROM_BROWSER={from_browser}")
if not lines_to_add:
return True # Nothing to write, already configured
# Ensure trailing newline before appending
with open(env_path, "a", encoding="utf-8") as f:
if existing_content and not existing_content.endswith("\n"):
f.write("\n")
f.write("\n".join(lines_to_add) + "\n")
return True
except OSError as exc:
logger.error("Failed to write setup config to %s: %s", env_path, exc)
return False
def get_setup_status_text(results: Dict[str, Any]) -> str:
"""Return a human-readable summary of auto-setup results.
Args:
results: Dict from run_auto_setup()
Returns:
Multi-line status text.
"""
lines = []
lines.append("Setup complete! Here's what I found:")
lines.append("")
cookies_found = results.get("cookies_found", {})
if cookies_found:
for source, browser in cookies_found.items():
lines.append(f" - {source.upper()} cookies found in {browser}")
else:
lines.append(" - No browser cookies found for X/Twitter")
ytdlp_action = results.get("ytdlp_action", "")
if ytdlp_action == "installed":
lines.append(" - Installed yt-dlp via Homebrew")
elif ytdlp_action == "install_failed":
lines.append(" - yt-dlp install failed \u2014 run `brew install yt-dlp` manually")
elif ytdlp_action == "no_homebrew":
lines.append(" - yt-dlp not found. Install Homebrew first, then: brew install yt-dlp")
elif ytdlp_action == "already_installed":
lines.append(" - yt-dlp already installed")
elif results.get("ytdlp_installed", False):
lines.append(" - yt-dlp is installed (YouTube search ready)")
else:
lines.append(" - yt-dlp not found (install with: brew install yt-dlp)")
env_written = results.get("env_written", False)
if env_written:
lines.append("")
lines.append("Configuration saved. Future runs will auto-detect your browsers.")
return "\n".join(lines)
+1 -1
View File
@@ -3,7 +3,7 @@
Uses ScrapeCreators REST API to search TikTok by keyword, extract engagement Uses ScrapeCreators REST API to search TikTok by keyword, extract engagement
metrics (views, likes, comments, shares), and fetch video transcripts. metrics (views, likes, comments, shares), and fetch video transcripts.
Requires SCRAPECREATORS_API_KEY in config. 100 free credits, then PAYG. Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
API docs: https://scrapecreators.com/docs API docs: https://scrapecreators.com/docs
""" """
+177 -107
View File
@@ -417,120 +417,190 @@ class ProgressDisplay:
sys.stderr.flush() sys.stderr.flush()
def show_diagnostic_banner(diag: dict): def _build_status_banner(diag: dict) -> list[str]:
"""Show pre-flight source status banner when sources are missing. """Build the status banner lines (plain text, no ANSI).
Returns a list of strings, each being a line of the banner box.
Args: Args:
diag: Dict from env diagnostics with keys: diag: Dict with keys:
openai, xai, x_source, bird_installed, bird_authenticated, setup_complete, reddit_source, x_source, x_method,
bird_username, youtube, web_search_backend youtube, tiktok, instagram, hackernews, polymarket,
bluesky, truthsocial, xiaohongshu, scrapecreators,
web_search_backend
""" """
has_openai = diag.get("openai", False) setup_complete = diag.get("setup_complete", False)
has_reddit_public = diag.get("reddit_public", False) has_sc = diag.get("scrapecreators", False)
has_reddit = has_openai or has_reddit_public
has_x = diag.get("x_source") is not None
has_youtube = diag.get("youtube", False)
has_xiaohongshu = diag.get("xiaohongshu", False)
has_web = diag.get("web_search_backend") is not None
# If everything is available, no banner needed # --- Build active sources list: (label, method_label) ---
if has_reddit and has_x and has_youtube and has_web: active: list[str] = []
return
lines = [] # Reddit — always available; what matters to users is comments or not
reddit_src = diag.get("reddit_source")
if reddit_src == "scrapecreators":
active.append("Reddit (with comments)")
else:
active.append("Reddit (threads only)")
# X/Twitter
x_source = diag.get("x_source")
x_method = diag.get("x_method")
if x_source:
if x_method and x_method.startswith("browser-"):
browser = x_method.split("-", 1)[1].capitalize()
active.append(f"X ({browser})")
elif x_method == "env":
active.append("X (env)")
elif x_method == "api":
active.append("X (xAI)")
else:
active.append("X")
# YouTube
if diag.get("youtube"):
active.append("YouTube")
# HN — always available
if diag.get("hackernews"):
active.append("HN")
# Polymarket — always available
if diag.get("polymarket"):
active.append("Polymarket")
# TikTok (requires SC or Apify)
if diag.get("tiktok"):
active.append("TikTok")
# Instagram (requires SC)
if diag.get("instagram"):
active.append("Instagram")
# Bluesky
if diag.get("bluesky"):
active.append("Bluesky")
# Truth Social
if diag.get("truthsocial"):
active.append("Truth Social")
# Xiaohongshu
if diag.get("xiaohongshu"):
active.append("Xiaohongshu")
# --- Format active sources into wrapped lines ---
BOX_INNER = 53 # characters inside the box (between │ and │)
PREFIX = " " # 2-space indent inside box
def _wrap_sources(sources: list[str]) -> list[str]:
"""Wrap source labels into lines that fit the box width."""
result_lines: list[str] = []
current = PREFIX
for i, s in enumerate(sources):
token = f"{s}"
sep = " " if current != PREFIX else ""
if len(current) + len(sep) + len(token) > BOX_INNER:
result_lines.append(current)
current = PREFIX + token
else:
current += sep + token
if current.strip():
result_lines.append(current)
return result_lines
source_lines = _wrap_sources(active)
# --- Title ---
if not setup_complete:
title = "/last30days v3.0 — First Run"
else:
title = "/last30days v3.0 — Source Status"
# --- Build upgrade suggestions ---
suggestions: list[str] = []
if not setup_complete:
suggestions.append("Run /last30days setup to unlock more sources")
else:
# Recommend ScrapeCreators if missing
if not has_sc:
suggestions.append("⭐ Add SCRAPECREATORS_API_KEY for Reddit comments")
suggestions.append(" + TikTok + Instagram")
suggestions.append(" 100 free calls, no CC — scrapecreators.com (no affiliation)")
# --- Assemble box lines ---
# Collect all content lines, then determine box width dynamically.
content: list[str] = []
content.append(f" {title}")
content.append("") # blank line
for sl in source_lines:
content.append(sl)
if suggestions:
content.append("") # blank line
for sg in suggestions:
content.append(f" {sg}")
content.append("") # blank line
content.append(" Config: ~/.config/last30days/.env")
# Width = widest content line + 1 for right margin
width = max(len(line) for line in content) + 1
if width < 53:
width = 53
lines: list[str] = []
lines.append("\u250c" + "\u2500" * width + "\u2510")
for c in content:
lines.append("\u2502" + c.ljust(width) + "\u2502")
lines.append("\u2514" + "\u2500" * width + "\u2518")
return lines
def _colorize_banner(lines: list[str]) -> list[str]:
"""Apply ANSI colors to plain-text banner lines for TTY output."""
colored: list[str] = []
for line in lines:
if line.startswith("\u250c") or line.startswith("\u2514"):
colored.append(f"{Colors.DIM}{line}{Colors.RESET}")
elif line.startswith("\u2502"):
inner = line[1:-1] # strip box chars on both sides
inner_width = len(inner)
# Colorize check marks green, star yellow
inner = inner.replace("\u2705", f"{Colors.GREEN}\u2705{Colors.RESET}")
inner = inner.replace("\u2b50", f"{Colors.YELLOW}\u2b50{Colors.RESET}")
# Bold the title line
if "/last30days v3.0" in inner:
stripped = inner.strip()
inner = f" {Colors.BOLD}{stripped}{Colors.RESET}"
# Re-pad to original width (ANSI codes are zero-width)
visible_len = 1 + len(stripped)
inner = inner + " " * max(0, inner_width - visible_len)
colored.append(f"{Colors.DIM}\u2502{Colors.RESET}{inner}{Colors.DIM}\u2502{Colors.RESET}")
else:
colored.append(line)
return colored
def show_diagnostic_banner(diag: dict):
"""Show pre-flight source status banner.
Free-first design: leads with what's working (✅), not what's broken.
Shows upgrade suggestions only when relevant.
Args:
diag: Dict with keys:
setup_complete, reddit_source, x_source, x_method,
youtube, tiktok, instagram, hackernews, polymarket,
bluesky, truthsocial, xiaohongshu, scrapecreators,
web_search_backend
"""
lines = _build_status_banner(diag)
if IS_TTY: if IS_TTY:
lines.append(f"{Colors.DIM}┌─────────────────────────────────────────────────────┐{Colors.RESET}") lines = _colorize_banner(lines)
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.BOLD}/last30days v2.1 — Source Status{Colors.RESET} {Colors.DIM}{Colors.RESET}")
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.DIM}{Colors.RESET}")
# Reddit
if has_openai:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.GREEN}✅ Reddit{Colors.RESET} — OpenAI/Codex auth found {Colors.DIM}{Colors.RESET}")
elif has_reddit_public:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.GREEN}✅ Reddit{Colors.RESET} — Public Reddit search (no key) {Colors.DIM}{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.RED}❌ Reddit{Colors.RESET} — No OPENAI_API_KEY {Colors.DIM}{Colors.RESET}")
lines.append(f"{Colors.DIM}{Colors.RESET} └─ Add to ~/.config/last30days/.env {Colors.DIM}{Colors.RESET}")
# X/Twitter
if has_x:
source = diag.get("x_source", "")
username = diag.get("bird_username", "")
label = f"Bird ({username})" if source == "bird" and username else source.upper()
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.GREEN}✅ X/Twitter{Colors.RESET}{label} {Colors.DIM}{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.RED}❌ X/Twitter{Colors.RESET} — No X auth or fallback key {Colors.DIM}{Colors.RESET}")
if diag.get("bird_installed"):
lines.append(f"{Colors.DIM}{Colors.RESET} └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY {Colors.DIM}{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}{Colors.RESET} └─ Needs Node.js 22+ (Bird is bundled) {Colors.DIM}{Colors.RESET}")
# YouTube
if has_youtube:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.GREEN}✅ YouTube{Colors.RESET} — yt-dlp found {Colors.DIM}{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.RED}❌ YouTube{Colors.RESET} — yt-dlp not installed {Colors.DIM}{Colors.RESET}")
lines.append(f"{Colors.DIM}{Colors.RESET} └─ Fix: brew install yt-dlp (free) {Colors.DIM}{Colors.RESET}")
# Xiaohongshu
if has_xiaohongshu:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.GREEN}✅ Xiaohongshu{Colors.RESET} — API connected + logged in {Colors.DIM}{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.YELLOW}⚡ Xiaohongshu{Colors.RESET} — API not connected/logged in {Colors.DIM}{Colors.RESET}")
# Web
if has_web:
backend = diag.get("web_search_backend", "")
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.GREEN}✅ Web{Colors.RESET}{backend} API {Colors.DIM}{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.YELLOW}⚡ Web{Colors.RESET} — Using assistant's search tool {Colors.DIM}{Colors.RESET}")
lines.append(f"{Colors.DIM}{Colors.RESET} {Colors.DIM}{Colors.RESET}")
lines.append(f"{Colors.DIM}{Colors.RESET} Config: {Colors.BOLD}~/.config/last30days/.env{Colors.RESET} {Colors.DIM}{Colors.RESET}")
lines.append(f"{Colors.DIM}└─────────────────────────────────────────────────────┘{Colors.RESET}")
else:
# Plain text for non-TTY (Claude Code / Codex)
lines.append("┌─────────────────────────────────────────────────────┐")
lines.append("│ /last30days v2.1 — Source Status │")
lines.append("│ │")
if has_openai:
lines.append("│ ✅ Reddit — OpenAI/Codex auth found │")
elif has_reddit_public:
lines.append("│ ✅ Reddit — Public Reddit search (no key) │")
else:
lines.append("│ ❌ Reddit — No OPENAI_API_KEY │")
lines.append("│ └─ Add to ~/.config/last30days/.env │")
if has_x:
lines.append("│ ✅ X/Twitter — available │")
else:
lines.append("│ ❌ X/Twitter — No X auth or fallback key │")
if diag.get("bird_installed"):
lines.append("│ └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY │")
else:
lines.append("│ └─ Needs Node.js 22+ (Bird is bundled) │")
if has_youtube:
lines.append("│ ✅ YouTube — yt-dlp found │")
else:
lines.append("│ ❌ YouTube — yt-dlp not installed │")
lines.append("│ └─ Fix: brew install yt-dlp (free) │")
if has_xiaohongshu:
lines.append("│ ✅ Xiaohongshu — API connected + logged in │")
else:
lines.append("│ ⚡ Xiaohongshu — API not connected/logged in │")
if has_web:
lines.append("│ ✅ Web — API search available │")
else:
lines.append("│ ⚡ Web — Using assistant's search tool │")
lines.append("│ │")
lines.append("│ Config: ~/.config/last30days/.env │")
lines.append("└─────────────────────────────────────────────────────┘")
sys.stderr.write("\n".join(lines) + "\n\n") sys.stderr.write("\n".join(lines) + "\n\n")
sys.stderr.flush() sys.stderr.flush()
+182 -7
View File
@@ -15,6 +15,8 @@ import shutil
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple from typing import Any, Dict, List, Optional, Set, Tuple
@@ -33,11 +35,57 @@ TRANSCRIPT_LIMITS = {
} }
# Max words to keep from each transcript # Max words to keep from each transcript
TRANSCRIPT_MAX_WORDS = 500 TRANSCRIPT_MAX_WORDS = 5000
from .relevance import token_overlap_relevance as _compute_relevance from .relevance import token_overlap_relevance as _compute_relevance
def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -> List[str]:
"""Extract quotable highlights from a YouTube transcript.
Similar to reddit_enrich.extract_comment_insights() but for
continuous speech-to-text rather than threaded comments.
"""
if not transcript:
return []
sentences = re.split(r'(?<=[.!?])\s+', transcript)
filler = [
r"^(hey |hi |what's up|welcome back|in today's video|don't forget to)",
r"(subscribe|like and comment|hit the bell|check out the link|down below)",
r"^(so |and |but |okay |alright |um |uh )",
r"(thanks for watching|see you (next|in the)|bye)",
]
topic_words = [w.lower() for w in topic.lower().split() if len(w) > 2]
candidates = []
for sent in sentences:
sent = sent.strip()
words = sent.split()
if len(words) < 8 or len(words) > 50:
continue
if any(re.search(p, sent, re.IGNORECASE) for p in filler):
continue
score = 0
if re.search(r'\d', sent):
score += 2
if re.search(r'[A-Z][a-z]+', sent):
score += 1
if '?' in sent:
score += 1
sent_lower = sent.lower()
if any(w in sent_lower for w in topic_words):
score += 2
candidates.append((score, sent))
candidates.sort(key=lambda x: -x[0])
return [sent for _, sent in candidates[:limit]]
def _log(msg: str): def _log(msg: str):
"""Log to stderr.""" """Log to stderr."""
sys.stderr.write(f"[YouTube] {msg}\n") sys.stderr.write(f"[YouTube] {msg}\n")
@@ -209,15 +257,114 @@ def _clean_vtt(vtt_text: str) -> str:
return re.sub(r'\s+', ' ', ' '.join(unique)).strip() return re.sub(r'\s+', ' ', ' '.join(unique)).strip()
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: _YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
"""Fetch auto-generated transcript for a YouTube video.
def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]:
"""Fetch YouTube transcript via direct HTTP without yt-dlp.
Scrapes the watch page HTML for the captions track URL in
ytInitialPlayerResponse, then fetches the VTT subtitle file.
Args:
video_id: YouTube video ID
timeout: HTTP request timeout in seconds
Returns:
Raw VTT text, or None if captions are unavailable.
"""
watch_url = f"https://www.youtube.com/watch?v={video_id}"
headers = {
"User-Agent": _YT_USER_AGENT,
"Accept-Language": "en-US,en;q=0.9",
}
# Step 1: Fetch the watch page HTML
req = urllib.request.Request(watch_url, headers=headers)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
html = resp.read().decode("utf-8", errors="replace")
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
_log(f"Direct transcript: failed to fetch watch page for {video_id}: {exc}")
return None
# Step 2: Extract captions URL from ytInitialPlayerResponse
# YouTube embeds this as a JS variable in the page HTML
match = re.search(
r'ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;(?:\s*var\s|\s*<\/script>)',
html,
)
if not match:
# Fallback: try the JSON embedded in the script tag
match = re.search(
r'var\s+ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;',
html,
)
if not match:
_log(f"Direct transcript: no ytInitialPlayerResponse found for {video_id}")
return None
try:
player_response = json.loads(match.group(1))
except json.JSONDecodeError:
_log(f"Direct transcript: failed to parse ytInitialPlayerResponse for {video_id}")
return None
# Navigate to caption tracks
captions = player_response.get("captions", {})
renderer = captions.get("playerCaptionsTracklistRenderer", {})
caption_tracks = renderer.get("captionTracks", [])
if not caption_tracks:
_log(f"Direct transcript: no caption tracks for {video_id}")
return None
# Find English track (prefer exact 'en', then any en variant, then first track)
base_url = None
for track in caption_tracks:
lang = track.get("languageCode", "")
if lang == "en":
base_url = track.get("baseUrl")
break
if not base_url:
for track in caption_tracks:
lang = track.get("languageCode", "")
if lang.startswith("en"):
base_url = track.get("baseUrl")
break
if not base_url:
# Fall back to first available track
base_url = caption_tracks[0].get("baseUrl")
if not base_url:
_log(f"Direct transcript: no baseUrl in caption tracks for {video_id}")
return None
# Step 3: Fetch the VTT subtitle file
sep = "&" if "?" in base_url else "?"
vtt_url = f"{base_url}{sep}fmt=vtt"
vtt_req = urllib.request.Request(vtt_url, headers=headers)
try:
with urllib.request.urlopen(vtt_req, timeout=timeout) as resp:
vtt_text = resp.read().decode("utf-8", errors="replace")
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
_log(f"Direct transcript: failed to fetch VTT for {video_id}: {exc}")
return None
if not vtt_text or not vtt_text.strip():
return None
return vtt_text
def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:
"""Fetch transcript using yt-dlp (original implementation).
Args: Args:
video_id: YouTube video ID video_id: YouTube video ID
temp_dir: Temporary directory for subtitle files temp_dir: Temporary directory for subtitle files
Returns: Returns:
Plaintext transcript string, or None if no captions available. Raw VTT text, or None if no captions available.
""" """
cmd = [ cmd = [
"yt-dlp", "yt-dlp",
@@ -265,11 +412,35 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
return None return None
try: try:
raw = vtt_path.read_text(encoding="utf-8", errors="replace") return vtt_path.read_text(encoding="utf-8", errors="replace")
except OSError: except OSError:
return None return None
transcript = _clean_vtt(raw)
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
"""Fetch auto-generated transcript for a YouTube video.
Uses yt-dlp when available (preferred, more robust). Falls back to
direct HTTP transcript fetching when yt-dlp is not installed.
Args:
video_id: YouTube video ID
temp_dir: Temporary directory for subtitle files
Returns:
Plaintext transcript string, or None if no captions available.
"""
raw_vtt = None
if is_ytdlp_installed():
raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
else:
_log("yt-dlp not installed, using direct HTTP transcript fetch")
raw_vtt = _fetch_transcript_direct(video_id)
if not raw_vtt:
return None
transcript = _clean_vtt(raw_vtt)
# Truncate to max words # Truncate to max words
words = transcript.split() words = transcript.split()
@@ -345,11 +516,15 @@ def search_and_transcribe(
top_ids = [item["video_id"] for item in items[:transcript_limit]] top_ids = [item["video_id"] for item in items[:transcript_limit]]
transcripts = fetch_transcripts_parallel(top_ids) transcripts = fetch_transcripts_parallel(top_ids)
# Step 3: Attach transcripts to items # Step 3: Attach transcripts and extract highlights
core_topic = _extract_core_subject(topic)
for item in items: for item in items:
vid = item["video_id"] vid = item["video_id"]
transcript = transcripts.get(vid) transcript = transcripts.get(vid)
item["transcript_snippet"] = transcript or "" item["transcript_snippet"] = transcript or ""
item["transcript_highlights"] = extract_transcript_highlights(
transcript or "", core_topic,
)
return {"items": items} return {"items": items}
+1 -1
View File
@@ -6,7 +6,7 @@ set -euo pipefail
# using `claude --print` to capture real end-to-end output. # using `claude --print` to capture real end-to-end output.
SKILL_DIR="$HOME/.claude/skills/last30days" SKILL_DIR="$HOME/.claude/skills/last30days"
REPO_DIR="/Users/mvanhorn/last30days-skill-private" REPO_DIR="/Users/mvanhorn/last30days-skill"
# Safety: always restore V2 SKILL.md on exit/crash # Safety: always restore V2 SKILL.md on exit/crash
cleanup() { cleanup() {
+350
View File
@@ -0,0 +1,350 @@
"""Tests for Chrome cookie extraction on macOS."""
import hashlib
import os
import sqlite3
import subprocess
import tempfile
from pathlib import Path
from unittest import mock
import pytest
from scripts.lib.chrome_cookies import (
CHROME_COOKIES_DB,
CHROME_IV_HEX,
CHROME_KEY_LENGTH,
CHROME_PBKDF2_ITERATIONS,
CHROME_SALT,
_derive_aes_key,
_get_chrome_encryption_key,
_get_db_version,
_remove_pkcs7_padding,
_decrypt_v10_value,
extract_chrome_cookies_macos,
)
# ---------------------------------------------------------------------------
# Helpers — create real encrypted cookie values using known key + system openssl
# ---------------------------------------------------------------------------
KNOWN_PASSPHRASE = b"test_passphrase_for_unit_tests"
KNOWN_AES_KEY = _derive_aes_key(KNOWN_PASSPHRASE)
def _encrypt_value_v10(plaintext: str, aes_key: bytes) -> bytes:
"""Encrypt a value the same way Chrome v10 does, using system openssl.
Returns b'v10' + AES-128-CBC ciphertext with PKCS7 padding.
"""
hex_key = aes_key.hex()
result = subprocess.run(
[
"openssl", "enc", "-aes-128-cbc", "-e",
"-K", hex_key,
"-iv", CHROME_IV_HEX,
],
input=plaintext.encode("utf-8"),
capture_output=True,
timeout=5,
)
assert result.returncode == 0, f"openssl encrypt failed: {result.stderr}"
return b"v10" + result.stdout
def _encrypt_value_v10_with_sha_prefix(plaintext: str, aes_key: bytes) -> bytes:
"""Encrypt with a 32-byte SHA-256 prefix (Chrome 130+ style)."""
raw = b"\x00" * 32 + plaintext.encode("utf-8")
hex_key = aes_key.hex()
result = subprocess.run(
[
"openssl", "enc", "-aes-128-cbc", "-e",
"-K", hex_key,
"-iv", CHROME_IV_HEX,
],
input=raw,
capture_output=True,
timeout=5,
)
assert result.returncode == 0, f"openssl encrypt failed: {result.stderr}"
return b"v10" + result.stdout
def _create_chrome_cookies_db(path: str, cookies: list[tuple], db_version: int = 20) -> None:
"""Create a minimal Chrome Cookies SQLite database.
cookies: list of (host_key, name, value, encrypted_value) tuples
"""
conn = sqlite3.connect(path)
c = conn.cursor()
c.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)")
c.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('version', ?)", (str(db_version),))
c.execute(
"CREATE TABLE IF NOT EXISTS cookies ("
" host_key TEXT NOT NULL,"
" name TEXT NOT NULL,"
" value TEXT NOT NULL DEFAULT '',"
" encrypted_value BLOB NOT NULL DEFAULT x'',"
" path TEXT NOT NULL DEFAULT '/',"
" expires_utc INTEGER NOT NULL DEFAULT 0,"
" is_secure INTEGER NOT NULL DEFAULT 1,"
" is_httponly INTEGER NOT NULL DEFAULT 1,"
" creation_utc INTEGER NOT NULL DEFAULT 0,"
" last_access_utc INTEGER NOT NULL DEFAULT 0,"
" has_expires INTEGER NOT NULL DEFAULT 1,"
" is_persistent INTEGER NOT NULL DEFAULT 1,"
" priority INTEGER NOT NULL DEFAULT 1,"
" samesite INTEGER NOT NULL DEFAULT 0,"
" source_scheme INTEGER NOT NULL DEFAULT 2,"
" source_port INTEGER NOT NULL DEFAULT 443,"
" last_update_utc INTEGER NOT NULL DEFAULT 0"
")"
)
for host_key, name, value, encrypted_value in cookies:
c.execute(
"INSERT INTO cookies (host_key, name, value, encrypted_value) VALUES (?, ?, ?, ?)",
(host_key, name, value, encrypted_value),
)
conn.commit()
conn.close()
# ---------------------------------------------------------------------------
# PKCS7 padding tests
# ---------------------------------------------------------------------------
class TestPkcs7Padding:
def test_valid_padding_1(self):
# 1 byte of padding
data = b"hello world!!!!!" + b"\x01"
assert _remove_pkcs7_padding(data) == b"hello world!!!!!"
def test_valid_padding_5(self):
data = b"hello world" + b"\x05\x05\x05\x05\x05"
assert _remove_pkcs7_padding(data) == b"hello world"
def test_valid_padding_16(self):
# Full block of padding
data = b"\x10" * 16
assert _remove_pkcs7_padding(data) == b""
def test_invalid_padding_zero(self):
data = b"hello\x00"
assert _remove_pkcs7_padding(data) is None
def test_invalid_padding_mismatch(self):
data = b"hello\x03\x03\x02"
assert _remove_pkcs7_padding(data) is None
def test_empty_data(self):
assert _remove_pkcs7_padding(b"") is None
# ---------------------------------------------------------------------------
# Key derivation test
# ---------------------------------------------------------------------------
class TestKeyDerivation:
def test_derive_aes_key_deterministic(self):
key1 = _derive_aes_key(b"my_passphrase")
key2 = _derive_aes_key(b"my_passphrase")
assert key1 == key2
assert len(key1) == 16
def test_derive_aes_key_different_passphrases(self):
key1 = _derive_aes_key(b"passphrase_a")
key2 = _derive_aes_key(b"passphrase_b")
assert key1 != key2
# ---------------------------------------------------------------------------
# Decryption test (real openssl, known key)
# ---------------------------------------------------------------------------
class TestDecryption:
def test_decrypt_v10_roundtrip(self):
"""Encrypt then decrypt — verifies the full pipeline works."""
original = "my_secret_cookie_value_12345"
encrypted = _encrypt_value_v10(original, KNOWN_AES_KEY)
assert encrypted[:3] == b"v10"
decrypted = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=20)
assert decrypted == original
def test_decrypt_v10_chrome130_with_sha_prefix(self):
"""Chrome 130+ (db_version >= 24) strips 32-byte SHA-256 prefix."""
original = "session_token_abc"
encrypted = _encrypt_value_v10_with_sha_prefix(original, KNOWN_AES_KEY)
decrypted = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=24)
assert decrypted == original
def test_decrypt_wrong_key_returns_none_or_garbage(self):
"""Wrong key should either fail decryption or produce garbage."""
original = "secret"
encrypted = _encrypt_value_v10(original, KNOWN_AES_KEY)
wrong_key = _derive_aes_key(b"wrong_passphrase")
result = _decrypt_v10_value(encrypted, wrong_key, db_version=20)
# Either None (padding check fails) or garbage (not the original)
assert result is None or result != original
def test_decrypt_empty_ciphertext(self):
"""v10 prefix with no ciphertext should return None."""
assert _decrypt_v10_value(b"v10", KNOWN_AES_KEY, db_version=20) is None
# ---------------------------------------------------------------------------
# Chrome not installed → returns None
# ---------------------------------------------------------------------------
class TestChromeNotInstalled:
def test_db_not_found(self):
with mock.patch(
"scripts.lib.chrome_cookies.CHROME_COOKIES_DB",
Path("/nonexistent/path/Cookies"),
):
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
assert result is None
# ---------------------------------------------------------------------------
# Keychain access denied → returns None
# ---------------------------------------------------------------------------
class TestKeychainDenied:
def test_security_command_fails(self):
with mock.patch("scripts.lib.chrome_cookies.subprocess.run") as mock_run:
mock_run.return_value = subprocess.CompletedProcess(
args=[], returncode=44, stdout="", stderr="security: SecKeychainSearchCopyNext: The specified item could not be found in the keychain."
)
result = _get_chrome_encryption_key()
assert result is None
def test_security_command_not_found(self):
with mock.patch("scripts.lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
result = _get_chrome_encryption_key()
assert result is None
# ---------------------------------------------------------------------------
# openssl not found → returns None
# ---------------------------------------------------------------------------
class TestOpensslNotFound:
def test_openssl_missing(self):
encrypted = _encrypt_value_v10("test", KNOWN_AES_KEY)
with mock.patch("scripts.lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
result = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=20)
assert result is None
# ---------------------------------------------------------------------------
# Unencrypted cookie values → returned as-is
# ---------------------------------------------------------------------------
class TestUnencryptedCookies:
def test_plain_value_returned(self, tmp_path):
"""Unencrypted cookies (value column populated) returned without decryption."""
db_path = str(tmp_path / "Cookies")
_create_chrome_cookies_db(db_path, [
(".x.com", "auth_token", "plain_token_value", b""),
(".x.com", "ct0", "plain_ct0_value", b""),
])
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
# No keychain needed for unencrypted values
with mock.patch("scripts.lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
assert result == {"auth_token": "plain_token_value", "ct0": "plain_ct0_value"}
# ---------------------------------------------------------------------------
# Full integration: mock DB with real v10 encryption, mock Keychain
# ---------------------------------------------------------------------------
class TestFullExtraction:
def test_encrypted_cookies_extracted(self, tmp_path):
"""End-to-end: create DB with real v10-encrypted values, extract them."""
auth_val = "my_auth_token_123"
ct0_val = "my_ct0_csrf_456"
encrypted_auth = _encrypt_value_v10(auth_val, KNOWN_AES_KEY)
encrypted_ct0 = _encrypt_value_v10(ct0_val, KNOWN_AES_KEY)
db_path = str(tmp_path / "Cookies")
_create_chrome_cookies_db(db_path, [
(".x.com", "auth_token", "", encrypted_auth),
(".x.com", "ct0", "", encrypted_ct0),
(".other.com", "other", "", b""), # unrelated cookie
])
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
with mock.patch(
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
return_value=KNOWN_PASSPHRASE,
):
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
assert result is not None
assert result["auth_token"] == auth_val
assert result["ct0"] == ct0_val
def test_no_matching_cookies_returns_none(self, tmp_path):
db_path = str(tmp_path / "Cookies")
_create_chrome_cookies_db(db_path, [
(".other.com", "session", "val", b""),
])
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
with mock.patch("scripts.lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
assert result is None
def test_chrome130_db_version_24(self, tmp_path):
"""Chrome 130+ with db_version >= 24 strips SHA-256 prefix."""
auth_val = "token_for_chrome130"
encrypted_auth = _encrypt_value_v10_with_sha_prefix(auth_val, KNOWN_AES_KEY)
db_path = str(tmp_path / "Cookies")
_create_chrome_cookies_db(db_path, [
(".x.com", "auth_token", "", encrypted_auth),
], db_version=24)
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
with mock.patch(
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
return_value=KNOWN_PASSPHRASE,
):
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
assert result is not None
assert result["auth_token"] == auth_val
# ---------------------------------------------------------------------------
# DB version detection
# ---------------------------------------------------------------------------
class TestDbVersion:
def test_reads_version_from_meta(self, tmp_path):
db_path = str(tmp_path / "test.db")
conn = sqlite3.connect(db_path)
c = conn.cursor()
c.execute("CREATE TABLE meta (key TEXT, value TEXT)")
c.execute("INSERT INTO meta VALUES ('version', '24')")
conn.commit()
assert _get_db_version(c) == 24
conn.close()
def test_no_meta_table_returns_zero(self, tmp_path):
db_path = str(tmp_path / "test.db")
conn = sqlite3.connect(db_path)
c = conn.cursor()
c.execute("CREATE TABLE dummy (x TEXT)")
conn.commit()
assert _get_db_version(c) == 0
conn.close()
+4 -2
View File
@@ -124,7 +124,8 @@ class TestLoadCodexAuth(unittest.TestCase):
class TestGetAvailableSourcesWithAuth(unittest.TestCase): class TestGetAvailableSourcesWithAuth(unittest.TestCase):
def test_codex_auth_ok_counts_as_openai(self): @patch("lib.bird_x.is_bird_installed", return_value=False)
def test_codex_auth_ok_counts_as_openai(self, _mock_bird):
config = { config = {
"OPENAI_API_KEY": "codex-token", "OPENAI_API_KEY": "codex-token",
"OPENAI_AUTH_STATUS": "ok", "OPENAI_AUTH_STATUS": "ok",
@@ -133,7 +134,8 @@ class TestGetAvailableSourcesWithAuth(unittest.TestCase):
result = env.get_available_sources(config) result = env.get_available_sources(config)
self.assertIn("reddit", result) self.assertIn("reddit", result)
def test_codex_auth_expired_not_counted(self): @patch("lib.bird_x.is_bird_installed", return_value=False)
def test_codex_auth_expired_not_counted(self, _mock_bird):
config = { config = {
"OPENAI_API_KEY": None, "OPENAI_API_KEY": None,
"OPENAI_AUTH_STATUS": "expired", "OPENAI_AUTH_STATUS": "expired",
+293
View File
@@ -0,0 +1,293 @@
"""Tests for browser cookie extraction module."""
import configparser
import sqlite3
import textwrap
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from unittest.mock import patch
import pytest
from scripts.lib.cookie_extract import (
extract_cookies,
extract_firefox_cookies,
_find_default_profile,
_get_firefox_profiles_dir,
)
@pytest.fixture
def mock_firefox_env(tmp_path):
"""Create a mock Firefox profiles directory with cookies.sqlite.
Returns (profiles_dir, profile_dir) for patching.
"""
def _make(
*,
profiles_ini=None, # type: Optional[str]
profiles=None, # type: Optional[Dict[str, List[Tuple[str, str, str]]]]
default_profile="abc123.default-release", # type: str
):
profiles_dir = tmp_path / "Firefox"
profiles_dir.mkdir(parents=True, exist_ok=True)
# Default: one profile with X cookies
if profiles is None:
profiles = {
default_profile: [
(".x.com", "auth_token", "tok_abc123"),
(".x.com", "ct0", "ct0_xyz789"),
(".example.com", "session", "sess_other"),
],
}
# Create profile directories with cookies databases
for profile_name, cookies in profiles.items():
profile_dir = profiles_dir / profile_name
profile_dir.mkdir(parents=True, exist_ok=True)
db_path = profile_dir / "cookies.sqlite"
conn = sqlite3.connect(str(db_path))
conn.execute(
"CREATE TABLE moz_cookies ("
" id INTEGER PRIMARY KEY,"
" name TEXT NOT NULL,"
" value TEXT NOT NULL,"
" host TEXT NOT NULL,"
" path TEXT DEFAULT '/',"
" expiry INTEGER DEFAULT 0,"
" isSecure INTEGER DEFAULT 1,"
" isHttpOnly INTEGER DEFAULT 1,"
" sameSite INTEGER DEFAULT 0,"
" schemeMap INTEGER DEFAULT 0"
")"
)
for host, name, value in cookies:
conn.execute(
"INSERT INTO moz_cookies (name, value, host) VALUES (?, ?, ?)",
(name, value, host),
)
conn.commit()
conn.close()
# Write profiles.ini
if profiles_ini is None:
profiles_ini = textwrap.dedent(f"""\
[General]
StartWithLastProfile=1
[Profile0]
Name=default-release
IsRelative=1
Path={default_profile}
Default=1
""")
(profiles_dir / "profiles.ini").write_text(profiles_ini)
return profiles_dir
return _make
class TestExtractFirefoxCookies:
"""Tests for extract_firefox_cookies."""
def test_valid_cookies_extracted(self, mock_firefox_env):
"""Cookies for the target domain are returned correctly."""
profiles_dir = mock_firefox_env()
with patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
):
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
assert result is not None
assert result["auth_token"] == "tok_abc123"
assert result["ct0"] == "ct0_xyz789"
assert "session" not in result # different domain cookie not included
def test_multiple_profiles_selects_default(self, mock_firefox_env):
"""When multiple profiles exist, the one with Default=1 is used."""
profiles_dir = mock_firefox_env(
profiles={
"aaa111.other": [
(".x.com", "auth_token", "wrong_token"),
],
"bbb222.default-release": [
(".x.com", "auth_token", "correct_token"),
(".x.com", "ct0", "correct_ct0"),
],
},
profiles_ini=textwrap.dedent("""\
[General]
StartWithLastProfile=1
[Profile0]
Name=other
IsRelative=1
Path=aaa111.other
[Profile1]
Name=default-release
IsRelative=1
Path=bbb222.default-release
Default=1
"""),
)
with patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
):
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
assert result is not None
assert result["auth_token"] == "correct_token"
assert result["ct0"] == "correct_ct0"
def test_firefox_not_installed(self):
"""Returns None when Firefox profiles directory doesn't exist."""
with patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=None,
):
result = extract_firefox_cookies(".x.com", ["auth_token"])
assert result is None
def test_cookies_sqlite_empty(self, mock_firefox_env):
"""Returns None when cookies.sqlite has no rows."""
profiles_dir = mock_firefox_env(
profiles={"abc123.default-release": []}, # no cookies
)
with patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
):
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
assert result is None
def test_domain_has_no_cookies(self, mock_firefox_env):
"""Returns None when cookies exist but not for the target domain."""
profiles_dir = mock_firefox_env(
profiles={
"abc123.default-release": [
(".example.com", "session", "sess_123"),
],
},
)
with patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
):
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
assert result is None
def test_malformed_profiles_ini_falls_back(self, mock_firefox_env):
"""Falls back to first profile on disk when profiles.ini is garbage."""
profiles_dir = mock_firefox_env(
profiles={
"zzz999.fallback": [
(".x.com", "auth_token", "fallback_token"),
],
},
profiles_ini="this is not valid ini content\n[[[broken",
)
with patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
):
result = extract_firefox_cookies(".x.com", ["auth_token"])
assert result is not None
assert result["auth_token"] == "fallback_token"
class TestExtractCookiesAuto:
"""Tests for extract_cookies with browser='auto'."""
def test_auto_macos_tries_chrome_then_firefox(self, mock_firefox_env):
"""On macOS, auto tries Chrome first, falls back to Firefox if Chrome fails."""
profiles_dir = mock_firefox_env()
with (
patch("scripts.lib.cookie_extract.platform.system", return_value="Darwin"),
patch(
"scripts.lib.cookie_extract.extract_chrome_cookies",
return_value=None,
),
patch(
"scripts.lib.cookie_extract.extract_safari_cookies",
return_value=None,
),
patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
),
):
result = extract_cookies("auto", ".x.com", ["auth_token", "ct0"])
# Chrome and Safari return None, Firefox succeeds
assert result is not None
assert result["auth_token"] == "tok_abc123"
assert result["ct0"] == "ct0_xyz789"
def test_auto_linux_tries_firefox_only(self, mock_firefox_env):
"""On Linux, auto only tries Firefox."""
profiles_dir = mock_firefox_env()
with (
patch("scripts.lib.cookie_extract.platform.system", return_value="Linux"),
patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
),
):
result = extract_cookies("auto", ".x.com", ["auth_token", "ct0"])
assert result is not None
assert result["auth_token"] == "tok_abc123"
def test_explicit_firefox(self, mock_firefox_env):
"""Explicit browser='firefox' goes directly to Firefox."""
profiles_dir = mock_firefox_env()
with patch(
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
return_value=profiles_dir,
):
result = extract_cookies("firefox", ".x.com", ["auth_token"])
assert result is not None
assert result["auth_token"] == "tok_abc123"
def test_unknown_browser_returns_none(self):
"""Unknown browser name returns None."""
result = extract_cookies("netscape", ".x.com", ["auth_token"])
assert result is None
def test_chrome_delegates_to_chrome_module(self):
"""Chrome extraction delegates to chrome_cookies module."""
with patch(
"scripts.lib.cookie_extract.extract_chrome_cookies",
return_value={"auth_token": "chrome_tok"},
):
result = extract_cookies("chrome", ".x.com", ["auth_token"])
assert result == {"auth_token": "chrome_tok"}
def test_safari_delegates_to_safari_module(self):
"""Safari extraction delegates to safari_cookies module."""
with patch(
"scripts.lib.cookie_extract.extract_safari_cookies",
return_value={"auth_token": "safari_tok"},
):
result = extract_cookies("safari", ".x.com", ["auth_token"])
assert result == {"auth_token": "safari_tok"}
+185
View File
@@ -0,0 +1,185 @@
"""Tests for browser cookie extraction integration in env.py."""
from unittest.mock import patch, MagicMock
import pytest
from scripts.lib.env import extract_browser_credentials, COOKIE_DOMAINS
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _base_config(**overrides):
"""Return a minimal config dict with common defaults."""
cfg = {
"AUTH_TOKEN": None,
"CT0": None,
"TRUTHSOCIAL_TOKEN": None,
"FROM_BROWSER": None,
"SETUP_COMPLETE": None,
}
cfg.update(overrides)
return cfg
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestExtractBrowserCredentials:
"""Unit tests for extract_browser_credentials()."""
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_auto_with_setup_complete_populates_credentials(self, mock_extract):
"""FROM_BROWSER=auto, SETUP_COMPLETE=true, mock returns valid cookies
-> config now contains AUTH_TOKEN and CT0."""
mock_extract.return_value = ({"auth_token": "tok123", "ct0": "ct0val"}, "chrome")
config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
result = extract_browser_credentials(config)
assert result["AUTH_TOKEN"] == "tok123"
assert result["CT0"] == "ct0val"
# Should have been called for x domain
mock_extract.assert_any_call("auto", ".x.com", ["auth_token", "ct0"])
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_explicit_auth_token_skips_x_extraction(self, mock_extract):
"""Config already has AUTH_TOKEN and CT0 from env var
-> cookie extraction skipped for X (explicit takes priority)."""
# Return None for any non-X domains that still get checked (e.g. Truth Social)
mock_extract.return_value = None
config = _base_config(
AUTH_TOKEN="explicit_token",
CT0="explicit_ct0",
FROM_BROWSER="auto",
SETUP_COMPLETE="true",
)
result = extract_browser_credentials(config)
# X cookies should not appear in result (already set)
assert "AUTH_TOKEN" not in result
assert "CT0" not in result
# extract_cookies should NOT have been called for .x.com
for call in mock_extract.call_args_list:
assert call[0][1] != ".x.com", "Should not extract cookies for X when credentials already set"
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_from_browser_off_skips_all(self, mock_extract):
"""FROM_BROWSER=off -> no cookie extraction attempted."""
config = _base_config(FROM_BROWSER="off", SETUP_COMPLETE="true")
result = extract_browser_credentials(config)
assert result == {}
mock_extract.assert_not_called()
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_no_setup_complete_no_from_browser_defaults_off(self, mock_extract):
"""FROM_BROWSER not set and SETUP_COMPLETE not set
-> no extraction (wizard hasn't run)."""
config = _base_config() # both None
result = extract_browser_credentials(config)
assert result == {}
mock_extract.assert_not_called()
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_setup_complete_no_from_browser_defaults_auto(self, mock_extract):
"""SETUP_COMPLETE is set but FROM_BROWSER is not
-> defaults to 'auto'."""
mock_extract.return_value = ({"auth_token": "found", "ct0": "found_ct0"}, "firefox")
config = _base_config(SETUP_COMPLETE="true") # FROM_BROWSER=None
result = extract_browser_credentials(config)
assert result["AUTH_TOKEN"] == "found"
# extract_cookies should have been called with 'auto'
mock_extract.assert_any_call("auto", ".x.com", ["auth_token", "ct0"])
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_from_browser_firefox_only(self, mock_extract):
"""FROM_BROWSER=firefox -> only Firefox extraction attempted."""
mock_extract.return_value = ({"auth_token": "ff_tok", "ct0": "ff_ct0"}, "firefox")
config = _base_config(FROM_BROWSER="firefox", SETUP_COMPLETE="true")
result = extract_browser_credentials(config)
assert result["AUTH_TOKEN"] == "ff_tok"
# All calls should use 'firefox' as the browser arg
for call in mock_extract.call_args_list:
assert call[0][0] == "firefox"
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_extraction_returns_none_config_unchanged(self, mock_extract):
"""Cookie extraction returns None for X -> config unchanged for AUTH_TOKEN."""
mock_extract.return_value = None
config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
result = extract_browser_credentials(config)
assert "AUTH_TOKEN" not in result
assert "CT0" not in result
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_extraction_raises_exception_config_unchanged(self, mock_extract):
"""Cookie extraction raises exception -> caught, config unchanged."""
mock_extract.side_effect = RuntimeError("database locked")
config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
result = extract_browser_credentials(config)
# Should not raise, and no credentials populated
assert "AUTH_TOKEN" not in result
assert "CT0" not in result
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_partial_credentials_only_fills_missing(self, mock_extract):
"""If AUTH_TOKEN is set but CT0 is not, only CT0 gets filled."""
mock_extract.return_value = ({"auth_token": "cookie_tok", "ct0": "cookie_ct0"}, "chrome")
config = _base_config(
AUTH_TOKEN="explicit",
CT0=None,
FROM_BROWSER="auto",
SETUP_COMPLETE="true",
)
result = extract_browser_credentials(config)
# AUTH_TOKEN already set, should not be overridden
assert "AUTH_TOKEN" not in result
# CT0 was missing, should be filled
assert result["CT0"] == "cookie_ct0"
class TestGetConfigCookieIntegration:
"""Integration test: get_config() calls extract_browser_credentials."""
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
@patch("scripts.lib.env._find_project_env", return_value=None)
@patch("scripts.lib.env.load_env_file", return_value={})
@patch("scripts.lib.env.get_openai_auth")
def test_get_config_injects_cookies(
self, mock_openai, mock_load, mock_proj, mock_extract
):
"""get_config merges browser cookies into the returned config."""
from scripts.lib.env import get_config, OpenAIAuth
mock_openai.return_value = OpenAIAuth(
token=None, source="none", status="missing",
account_id=None, codex_auth_file="/fake",
)
mock_extract.return_value = ({"auth_token": "browser_tok", "ct0": "browser_ct0"}, "firefox")
import os
env_patch = {
"SETUP_COMPLETE": "true",
"FROM_BROWSER": "auto",
"LAST30DAYS_CONFIG_DIR": "",
}
with patch.dict(os.environ, env_patch, clear=False):
config = get_config()
assert config["AUTH_TOKEN"] == "browser_tok"
assert config["CT0"] == "browser_ct0"
+3 -1
View File
@@ -223,7 +223,9 @@ class TestXSourceSelection(unittest.TestCase):
'can_install': False, 'can_install': False,
} }
with patch('lib.bird_x.get_bird_status', return_value=bird_status): with patch('lib.bird_x.get_bird_status', return_value=bird_status), \
patch('lib.bird_x.is_bird_installed', return_value=True), \
patch('lib.bird_x.is_bird_authenticated', return_value=None):
status = env.get_x_source_status(config) status = env.get_x_source_status(config)
self.assertIsNone(status['source']) self.assertIsNone(status['source'])
+249
View File
@@ -0,0 +1,249 @@
"""Tests for Exa Search module."""
import sys
import os
import unittest
from unittest.mock import patch, MagicMock
# Ensure scripts/ is on path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts'))
# Force clean config mode for tests
os.environ['LAST30DAYS_CONFIG_DIR'] = ''
from lib.exa_search import search_web, _normalize_results, _parse_exa_date, EXCLUDED_DOMAINS
from lib import env, http
class TestNormalizeResults(unittest.TestCase):
"""Test result normalization from Exa API responses."""
def test_valid_results(self):
response = {
"results": [
{
"title": "AI Trends in 2026",
"url": "https://blog.example.com/ai-trends",
"text": "This article covers the latest AI trends...",
"publishedDate": "2026-03-15T00:00:00.000Z",
"score": 0.85,
},
{
"title": "Machine Learning Updates",
"url": "https://ml.example.com/updates",
"text": "New ML frameworks released this month...",
"publishedDate": "2026-03-10T12:30:00.000Z",
"score": 0.72,
},
]
}
items = _normalize_results(response)
self.assertEqual(len(items), 2)
# Check first item structure
item = items[0]
self.assertEqual(item["title"], "AI Trends in 2026")
self.assertEqual(item["url"], "https://blog.example.com/ai-trends")
self.assertIn("AI trends", item["snippet"])
self.assertEqual(item["date"], "2026-03-15")
self.assertEqual(item["date_confidence"], "med")
self.assertAlmostEqual(item["relevance"], 0.85)
self.assertEqual(item["id"], "W1")
self.assertEqual(item["source_domain"], "blog.example.com")
def test_excludes_reddit_and_x(self):
response = {
"results": [
{"title": "Reddit post", "url": "https://www.reddit.com/r/test/123", "text": "Post"},
{"title": "X post", "url": "https://x.com/user/status/456", "text": "Tweet"},
{"title": "Good result", "url": "https://example.com/article", "text": "Content"},
]
}
items = _normalize_results(response)
self.assertEqual(len(items), 1)
self.assertEqual(items[0]["title"], "Good result")
def test_empty_results(self):
items = _normalize_results({"results": []})
self.assertEqual(items, [])
def test_missing_results_key(self):
items = _normalize_results({})
self.assertEqual(items, [])
def test_skips_items_without_url(self):
response = {
"results": [
{"title": "No URL", "text": "Content"},
{"title": "Has URL", "url": "https://example.com/a", "text": "Content"},
]
}
items = _normalize_results(response)
self.assertEqual(len(items), 1)
def test_skips_items_without_title_and_snippet(self):
response = {
"results": [
{"url": "https://example.com/a", "title": "", "text": ""},
{"url": "https://example.com/b", "title": "Valid", "text": "Content"},
]
}
items = _normalize_results(response)
self.assertEqual(len(items), 1)
def test_no_date_gives_low_confidence(self):
response = {
"results": [
{"title": "No date", "url": "https://example.com/a", "text": "Content"},
]
}
items = _normalize_results(response)
self.assertEqual(items[0]["date"], None)
self.assertEqual(items[0]["date_confidence"], "low")
def test_truncates_long_fields(self):
response = {
"results": [
{
"title": "T" * 300,
"url": "https://example.com/a",
"text": "S" * 1000,
},
]
}
items = _normalize_results(response)
self.assertLessEqual(len(items[0]["title"]), 200)
self.assertLessEqual(len(items[0]["snippet"]), 500)
def test_relevance_clamped(self):
response = {
"results": [
{"title": "High", "url": "https://example.com/a", "text": "C", "score": 1.5},
{"title": "Low", "url": "https://example.com/b", "text": "C", "score": -0.5},
]
}
items = _normalize_results(response)
self.assertEqual(items[0]["relevance"], 1.0)
self.assertEqual(items[1]["relevance"], 0.0)
class TestParseExaDate(unittest.TestCase):
def test_iso_datetime(self):
self.assertEqual(_parse_exa_date("2026-03-15T00:00:00.000Z"), "2026-03-15")
def test_date_only(self):
self.assertEqual(_parse_exa_date("2026-03-15"), "2026-03-15")
def test_none(self):
self.assertIsNone(_parse_exa_date(None))
def test_empty_string(self):
self.assertIsNone(_parse_exa_date(""))
class TestSearchWebIntegration(unittest.TestCase):
"""Test search_web function with mocked HTTP calls."""
@patch("lib.exa_search.http.post")
def test_valid_search(self, mock_post):
mock_post.return_value = {
"results": [
{
"title": "Test Result",
"url": "https://example.com/test",
"text": "Test content here",
"publishedDate": "2026-03-20T00:00:00.000Z",
"score": 0.9,
},
]
}
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-api-key")
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["title"], "Test Result")
self.assertEqual(results[0]["url"], "https://example.com/test")
self.assertEqual(results[0]["snippet"], "Test content here")
# Verify API call
mock_post.assert_called_once()
call_args = mock_post.call_args
self.assertEqual(call_args[0][0], "https://api.exa.ai/search")
self.assertEqual(call_args[1]["headers"]["x-api-key"], "test-api-key")
@patch("lib.exa_search.http.post")
def test_401_invalid_key(self, mock_post):
mock_post.side_effect = http.HTTPError("HTTP 401: Unauthorized", status_code=401)
results = search_web("AI news", "2026-03-01", "2026-03-29", "bad-key")
self.assertEqual(results, [])
@patch("lib.exa_search.http.post")
def test_429_rate_limit(self, mock_post):
mock_post.side_effect = http.HTTPError("HTTP 429: Too Many Requests", status_code=429)
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-key")
self.assertEqual(results, [])
@patch("lib.exa_search.http.post")
def test_empty_results(self, mock_post):
mock_post.return_value = {"results": []}
results = search_web("obscure query", "2026-03-01", "2026-03-29", "test-key")
self.assertEqual(results, [])
@patch("lib.exa_search.http.post")
def test_network_timeout(self, mock_post):
mock_post.side_effect = http.HTTPError("Connection error: TimeoutError: timed out")
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-key")
self.assertEqual(results, [])
@patch("lib.exa_search.http.post")
def test_generic_exception(self, mock_post):
mock_post.side_effect = Exception("Something unexpected")
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-key")
self.assertEqual(results, [])
class TestEnvExaPriority(unittest.TestCase):
"""Test that Exa is prioritized correctly in env.py."""
def test_no_exa_key_not_selected(self):
config = {}
self.assertIsNone(env.get_web_search_source(config))
def test_exa_key_selected(self):
config = {"EXA_API_KEY": "exa-test-key"}
self.assertEqual(env.get_web_search_source(config), "exa")
def test_exa_takes_priority_over_brave(self):
config = {"EXA_API_KEY": "exa-key", "BRAVE_API_KEY": "brave-key"}
self.assertEqual(env.get_web_search_source(config), "exa")
def test_exa_takes_priority_over_parallel(self):
config = {"EXA_API_KEY": "exa-key", "PARALLEL_API_KEY": "parallel-key"}
self.assertEqual(env.get_web_search_source(config), "exa")
def test_exa_takes_priority_over_openrouter(self):
config = {"EXA_API_KEY": "exa-key", "OPENROUTER_API_KEY": "or-key"}
self.assertEqual(env.get_web_search_source(config), "exa")
def test_exa_takes_priority_over_all(self):
config = {
"EXA_API_KEY": "exa-key",
"PARALLEL_API_KEY": "parallel-key",
"BRAVE_API_KEY": "brave-key",
"OPENROUTER_API_KEY": "or-key",
}
self.assertEqual(env.get_web_search_source(config), "exa")
def test_fallback_to_parallel_without_exa(self):
config = {"PARALLEL_API_KEY": "parallel-key", "BRAVE_API_KEY": "brave-key"}
self.assertEqual(env.get_web_search_source(config), "parallel")
def test_has_web_search_keys_with_exa(self):
config = {"EXA_API_KEY": "exa-key"}
self.assertTrue(env.has_web_search_keys(config))
def test_has_web_search_keys_without_any(self):
config = {}
self.assertFalse(env.has_web_search_keys(config))
if __name__ == "__main__":
unittest.main()
+411
View File
@@ -0,0 +1,411 @@
"""End-to-end NUX integration tests.
Verifies that setup wizard, status banner, quality nudge, and SKILL.md
first-run flow work together coherently:
- first_run flag emitted / not emitted based on SETUP_COMPLETE
- setup subcommand runs auto_setup and writes config
- quality score is consistent with source configuration
- quality nudge disappears at 100%
- banner and quality nudge don't contradict each other
"""
import sys
import tempfile
from pathlib import Path
from unittest.mock import patch, MagicMock
import pytest
# Add scripts dir to path
SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
sys.path.insert(0, str(SCRIPTS_DIR))
from lib import setup_wizard, quality_nudge, ui
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _base_config(**overrides):
"""Return a minimal config dict."""
config = {
"AUTH_TOKEN": None,
"CT0": None,
"XAI_API_KEY": None,
"SCRAPECREATORS_API_KEY": None,
}
config.update(overrides)
return config
def _base_results(**overrides):
"""Return a minimal research_results dict with no errors."""
results = {
"x_error": None,
"youtube_error": None,
"reddit_error": None,
}
results.update(overrides)
return results
def _base_diag(**overrides):
"""Return a minimal diag dict for banner testing."""
diag = {
"setup_complete": False,
"reddit_source": None,
"x_source": None,
"x_method": None,
"youtube": False,
"tiktok": False,
"instagram": False,
"hackernews": True,
"polymarket": True,
"bluesky": False,
"truthsocial": False,
"xiaohongshu": False,
"scrapecreators": False,
"web_search_backend": None,
}
diag.update(overrides)
return diag
def _compute(config_overrides=None, result_overrides=None, ytdlp_installed=False):
"""Helper to call compute_quality_score with mocked yt-dlp check."""
from lib import youtube_yt
config = _base_config(**(config_overrides or {}))
results = _base_results(**(result_overrides or {}))
with patch.object(youtube_yt, "is_ytdlp_installed", return_value=ytdlp_installed):
return quality_nudge.compute_quality_score(config, results)
# ---------------------------------------------------------------------------
# First-run flag detection
# ---------------------------------------------------------------------------
class TestFirstRunDetection:
"""first_run flag is emitted based on SETUP_COMPLETE in config."""
def test_first_run_when_setup_not_complete(self):
"""SETUP_COMPLETE missing -> is_first_run returns True."""
config = _base_config()
assert setup_wizard.is_first_run(config) is True
def test_first_run_when_setup_complete_empty(self):
"""SETUP_COMPLETE='' -> is_first_run returns True."""
config = _base_config(SETUP_COMPLETE="")
assert setup_wizard.is_first_run(config) is True
def test_not_first_run_when_setup_complete(self):
"""SETUP_COMPLETE=true -> is_first_run returns False."""
config = _base_config(SETUP_COMPLETE="true")
assert setup_wizard.is_first_run(config) is False
def test_not_first_run_any_truthy_value(self):
"""SETUP_COMPLETE=1 -> is_first_run returns False."""
config = _base_config(SETUP_COMPLETE="1")
assert setup_wizard.is_first_run(config) is False
# ---------------------------------------------------------------------------
# Setup subcommand writes config
# ---------------------------------------------------------------------------
class TestSetupSubcommandWritesConfig:
"""setup subcommand runs auto_setup and writes SETUP_COMPLETE."""
@patch("lib.cookie_extract.extract_cookies_with_source")
@patch("shutil.which")
def test_auto_setup_and_write(self, mock_which, mock_extract):
"""run_auto_setup + write_setup_config creates valid config."""
mock_extract.return_value = ({"auth_token": "abc", "ct0": "xyz"}, "chrome")
mock_which.return_value = "/usr/local/bin/yt-dlp"
config = _base_config()
results = setup_wizard.run_auto_setup(config)
assert results["cookies_found"]["x"] == "chrome"
assert results["ytdlp_installed"] is True
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / ".env"
written = setup_wizard.write_setup_config(env_path)
assert written is True
content = env_path.read_text()
assert "SETUP_COMPLETE=true" in content
assert "FROM_BROWSER=auto" in content
@patch("lib.cookie_extract.extract_cookies_with_source")
@patch("shutil.which")
def test_after_setup_not_first_run(self, mock_which, mock_extract):
"""After write_setup_config, is_first_run should return False."""
mock_extract.return_value = None
mock_which.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / ".env"
setup_wizard.write_setup_config(env_path)
# Simulate reading the config back
config = {"SETUP_COMPLETE": "true"}
assert setup_wizard.is_first_run(config) is False
# ---------------------------------------------------------------------------
# Quality score consistency with source config
# ---------------------------------------------------------------------------
class TestQualityScoreConsistency:
"""Quality score matches the source configuration state."""
def test_zero_config_is_40_pct(self):
"""No X, no yt-dlp, no SC -> 40% (HN + Polymarket only)."""
q = _compute()
assert q["score_pct"] == 40
assert set(q["core_active"]) == {"hn", "polymarket"}
def test_x_cookies_is_60_pct(self):
"""X cookies active -> 60%."""
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
assert q["score_pct"] == 60
assert "x" in q["core_active"]
def test_x_plus_ytdlp_is_80_pct(self):
"""X cookies + yt-dlp -> 80%."""
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert q["score_pct"] == 80
assert "x" in q["core_active"]
assert "youtube" in q["core_active"]
def test_full_config_is_100_pct(self):
"""X + yt-dlp + ScrapeCreators -> 100%."""
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
assert q["score_pct"] == 100
assert len(q["core_active"]) == 5
def test_xai_key_also_enables_x(self):
"""XAI_API_KEY activates X source same as cookies."""
q = _compute(config_overrides={"XAI_API_KEY": "xai_key"})
assert q["score_pct"] == 60
assert "x" in q["core_active"]
# ---------------------------------------------------------------------------
# Quality nudge disappears at 100%
# ---------------------------------------------------------------------------
class TestQualityNudgeDisappears:
"""Quality nudge is None at 100% and present below 100%."""
def test_nudge_present_at_40(self):
q = _compute()
assert q["nudge_text"] is not None
assert len(q["nudge_text"]) > 0
def test_nudge_present_at_60(self):
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
assert q["nudge_text"] is not None
def test_nudge_present_at_80(self):
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert q["nudge_text"] is not None
def test_nudge_none_at_100(self):
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
assert q["nudge_text"] is None
assert q["core_missing"] == []
# ---------------------------------------------------------------------------
# Banner and quality nudge consistency
# ---------------------------------------------------------------------------
class TestBannerQualityNudgeConsistency:
"""Banner source count and quality nudge percentage don't contradict."""
def test_zero_config_banner_3_sources_nudge_40(self):
"""Banner shows 3 sources (Reddit, HN, PM), nudge says 40%."""
diag = _base_diag()
banner = "\n".join(ui._build_status_banner(diag))
# Banner shows 3 active sources
assert "Reddit (threads only)" in banner
assert "HN" in banner
assert "Polymarket" in banner
# X and YouTube should NOT be in the banner
assert "X (" not in banner
assert "YouTube" not in banner
# Quality nudge is 40%
q = _compute()
assert q["score_pct"] == 40
def test_after_wizard_banner_5_sources_nudge_80(self):
"""After wizard (X cookies + yt-dlp): banner shows 5+ sources, nudge ~80%."""
diag = _base_diag(
setup_complete=True,
x_source="bird",
x_method="browser-chrome",
youtube=True,
)
banner = "\n".join(ui._build_status_banner(diag))
# Banner shows X and YouTube
assert "X (Chrome)" in banner
assert "YouTube" in banner
assert "Reddit (threads only)" in banner
assert "HN" in banner
assert "Polymarket" in banner
# Quality nudge is 80%
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert q["score_pct"] == 80
def test_full_config_banner_all_nudge_gone(self):
"""Full config: banner shows all sources, nudge disappears (100%)."""
diag = _base_diag(
setup_complete=True,
reddit_source="scrapecreators",
x_source="bird",
x_method="browser-chrome",
youtube=True,
tiktok=True,
instagram=True,
scrapecreators=True,
)
banner = "\n".join(ui._build_status_banner(diag))
assert "Reddit (with comments)" in banner
assert "X (Chrome)" in banner
assert "YouTube" in banner
assert "TikTok" in banner
assert "Instagram" in banner
# Quality nudge is 100% / gone
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
assert q["score_pct"] == 100
assert q["nudge_text"] is None
def test_banner_first_run_suggests_setup(self):
"""First-run banner suggests running setup wizard."""
diag = _base_diag(setup_complete=False)
banner = "\n".join(ui._build_status_banner(diag))
assert "First Run" in banner
assert "/last30days setup" in banner
def test_banner_partial_suggests_scrapecreators(self):
"""After setup, missing SC is the only recommendation."""
diag = _base_diag(
setup_complete=True,
x_source="bird",
x_method="browser-chrome",
youtube=True,
scrapecreators=False,
)
banner = "\n".join(ui._build_status_banner(diag))
assert "SCRAPECREATORS_API_KEY" in banner
assert "100 free calls, no CC" in banner
# ---------------------------------------------------------------------------
# Progressive narrowing of nudge suggestions
# ---------------------------------------------------------------------------
class TestProgressiveNarrowing:
"""As users configure more, nudge suggestions narrow."""
def test_baseline_mentions_all_three_missing(self):
q = _compute()
assert "X/Twitter" in q["nudge_text"]
assert "YouTube" in q["nudge_text"]
assert "Reddit with comments" in q["nudge_text"]
def test_with_x_mentions_yt_and_sc(self):
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
assert "X/Twitter" not in q["nudge_text"]
assert "YouTube" in q["nudge_text"]
assert "Reddit with comments" in q["nudge_text"]
def test_with_x_and_yt_mentions_sc_only(self):
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert "X/Twitter" not in q["nudge_text"]
assert "YouTube" not in q["nudge_text"]
# SC nudge is present
assert "Reddit" in q["nudge_text"] or "ScrapeCreators" in q["nudge_text"].lower() or "scrapecreators" in q["nudge_text"]
def test_full_coverage_no_nudge(self):
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
assert q["nudge_text"] is None
# ---------------------------------------------------------------------------
# Render quality nudge integration
# ---------------------------------------------------------------------------
class TestRenderQualityNudge:
"""render_quality_nudge produces correct output."""
def test_render_at_80_pct(self):
from lib.render import render_quality_nudge
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
rendered = render_quality_nudge(q)
assert "80%" in rendered
assert "Research Coverage" in rendered
def test_render_at_100_pct_is_empty(self):
from lib.render import render_quality_nudge
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
rendered = render_quality_nudge(q)
assert rendered == ""
+203
View File
@@ -0,0 +1,203 @@
"""Tests for post-research quality score and upgrade nudge."""
import pytest
from unittest.mock import patch
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _base_config(**overrides):
"""Return a minimal config dict."""
config = {
"AUTH_TOKEN": None,
"CT0": None,
"XAI_API_KEY": None,
"SCRAPECREATORS_API_KEY": None,
}
config.update(overrides)
return config
def _base_results(**overrides):
"""Return a minimal research_results dict with no errors."""
results = {
"x_error": None,
"youtube_error": None,
"reddit_error": None,
}
results.update(overrides)
return results
def _compute(config_overrides=None, result_overrides=None, ytdlp_installed=False):
"""Helper to call compute_quality_score with mocked yt-dlp check."""
from scripts.lib.quality_nudge import compute_quality_score
from scripts.lib import youtube_yt
config = _base_config(**(config_overrides or {}))
results = _base_results(**(result_overrides or {}))
with patch.object(youtube_yt, "is_ytdlp_installed", return_value=ytdlp_installed):
return compute_quality_score(config, results)
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestBaseline:
"""HN + Polymarket only (no X, no YT, no SC) -> 40%."""
def test_score_40(self):
q = _compute()
assert q["score_pct"] == 40
def test_active_sources(self):
q = _compute()
assert "hn" in q["core_active"]
assert "polymarket" in q["core_active"]
assert len(q["core_active"]) == 2
def test_missing_all_three(self):
q = _compute()
assert set(q["core_missing"]) == {"x", "youtube", "reddit_comments"}
def test_nudge_mentions_all_missing(self):
q = _compute()
assert q["nudge_text"] is not None
assert "X/Twitter" in q["nudge_text"]
assert "YouTube" in q["nudge_text"]
assert "Reddit with comments" in q["nudge_text"]
class TestXCookies:
"""+X cookies -> 60%."""
def test_score_60(self):
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
assert q["score_pct"] == 60
def test_nudge_mentions_yt_and_sc(self):
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
assert "YouTube" in q["nudge_text"]
assert "Reddit with comments" in q["nudge_text"]
assert "X/Twitter" not in q["nudge_text"]
class TestXPlusYtdlp:
"""+X + yt-dlp -> 80%."""
def test_score_80(self):
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert q["score_pct"] == 80
def test_nudge_mentions_sc_only(self):
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert "ScrapeCreators" in q["nudge_text"] or "scrapecreators" in q["nudge_text"]
assert "YouTube" not in q["nudge_text"]
assert "X/Twitter" not in q["nudge_text"]
class TestFullCoverage:
"""+X + yt-dlp + SC -> 100%, no nudge."""
def test_score_100(self):
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
assert q["score_pct"] == 100
def test_nudge_is_none(self):
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
assert q["nudge_text"] is None
class TestSCActiveNoX:
"""SC active but no X -> 80%, nudge suggests browser cookies (free)."""
def test_score_80(self):
q = _compute(
config_overrides={"SCRAPECREATORS_API_KEY": "sc_key"},
ytdlp_installed=True,
)
assert q["score_pct"] == 80
def test_nudge_suggests_browser_cookies(self):
q = _compute(
config_overrides={"SCRAPECREATORS_API_KEY": "sc_key"},
ytdlp_installed=True,
)
assert q["nudge_text"] is not None
assert "browser" in q["nudge_text"].lower()
assert "x.com" in q["nudge_text"].lower()
class TestRedditErrored:
"""SC is configured but Reddit errored this run."""
def test_nudge_mentions_error(self):
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
result_overrides={"reddit_error": "ScrapeCreators: 500 Internal Server Error"},
ytdlp_installed=True,
)
assert "reddit_comments" in q["core_errored"]
assert "errored" in q["nudge_text"].lower()
class TestDisclaimerAlwaysPresent:
"""Nudge always includes no-affiliate disclaimer when present."""
def test_disclaimer_baseline(self):
q = _compute()
assert "no affiliation" in q["nudge_text"]
def test_disclaimer_partial(self):
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
assert "no affiliation" in q["nudge_text"]
def test_disclaimer_not_present_at_100(self):
q = _compute(
config_overrides={
"AUTH_TOKEN": "tok123",
"SCRAPECREATORS_API_KEY": "sc_key",
},
ytdlp_installed=True,
)
assert q["nudge_text"] is None
class TestSCNudgeContent:
"""SC nudge always includes '100 free API calls, no credit card'."""
def test_sc_nudge_content(self):
q = _compute()
assert "100 free API calls, no credit card" in q["nudge_text"]
def test_sc_nudge_content_when_only_missing_sc(self):
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert "100 free API calls, no credit card" in q["nudge_text"]
+330
View File
@@ -0,0 +1,330 @@
"""Tests for scripts/lib/reddit_public.py — standalone Reddit public JSON search."""
import json
import urllib.error
from unittest import mock
import pytest
import sys
import os
# Ensure lib is importable
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
from lib import reddit_public
# ---------------------------------------------------------------------------
# Fixtures / helpers
# ---------------------------------------------------------------------------
def _make_reddit_listing(posts):
"""Build a Reddit listing JSON structure from a list of post dicts."""
children = []
for p in posts:
children.append({
"kind": "t3",
"data": {
"title": p.get("title", "Test Post"),
"permalink": p.get("permalink", "/r/test/comments/abc123/test_post/"),
"subreddit": p.get("subreddit", "test"),
"score": p.get("score", 42),
"num_comments": p.get("num_comments", 10),
"created_utc": p.get("created_utc", 1711670400), # 2024-03-29
"author": p.get("author", "testuser"),
"selftext": p.get("selftext", "Some body text"),
"upvote_ratio": p.get("upvote_ratio", 0.95),
},
})
return {"data": {"children": children}}
SAMPLE_LISTING = _make_reddit_listing([
{
"title": "Claude Code is amazing",
"permalink": "/r/ClaudeAI/comments/abc123/claude_code_is_amazing/",
"subreddit": "ClaudeAI",
"score": 250,
"num_comments": 45,
"created_utc": 1711670400,
"author": "ai_fan",
"selftext": "I've been using Claude Code for a week and it changed my workflow.",
},
{
"title": "Tips for Claude Code prompting",
"permalink": "/r/ClaudeAI/comments/def456/tips_for_claude_code/",
"subreddit": "ClaudeAI",
"score": 120,
"num_comments": 22,
"created_utc": 1711584000,
"author": "prompt_engineer",
"selftext": "Here are my top tips for getting the most out of Claude Code.",
},
])
def _mock_urlopen_ok(listing_data):
"""Return a context-manager mock for urllib.request.urlopen that returns listing_data."""
resp = mock.MagicMock()
resp.read.return_value = json.dumps(listing_data).encode("utf-8")
resp.headers = {"Content-Type": "application/json"}
resp.__enter__ = mock.MagicMock(return_value=resp)
resp.__exit__ = mock.MagicMock(return_value=False)
return resp
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestSearchReturnsCorrectFields:
"""Search query returns parsed results with correct fields."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_search_returns_parsed_results(self, mock_urlopen):
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
results = reddit_public.search("Claude Code", depth="quick")
assert len(results) == 2
first = results[0]
# Check all required fields exist
assert "id" in first
assert "title" in first
assert "url" in first
assert "score" in first
assert "num_comments" in first
assert "subreddit" in first
assert "created_utc" in first
assert "author" in first
assert "selftext" in first
# Check values
assert first["title"] == "Claude Code is amazing"
assert first["subreddit"] == "ClaudeAI"
assert first["score"] == 250
assert first["num_comments"] == 45
assert first["author"] == "ai_fan"
assert "/comments/" in first["url"]
assert first["id"] == "R1"
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_search_includes_normalized_fields(self, mock_urlopen):
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
results = reddit_public.search("Claude Code")
first = results[0]
# Normalized fields matching ScrapeCreators format
assert "date" in first
assert "engagement" in first
assert "relevance" in first
assert "why_relevant" in first
assert isinstance(first["engagement"], dict)
assert "score" in first["engagement"]
assert "num_comments" in first["engagement"]
class TestSubredditScopedSearch:
"""Subreddit-scoped search works."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_subreddit_search_builds_correct_url(self, mock_urlopen):
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
reddit_public.search("Claude Code", subreddit="ClaudeAI")
# Check the URL passed to urlopen
call_args = mock_urlopen.call_args
req = call_args[0][0] # First positional arg is the Request object
assert "/r/ClaudeAI/search.json" in req.full_url
assert "restrict_sr=on" in req.full_url
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_subreddit_search_strips_prefix(self, mock_urlopen):
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
reddit_public.search("Claude Code", subreddit="r/ClaudeAI")
req = mock_urlopen.call_args[0][0]
# Should strip the r/ prefix, not double it
assert "/r/ClaudeAI/search.json" in req.full_url
assert "/r/r/" not in req.full_url
class TestRetryOn429:
"""429 response triggers retries, eventually returns partial results."""
@mock.patch("lib.reddit_public.time.sleep")
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_429_retries_then_returns_empty(self, mock_urlopen, mock_sleep):
error = urllib.error.HTTPError(
"https://reddit.com/search.json", 429, "Too Many Requests",
{"Retry-After": "1"}, None,
)
mock_urlopen.side_effect = error
results = reddit_public.search("test query")
assert results == []
# Should have retried (MAX_RETRIES = 3, sleeps happen between attempts)
assert mock_sleep.call_count == reddit_public.MAX_RETRIES - 1
@mock.patch("lib.reddit_public.time.sleep")
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_429_then_success(self, mock_urlopen, mock_sleep):
error = urllib.error.HTTPError(
"https://reddit.com/search.json", 429, "Too Many Requests",
{}, None,
)
success_resp = _mock_urlopen_ok(SAMPLE_LISTING)
mock_urlopen.side_effect = [error, success_resp]
results = reddit_public.search("test query")
assert len(results) == 2
assert mock_sleep.call_count == 1
class TestHtmlAntiBot:
"""HTML response (anti-bot) is detected, returns empty."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_html_response_returns_empty(self, mock_urlopen):
resp = mock.MagicMock()
resp.read.return_value = b"<html><body>Please verify you are human</body></html>"
resp.headers = {"Content-Type": "text/html; charset=utf-8"}
resp.__enter__ = mock.MagicMock(return_value=resp)
resp.__exit__ = mock.MagicMock(return_value=False)
mock_urlopen.return_value = resp
results = reddit_public.search("test query")
assert results == []
class TestNetworkTimeout:
"""Network timeout returns empty."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_timeout_returns_empty(self, mock_urlopen):
mock_urlopen.side_effect = TimeoutError("Connection timed out")
results = reddit_public.search("test query")
assert results == []
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_url_error_returns_empty(self, mock_urlopen):
mock_urlopen.side_effect = urllib.error.URLError("Connection refused")
results = reddit_public.search("test query")
assert results == []
class TestNormalizationMatchesScrapeCreators:
"""Results normalize to same schema as ScrapeCreators (field names match)."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_field_names_match_scrapecreators(self, mock_urlopen):
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
results = reddit_public.search("Claude Code")
assert len(results) > 0
item = results[0]
# These fields must exist to match ScrapeCreators _normalize_post output
sc_required_fields = {"id", "title", "url", "subreddit", "date",
"engagement", "relevance", "why_relevant"}
actual_fields = set(item.keys())
assert sc_required_fields.issubset(actual_fields), (
f"Missing fields: {sc_required_fields - actual_fields}"
)
# Engagement sub-fields
eng = item["engagement"]
assert "score" in eng
assert "num_comments" in eng
assert "upvote_ratio" in eng
class TestDepthLimits:
"""Depth-aware limits are respected."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_quick_limit(self, mock_urlopen):
# Create more posts than the quick limit
many_posts = _make_reddit_listing([
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/"}
for i in range(20)
])
mock_urlopen.return_value = _mock_urlopen_ok(many_posts)
results = reddit_public.search("test", depth="quick")
assert len(results) <= 10
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_default_limit(self, mock_urlopen):
many_posts = _make_reddit_listing([
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/"}
for i in range(50)
])
mock_urlopen.return_value = _mock_urlopen_ok(many_posts)
results = reddit_public.search("test", depth="default")
assert len(results) <= 25
class TestSearchRedditPublicHighLevel:
"""Test the high-level search_reddit_public interface."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_date_filtering(self, mock_urlopen):
listing = _make_reddit_listing([
{
"title": "In range",
"permalink": "/r/test/comments/aaa/in_range/",
"created_utc": 1711670400, # 2024-03-29
},
{
"title": "Out of range",
"permalink": "/r/test/comments/bbb/out_of_range/",
"created_utc": 1609459200, # 2021-01-01
},
])
mock_urlopen.return_value = _mock_urlopen_ok(listing)
results = reddit_public.search_reddit_public(
"test", "2024-03-01", "2024-03-31"
)
titles = [r["title"] for r in results]
assert "In range" in titles
assert "Out of range" not in titles
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_user_agent_header(self, mock_urlopen):
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
reddit_public.search("test")
req = mock_urlopen.call_args[0][0]
assert req.get_header("User-agent") == "last30days/3.0 (research tool)"
class TestMissingSubreddit:
"""Subreddit doesn't exist returns empty."""
@mock.patch("lib.reddit_public.urllib.request.urlopen")
def test_404_subreddit_returns_empty(self, mock_urlopen):
mock_urlopen.side_effect = urllib.error.HTTPError(
"https://reddit.com/r/nonexistent/search.json",
404, "Not Found", {}, None,
)
results = reddit_public.search("test", subreddit="nonexistent")
assert results == []
+219
View File
@@ -0,0 +1,219 @@
"""Tests for Safari binary cookie extraction."""
from __future__ import annotations
import struct
import sys
from pathlib import Path
from unittest.mock import patch
import pytest
# Import the internal parser directly for testability (avoids platform check)
from scripts.lib.safari_cookies import (
_parse_binary_cookies,
extract_safari_cookies_macos,
)
def _build_cookie_record(url: str, name: str, value: str, path: str = "/") -> bytes:
"""Build a single binary cookie record."""
# Fixed header: size(4) + flags(4) + padding(8) + url_off(4) + name_off(4) + path_off(4) + value_off(4) + comment(8) + expiry(8) + creation(8)
# Total fixed = 4 + 4 + 8 + 4 + 4 + 4 + 4 + 8 + 8 + 8 = 56 bytes
# String data starts at offset 56
url_b = url.encode("utf-8") + b"\x00"
name_b = name.encode("utf-8") + b"\x00"
path_b = path.encode("utf-8") + b"\x00"
value_b = value.encode("utf-8") + b"\x00"
str_offset_base = 56
url_offset = str_offset_base
name_offset = url_offset + len(url_b)
path_offset = name_offset + len(name_b)
value_offset = path_offset + len(path_b)
total_size = value_offset + len(value_b)
record = struct.pack("<I", total_size) # size
record += struct.pack("<I", 0) # flags
record += b"\x00" * 8 # padding/unknown
record += struct.pack("<I", url_offset) # url offset
record += struct.pack("<I", name_offset) # name offset
record += struct.pack("<I", path_offset) # path offset
record += struct.pack("<I", value_offset) # value offset
record += b"\x00" * 8 # comment (unused)
record += struct.pack("<d", 700000000.0) # expiry (Mac epoch)
record += struct.pack("<d", 690000000.0) # creation (Mac epoch)
record += url_b + name_b + path_b + value_b
return record
def _build_page(cookie_records: list[bytes]) -> bytes:
"""Build a binary cookies page from a list of cookie records."""
num_cookies = len(cookie_records)
# Page header: 4-byte marker + 4-byte cookie count + offset array
header_size = 4 + 4 + num_cookies * 4
# Also add 4 bytes for end-of-page marker
offsets_start = header_size
# Calculate offsets for each cookie record
offsets = []
current_offset = offsets_start
for rec in cookie_records:
offsets.append(current_offset)
current_offset += len(rec)
page = b"\x00\x00\x01\x00" # page header marker
page += struct.pack("<I", num_cookies)
for off in offsets:
page += struct.pack("<I", off)
for rec in cookie_records:
page += rec
return page
def _build_binary_cookies_file(pages: list[bytes]) -> bytes:
"""Build a complete Cookies.binarycookies file from pages."""
num_pages = len(pages)
data = b"cook" # magic
data += struct.pack(">I", num_pages) # page count (big-endian)
# Page sizes (big-endian)
for page in pages:
data += struct.pack(">I", len(page))
# Page data
for page in pages:
data += page
return data
@pytest.fixture
def x_cookies_file() -> bytes:
"""Build a minimal valid binary cookies file with .x.com cookies."""
rec1 = _build_cookie_record(".x.com", "auth_token", "test_auth_abc123")
rec2 = _build_cookie_record(".x.com", "ct0", "test_ct0_xyz789")
rec3 = _build_cookie_record(".google.com", "NID", "google_nid_value")
page = _build_page([rec1, rec2, rec3])
return _build_binary_cookies_file([page])
class TestParseValidCookies:
def test_extracts_matching_cookies(self, x_cookies_file: bytes):
result = _parse_binary_cookies(x_cookies_file, "x.com", ["auth_token", "ct0"])
assert result is not None
assert result["auth_token"] == "test_auth_abc123"
assert result["ct0"] == "test_ct0_xyz789"
def test_ignores_other_domains(self, x_cookies_file: bytes):
result = _parse_binary_cookies(x_cookies_file, "google.com", ["auth_token"])
assert result is None
def test_partial_match_returns_found_only(self, x_cookies_file: bytes):
result = _parse_binary_cookies(
x_cookies_file, "x.com", ["auth_token", "nonexistent"]
)
assert result is not None
assert result["auth_token"] == "test_auth_abc123"
assert "nonexistent" not in result
def test_no_matching_cookie_names(self, x_cookies_file: bytes):
result = _parse_binary_cookies(x_cookies_file, "x.com", ["bogus"])
assert result is None
def test_domain_substring_match_with_leading_dot(self, x_cookies_file: bytes):
"""Domain '.x.com' in cookie should match search for 'x.com'."""
result = _parse_binary_cookies(x_cookies_file, "x.com", ["auth_token"])
assert result is not None
assert result["auth_token"] == "test_auth_abc123"
class TestMultiplePages:
def test_cookies_across_pages(self):
rec1 = _build_cookie_record(".x.com", "auth_token", "page1_auth")
rec2 = _build_cookie_record(".x.com", "ct0", "page2_ct0")
page1 = _build_page([rec1])
page2 = _build_page([rec2])
data = _build_binary_cookies_file([page1, page2])
result = _parse_binary_cookies(data, "x.com", ["auth_token", "ct0"])
assert result is not None
assert result["auth_token"] == "page1_auth"
assert result["ct0"] == "page2_ct0"
class TestErrorPaths:
def test_file_not_found(self, tmp_path: Path):
with patch(
"scripts.lib.safari_cookies.Path.home", return_value=tmp_path
), patch("scripts.lib.safari_cookies.sys") as mock_sys:
mock_sys.platform = "darwin"
mock_sys.stderr = sys.stderr
result = extract_safari_cookies_macos("x.com", ["auth_token"])
assert result is None
def test_permission_denied(self, tmp_path: Path, capsys):
cookie_dir = tmp_path / "Library" / "Cookies"
cookie_dir.mkdir(parents=True)
cookie_file = cookie_dir / "Cookies.binarycookies"
cookie_file.write_bytes(b"cook")
cookie_file.chmod(0o000)
try:
with patch(
"scripts.lib.safari_cookies.Path.home", return_value=tmp_path
), patch("scripts.lib.safari_cookies.sys") as mock_sys:
mock_sys.platform = "darwin"
mock_sys.stderr = sys.stderr
result = extract_safari_cookies_macos("x.com", ["auth_token"])
assert result is None
captured = capsys.readouterr()
assert "Full Disk Access" in captured.err
finally:
cookie_file.chmod(0o644)
def test_truncated_magic_only(self):
result = _parse_binary_cookies(b"cook", "x.com", ["auth_token"])
assert result is None
def test_empty_file(self):
result = _parse_binary_cookies(b"", "x.com", ["auth_token"])
assert result is None
def test_wrong_magic(self):
result = _parse_binary_cookies(b"notcook!", "x.com", ["auth_token"])
assert result is None
def test_truncated_page_sizes(self):
# Header says 5 pages but data is too short
data = b"cook" + struct.pack(">I", 5) + b"\x00" * 4
result = _parse_binary_cookies(data, "x.com", ["auth_token"])
assert result is None
def test_truncated_page_data(self):
# Valid header with 1 page of size 1000, but no actual page data
data = b"cook" + struct.pack(">I", 1) + struct.pack(">I", 1000)
result = _parse_binary_cookies(data, "x.com", ["auth_token"])
assert result is None
def test_non_darwin_returns_none(self):
with patch("scripts.lib.safari_cookies.sys") as mock_sys:
mock_sys.platform = "linux"
result = extract_safari_cookies_macos("x.com", ["auth_token"])
assert result is None
def test_garbage_data_no_crash(self):
"""Random bytes after valid magic should not crash."""
import os
data = b"cook" + os.urandom(200)
# Should not raise — may return None or a dict
result = _parse_binary_cookies(data, "x.com", ["auth_token"])
# Just verify no exception; result is either None or dict
assert result is None or isinstance(result, dict)
+342
View File
@@ -0,0 +1,342 @@
"""Tests for the first-run setup wizard module."""
import os
import sys
import tempfile
from pathlib import Path
from unittest.mock import patch, MagicMock
import pytest
# Add scripts dir to path
SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
sys.path.insert(0, str(SCRIPTS_DIR))
from lib import setup_wizard
class TestIsFirstRun:
"""Tests for is_first_run()."""
def test_first_run_when_setup_complete_not_set(self):
"""SETUP_COMPLETE not in config -> first run."""
config = {"AUTH_TOKEN": "abc", "CT0": "xyz"}
assert setup_wizard.is_first_run(config) is True
def test_first_run_when_setup_complete_is_none(self):
"""SETUP_COMPLETE=None -> first run."""
config = {"SETUP_COMPLETE": None}
assert setup_wizard.is_first_run(config) is True
def test_first_run_when_setup_complete_is_empty(self):
"""SETUP_COMPLETE="" -> first run."""
config = {"SETUP_COMPLETE": ""}
assert setup_wizard.is_first_run(config) is True
def test_not_first_run_when_setup_complete_true(self):
"""SETUP_COMPLETE=true -> not first run."""
config = {"SETUP_COMPLETE": "true"}
assert setup_wizard.is_first_run(config) is False
def test_not_first_run_when_setup_complete_any_value(self):
"""SETUP_COMPLETE set to any truthy value -> not first run."""
config = {"SETUP_COMPLETE": "yes"}
assert setup_wizard.is_first_run(config) is False
class TestRunAutoSetup:
"""Tests for run_auto_setup()."""
@patch("lib.cookie_extract.extract_cookies_with_source")
@patch("shutil.which")
def test_cookies_found(self, mock_which, mock_extract):
"""When cookies are found, results dict includes them."""
mock_extract.return_value = ({"auth_token": "abc", "ct0": "xyz"}, "chrome")
mock_which.return_value = "/usr/local/bin/yt-dlp"
config = {}
results = setup_wizard.run_auto_setup(config)
assert "x" in results["cookies_found"]
assert results["cookies_found"]["x"] == "chrome"
assert results["ytdlp_installed"] is True
assert results["ytdlp_action"] == "already_installed"
assert results["env_written"] is False
@patch("lib.cookie_extract.extract_cookies_with_source")
@patch("shutil.which")
def test_no_cookies_found(self, mock_which, mock_extract):
"""When no cookies found, results dict has empty cookies_found."""
mock_extract.return_value = None
mock_which.return_value = None
config = {}
results = setup_wizard.run_auto_setup(config)
assert results["cookies_found"] == {}
assert results["ytdlp_installed"] is False
assert results["ytdlp_action"] == "no_homebrew"
@patch("lib.cookie_extract.extract_cookies_with_source")
@patch("shutil.which")
def test_cookie_extraction_exception(self, mock_which, mock_extract):
"""Cookie extraction raising an exception is handled gracefully."""
mock_extract.side_effect = Exception("DB locked")
mock_which.return_value = None
config = {}
results = setup_wizard.run_auto_setup(config)
assert results["cookies_found"] == {}
@patch("lib.cookie_extract.extract_cookies_with_source")
@patch("shutil.which")
def test_multiple_sources(self, mock_which, mock_extract):
"""Multiple cookie sources can be found."""
def side_effect(browser, domain, cookie_names):
if domain == ".x.com":
return ({"auth_token": "abc", "ct0": "xyz"}, "firefox")
elif domain == ".truthsocial.com":
return ({"_session_id": "sess123"}, "firefox")
return None
mock_extract.side_effect = side_effect
mock_which.return_value = None
config = {}
results = setup_wizard.run_auto_setup(config)
assert results["cookies_found"]["x"] == "firefox"
assert results["cookies_found"]["truthsocial"] == "firefox"
class TestYtdlpAutoInstall:
"""Tests for yt-dlp auto-install via Homebrew in run_auto_setup()."""
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
@patch("subprocess.run")
@patch("shutil.which")
def test_ytdlp_missing_brew_available_installs(self, mock_which, mock_subproc, mock_extract):
"""yt-dlp missing + brew available -> installs via brew."""
def which_side_effect(cmd):
if cmd == "yt-dlp":
return None
if cmd == "brew":
return "/opt/homebrew/bin/brew"
return None
mock_which.side_effect = which_side_effect
mock_subproc.return_value = MagicMock(returncode=0, stderr="")
results = setup_wizard.run_auto_setup({})
mock_subproc.assert_called_once_with(
["brew", "install", "yt-dlp"],
capture_output=True, text=True, timeout=120,
)
assert results["ytdlp_installed"] is True
assert results["ytdlp_action"] == "installed"
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
@patch("shutil.which")
def test_ytdlp_missing_brew_missing(self, mock_which, mock_extract):
"""yt-dlp missing + brew missing -> no_homebrew."""
mock_which.return_value = None
results = setup_wizard.run_auto_setup({})
assert results["ytdlp_installed"] is False
assert results["ytdlp_action"] == "no_homebrew"
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
@patch("shutil.which")
def test_ytdlp_already_installed(self, mock_which, mock_extract):
"""yt-dlp already installed -> already_installed."""
mock_which.return_value = "/usr/local/bin/yt-dlp"
results = setup_wizard.run_auto_setup({})
assert results["ytdlp_installed"] is True
assert results["ytdlp_action"] == "already_installed"
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
@patch("subprocess.run")
@patch("shutil.which")
def test_brew_install_fails(self, mock_which, mock_subproc, mock_extract):
"""brew install yt-dlp fails -> install_failed with stderr."""
def which_side_effect(cmd):
if cmd == "yt-dlp":
return None
if cmd == "brew":
return "/opt/homebrew/bin/brew"
return None
mock_which.side_effect = which_side_effect
mock_subproc.return_value = MagicMock(returncode=1, stderr="Error: something broke")
results = setup_wizard.run_auto_setup({})
assert results["ytdlp_installed"] is False
assert results["ytdlp_action"] == "install_failed"
assert "something broke" in results["ytdlp_stderr"]
class TestWriteSetupConfig:
"""Tests for write_setup_config()."""
def test_creates_new_env_file(self):
"""Creates .env file with SETUP_COMPLETE and FROM_BROWSER."""
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / "subdir" / ".env"
result = setup_wizard.write_setup_config(env_path)
assert result is True
assert env_path.exists()
content = env_path.read_text()
assert "SETUP_COMPLETE=true" in content
assert "FROM_BROWSER=auto" in content
def test_appends_to_existing_file(self):
"""Appends to existing .env without overwriting keys."""
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / ".env"
env_path.write_text("XAI_API_KEY=my-key\nAUTH_TOKEN=tok123\n")
result = setup_wizard.write_setup_config(env_path)
assert result is True
content = env_path.read_text()
# Original keys preserved
assert "XAI_API_KEY=my-key" in content
assert "AUTH_TOKEN=tok123" in content
# New keys appended
assert "SETUP_COMPLETE=true" in content
assert "FROM_BROWSER=auto" in content
def test_does_not_overwrite_existing_keys(self):
"""If SETUP_COMPLETE or FROM_BROWSER already exist, don't duplicate."""
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / ".env"
env_path.write_text("SETUP_COMPLETE=true\nFROM_BROWSER=firefox\n")
result = setup_wizard.write_setup_config(env_path)
assert result is True
content = env_path.read_text()
# Should only appear once
assert content.count("SETUP_COMPLETE") == 1
assert content.count("FROM_BROWSER") == 1
# Original value preserved
assert "FROM_BROWSER=firefox" in content
def test_custom_from_browser_value(self):
"""Custom from_browser value is written."""
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / ".env"
result = setup_wizard.write_setup_config(env_path, from_browser="chrome")
assert result is True
content = env_path.read_text()
assert "FROM_BROWSER=chrome" in content
def test_creates_parent_directories(self):
"""Creates parent directories if they don't exist."""
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / "a" / "b" / "c" / ".env"
result = setup_wizard.write_setup_config(env_path)
assert result is True
assert env_path.exists()
def test_handles_file_without_trailing_newline(self):
"""Appends correctly when existing file has no trailing newline."""
with tempfile.TemporaryDirectory() as tmpdir:
env_path = Path(tmpdir) / ".env"
env_path.write_text("EXISTING_KEY=value") # no trailing newline
result = setup_wizard.write_setup_config(env_path)
assert result is True
content = env_path.read_text()
# Should have newline separator
lines = content.strip().split("\n")
assert len(lines) == 3
assert lines[0] == "EXISTING_KEY=value"
assert "SETUP_COMPLETE=true" in lines[1]
class TestGetSetupStatusText:
"""Tests for get_setup_status_text()."""
def test_with_cookies_and_ytdlp(self):
"""Status text mentions found cookies and yt-dlp."""
results = {
"cookies_found": {"x": "chrome"},
"ytdlp_installed": True,
"ytdlp_action": "already_installed",
"env_written": True,
}
text = setup_wizard.get_setup_status_text(results)
assert "X cookies found in chrome" in text
assert "yt-dlp already installed" in text
assert "Configuration saved" in text
def test_with_no_cookies_no_ytdlp(self):
"""Status text shows no cookies and suggests yt-dlp install."""
results = {
"cookies_found": {},
"ytdlp_installed": False,
"ytdlp_action": "no_homebrew",
"env_written": False,
}
text = setup_wizard.get_setup_status_text(results)
assert "No browser cookies found" in text
assert "Install Homebrew first" in text
def test_status_text_installed(self):
"""Status text for freshly installed yt-dlp."""
results = {
"cookies_found": {},
"ytdlp_installed": True,
"ytdlp_action": "installed",
"env_written": False,
}
text = setup_wizard.get_setup_status_text(results)
assert "Installed yt-dlp via Homebrew" in text
def test_status_text_install_failed(self):
"""Status text for failed yt-dlp install."""
results = {
"cookies_found": {},
"ytdlp_installed": False,
"ytdlp_action": "install_failed",
"env_written": False,
}
text = setup_wizard.get_setup_status_text(results)
assert "yt-dlp install failed" in text
assert "manually" in text
class TestSetupSubcommand:
"""Tests for setup subcommand detection in argument parsing."""
def test_setup_detected_as_topic(self):
"""The word 'setup' is treated as the setup subcommand."""
# Simulate what argparse produces
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("topic", nargs="*")
args = parser.parse_args(["setup"])
topic = " ".join(args.topic) if args.topic else None
assert topic is not None
assert topic.strip().lower() == "setup"
def test_normal_topic_not_setup(self):
"""A normal topic is not confused with setup."""
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("topic", nargs="*")
args = parser.parse_args(["AI", "video", "tools"])
topic = " ".join(args.topic) if args.topic else None
assert topic.strip().lower() != "setup"
+737
View File
@@ -0,0 +1,737 @@
"""Tests for source resolution priority hierarchy (Unit 4).
Validates the free-first priority chain:
env AUTH_TOKEN/CT0 -> browser cookies -> XAI_API_KEY -> None
"""
import os
from unittest.mock import patch, MagicMock
import pytest
from scripts.lib import env
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _base_config(**overrides):
"""Return a minimal config dict with typical defaults."""
cfg = {
"AUTH_TOKEN": None,
"CT0": None,
"XAI_API_KEY": None,
"SCRAPECREATORS_API_KEY": None,
"OPENAI_API_KEY": None,
"OPENAI_AUTH_STATUS": "missing",
"OPENROUTER_API_KEY": None,
"PARALLEL_API_KEY": None,
"BRAVE_API_KEY": None,
"BSKY_HANDLE": None,
"BSKY_APP_PASSWORD": None,
"TRUTHSOCIAL_TOKEN": None,
"FROM_BROWSER": None,
"SETUP_COMPLETE": None,
"_AUTH_TOKEN_SOURCE": None,
}
cfg.update(overrides)
return cfg
def _mock_bird_installed(installed=True):
"""Patch bird_x.is_bird_installed to return the given value."""
return patch("scripts.lib.bird_x.is_bird_installed", return_value=installed)
def _mock_bird_authenticated(username=None):
"""Patch bird_x.is_bird_authenticated to return the given value."""
return patch("scripts.lib.bird_x.is_bird_authenticated", return_value=username)
def _mock_bird_status(installed=True, authenticated=True, username="env AUTH_TOKEN"):
"""Patch bird_x.get_bird_status to return a status dict."""
return patch("scripts.lib.bird_x.get_bird_status", return_value={
"installed": installed,
"authenticated": authenticated,
"username": username,
"can_install": True,
})
# ---------------------------------------------------------------------------
# Tests: X source resolution priority
# ---------------------------------------------------------------------------
class TestXSourcePriority:
"""Test the X source priority chain: env -> browser cookies -> xAI -> None."""
def test_no_env_cookies_found_resolves_bird_browser(self):
"""No env vars, SETUP_COMPLETE=true, cookies found -> Bird with browser method."""
config = _base_config(
AUTH_TOKEN="cookie_tok",
CT0="cookie_ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-firefox",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
source, method = env.get_x_source_with_method(config)
assert source == "bird"
assert method == "browser-firefox"
def test_env_auth_token_plus_cookies_env_wins(self):
"""AUTH_TOKEN in .env + cookies available -> env wins (method='env')."""
config = _base_config(
AUTH_TOKEN="explicit_token",
CT0="explicit_ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="env",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
source, method = env.get_x_source_with_method(config)
assert source == "bird"
assert method == "env"
def test_no_env_no_cookies_xai_key_resolves_xai(self):
"""No env vars, no cookies, XAI_API_KEY set -> xAI with method 'api'."""
config = _base_config(XAI_API_KEY="xai-key-123")
with _mock_bird_installed(True), _mock_bird_authenticated(None):
source, method = env.get_x_source_with_method(config)
assert source == "xai"
assert method == "api"
def test_no_env_no_cookies_no_api_keys_none(self):
"""No env vars, no cookies, no API keys -> X not available."""
config = _base_config()
with _mock_bird_installed(True), _mock_bird_authenticated(None):
source, method = env.get_x_source_with_method(config)
assert source is None
assert method is None
def test_bird_not_installed_falls_to_xai(self):
"""Bird not installed, XAI_API_KEY set -> xAI."""
config = _base_config(XAI_API_KEY="xai-key")
with _mock_bird_installed(False), _mock_bird_authenticated(None):
source, method = env.get_x_source_with_method(config)
assert source == "xai"
assert method == "api"
def test_bird_not_installed_no_xai_none(self):
"""Bird not installed, no XAI_API_KEY -> None."""
config = _base_config()
with _mock_bird_installed(False), _mock_bird_authenticated(None):
source, method = env.get_x_source_with_method(config)
assert source is None
assert method is None
def test_browser_chrome_method_tracked(self):
"""Cookies from Chrome -> method is 'browser-chrome'."""
config = _base_config(
AUTH_TOKEN="chrome_tok",
CT0="chrome_ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-chrome",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
source, method = env.get_x_source_with_method(config)
assert source == "bird"
assert method == "browser-chrome"
def test_browser_safari_method_tracked(self):
"""Cookies from Safari -> method is 'browser-safari'."""
config = _base_config(
AUTH_TOKEN="safari_tok",
CT0="safari_ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-safari",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
source, method = env.get_x_source_with_method(config)
assert source == "bird"
assert method == "browser-safari"
# ---------------------------------------------------------------------------
# Tests: get_x_source() backward compat
# ---------------------------------------------------------------------------
class TestGetXSourceBackwardCompat:
"""Ensure get_x_source() returns the same string as before."""
def test_bird_returns_bird(self):
config = _base_config(AUTH_TOKEN="tok", CT0="ct0", _AUTH_TOKEN_SOURCE="env")
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
assert env.get_x_source(config) == "bird"
def test_xai_returns_xai(self):
config = _base_config(XAI_API_KEY="key")
with _mock_bird_installed(True), _mock_bird_authenticated(None):
assert env.get_x_source(config) == "xai"
def test_none_returns_none(self):
config = _base_config()
with _mock_bird_installed(False):
assert env.get_x_source(config) is None
# ---------------------------------------------------------------------------
# Tests: get_x_source_status() method field
# ---------------------------------------------------------------------------
class TestGetXSourceStatusMethod:
"""Test that get_x_source_status() includes the method field."""
def test_bird_env_method(self):
config = _base_config(AUTH_TOKEN="tok", CT0="ct0", _AUTH_TOKEN_SOURCE="env")
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"), \
_mock_bird_status(installed=True, authenticated=True, username="env AUTH_TOKEN"):
status = env.get_x_source_status(config)
assert status["source"] == "bird"
assert status["method"] == "env"
assert "bird_installed" in status
assert "xai_available" in status
def test_bird_browser_method(self):
config = _base_config(
AUTH_TOKEN="tok", CT0="ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-firefox",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"), \
_mock_bird_status(installed=True, authenticated=True):
status = env.get_x_source_status(config)
assert status["source"] == "bird"
assert status["method"] == "browser-firefox"
def test_xai_api_method(self):
config = _base_config(XAI_API_KEY="key")
with _mock_bird_installed(True), _mock_bird_authenticated(None), \
_mock_bird_status(installed=True, authenticated=False, username=None):
status = env.get_x_source_status(config)
assert status["source"] == "xai"
assert status["method"] == "api"
def test_no_source_method_none(self):
config = _base_config()
with _mock_bird_installed(False), \
_mock_bird_status(installed=False, authenticated=False, username=None):
status = env.get_x_source_status(config)
assert status["source"] is None
assert status["method"] is None
# ---------------------------------------------------------------------------
# Tests: get_available_sources() with various configs
# ---------------------------------------------------------------------------
class TestGetAvailableSources:
"""Test get_available_sources() returns correct strings."""
def test_no_x_no_web_reddit_only(self):
"""No X source, no web keys -> 'reddit' (Reddit always available)."""
config = _base_config()
with _mock_bird_installed(False):
result = env.get_available_sources(config)
assert result == "reddit"
def test_xai_key_no_web(self):
"""XAI_API_KEY set, no web keys -> 'both'."""
config = _base_config(XAI_API_KEY="key")
with _mock_bird_installed(True), _mock_bird_authenticated(None):
result = env.get_available_sources(config)
assert result == "both"
def test_bird_auth_no_web(self):
"""Bird authenticated (cookies), SETUP_COMPLETE=true, no web keys -> 'both'."""
config = _base_config(
AUTH_TOKEN="tok", CT0="ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-firefox",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
result = env.get_available_sources(config)
assert result == "both"
def test_bird_auth_with_web(self):
"""Bird authenticated + web keys -> 'all'."""
config = _base_config(
AUTH_TOKEN="tok", CT0="ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-firefox",
BRAVE_API_KEY="brave-key",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
result = env.get_available_sources(config)
assert result == "all"
def test_no_x_with_web(self):
"""No X source, web keys -> 'reddit-web'."""
config = _base_config(BRAVE_API_KEY="brave-key")
with _mock_bird_installed(False):
result = env.get_available_sources(config)
assert result == "reddit-web"
def test_reddit_hn_polymarket_always_available(self):
"""Reddit, HN, and Polymarket are always available regardless of config."""
config = _base_config()
# These functions don't depend on config
assert env.is_hackernews_available() is True
assert env.is_polymarket_available() is True
# Reddit: get_available_sources always includes it
with _mock_bird_installed(False):
result = env.get_available_sources(config)
assert result in ("reddit", "reddit-web") # never 'none' when Reddit is always True
# ---------------------------------------------------------------------------
# Tests: Full resolution with mixed config
# ---------------------------------------------------------------------------
class TestFullResolution:
"""Test that each source resolves independently with mixed config."""
def test_mixed_config_all_sources(self):
"""Bird for X, public Reddit, web keys, YouTube/TikTok available."""
config = _base_config(
AUTH_TOKEN="tok", CT0="ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-chrome",
BRAVE_API_KEY="brave-key",
SCRAPECREATORS_API_KEY="sc-key",
BSKY_HANDLE="user.bsky.social",
BSKY_APP_PASSWORD="app-pw",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
x_source, x_method = env.get_x_source_with_method(config)
available = env.get_available_sources(config)
assert x_source == "bird"
assert x_method == "browser-chrome"
assert available == "all" # reddit + x + web
assert env.is_bluesky_available(config) is True
assert env.is_tiktok_available(config) is True
assert env.is_hackernews_available() is True
assert env.is_polymarket_available() is True
def test_no_config_minimal_sources(self):
"""No API keys, no cookies -> Reddit + HN + Polymarket only."""
config = _base_config()
with _mock_bird_installed(False):
x_source = env.get_x_source(config)
available = env.get_available_sources(config)
assert x_source is None
assert available == "reddit"
assert env.is_hackernews_available() is True
assert env.is_polymarket_available() is True
assert env.is_bluesky_available(config) is False
assert env.is_tiktok_available(config) is False
# ---------------------------------------------------------------------------
# Tests: extract_browser_credentials tracks browser source
# ---------------------------------------------------------------------------
class TestExtractBrowserCredentialsSource:
"""Test that extract_browser_credentials tracks __X_BROWSER."""
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_tracks_firefox_source(self, mock_extract):
"""Cookies from Firefox -> __X_BROWSER set to 'firefox'."""
mock_extract.return_value = (
{"auth_token": "tok", "ct0": "ct0val"},
"firefox",
)
config = {
"AUTH_TOKEN": None,
"CT0": None,
"TRUTHSOCIAL_TOKEN": None,
"FROM_BROWSER": "auto",
"SETUP_COMPLETE": "true",
}
result = env.extract_browser_credentials(config)
assert result["AUTH_TOKEN"] == "tok"
assert result["CT0"] == "ct0val"
assert result["__X_BROWSER"] == "firefox"
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_tracks_chrome_source(self, mock_extract):
"""Cookies from Chrome -> __X_BROWSER set to 'chrome'."""
mock_extract.return_value = (
{"auth_token": "tok", "ct0": "ct0val"},
"chrome",
)
config = {
"AUTH_TOKEN": None,
"CT0": None,
"TRUTHSOCIAL_TOKEN": None,
"FROM_BROWSER": "chrome",
"SETUP_COMPLETE": "true",
}
result = env.extract_browser_credentials(config)
assert result["__X_BROWSER"] == "chrome"
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
def test_no_cookies_no_browser_key(self, mock_extract):
"""No cookies found -> no __X_BROWSER key."""
mock_extract.return_value = None
config = {
"AUTH_TOKEN": None,
"CT0": None,
"TRUTHSOCIAL_TOKEN": None,
"FROM_BROWSER": "auto",
"SETUP_COMPLETE": "true",
}
result = env.extract_browser_credentials(config)
assert "__X_BROWSER" not in result
# ---------------------------------------------------------------------------
# Tests: get_config() _AUTH_TOKEN_SOURCE tracking
# ---------------------------------------------------------------------------
class TestGetConfigAuthTokenSource:
"""Test that get_config() sets _AUTH_TOKEN_SOURCE correctly."""
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
@patch("scripts.lib.env._find_project_env", return_value=None)
@patch("scripts.lib.env.load_env_file", return_value={})
@patch("scripts.lib.env.get_openai_auth")
def test_env_var_auth_token_source_env(
self, mock_openai, mock_load, mock_proj, mock_extract
):
"""AUTH_TOKEN from env var -> _AUTH_TOKEN_SOURCE='env'."""
from scripts.lib.env import get_config, OpenAIAuth
mock_openai.return_value = OpenAIAuth(
token=None, source="none", status="missing",
account_id=None, codex_auth_file="/fake",
)
mock_extract.return_value = None
env_patch = {
"AUTH_TOKEN": "env_token",
"CT0": "env_ct0",
"LAST30DAYS_CONFIG_DIR": "",
}
with patch.dict(os.environ, env_patch, clear=False):
config = get_config()
assert config["_AUTH_TOKEN_SOURCE"] == "env"
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
@patch("scripts.lib.env._find_project_env", return_value=None)
@patch("scripts.lib.env.load_env_file", return_value={})
@patch("scripts.lib.env.get_openai_auth")
def test_browser_cookies_auth_token_source_browser(
self, mock_openai, mock_load, mock_proj, mock_extract
):
"""AUTH_TOKEN from cookies -> _AUTH_TOKEN_SOURCE='browser-firefox'."""
from scripts.lib.env import get_config, OpenAIAuth
mock_openai.return_value = OpenAIAuth(
token=None, source="none", status="missing",
account_id=None, codex_auth_file="/fake",
)
mock_extract.return_value = (
{"auth_token": "cookie_tok", "ct0": "cookie_ct0"},
"firefox",
)
env_patch = {
"SETUP_COMPLETE": "true",
"FROM_BROWSER": "auto",
"LAST30DAYS_CONFIG_DIR": "",
}
# Ensure AUTH_TOKEN is NOT in env
clean_env = {k: v for k, v in os.environ.items()
if k not in ("AUTH_TOKEN", "CT0")}
clean_env.update(env_patch)
with patch.dict(os.environ, clean_env, clear=True):
config = get_config()
assert config["AUTH_TOKEN"] == "cookie_tok"
assert config["_AUTH_TOKEN_SOURCE"] == "browser-firefox"
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
@patch("scripts.lib.env._find_project_env", return_value=None)
@patch("scripts.lib.env.load_env_file", return_value={})
@patch("scripts.lib.env.get_openai_auth")
def test_no_auth_token_source_none(
self, mock_openai, mock_load, mock_proj, mock_extract
):
"""No AUTH_TOKEN at all -> _AUTH_TOKEN_SOURCE=None."""
from scripts.lib.env import get_config, OpenAIAuth
mock_openai.return_value = OpenAIAuth(
token=None, source="none", status="missing",
account_id=None, codex_auth_file="/fake",
)
mock_extract.return_value = None
clean_env = {k: v for k, v in os.environ.items()
if k not in ("AUTH_TOKEN", "CT0")}
clean_env["LAST30DAYS_CONFIG_DIR"] = ""
with patch.dict(os.environ, clean_env, clear=True):
config = get_config()
assert config["_AUTH_TOKEN_SOURCE"] is None
# ---------------------------------------------------------------------------
# Tests: SETUP_COMPLETE gate — Bird cookie probing blocked before consent
# ---------------------------------------------------------------------------
class TestSetupCompleteGate:
"""Test that Bird cookie probing is gated behind SETUP_COMPLETE consent."""
def test_no_setup_complete_bird_has_cookies_returns_none(self):
"""SETUP_COMPLETE not set, Bird has browser cookies -> returns None (not 'bird').
Bird's is_bird_authenticated() should NOT be called at all because
there is no consent yet. Cookie-sourced AUTH_TOKEN must be ignored.
"""
config = _base_config(
AUTH_TOKEN="cookie_tok",
CT0="cookie_ct0",
SETUP_COMPLETE=None, # not set — first run
_AUTH_TOKEN_SOURCE="browser-chrome",
)
with _mock_bird_installed(True), \
_mock_bird_authenticated("Chrome") as mock_auth:
source = env.get_x_source(config)
assert source is None
# is_bird_authenticated should NOT have been called (no cookie probing)
mock_auth.assert_not_called()
def test_setup_complete_bird_has_cookies_returns_bird(self):
"""SETUP_COMPLETE=true, Bird has browser cookies -> returns 'bird'.
After user consent, cookie probing should work normally.
"""
config = _base_config(
AUTH_TOKEN="cookie_tok",
CT0="cookie_ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-chrome",
)
with _mock_bird_installed(True), _mock_bird_authenticated("Chrome"):
source = env.get_x_source(config)
assert source == "bird"
def test_no_setup_complete_explicit_auth_token_returns_bird(self):
"""SETUP_COMPLETE not set, AUTH_TOKEN from env var -> returns 'bird' with method 'env'.
Explicit env var credentials must ALWAYS work regardless of SETUP_COMPLETE.
"""
config = _base_config(
AUTH_TOKEN="explicit_token",
CT0="explicit_ct0",
SETUP_COMPLETE=None, # not set
_AUTH_TOKEN_SOURCE="env",
)
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
source, method = env.get_x_source_with_method(config)
assert source == "bird"
assert method == "env"
def test_no_setup_complete_xai_key_returns_xai(self):
"""SETUP_COMPLETE not set, XAI_API_KEY configured -> returns 'xai'.
API keys always work without setup consent.
"""
config = _base_config(
XAI_API_KEY="xai-key-123",
SETUP_COMPLETE=None, # not set
)
with _mock_bird_installed(True), _mock_bird_authenticated(None):
source, method = env.get_x_source_with_method(config)
assert source == "xai"
assert method == "api"
def test_no_setup_complete_status_reports_not_configured(self):
"""First-run status banner should show X as not configured when only cookies exist."""
config = _base_config(
AUTH_TOKEN="cookie_tok",
CT0="cookie_ct0",
SETUP_COMPLETE=None,
_AUTH_TOKEN_SOURCE="browser-firefox",
)
with _mock_bird_installed(True):
status = env.get_x_source_status(config)
assert status["source"] is None
assert status["method"] is None
assert status["bird_authenticated"] is False
def test_setup_complete_status_reports_bird(self):
"""After consent, status banner should show Bird as configured."""
config = _base_config(
AUTH_TOKEN="cookie_tok",
CT0="cookie_ct0",
SETUP_COMPLETE="true",
_AUTH_TOKEN_SOURCE="browser-firefox",
)
with _mock_bird_installed(True), _mock_bird_authenticated("Firefox"), \
_mock_bird_status(installed=True, authenticated=True, username="Firefox"):
status = env.get_x_source_status(config)
assert status["source"] == "bird"
assert status["method"] == "browser-firefox"
assert status["bird_authenticated"] is True
# ---------------------------------------------------------------------------
# Tests: BIRD_DISABLE_BROWSER_COOKIES env var on first run
# ---------------------------------------------------------------------------
class TestBirdDisableBrowserCookiesEnvVar:
"""Test that BIRD_DISABLE_BROWSER_COOKIES is set to block Bird's Node.js
sweet-cookie scanner on first run before user consent."""
def _run_main_flow_env_setup(self, config, first_run):
"""Simulate the env-var-setting logic from last30days.py main flow.
Mirrors the block right after first_run detection in main().
"""
if first_run and config.get('_AUTH_TOKEN_SOURCE') != 'env':
os.environ['BIRD_DISABLE_BROWSER_COOKIES'] = '1'
else:
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
def test_first_run_no_auth_token_sets_env_var(self):
"""first_run=True, no AUTH_TOKEN -> BIRD_DISABLE_BROWSER_COOKIES is set."""
config = _base_config(SETUP_COMPLETE=None, _AUTH_TOKEN_SOURCE=None)
try:
self._run_main_flow_env_setup(config, first_run=True)
assert os.environ.get('BIRD_DISABLE_BROWSER_COOKIES') == '1'
finally:
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
def test_not_first_run_no_env_var(self):
"""first_run=False -> BIRD_DISABLE_BROWSER_COOKIES is NOT set."""
config = _base_config(SETUP_COMPLETE="true", _AUTH_TOKEN_SOURCE="browser-firefox")
try:
self._run_main_flow_env_setup(config, first_run=False)
assert 'BIRD_DISABLE_BROWSER_COOKIES' not in os.environ
finally:
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
def test_first_run_explicit_auth_token_no_env_var(self):
"""first_run=True, AUTH_TOKEN explicitly set -> BIRD_DISABLE_BROWSER_COOKIES is NOT set."""
config = _base_config(
AUTH_TOKEN="explicit_token",
CT0="explicit_ct0",
SETUP_COMPLETE=None,
_AUTH_TOKEN_SOURCE="env",
)
try:
self._run_main_flow_env_setup(config, first_run=True)
assert 'BIRD_DISABLE_BROWSER_COOKIES' not in os.environ
finally:
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
# ---------------------------------------------------------------------------
# Tests: INCLUDE_SOURCES config override
# ---------------------------------------------------------------------------
class TestIncludeSourcesOverride:
"""Test that INCLUDE_SOURCES forces sources on regardless of tier."""
def _simulate_source_decisions(self, config, query_type="breaking_news"):
"""Simulate the source decision logic from last30days.py main flow.
Returns (search_run_tiktok, search_run_instagram) after tier + override.
"""
from scripts.lib import query_type as qt
has_tiktok = env.is_tiktok_available(config)
has_instagram = env.is_instagram_available(config)
# Tier system decision
search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type)
search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type)
# INCLUDE_SOURCES override (mirrors last30days.py logic)
_include_sources = {
s.strip().lower()
for s in config.get('INCLUDE_SOURCES', '').split(',')
if s.strip()
}
if _include_sources:
if 'tiktok' in _include_sources and has_tiktok:
if not search_run_tiktok:
search_run_tiktok = True
if 'instagram' in _include_sources and has_instagram:
if not search_run_instagram:
search_run_instagram = True
return search_run_tiktok, search_run_instagram
def test_include_sources_forces_tiktok_and_instagram_on(self):
"""INCLUDE_SOURCES=tiktok,instagram + SC key + GENERAL query -> both forced on."""
config = _base_config(
SCRAPECREATORS_API_KEY="sc-key",
INCLUDE_SOURCES="tiktok,instagram",
)
run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
assert run_tiktok is True
assert run_instagram is True
def test_no_include_sources_tier_controls(self):
"""No INCLUDE_SOURCES + SC key + GENERAL query -> tier system controls (both off)."""
config = _base_config(
SCRAPECREATORS_API_KEY="sc-key",
)
run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
# breaking_news tier doesn't include tiktok or instagram
assert run_tiktok is False
assert run_instagram is False
def test_include_sources_no_sc_key_still_off(self):
"""INCLUDE_SOURCES=tiktok but no SC key -> TikTok still off (no backend)."""
config = _base_config(
INCLUDE_SOURCES="tiktok",
)
run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
assert run_tiktok is False
assert run_instagram is False
+283
View File
@@ -0,0 +1,283 @@
"""Tests for the redesigned status banner (free-first design)."""
import pytest
from scripts.lib.ui import _build_status_banner
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _base_diag(**overrides):
"""Return a minimal diag dict with common defaults."""
diag = {
"setup_complete": False,
"reddit_source": None, # None = public fallback
"x_source": None,
"x_method": None,
"youtube": False,
"tiktok": False,
"instagram": False,
"hackernews": True,
"polymarket": True,
"bluesky": False,
"truthsocial": False,
"xiaohongshu": False,
"scrapecreators": False,
"web_search_backend": None,
}
diag.update(overrides)
return diag
def _banner_text(diag):
"""Return full banner as a single string."""
return "\n".join(_build_status_banner(diag))
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestZeroConfig:
"""Zero-config state: SETUP_COMPLETE not set, wizard hasn't run."""
def test_shows_first_run_title(self):
banner = _banner_text(_base_diag())
assert "First Run" in banner
def test_shows_three_free_sources(self):
banner = _banner_text(_base_diag())
assert "Reddit (threads only)" in banner
assert "HN" in banner
assert "Polymarket" in banner
def test_shows_setup_prompt(self):
banner = _banner_text(_base_diag())
assert "/last30days setup" in banner
def test_does_not_show_source_status_title(self):
banner = _banner_text(_base_diag())
assert "Source Status" not in banner
class TestFullConfig:
"""Fully configured: X + yt-dlp + ScrapeCreators = everything active."""
def _full_diag(self):
return _base_diag(
setup_complete=True,
reddit_source="scrapecreators",
x_source="bird",
x_method="browser-chrome",
youtube=True,
tiktok=True,
instagram=True,
hackernews=True,
polymarket=True,
bluesky=True,
truthsocial=True,
xiaohongshu=True,
scrapecreators=True,
web_search_backend="parallel",
)
def test_shows_source_status_title(self):
banner = _banner_text(self._full_diag())
assert "Source Status" in banner
def test_shows_all_sources(self):
banner = _banner_text(self._full_diag())
assert "Reddit (with comments)" in banner
assert "X (Chrome)" in banner
assert "YouTube" in banner
assert "HN" in banner
assert "Polymarket" in banner
assert "TikTok" in banner
assert "Instagram" in banner
assert "Bluesky" in banner
assert "Truth Social" in banner
assert "Xiaohongshu" in banner
def test_no_recommendations(self):
banner = _banner_text(self._full_diag())
assert "" not in banner
assert "scrapecreators.com" not in banner
assert "/last30days setup" not in banner
class TestPartialConfig:
"""Partially configured: X + yt-dlp but no ScrapeCreators."""
def _partial_diag(self):
return _base_diag(
setup_complete=True,
reddit_source=None, # public fallback
x_source="bird",
x_method="browser-chrome",
youtube=True,
scrapecreators=False,
)
def test_shows_active_sources(self):
banner = _banner_text(self._partial_diag())
assert "Reddit (threads only)" in banner
assert "X (Chrome)" in banner
assert "YouTube" in banner
assert "HN" in banner
assert "Polymarket" in banner
def test_recommends_scrapecreators(self):
banner = _banner_text(self._partial_diag())
assert "SCRAPECREATORS_API_KEY" in banner
def test_scrapecreators_free_calls_copy(self):
banner = _banner_text(self._partial_diag())
assert "100 free calls, no CC" in banner
assert "scrapecreators.com" in banner
def test_shows_tiktok_instagram_unlock(self):
banner = _banner_text(self._partial_diag())
assert "TikTok" in banner
assert "Instagram" in banner
class TestScrapeCreatorsRecommendation:
"""ScrapeCreators recommendation always includes key copy."""
def test_always_includes_free_calls_no_cc(self):
"""Any config missing SC should show the free-calls copy."""
# Zero config
banner_zero = _banner_text(_base_diag())
# Zero config doesn't recommend SC directly (recommends setup wizard)
# But after setup, missing SC should always include the copy
banner_partial = _banner_text(_base_diag(
setup_complete=True,
scrapecreators=False,
))
assert "100 free calls, no CC" in banner_partial
def test_present_when_x_available_but_no_sc(self):
banner = _banner_text(_base_diag(
setup_complete=True,
x_source="xai",
x_method="api",
scrapecreators=False,
))
assert "100 free calls, no CC" in banner
assert "scrapecreators.com" in banner
class TestRedditLabelDisplay:
"""Reddit label reflects comment availability, not implementation details."""
def test_no_scrapecreators_shows_threads_only(self):
"""Without SC, Reddit label should say 'threads only' regardless of OpenAI auth."""
banner = _banner_text(_base_diag(
setup_complete=True,
reddit_source="openai",
scrapecreators=False,
))
assert "Reddit (threads only)" in banner
assert "OpenAI" not in banner
assert "Codex" not in banner
def test_no_scrapecreators_public_shows_threads_only(self):
"""Public fallback also shows 'threads only'."""
banner = _banner_text(_base_diag(
setup_complete=True,
reddit_source=None,
scrapecreators=False,
))
assert "Reddit (threads only)" in banner
def test_scrapecreators_shows_with_comments(self):
"""With SC configured, Reddit label should say 'with comments'."""
banner = _banner_text(_base_diag(
setup_complete=True,
reddit_source="scrapecreators",
scrapecreators=True,
))
assert "Reddit (with comments)" in banner
def test_no_openai_or_codex_in_banner(self):
"""Banner should never mention OpenAI or Codex — those are implementation details."""
for source in [None, "openai", "scrapecreators"]:
banner = _banner_text(_base_diag(
setup_complete=True,
reddit_source=source,
scrapecreators=(source == "scrapecreators"),
))
assert "OpenAI" not in banner, f"Found 'OpenAI' with reddit_source={source}"
assert "Codex" not in banner, f"Found 'Codex' with reddit_source={source}"
class TestXMethodDisplay:
"""X source shows auth method in parens."""
def test_browser_chrome(self):
banner = _banner_text(_base_diag(
setup_complete=True,
x_source="bird",
x_method="browser-chrome",
))
assert "X (Chrome)" in banner
def test_browser_firefox(self):
banner = _banner_text(_base_diag(
setup_complete=True,
x_source="bird",
x_method="browser-firefox",
))
assert "X (Firefox)" in banner
def test_env_method(self):
banner = _banner_text(_base_diag(
setup_complete=True,
x_source="bird",
x_method="env",
))
assert "X (env)" in banner
def test_xai_api(self):
banner = _banner_text(_base_diag(
setup_complete=True,
x_source="xai",
x_method="api",
))
assert "X (xAI)" in banner
class TestBannerStructure:
"""Banner formatting and structure tests."""
def test_has_box_drawing(self):
lines = _build_status_banner(_base_diag())
assert lines[0].startswith("")
assert lines[-1].startswith("")
def test_shows_config_path(self):
banner = _banner_text(_base_diag())
assert "~/.config/last30days/.env" in banner
def test_max_10_inner_lines(self):
"""Banner should be compact — max 10 lines inside the box."""
# Full config (most lines)
diag = _base_diag(
setup_complete=True,
reddit_source="scrapecreators",
x_source="bird",
x_method="browser-chrome",
youtube=True,
tiktok=True,
instagram=True,
bluesky=True,
truthsocial=True,
xiaohongshu=True,
scrapecreators=True,
)
lines = _build_status_banner(diag)
# Subtract top and bottom border
inner_lines = [l for l in lines if l.startswith("")]
assert len(inner_lines) <= 10, f"Banner has {len(inner_lines)} inner lines, max is 10"
+169
View File
@@ -1,8 +1,10 @@
"""Tests for yt-dlp invocation safety flags.""" """Tests for yt-dlp invocation safety flags."""
import json
import sys import sys
import tempfile import tempfile
import unittest import unittest
import urllib.error
from pathlib import Path from pathlib import Path
from unittest import mock from unittest import mock
@@ -37,6 +39,7 @@ class TestYtDlpFlags(unittest.TestCase):
def test_transcript_fetch_ignores_global_config_and_browser_cookies(self): def test_transcript_fetch_ignores_global_config_and_browser_cookies(self):
proc = _DummyProc() proc = _DummyProc()
with tempfile.TemporaryDirectory() as temp_dir, \ with tempfile.TemporaryDirectory() as temp_dir, \
mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock: mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock:
youtube_yt.fetch_transcript("abc123", temp_dir) youtube_yt.fetch_transcript("abc123", temp_dir)
@@ -45,5 +48,171 @@ class TestYtDlpFlags(unittest.TestCase):
self.assertIn("--no-cookies-from-browser", cmd) self.assertIn("--no-cookies-from-browser", cmd)
class TestExtractTranscriptHighlights(unittest.TestCase):
def test_extracts_specific_sentences(self):
transcript = (
"Hey guys welcome back to the channel. "
"In today's video we're looking at something special. "
"The Lego Bugatti Chiron took 13,438 hours to build with over 1 million pieces. "
"Don't forget to subscribe and hit the bell. "
"The tolerance on each brick is 0.002 millimeters which is insane for injection molding. "
"So yeah that's pretty cool. "
"Thanks for watching see you next time."
)
highlights = youtube_yt.extract_transcript_highlights(transcript, "Lego")
self.assertTrue(len(highlights) > 0)
# Should pick the sentences with numbers and topic relevance, not filler
joined = " ".join(highlights)
self.assertIn("13,438", joined)
self.assertNotIn("subscribe", joined)
self.assertNotIn("welcome back", joined)
def test_empty_transcript(self):
self.assertEqual(youtube_yt.extract_transcript_highlights("", "test"), [])
def test_respects_limit(self):
sentences = ". ".join(
f"The model {i} has {i * 100} parameters and runs at {i * 10} tokens per second"
for i in range(20)
) + "."
highlights = youtube_yt.extract_transcript_highlights(sentences, "model", limit=3)
self.assertEqual(len(highlights), 3)
class TestFetchTranscriptDirect(unittest.TestCase):
"""Tests for _fetch_transcript_direct() — direct HTTP transcript fetching."""
# Minimal ytInitialPlayerResponse JSON with a caption track
_PLAYER_RESPONSE = json.dumps({
"captions": {
"playerCaptionsTracklistRenderer": {
"captionTracks": [
{
"baseUrl": "https://www.youtube.com/api/timedtext?v=abc123&lang=en",
"languageCode": "en",
}
]
}
}
})
_WATCH_HTML = (
'<html><script>var ytInitialPlayerResponse = '
+ _PLAYER_RESPONSE
+ ';</script></html>'
)
_SAMPLE_VTT = (
"WEBVTT\n\n"
"00:00:00.000 --> 00:00:02.000\n"
"Hello world this is a test sentence with enough words to pass.\n\n"
"00:00:02.000 --> 00:00:04.000\n"
"Another line of transcript text here for testing purposes.\n"
)
def _mock_urlopen(self, url_or_req, *, timeout=None):
"""Return watch HTML or VTT depending on URL."""
url = url_or_req.full_url if hasattr(url_or_req, 'full_url') else url_or_req
class _Resp:
def __init__(self, data):
self._data = data.encode("utf-8")
def read(self):
return self._data
def __enter__(self):
return self
def __exit__(self, *a):
pass
if "watch?" in url:
return _Resp(self._WATCH_HTML)
elif "timedtext" in url:
return _Resp(self._SAMPLE_VTT)
raise urllib.error.URLError("unexpected URL")
def test_extracts_vtt_from_mock_page(self):
"""Happy path: extracts VTT text from a page with captions."""
with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=self._mock_urlopen):
result = youtube_yt._fetch_transcript_direct("abc123")
self.assertIsNotNone(result)
self.assertIn("WEBVTT", result)
self.assertIn("Hello world", result)
def test_no_captions_returns_none(self):
"""Video with no caption tracks returns None."""
no_captions_response = json.dumps({"captions": {"playerCaptionsTracklistRenderer": {"captionTracks": []}}})
html = f'<html><script>var ytInitialPlayerResponse = {no_captions_response};</script></html>'
class _Resp:
def __init__(self, data):
self._data = data.encode("utf-8")
def read(self):
return self._data
def __enter__(self):
return self
def __exit__(self, *a):
pass
def mock_open(req, *, timeout=None):
return _Resp(html)
with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=mock_open):
result = youtube_yt._fetch_transcript_direct("nocaps")
self.assertIsNone(result)
def test_http_timeout_returns_none(self):
"""HTTP timeout on watch page returns None."""
def timeout_open(req, *, timeout=None):
raise TimeoutError("timed out")
with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=timeout_open):
result = youtube_yt._fetch_transcript_direct("timeout_vid")
self.assertIsNone(result)
def test_direct_vtt_feeds_into_clean_vtt(self):
"""VTT from direct fetch produces clean plaintext via _clean_vtt()."""
cleaned = youtube_yt._clean_vtt(self._SAMPLE_VTT)
self.assertNotIn("WEBVTT", cleaned)
self.assertNotIn("-->", cleaned)
self.assertIn("Hello world", cleaned)
self.assertIn("Another line", cleaned)
class TestFetchTranscriptFallback(unittest.TestCase):
"""Tests that fetch_transcript picks yt-dlp or direct path correctly."""
def test_uses_ytdlp_when_installed(self):
"""When yt-dlp is installed, uses _fetch_transcript_ytdlp."""
with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", return_value="WEBVTT\n\nfake") as yt_mock, \
mock.patch.object(youtube_yt, "_fetch_transcript_direct") as direct_mock:
result = youtube_yt.fetch_transcript("vid1", "/tmp/test")
yt_mock.assert_called_once_with("vid1", "/tmp/test")
direct_mock.assert_not_called()
def test_uses_direct_when_ytdlp_missing(self):
"""When yt-dlp is NOT installed, falls back to _fetch_transcript_direct."""
sample_vtt = (
"WEBVTT\n\n"
"00:00:00.000 --> 00:00:02.000\n"
"Direct transcript content with enough words for testing.\n"
)
with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \
mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp") as yt_mock, \
mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=sample_vtt) as direct_mock:
result = youtube_yt.fetch_transcript("vid2", "/tmp/test")
yt_mock.assert_not_called()
direct_mock.assert_called_once_with("vid2")
self.assertIsNotNone(result)
self.assertIn("Direct transcript content", result)
def test_returns_none_when_both_fail(self):
"""Returns None when the chosen path returns None."""
with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \
mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=None):
result = youtube_yt.fetch_transcript("novid", "/tmp/test")
self.assertIsNone(result)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()