Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b76655c52e | |||
| 86deed2844 | |||
| bb79d54b2b | |||
| 2eb9cd6fba | |||
| 775596ce21 | |||
| 4d6224f79a | |||
| 499074b564 | |||
| 6a5a0013c0 |
@@ -10,7 +10,7 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"description": "Research any topic from the last 30 days. Become an expert and write copy-paste-ready prompts.",
|
"description": "Research any topic from the last 30 days. Become an expert and write copy-paste-ready prompts.",
|
||||||
"version": "2.9.5",
|
"version": "2.9.6",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
"url": "https://github.com/mvanhorn"
|
"url": "https://github.com/mvanhorn"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "last30days",
|
"name": "last30days",
|
||||||
"version": "2.9.5",
|
"version": "2.9.6",
|
||||||
"description": "Research any topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
"description": "Research any topic from the last 30 days across Reddit, X, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web.",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "Matt Van Horn",
|
"name": "Matt Van Horn",
|
||||||
@@ -10,17 +10,7 @@
|
|||||||
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
"homepage": "https://github.com/mvanhorn/last30days-skill",
|
||||||
"repository": "https://github.com/mvanhorn/last30days-skill",
|
"repository": "https://github.com/mvanhorn/last30days-skill",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"keywords": [
|
"keywords": ["research", "reddit", "twitter", "youtube", "tiktok", "instagram", "trends", "prompts", "polymarket"],
|
||||||
"research",
|
|
||||||
"reddit",
|
|
||||||
"twitter",
|
|
||||||
"youtube",
|
|
||||||
"tiktok",
|
|
||||||
"instagram",
|
|
||||||
"trends",
|
|
||||||
"prompts",
|
|
||||||
"polymarket"
|
|
||||||
],
|
|
||||||
"skills": ["./"],
|
"skills": ["./"],
|
||||||
"hooks": {}
|
"hooks": {}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,12 +12,12 @@
|
|||||||
clawhub install last30days-official
|
clawhub install last30days-official
|
||||||
```
|
```
|
||||||
|
|
||||||
**The AI world reinvents itself every month. This skill keeps you current.** /last30days researches your topic across Reddit, X, Bluesky, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations. Whether it's Seedance 2.0 access, paper.design prompts, or the latest Nano Banana Pro techniques, you'll know what people who are paying attention already know.
|
**The AI world reinvents itself every month. This skill keeps you current.** /last30days researches your topic across Reddit, X, YouTube, and other sources from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations. Whether it's Seedance 2.0 access, paper.design prompts, or the latest Nano Banana Pro techniques, you'll know what people who are paying attention already know.
|
||||||
|
|
||||||
**New in v2.9.5 — Bluesky, Comparative Mode, and Config Improvements:**
|
**New in v2.9.5 — Bluesky, Comparative Mode, and Config Improvements:**
|
||||||
|
|
||||||
- **Bluesky/AT Protocol** is now a social source. Opt-in via `BSKY_HANDLE` + `BSKY_APP_PASSWORD` (create at bsky.app/settings/app-passwords). Full pipeline: search, score, dedupe, render.
|
- **Bluesky/AT Protocol** is now a social source. Opt-in via `BSKY_HANDLE` + `BSKY_APP_PASSWORD` (create at bsky.app/settings/app-passwords). Full pipeline: search, score, dedupe, render.
|
||||||
- **Comparative mode** - ask "X vs Y" (e.g., `/last30 cursor vs windsurf`) and get 3 parallel research passes with a side-by-side comparison: strengths, weaknesses, head-to-head table, and a data-driven verdict.
|
- **Comparative mode** - ask "X vs Y" (e.g., `/last30 Claude Code vs Codex`) and get 3 parallel research passes with a side-by-side comparison: strengths, weaknesses, head-to-head table, and a data-driven verdict.
|
||||||
- **Per-project .env config** - drop a `.claude/last30days.env` in your project root for per-project API keys.
|
- **Per-project .env config** - drop a `.claude/last30days.env` in your project root for per-project API keys.
|
||||||
- **SessionStart config check** - validates your config automatically when a Claude Code session starts.
|
- **SessionStart config check** - validates your config automatically when a Claude Code session starts.
|
||||||
- **Expanded test coverage** - 455+ tests across all modules.
|
- **Expanded test coverage** - 455+ tests across all modules.
|
||||||
@@ -63,44 +63,101 @@ gemini extensions install https://github.com/mvanhorn/last30days-skill.git
|
|||||||
|
|
||||||
### Manual Install (Claude Code / Codex)
|
### Manual Install (Claude Code / Codex)
|
||||||
```bash
|
```bash
|
||||||
# Clone the repo
|
|
||||||
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last30days
|
||||||
|
|
||||||
# Add your API keys (optional if signed in to Codex)
|
|
||||||
mkdir -p ~/.config/last30days
|
|
||||||
cat > ~/.config/last30days/.env << 'EOF'
|
|
||||||
SCRAPECREATORS_API_KEY=... # Reddit + TikTok + Instagram (one key, all three) - scrapecreators.com
|
|
||||||
OPENAI_API_KEY=sk-... # optional - legacy Reddit fallback if using `codex login`
|
|
||||||
AUTH_TOKEN=... # recommended for X search - copy once from x.com cookies
|
|
||||||
CT0=... # recommended for X search - copy once from x.com cookies
|
|
||||||
XAI_API_KEY=xai-... # optional - X fallback if you do not want cookie-based auth
|
|
||||||
BSKY_HANDLE=you.bsky.social # optional - Bluesky search (create app password below)
|
|
||||||
BSKY_APP_PASSWORD=xxxx-xxxx-xxxx # optional - bsky.app/settings/app-passwords
|
|
||||||
EOF
|
|
||||||
chmod 600 ~/.config/last30days/.env
|
|
||||||
```
|
```
|
||||||
|
|
||||||
If you're signed in to Codex (`codex login`), the skill will use your Codex credentials for the OpenAI Responses API and you can omit `OPENAI_API_KEY`. If you're not signed in, run `codex login` first.
|
That's it. Reddit, Hacker News, and Polymarket work immediately with zero configuration. Run `/last30days` to unlock more sources.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Setup: Progressive Source Unlocking
|
||||||
|
|
||||||
|
Start using /last30days immediately. Add sources when you want better results.
|
||||||
|
|
||||||
|
### 1. Zero Config (3 sources) — Just install
|
||||||
|
|
||||||
|
Reddit (public JSON), Hacker News, and Polymarket work out of the box. No API keys, no configuration.
|
||||||
|
|
||||||
|
### 2. Run the setup wizard (5+ sources)
|
||||||
|
|
||||||
|
```
|
||||||
|
/last30days setup
|
||||||
|
```
|
||||||
|
|
||||||
|
The setup wizard automatically extracts X/Twitter login cookies from your browsers (Chrome, Firefox, Safari) and checks for yt-dlp. Takes about 30 seconds. Your cookies stay in memory and are never saved to disk.
|
||||||
|
|
||||||
|
### 3. Add Exa (FREE — semantic web search)
|
||||||
|
|
||||||
|
Register at [exa.ai](https://exa.ai) for 1,000 free searches/month, no credit card required.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Add to ~/.config/last30days/.env
|
||||||
|
EXA_API_KEY=...
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Add ScrapeCreators (RECOMMENDED — Reddit comments + TikTok + Instagram)
|
||||||
|
|
||||||
|
**This is the single most impactful upgrade.** Reddit comments are often the highest-value research content — top-voted replies with real insights. ScrapeCreators unlocks comment enrichment plus TikTok and Instagram. Register at [scrapecreators.com](https://scrapecreators.com) for 100 free API calls (no credit card required). After that, pay-as-you-go. last30days receives no money from any API provider — no referrals, no kickbacks.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Add to ~/.config/last30days/.env
|
||||||
|
SCRAPECREATORS_API_KEY=...
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Add Bluesky (FREE — app password)
|
||||||
|
|
||||||
|
Create an app password at [bsky.app/settings/app-passwords](https://bsky.app/settings/app-passwords).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Add to ~/.config/last30days/.env
|
||||||
|
BSKY_HANDLE=you.bsky.social
|
||||||
|
BSKY_APP_PASSWORD=xxxx-xxxx-xxxx
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6. Optional paid web search backends
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Add to ~/.config/last30days/.env
|
||||||
|
PARALLEL_API_KEY=... # Parallel AI (preferred — LLM-optimized results)
|
||||||
|
BRAVE_API_KEY=... # Brave Search (free tier: 2,000 queries/month)
|
||||||
|
OPENROUTER_API_KEY=... # OpenRouter/Perplexity Sonar Pro
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Do I need API keys?
|
||||||
|
|
||||||
|
| Source | Free Method | API Key | Do you need the API key? |
|
||||||
|
|--------|------------|---------|--------------------------|
|
||||||
|
| Reddit | Public JSON (always works) | ScrapeCreators | **Yes, strongly recommended.** Unlocks top comments — often the most valuable content. |
|
||||||
|
| X/Twitter | Browser cookies (auto-extracted) | xAI API key (`XAI_API_KEY`) | **No.** Cookies give identical quality. The setup wizard handles this. |
|
||||||
|
| YouTube | yt-dlp (`brew install yt-dlp`) | N/A | **No API key exists.** Install yt-dlp for search; transcripts work without it. |
|
||||||
|
| Hacker News | Always free | N/A | **No.** Always works, no config needed. |
|
||||||
|
| Polymarket | Always free | N/A | **No.** Always works, no config needed. |
|
||||||
|
| Web search | N/A | Exa (`EXA_API_KEY`) | **Optional.** 1,000 free searches/month at exa.ai. |
|
||||||
|
| Bluesky | Free app password | N/A | **Optional.** Free app password at bsky.app. |
|
||||||
|
| TikTok | N/A | ScrapeCreators | **Optional.** Included with ScrapeCreators key. |
|
||||||
|
| Instagram | N/A | ScrapeCreators | **Optional.** Included with ScrapeCreators key. |
|
||||||
|
| Truth Social | Browser cookies | N/A | **Optional.** Auto-extracted if logged in. |
|
||||||
|
|
||||||
|
*last30days receives no money from any API provider — no referrals, no kickbacks.*
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Config file locations
|
||||||
|
|
||||||
For project-specific overrides, create `.claude/last30days.env` in the repo root. It overrides the global `~/.config/last30days/.env`.
|
For project-specific overrides, create `.claude/last30days.env` in the repo root. It overrides the global `~/.config/last30days/.env`.
|
||||||
|
|
||||||
### X Search Authentication
|
|
||||||
|
|
||||||
X search prefers explicit env auth. This keeps local runs headless and avoids browser-cookie and macOS Keychain prompts.
|
|
||||||
|
|
||||||
**Recommended setup:**
|
|
||||||
1. While logged into x.com once, open browser dev tools and copy the `auth_token` and `ct0` cookies for `x.com`.
|
|
||||||
2. Save them as `AUTH_TOKEN` and `CT0` in `~/.config/last30days/.env`, export them in your shell, or add them to `.claude/last30days.env` for a single project.
|
|
||||||
3. Re-run `/last30days`.
|
|
||||||
|
|
||||||
**xAI fallback:** If you do not want to provide `AUTH_TOKEN` and `CT0`, set `XAI_API_KEY` and the skill will use xAI's `x_search` backend instead.
|
|
||||||
|
|
||||||
**Verify it's working:**
|
|
||||||
```bash
|
```bash
|
||||||
node ~/.claude/skills/last30days/scripts/lib/vendor/bird-search/bird-search.mjs --whoami
|
# Global config
|
||||||
|
mkdir -p ~/.config/last30days
|
||||||
|
chmod 600 ~/.config/last30days/.env
|
||||||
|
|
||||||
|
# Project-specific config (optional)
|
||||||
|
# .claude/last30days.env
|
||||||
```
|
```
|
||||||
|
|
||||||
**Requirements:** Node.js 22+ (for the vendored Twitter GraphQL client).
|
Check source availability: `python3 scripts/last30days.py --diagnose`
|
||||||
|
|
||||||
### Codex CLI
|
### Codex CLI
|
||||||
|
|
||||||
@@ -144,21 +201,6 @@ The open variant adds four modes on top of one-shot research:
|
|||||||
|
|
||||||
Both variants use the same Python engine and scripts directory. The open variant adds command routing (`watch`, `briefing`, `history`) and references mode-specific instruction files.
|
Both variants use the same Python engine and scripts directory. The open variant adds command routing (`watch`, `briefing`, `history`) and references mode-specific instruction files.
|
||||||
|
|
||||||
**Optional web search API keys** (add to `~/.config/last30days/.env`):
|
|
||||||
```bash
|
|
||||||
PARALLEL_API_KEY=... # Parallel AI (preferred - LLM-optimized results)
|
|
||||||
BRAVE_API_KEY=... # Brave Search (free tier: 2,000 queries/month)
|
|
||||||
OPENROUTER_API_KEY=... # OpenRouter/Perplexity Sonar Pro
|
|
||||||
```
|
|
||||||
|
|
||||||
**Optional Bluesky credentials** (add to `~/.config/last30days/.env`):
|
|
||||||
```bash
|
|
||||||
BSKY_HANDLE=you.bsky.social # Your Bluesky handle
|
|
||||||
BSKY_APP_PASSWORD=xxxx-xxxx-xxxx # Create at bsky.app/settings/app-passwords
|
|
||||||
```
|
|
||||||
|
|
||||||
Check source availability: `python3 scripts/last30days.py --diagnose`
|
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -174,7 +216,7 @@ Examples:
|
|||||||
|
|
||||||
## What It Does
|
## What It Does
|
||||||
|
|
||||||
1. **Researches** - Scans Reddit, X, Bluesky, Truth Social, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web for discussions from the last 30 days
|
1. **Researches** - Scans Reddit, X, YouTube, and other sources for discussions from the last 30 days
|
||||||
2. **Synthesizes** - Identifies patterns, best practices, and what actually works
|
2. **Synthesizes** - Identifies patterns, best practices, and what actually works
|
||||||
3. **Delivers** - Either writes copy-paste-ready prompts for your target tool, or gives you a curated expert-level answer
|
3. **Delivers** - Either writes copy-paste-ready prompts for your target tool, or gives you a curated expert-level answer
|
||||||
|
|
||||||
@@ -928,13 +970,11 @@ This example shows /last30days discovering **emerging developer workflows** - re
|
|||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
- **OpenAI auth** - For Reddit research (uses web search via Responses API). Use `OPENAI_API_KEY` or `codex login`.
|
- **Python 3** - For the research engine
|
||||||
- **Node.js 22+** - For X search (bundled Twitter GraphQL client)
|
- **Node.js 22+** - For X search (bundled Twitter GraphQL client)
|
||||||
- **Bundled X auth** - Set `AUTH_TOKEN` and `CT0` for popup-free local X search
|
- **yt-dlp** (recommended) - For YouTube search. Install via `brew install yt-dlp` or `pip install yt-dlp`. Transcripts work without it.
|
||||||
- **Alternate X backend** - Set `XAI_API_KEY` if bundled X auth is not configured
|
|
||||||
- **yt-dlp** (optional) - For YouTube search + transcript extraction. Install via `brew install yt-dlp` or `pip install yt-dlp`. When present, automatically searches YouTube and extracts video transcripts as an additional source.
|
|
||||||
|
|
||||||
At least one auth path is required. Reddit needs OpenAI auth. X needs either `AUTH_TOKEN` plus `CT0` or `XAI_API_KEY`. YouTube search activates automatically when yt-dlp is in your PATH.
|
No API keys are required to start. Reddit, Hacker News, and Polymarket work out of the box. Run `/last30days setup` to unlock X/Twitter via browser cookies and configure additional sources. See [Setup: Progressive Source Unlocking](#setup-progressive-source-unlocking) for the full progression.
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
@@ -986,7 +1026,7 @@ If your OpenAI org doesn't have access to a model (e.g., unverified for gpt-4.1)
|
|||||||
|
|
||||||
### ScrapeCreators Reddit as default
|
### ScrapeCreators Reddit as default
|
||||||
|
|
||||||
Reddit now runs on [ScrapeCreators](https://scrapecreators.com) by default. One `SCRAPECREATORS_API_KEY` powers Reddit, TikTok, and Instagram — three sources, one key. No more `OPENAI_API_KEY` required for Reddit search.
|
Reddit now runs on [ScrapeCreators](https://scrapecreators.com) by default. One `SCRAPECREATORS_API_KEY` powers Reddit, TikTok, and Instagram — three sources, one key. No more `OPENAI_API_KEY` required for Reddit search. 100 free API calls, no credit card required — just register at [scrapecreators.com](https://scrapecreators.com), then pay-as-you-go. last30days receives no money from any API provider — no referrals, no kickbacks.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env
|
echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env
|
||||||
@@ -1039,7 +1079,7 @@ Search "AI tools" and you get:
|
|||||||
|
|
||||||
### TikTok + Instagram on ScrapeCreators
|
### TikTok + Instagram on ScrapeCreators
|
||||||
|
|
||||||
Both TikTok and Instagram are powered by [ScrapeCreators](https://scrapecreators.com) — one API key covers both sources. 100 free credits, then pay-as-you-go.
|
Both TikTok and Instagram are powered by [ScrapeCreators](https://scrapecreators.com) — one API key covers both sources. Register at [scrapecreators.com](https://scrapecreators.com) for 100 free API calls (no credit card required). After that, pay-as-you-go. last30days receives no money from any API provider — no referrals, no kickbacks.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env
|
echo 'SCRAPECREATORS_API_KEY=your_key_here' >> ~/.config/last30days/.env
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
---
|
---
|
||||||
name: last30days
|
name: last30days
|
||||||
version: "2.9.5"
|
version: "2.9.6"
|
||||||
description: "Deep research engine covering the last 30 days across 10+ sources - Reddit, X/Twitter, YouTube, TikTok, Instagram, Hacker News, Polymarket, Bluesky, Truth Social, web. AI synthesizes findings into grounded, cited reports. The most comprehensive recency research skill on ClawHub."
|
description: "Deep research engine covering the last 30 days across 10+ sources - Reddit, X/Twitter, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web. AI synthesizes findings into grounded, cited reports."
|
||||||
argument-hint: 'last30 AI video tools, last30 best project management tools'
|
argument-hint: 'last30 AI video tools, last30 best project management tools'
|
||||||
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
|
||||||
homepage: https://github.com/mvanhorn/last30days-skill
|
homepage: https://github.com/mvanhorn/last30days-skill
|
||||||
@@ -61,9 +61,139 @@ metadata:
|
|||||||
|
|
||||||
# last30days v2.9.5: Research Any Topic from the Last 30 Days
|
# last30days v2.9.5: Research Any Topic from the Last 30 Days
|
||||||
|
|
||||||
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). Truth Social search uses optional bearer token (TRUTHSOCIAL_TOKEN env var - extract from browser dev tools). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section.
|
||||||
|
|
||||||
Research ANY topic across Reddit, X, Bluesky, Truth Social, YouTube, TikTok, Hacker News, Polymarket, and the web. Surface what people are actually discussing, recommending, betting on, and debating right now.
|
Research ANY topic across Reddit, X, YouTube, and other sources. Surface what people are actually discussing, recommending, betting on, and debating right now.
|
||||||
|
|
||||||
|
## Step 0: First-Run Setup Wizard
|
||||||
|
|
||||||
|
**CRITICAL: ALWAYS execute Step 0 BEFORE Step 1, even if the user provided a topic.** If the user typed `/last30days Mercer Island`, you MUST check for FIRST_RUN and present the wizard BEFORE running research. The topic "Mercer Island" is preserved — research runs immediately after the wizard completes. Do NOT skip the wizard because a topic was provided. The wizard takes 10 seconds and only runs once ever.
|
||||||
|
|
||||||
|
To detect first run: check if `~/.config/last30days/.env` exists. If it does NOT exist, this is a first run. **Do NOT run any Bash commands or show any command output to detect this — just check the file existence silently.** If the file exists and contains `SETUP_COMPLETE=true`, skip this section and proceed to Step 1.
|
||||||
|
|
||||||
|
**When first run is detected, you MUST follow these steps IN ORDER. Do NOT skip ahead to the topic picker or research. The sequence is: (1) welcome text → (2) setup modal → (3) run setup if chosen → (4) ScrapeCreators modal → (5) topic picker. You MUST start at step 1.**
|
||||||
|
|
||||||
|
**Step 1: Display the following welcome text ONCE as a normal message (not blockquoted). Then IMMEDIATELY call AskUserQuestion — do NOT repeat any of the welcome text inside the AskUserQuestion call.**
|
||||||
|
|
||||||
|
👋 Welcome to /last30days!
|
||||||
|
|
||||||
|
I research any topic across Reddit, X, YouTube, and other sources — synthesizing what people are actually saying right now.
|
||||||
|
|
||||||
|
To get the best results, I can:
|
||||||
|
🔍 **Scan your browser** for X/Twitter cookies (free X search — reads x.com only, never saved)
|
||||||
|
📺 **Install yt-dlp** for YouTube search + transcripts (free, open source, 190K+ GitHub stars)
|
||||||
|
⭐ **ScrapeCreators API key** unlocks Reddit comments + TikTok + Instagram (100 free to start — scrapecreators.com)
|
||||||
|
|
||||||
|
We recommend all 3 before your first run — it's what makes the magic. More community sources available later. We get no kickbacks from any of these.
|
||||||
|
|
||||||
|
**Then call AskUserQuestion with ONLY this question and these options — no additional text:**
|
||||||
|
|
||||||
|
Question: "How would you like to set up?"
|
||||||
|
Options:
|
||||||
|
- "Auto setup (~30 seconds) — scans browser for X cookies (free X search) and installs yt-dlp (free YouTube transcripts)"
|
||||||
|
- "Manual setup — show me what to configure"
|
||||||
|
- "Skip for now — Reddit (threads only), HN, Polymarket, Web"
|
||||||
|
|
||||||
|
**If the user picks 1 (Auto setup):**
|
||||||
|
Run the setup subcommand:
|
||||||
|
```bash
|
||||||
|
cd {SKILL_DIR} && python3 scripts/last30days.py setup
|
||||||
|
```
|
||||||
|
Show the user the results (what cookies were found, whether yt-dlp was installed).
|
||||||
|
|
||||||
|
**Then show the ScrapeCreators push (plain text, then modal):**
|
||||||
|
|
||||||
|
⭐ One more thing — Reddit comments are some of the best content on the internet. The top-voted replies often have sharper insights than the posts themselves. ScrapeCreators unlocks these (plus TikTok + Instagram) — 100 free to start, no credit card.
|
||||||
|
|
||||||
|
**Call AskUserQuestion:**
|
||||||
|
Question: "Want to add Reddit comments to your research?"
|
||||||
|
Options:
|
||||||
|
- "Open scrapecreators.com to get my free key" — run `open https://scrapecreators.com` via Bash to open in the user's browser. Then ask them to paste the API key they get. When they paste it, write SCRAPECREATORS_API_KEY={key} to ~/.config/last30days/.env
|
||||||
|
- "I have a key — let me paste it" — accept the key, write to .env
|
||||||
|
- "Skip for now — start researching" — proceed without ScrapeCreators
|
||||||
|
|
||||||
|
**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:**
|
||||||
|
|
||||||
|
Your ScrapeCreators key also powers TikTok and Instagram search. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.)
|
||||||
|
|
||||||
|
**Call AskUserQuestion:**
|
||||||
|
Question: "Also search TikTok and Instagram?"
|
||||||
|
Options:
|
||||||
|
- "Yes, search everything — TikTok + Instagram on every run" — append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "All set — Reddit comments, TikTok, and Instagram will run on every search."
|
||||||
|
- "Just Reddit comments for now" — don't write the flag. Confirm: "Got it — Reddit comments are on. TikTok and Instagram will kick in automatically for relevant topics like product reviews and trends. Add INCLUDE_SOURCES=tiktok,instagram to your .env anytime to force them on."
|
||||||
|
|
||||||
|
**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:**
|
||||||
|
|
||||||
|
**Call AskUserQuestion:**
|
||||||
|
Question: "What do you want to research first?"
|
||||||
|
Options:
|
||||||
|
- "Claude Code vs Codex" — tech comparison
|
||||||
|
- "Sam Altman" — person in the news
|
||||||
|
- "Warriors Basketball" — sports
|
||||||
|
- "AI Legal Prompting Techniques" — niche/professional
|
||||||
|
- "Type my own topic"
|
||||||
|
|
||||||
|
If user picks an example, run research with that topic. If they pick "Type my own", ask them what they want to research. If the user originally provided a topic with the command (e.g., `/last30days Mercer Island`), skip this modal and use their topic directly.
|
||||||
|
|
||||||
|
**END OF FIRST-RUN WIZARD. Everything above in Step 0 ONLY runs on first run. If SETUP_COMPLETE=true exists in .env, skip ALL of Step 0 — no welcome, no setup, no ScrapeCreators modal, no topic picker. Go directly to Step 1 (Parse User Intent). The topic picker is ONLY for first-time users who haven't run /last30days before.**
|
||||||
|
|
||||||
|
**If the user picks 2 (Manual setup):**
|
||||||
|
Show them this guide (present as plain text, not blockquoted):
|
||||||
|
|
||||||
|
**The magic of /last30days is Reddit comments + X posts together.** Here's how to unlock each source.
|
||||||
|
|
||||||
|
Add these to `~/.config/last30days/.env`:
|
||||||
|
|
||||||
|
**🔍 X/Twitter** (pick one — this is the most important):
|
||||||
|
- `FROM_BROWSER=auto` — easiest, free. Scans your browser cookies for x.com login.
|
||||||
|
- `AUTH_TOKEN=xxx` + `CT0=xxx` — paste your X cookies manually (x.com → F12 → Application → Cookies)
|
||||||
|
- `XAI_API_KEY=xxx` — use an xAI/Grok API key instead (api.x.ai)
|
||||||
|
|
||||||
|
**⭐ Reddit Comments + TikTok + Instagram + YouTube backup** (one key, 5 platforms):
|
||||||
|
- `SCRAPECREATORS_API_KEY=xxx` — 100 free to start at scrapecreators.com. This is the single most impactful key — Reddit comments are where the sharpest insights live.
|
||||||
|
|
||||||
|
**💡 Reddit discovery fallback** (optional):
|
||||||
|
- `OPENAI_API_KEY=xxx` — ChatGPT web search for Reddit threads. Only used if ScrapeCreators isn't configured — SC is better.
|
||||||
|
|
||||||
|
**📺 YouTube** (primary engine):
|
||||||
|
- Run `brew install yt-dlp` — free, open source. ScrapeCreators covers YouTube as backup if you skip this.
|
||||||
|
|
||||||
|
**Bonus sources** (add anytime):
|
||||||
|
- `EXA_API_KEY=xxx` — semantic web search, 1K free/month (exa.ai)
|
||||||
|
- `BSKY_HANDLE=you.bsky.social` + `BSKY_APP_PASSWORD=xxx` — Bluesky (free app password)
|
||||||
|
- `BRAVE_API_KEY=xxx` — Brave web search
|
||||||
|
- `OPENROUTER_API_KEY=xxx` — Perplexity Sonar Pro search
|
||||||
|
|
||||||
|
Always add this last line: `SETUP_COMPLETE=true`
|
||||||
|
|
||||||
|
You can paste your API keys right here and I'll write the file for you. Some people say giving LLMs your keys is risky, others live in YOLO mode. Your call.
|
||||||
|
|
||||||
|
**If the user pastes keys**, write them to `~/.config/last30days/.env` (create the file and parent dirs if needed, append without overwriting existing keys, always include `SETUP_COMPLETE=true`). Then proceed with research.
|
||||||
|
|
||||||
|
**If the user says they'll do it themselves**, tell them: "Save the file, then run `/last30days <topic>` to research anything." Then proceed with research using whatever sources are currently available.
|
||||||
|
|
||||||
|
**If the user picks Skip:**
|
||||||
|
Proceed with research immediately using the user's original topic. Note: without setup, sources are limited to Reddit (threads only, no comments), HN, and Polymarket. X/Twitter and YouTube require setup.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Do I Need API Keys?
|
||||||
|
|
||||||
|
When users ask about API keys, setup, or how to unlock more sources, reference this:
|
||||||
|
|
||||||
|
**You do NOT need API keys to use last30days.** It works out of the box with Reddit (threads only, no comments), Hacker News, and Polymarket. Browser cookies for X/Twitter are equivalent to an API key — just log into x.com in any browser and last30days will find your session automatically.
|
||||||
|
|
||||||
|
**Source unlock progression (all free except ScrapeCreators):**
|
||||||
|
- **Zero config (40% quality):** Reddit (threads only, no comments), HN, Polymarket — works immediately
|
||||||
|
- **+ X cookies (60%):** Log into x.com in any browser. last30days scans your cookies automatically. This is the same as having an API key — no signup required.
|
||||||
|
- **+ yt-dlp (80%):** `brew install yt-dlp` — open source, 190K+ GitHub stars. Enables YouTube search and transcripts.
|
||||||
|
- **+ ScrapeCreators (100%):** The only paid option worth mentioning. Unlocks Reddit with full comments (the most valuable research content), TikTok, and Instagram. 100 free API calls, no credit card — scrapecreators.com
|
||||||
|
|
||||||
|
**Key comparison:** X browser cookies = same access as an API key (free, no signup). ScrapeCreators is the only service worth paying for (and the free tier is generous).
|
||||||
|
|
||||||
|
last30days has no affiliation with any API provider — no referrals, no kickbacks.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## CRITICAL: Parse User Intent
|
## CRITICAL: Parse User Intent
|
||||||
|
|
||||||
@@ -100,7 +230,7 @@ Common patterns:
|
|||||||
**DISPLAY your parsing to the user.** Before running any tools, output:
|
**DISPLAY your parsing to the user.** Before running any tools, output:
|
||||||
|
|
||||||
```
|
```
|
||||||
I'll research {TOPIC} across Reddit, X, Bluesky, Truth Social, TikTok, and the web to find what's been discussed in the last 30 days.
|
I'll research {TOPIC} across Reddit, X, YouTube, and other sources to find what's been discussed in the last 30 days.
|
||||||
|
|
||||||
Parsed intent:
|
Parsed intent:
|
||||||
- TOPIC = {TOPIC}
|
- TOPIC = {TOPIC}
|
||||||
@@ -164,7 +294,7 @@ Agent mode report format:
|
|||||||
|
|
||||||
```
|
```
|
||||||
## Research Report: {TOPIC}
|
## Research Report: {TOPIC}
|
||||||
Generated: {date} | Sources: Reddit, X, Bluesky, Truth Social, YouTube, TikTok, HN, Polymarket, Web
|
Generated: {date} | Sources: Reddit, X, Bluesky, YouTube, TikTok, HN, Polymarket, Web
|
||||||
|
|
||||||
### Key Findings
|
### Key Findings
|
||||||
[3-5 bullet points, highest-signal insights with citations]
|
[3-5 bullet points, highest-signal insights with citations]
|
||||||
@@ -240,7 +370,7 @@ The script will automatically:
|
|||||||
|
|
||||||
**Read the ENTIRE output.** It contains EIGHT data sections in this order: Reddit items, X items, YouTube items, TikTok items, Instagram Reels items, Hacker News items, Polymarket items, and WebSearch items. If you miss sections, you will produce incomplete stats.
|
**Read the ENTIRE output.** It contains EIGHT data sections in this order: Reddit items, X items, YouTube items, TikTok items, Instagram Reels items, Hacker News items, Polymarket items, and WebSearch items. If you miss sections, you will produce incomplete stats.
|
||||||
|
|
||||||
**YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, and optional transcript snippet. Count them and include them in your synthesis and stats block.
|
**YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, **transcript highlights** (pre-extracted quotable excerpts from the video), and an optional full transcript in a collapsible section. **Quote the highlights directly in your synthesis** - they are the YouTube equivalent of Reddit top comments. Attribute quotes to the channel name. Count them and include them in your synthesis and stats block.
|
||||||
|
|
||||||
**TikTok items in the output look like:** `**{TK_id}** (score:N) @{creator} [N views, N likes]` followed by a caption, URL, hashtags, and optional caption snippet. Count them and include them in your synthesis and stats block.
|
**TikTok items in the output look like:** `**{TK_id}** (score:N) @{creator} [N views, N likes]` followed by a caption, URL, hashtags, and optional caption snippet. Count them and include them in your synthesis and stats block.
|
||||||
|
|
||||||
@@ -303,9 +433,10 @@ The Judge Agent must:
|
|||||||
3. Weight TikTok sources HIGH (they have views, likes, and caption content — viral signal)
|
3. Weight TikTok sources HIGH (they have views, likes, and caption content — viral signal)
|
||||||
4. Weight WebSearch sources LOWER (no engagement data)
|
4. Weight WebSearch sources LOWER (no engagement data)
|
||||||
5. **For Reddit: Pay special attention to top comments** — they often contain the wittiest, most insightful, or funniest take. When a top comment has high upvotes (shown as `💬 Top comment (N upvotes)`), quote it directly in your synthesis. Reddit's value is in the comments.
|
5. **For Reddit: Pay special attention to top comments** — they often contain the wittiest, most insightful, or funniest take. When a top comment has high upvotes (shown as `💬 Top comment (N upvotes)`), quote it directly in your synthesis. Reddit's value is in the comments.
|
||||||
6. Identify patterns that appear across ALL sources (strongest signals)
|
6. **For YouTube: Quote transcript highlights directly in your synthesis.** These are pre-extracted key moments from the video - treat them like Reddit top comments. Attribute to the channel name and include the actual quote. YouTube's value is in what creators SAY, not just their view counts.
|
||||||
7. Note any contradictions between sources
|
7. Identify patterns that appear across ALL sources (strongest signals)
|
||||||
8. Extract the top 3-5 actionable insights
|
8. Note any contradictions between sources
|
||||||
|
9. Extract the top 3-5 actionable insights
|
||||||
|
|
||||||
7. **Cross-platform signals are the strongest evidence.** When items have `[also on: Reddit, HN]` or similar tags, it means the same story appears across multiple platforms. Lead with these cross-platform findings - they're the most important signals in the research.
|
7. **Cross-platform signals are the strongest evidence.** When items have `[also on: Reddit, HN]` or similar tags, it means the same story appears across multiple platforms. Lead with these cross-platform findings - they're the most important signals in the research.
|
||||||
|
|
||||||
@@ -498,6 +629,10 @@ KEY PATTERNS from the research:
|
|||||||
3. [Pattern] — per @handle
|
3. [Pattern] — per @handle
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**THEN - Quality Nudge (if present in the output):**
|
||||||
|
|
||||||
|
If the research output contains a `**🔍 Research Coverage:**` block, render it verbatim right before the stats block. This tells the user which core sources are missing and how to unlock them. Do NOT render this block if it is absent from the output (100% coverage = no nudge).
|
||||||
|
|
||||||
**THEN - Stats (right before invitation):**
|
**THEN - Stats (right before invitation):**
|
||||||
|
|
||||||
**CRITICAL: Calculate actual totals from the research output.**
|
**CRITICAL: Calculate actual totals from the research output.**
|
||||||
@@ -725,7 +860,7 @@ Want another prompt? Just tell me what you're creating next.
|
|||||||
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth)
|
||||||
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
- Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth)
|
||||||
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
- Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data)
|
||||||
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (same SCRAPECREATORS_API_KEY as Reddit, PAYG after 100 free credits)
|
- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (same SCRAPECREATORS_API_KEY as Reddit, PAYG after 100 free API calls)
|
||||||
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
- Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search
|
||||||
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
- Fetches public Reddit thread data from `reddit.com` for engagement metrics
|
||||||
- Stores research findings in local SQLite database (watchlist mode only)
|
- Stores research findings in local SQLite database (watchlist mode only)
|
||||||
@@ -738,7 +873,7 @@ Want another prompt? Just tell me what you're creating next.
|
|||||||
- Does not log, cache, or write API keys to output files
|
- Does not log, cache, or write API keys to output files
|
||||||
- Does not send data to any endpoint not listed above
|
- Does not send data to any endpoint not listed above
|
||||||
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
- Hacker News and Polymarket sources are always available (no API key, no binary dependency)
|
||||||
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (same key covers both; 100 free credits, then PAYG)
|
- TikTok and Instagram sources require SCRAPECREATORS_API_KEY (same key covers both; 100 free API calls, then PAYG)
|
||||||
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
- Can be invoked autonomously by agents via the Skill tool (runs inline, not forked); pass `--agent` for non-interactive report output
|
||||||
|
|
||||||
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
**Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# Check if last30days has any configuration source available.
|
# Check last30days configuration status and show appropriate welcome message.
|
||||||
# Priority: .claude/last30days.env > ~/.config/last30days/.env > env vars
|
# Priority: .claude/last30days.env > ~/.config/last30days/.env > env vars
|
||||||
|
|
||||||
PROJECT_ENV=".claude/last30days.env"
|
PROJECT_ENV=".claude/last30days.env"
|
||||||
@@ -19,33 +19,90 @@ check_perms() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Check per-project config
|
# Load env file into variables for inspection (without exporting)
|
||||||
|
load_env_vars() {
|
||||||
|
local file="$1"
|
||||||
|
if [[ -f "$file" ]]; then
|
||||||
|
while IFS='=' read -r key value; do
|
||||||
|
# Skip comments, empty lines
|
||||||
|
[[ "$key" =~ ^[[:space:]]*# ]] && continue
|
||||||
|
[[ -z "$key" ]] && continue
|
||||||
|
key=$(echo "$key" | xargs)
|
||||||
|
value=$(echo "$value" | xargs | sed 's/^["'\''"]//;s/["'\''"]$//')
|
||||||
|
if [[ -n "$key" && -n "$value" ]]; then
|
||||||
|
eval "ENV_${key}=\"${value}\""
|
||||||
|
fi
|
||||||
|
done < "$file"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Determine which config file is active
|
||||||
|
CONFIG_FILE=""
|
||||||
if [[ -f "$PROJECT_ENV" ]]; then
|
if [[ -f "$PROJECT_ENV" ]]; then
|
||||||
|
CONFIG_FILE="$PROJECT_ENV"
|
||||||
check_perms "$PROJECT_ENV"
|
check_perms "$PROJECT_ENV"
|
||||||
exit 0
|
elif [[ -f "$GLOBAL_ENV" ]]; then
|
||||||
fi
|
CONFIG_FILE="$GLOBAL_ENV"
|
||||||
|
|
||||||
# Check global config
|
|
||||||
if [[ -f "$GLOBAL_ENV" ]]; then
|
|
||||||
check_perms "$GLOBAL_ENV"
|
check_perms "$GLOBAL_ENV"
|
||||||
exit 0
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check if OPENAI_API_KEY is set in environment (minimum requirement)
|
# Load config if found
|
||||||
if [[ -n "${OPENAI_API_KEY:-}" ]]; then
|
if [[ -n "$CONFIG_FILE" ]]; then
|
||||||
exit 0
|
load_env_vars "$CONFIG_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check if SCRAPECREATORS_API_KEY is set (also sufficient)
|
# Check SETUP_COMPLETE (from file or env)
|
||||||
if [[ -n "${SCRAPECREATORS_API_KEY:-}" ]]; then
|
SETUP_COMPLETE="${ENV_SETUP_COMPLETE:-${SETUP_COMPLETE:-}}"
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# No config found — inform user
|
# If setup has never been run, show welcome message for new users
|
||||||
cat <<'EOF'
|
if [[ -z "$SETUP_COMPLETE" && -z "$CONFIG_FILE" && -z "${OPENAI_API_KEY:-}" && -z "${SCRAPECREATORS_API_KEY:-}" && -z "${AUTH_TOKEN:-}" && -z "${XAI_API_KEY:-}" ]]; then
|
||||||
/last30days: No API keys configured.
|
cat <<'EOF'
|
||||||
|
/last30days: Ready to use. Run /last30days to get started — setup takes 30 seconds.
|
||||||
|
|
||||||
Create .claude/last30days.env with your API keys, or create
|
Reddit, Hacker News, and Polymarket work out of the box.
|
||||||
~/.config/last30days/.env globally. At minimum, SCRAPECREATORS_API_KEY
|
The setup wizard can unlock X/Twitter, YouTube, and more.
|
||||||
or OPENAI_API_KEY is required. See the README for setup instructions.
|
|
||||||
EOF
|
EOF
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Setup done but check for ScrapeCreators
|
||||||
|
HAS_SCRAPECREATORS="${ENV_SCRAPECREATORS_API_KEY:-${SCRAPECREATORS_API_KEY:-}}"
|
||||||
|
HAS_X="${ENV_AUTH_TOKEN:-${AUTH_TOKEN:-}}"
|
||||||
|
HAS_XAI="${ENV_XAI_API_KEY:-${XAI_API_KEY:-}}"
|
||||||
|
HAS_YTDLP=""
|
||||||
|
if command -v yt-dlp &>/dev/null; then
|
||||||
|
HAS_YTDLP="yes"
|
||||||
|
fi
|
||||||
|
HAS_BSKY="${ENV_BSKY_HANDLE:-${BSKY_HANDLE:-}}"
|
||||||
|
HAS_EXA="${ENV_EXA_API_KEY:-${EXA_API_KEY:-}}"
|
||||||
|
|
||||||
|
# Count active sources
|
||||||
|
SOURCE_COUNT=2 # HN + Polymarket are always free
|
||||||
|
if [[ -n "$HAS_X" || -n "$HAS_XAI" ]]; then
|
||||||
|
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||||
|
fi
|
||||||
|
# Reddit public JSON always works
|
||||||
|
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||||
|
if [[ -n "$HAS_YTDLP" ]]; then
|
||||||
|
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||||
|
fi
|
||||||
|
if [[ -n "$HAS_EXA" ]]; then
|
||||||
|
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||||
|
fi
|
||||||
|
if [[ -n "$HAS_BSKY" ]]; then
|
||||||
|
SOURCE_COUNT=$((SOURCE_COUNT + 1))
|
||||||
|
fi
|
||||||
|
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||||
|
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$HAS_SCRAPECREATORS" ]]; then
|
||||||
|
# Fully configured — compact ready message
|
||||||
|
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||||
|
else
|
||||||
|
# Setup done but missing ScrapeCreators — recommend it
|
||||||
|
echo "/last30days: Ready — ${SOURCE_COUNT} sources active."
|
||||||
|
echo " Tip: Add ScrapeCreators for Reddit comments + TikTok + Instagram."
|
||||||
|
echo " 100 free API calls, no credit card — scrapecreators.com"
|
||||||
|
echo " last30days has no affiliation with any API provider."
|
||||||
|
fi
|
||||||
|
|||||||
+136
-79
@@ -149,17 +149,20 @@ from lib import (
|
|||||||
normalize,
|
normalize,
|
||||||
openai_reddit,
|
openai_reddit,
|
||||||
reddit,
|
reddit,
|
||||||
|
reddit_public,
|
||||||
reddit_enrich,
|
reddit_enrich,
|
||||||
render,
|
render,
|
||||||
schema,
|
schema,
|
||||||
score,
|
score,
|
||||||
scrapecreators_x,
|
scrapecreators_x,
|
||||||
|
setup_wizard,
|
||||||
ui,
|
ui,
|
||||||
tiktok,
|
tiktok,
|
||||||
instagram,
|
instagram,
|
||||||
websearch,
|
websearch,
|
||||||
xai_x,
|
xai_x,
|
||||||
youtube_yt,
|
youtube_yt,
|
||||||
|
quality_nudge,
|
||||||
query_type as qt,
|
query_type as qt,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -184,8 +187,10 @@ def _search_reddit(
|
|||||||
) -> tuple:
|
) -> tuple:
|
||||||
"""Search Reddit (runs in thread).
|
"""Search Reddit (runs in thread).
|
||||||
|
|
||||||
Uses ScrapeCreators when SCRAPECREATORS_API_KEY is available (preferred).
|
Hierarchy:
|
||||||
Falls back to OpenAI Responses API otherwise.
|
1. ScrapeCreators (if SCRAPECREATORS_API_KEY exists) — premium, best quality
|
||||||
|
2. Public Reddit JSON (always available) — free, good for thread discovery
|
||||||
|
3. OpenAI Responses API — legacy fallback for backwards compatibility
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (reddit_items, raw_response, error, used_scrapecreators)
|
Tuple of (reddit_items, raw_response, error, used_scrapecreators)
|
||||||
@@ -199,7 +204,7 @@ def _search_reddit(
|
|||||||
if mock:
|
if mock:
|
||||||
raw_response = load_fixture("openai_sample.json")
|
raw_response = load_fixture("openai_sample.json")
|
||||||
elif sc_token:
|
elif sc_token:
|
||||||
# === ScrapeCreators path (preferred) ===
|
# === Tier 1: ScrapeCreators path (preferred) ===
|
||||||
used_scrapecreators = True
|
used_scrapecreators = True
|
||||||
try:
|
try:
|
||||||
sys.stderr.write("[Reddit] Using ScrapeCreators API\n")
|
sys.stderr.write("[Reddit] Using ScrapeCreators API\n")
|
||||||
@@ -216,61 +221,56 @@ def _search_reddit(
|
|||||||
reddit_error = f"ScrapeCreators: {type(e).__name__}: {e}"
|
reddit_error = f"ScrapeCreators: {type(e).__name__}: {e}"
|
||||||
sys.stderr.write(f"[Reddit] ScrapeCreators failed: {e}\n")
|
sys.stderr.write(f"[Reddit] ScrapeCreators failed: {e}\n")
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
# Fall through to OpenAI if we have that key
|
|
||||||
if not config.get("OPENAI_API_KEY"):
|
|
||||||
# No OpenAI either: try public Reddit fallback.
|
|
||||||
try:
|
|
||||||
reddit_items = openai_reddit.search_reddit_public(
|
|
||||||
topic, from_date, to_date, depth=depth,
|
|
||||||
)
|
|
||||||
raw_response = {"source": "reddit_public", "items": reddit_items}
|
|
||||||
return reddit_items, raw_response, None, False
|
|
||||||
except Exception as e2:
|
|
||||||
return [], {"error": str(e)}, reddit_error, used_scrapecreators
|
|
||||||
used_scrapecreators = False
|
used_scrapecreators = False
|
||||||
sys.stderr.write("[Reddit] Falling back to OpenAI\n")
|
# Fall through to Tier 2 (public JSON)
|
||||||
sys.stderr.flush()
|
|
||||||
|
|
||||||
# === OpenAI path (fallback) ===
|
# === Tier 2: Public Reddit JSON (free, always available) ===
|
||||||
if not mock:
|
if not mock:
|
||||||
if config.get("OPENAI_API_KEY"):
|
try:
|
||||||
try:
|
sys.stderr.write("[Reddit] Trying public Reddit JSON\n")
|
||||||
raw_response = openai_reddit.search_reddit(
|
sys.stderr.flush()
|
||||||
config["OPENAI_API_KEY"],
|
reddit_items = reddit_public.search_reddit_public(
|
||||||
selected_models["openai"],
|
topic, from_date, to_date, depth=depth,
|
||||||
topic,
|
)
|
||||||
from_date,
|
if reddit_items:
|
||||||
to_date,
|
|
||||||
depth=depth,
|
|
||||||
auth_source=config.get("OPENAI_AUTH_SOURCE", "api_key"),
|
|
||||||
account_id=config.get("OPENAI_CHATGPT_ACCOUNT_ID"),
|
|
||||||
)
|
|
||||||
except http.HTTPError as e:
|
|
||||||
raw_response = {"error": str(e)}
|
|
||||||
reddit_error = f"API error: {e}"
|
|
||||||
except Exception as e:
|
|
||||||
raw_response = {"error": str(e)}
|
|
||||||
reddit_error = f"{type(e).__name__}: {e}"
|
|
||||||
else:
|
|
||||||
# No OpenAI auth: direct Reddit public JSON fallback.
|
|
||||||
try:
|
|
||||||
reddit_items = openai_reddit.search_reddit_public(
|
|
||||||
topic, from_date, to_date, depth=depth,
|
|
||||||
)
|
|
||||||
raw_response = {"source": "reddit_public", "items": reddit_items}
|
raw_response = {"source": "reddit_public", "items": reddit_items}
|
||||||
except http.HTTPError as e:
|
sys.stderr.write(f"[Reddit] Public JSON returned {len(reddit_items)} results\n")
|
||||||
reddit_items = []
|
sys.stderr.flush()
|
||||||
raw_response = {"error": str(e), "source": "reddit_public"}
|
return reddit_items, raw_response, None, False
|
||||||
reddit_error = f"Reddit public API error: {e}"
|
# Empty results — fall through to Tier 3
|
||||||
except Exception as e:
|
sys.stderr.write("[Reddit] Public JSON returned 0 results, trying OpenAI\n")
|
||||||
reddit_items = []
|
sys.stderr.flush()
|
||||||
raw_response = {"error": str(e), "source": "reddit_public"}
|
except Exception as e:
|
||||||
reddit_error = f"Reddit public search error: {type(e).__name__}: {e}"
|
sys.stderr.write(f"[Reddit] Public JSON failed: {e}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
# Fall through to Tier 3
|
||||||
|
|
||||||
# Parse response
|
# === Tier 3: OpenAI Responses API (legacy fallback) ===
|
||||||
|
if not mock and config.get("OPENAI_API_KEY"):
|
||||||
|
try:
|
||||||
|
sys.stderr.write("[Reddit] Falling back to OpenAI Responses API\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
raw_response = openai_reddit.search_reddit(
|
||||||
|
config["OPENAI_API_KEY"],
|
||||||
|
selected_models["openai"],
|
||||||
|
topic,
|
||||||
|
from_date,
|
||||||
|
to_date,
|
||||||
|
depth=depth,
|
||||||
|
auth_source=config.get("OPENAI_AUTH_SOURCE", "api_key"),
|
||||||
|
account_id=config.get("OPENAI_CHATGPT_ACCOUNT_ID"),
|
||||||
|
)
|
||||||
|
except http.HTTPError as e:
|
||||||
|
raw_response = {"error": str(e)}
|
||||||
|
reddit_error = f"API error: {e}"
|
||||||
|
except Exception as e:
|
||||||
|
raw_response = {"error": str(e)}
|
||||||
|
reddit_error = f"{type(e).__name__}: {e}"
|
||||||
|
|
||||||
|
# Parse response (OpenAI path)
|
||||||
reddit_items = openai_reddit.parse_reddit_response(raw_response or {})
|
reddit_items = openai_reddit.parse_reddit_response(raw_response or {})
|
||||||
|
|
||||||
# Quick retry with simpler query if few results
|
# Quick retry with simpler query if few results (OpenAI path only)
|
||||||
if len(reddit_items) < 5 and not mock and not reddit_error and config.get("OPENAI_API_KEY"):
|
if len(reddit_items) < 5 and not mock and not reddit_error and config.get("OPENAI_API_KEY"):
|
||||||
core = openai_reddit._extract_core_subject(topic)
|
core = openai_reddit._extract_core_subject(topic)
|
||||||
if core.lower() != topic.lower():
|
if core.lower() != topic.lower():
|
||||||
@@ -292,7 +292,7 @@ def _search_reddit(
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Subreddit-targeted fallback if still < 3 results
|
# Subreddit-targeted fallback if still < 3 results (OpenAI path only)
|
||||||
if len(reddit_items) < 3 and not mock and not reddit_error and config.get("OPENAI_API_KEY"):
|
if len(reddit_items) < 3 and not mock and not reddit_error and config.get("OPENAI_API_KEY"):
|
||||||
sub_query = openai_reddit._build_subreddit_query(topic)
|
sub_query = openai_reddit._build_subreddit_query(topic)
|
||||||
try:
|
try:
|
||||||
@@ -617,7 +617,7 @@ def _search_web(
|
|||||||
Tuple of (web_items, web_error)
|
Tuple of (web_items, web_error)
|
||||||
web_items are raw dicts ready for websearch.normalize_websearch_items()
|
web_items are raw dicts ready for websearch.normalize_websearch_items()
|
||||||
"""
|
"""
|
||||||
from lib import brave_search, parallel_search, openrouter_search
|
from lib import brave_search, parallel_search, openrouter_search, exa_search
|
||||||
|
|
||||||
backend = env.get_web_search_source(config)
|
backend = env.get_web_search_source(config)
|
||||||
if not backend:
|
if not backend:
|
||||||
@@ -627,7 +627,11 @@ def _search_web(
|
|||||||
raw_results = []
|
raw_results = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if backend == "parallel":
|
if backend == "exa":
|
||||||
|
raw_results = exa_search.search_web(
|
||||||
|
topic, from_date, to_date, config["EXA_API_KEY"], depth=depth,
|
||||||
|
)
|
||||||
|
elif backend == "parallel":
|
||||||
raw_results = parallel_search.search_web(
|
raw_results = parallel_search.search_web(
|
||||||
topic, from_date, to_date, config["PARALLEL_API_KEY"], depth=depth,
|
topic, from_date, to_date, config["PARALLEL_API_KEY"], depth=depth,
|
||||||
)
|
)
|
||||||
@@ -1537,12 +1541,29 @@ def main():
|
|||||||
# Load config
|
# Load config
|
||||||
config = env.get_config()
|
config = env.get_config()
|
||||||
|
|
||||||
# Inject .env credentials into Bird module before auth check
|
# Detect first run (no SETUP_COMPLETE in config)
|
||||||
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0'))
|
first_run = setup_wizard.is_first_run(config)
|
||||||
|
|
||||||
|
# On first run, block Bird's Node.js sweet-cookie scanner from probing
|
||||||
|
# browser cookies before the user has given consent via the setup wizard.
|
||||||
|
# Explicit AUTH_TOKEN (from env var) is fine — only block browser scanning.
|
||||||
|
if first_run and config.get('_AUTH_TOKEN_SOURCE') != 'env':
|
||||||
|
os.environ['BIRD_DISABLE_BROWSER_COOKIES'] = '1'
|
||||||
|
|
||||||
|
# Inject .env credentials into Bird module before auth check.
|
||||||
|
# On first run (no SETUP_COMPLETE), only inject explicit env var credentials —
|
||||||
|
# skip if AUTH_TOKEN came from browser cookies (no consent yet).
|
||||||
|
if first_run:
|
||||||
|
auth_source = config.get('_AUTH_TOKEN_SOURCE')
|
||||||
|
if auth_source == 'env':
|
||||||
|
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0'))
|
||||||
|
else:
|
||||||
|
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0'))
|
||||||
|
|
||||||
# Auto-detect Bird (no prompts - just use it if available)
|
# Auto-detect Bird (no prompts - just use it if available)
|
||||||
x_source_status = env.get_x_source_status(config)
|
x_source_status = env.get_x_source_status(config)
|
||||||
x_source = x_source_status["source"] # 'bird', 'xai', or None
|
x_source = x_source_status["source"] # 'bird', 'xai', or None
|
||||||
|
x_method = x_source_status.get("method") # 'env', 'browser-firefox', 'api', etc.
|
||||||
|
|
||||||
# Auto-detect yt-dlp for YouTube search
|
# Auto-detect yt-dlp for YouTube search
|
||||||
has_ytdlp = env.is_ytdlp_available()
|
has_ytdlp = env.is_ytdlp_available()
|
||||||
@@ -1570,6 +1591,7 @@ def main():
|
|||||||
"reddit_public": True,
|
"reddit_public": True,
|
||||||
"xai": bool(config.get("XAI_API_KEY")),
|
"xai": bool(config.get("XAI_API_KEY")),
|
||||||
"x_source": x_source_status["source"],
|
"x_source": x_source_status["source"],
|
||||||
|
"x_method": x_source_status.get("method"),
|
||||||
"bird_installed": x_source_status["bird_installed"],
|
"bird_installed": x_source_status["bird_installed"],
|
||||||
"bird_authenticated": x_source_status["bird_authenticated"],
|
"bird_authenticated": x_source_status["bird_authenticated"],
|
||||||
"bird_username": x_source_status.get("bird_username"),
|
"bird_username": x_source_status.get("bird_username"),
|
||||||
@@ -1583,6 +1605,7 @@ def main():
|
|||||||
"truthsocial": has_truthsocial,
|
"truthsocial": has_truthsocial,
|
||||||
"polymarket": True,
|
"polymarket": True,
|
||||||
"web_search_backend": web_source,
|
"web_search_backend": web_source,
|
||||||
|
"exa": bool(config.get("EXA_API_KEY")),
|
||||||
"parallel_ai": bool(config.get("PARALLEL_API_KEY")),
|
"parallel_ai": bool(config.get("PARALLEL_API_KEY")),
|
||||||
"brave": bool(config.get("BRAVE_API_KEY")),
|
"brave": bool(config.get("BRAVE_API_KEY")),
|
||||||
"openrouter": bool(config.get("OPENROUTER_API_KEY")),
|
"openrouter": bool(config.get("OPENROUTER_API_KEY")),
|
||||||
@@ -1590,6 +1613,19 @@ def main():
|
|||||||
print(json.dumps(diag, indent=2))
|
print(json.dumps(diag, indent=2))
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
# Handle 'setup' subcommand
|
||||||
|
if args.topic and args.topic.strip().lower() == "setup":
|
||||||
|
results = setup_wizard.run_auto_setup(config)
|
||||||
|
# Write config
|
||||||
|
env_path = env.CONFIG_FILE
|
||||||
|
if env_path:
|
||||||
|
written = setup_wizard.write_setup_config(env_path)
|
||||||
|
results["env_written"] = written
|
||||||
|
else:
|
||||||
|
results["env_written"] = False
|
||||||
|
print(setup_wizard.get_setup_status_text(results))
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
# Validate topic (--diagnose doesn't need one)
|
# Validate topic (--diagnose doesn't need one)
|
||||||
if not args.topic:
|
if not args.topic:
|
||||||
print("Error: Please provide a topic to research.", file=sys.stderr)
|
print("Error: Please provide a topic to research.", file=sys.stderr)
|
||||||
@@ -1599,40 +1635,30 @@ def main():
|
|||||||
# Initialize progress display with topic
|
# Initialize progress display with topic
|
||||||
progress = ui.ProgressDisplay(args.topic, show_banner=True)
|
progress = ui.ProgressDisplay(args.topic, show_banner=True)
|
||||||
|
|
||||||
# Show diagnostic banner when sources are missing
|
# Show status banner (free-first design — lead with what works)
|
||||||
web_source = env.get_web_search_source(config)
|
web_source = env.get_web_search_source(config)
|
||||||
|
reddit_source = env.get_reddit_source(config)
|
||||||
diag = {
|
diag = {
|
||||||
"openai": bool(config.get("OPENAI_API_KEY")),
|
"setup_complete": bool(config.get("SETUP_COMPLETE")),
|
||||||
"reddit_public": True,
|
"reddit_source": reddit_source, # 'scrapecreators', 'openai', or None
|
||||||
"xai": bool(config.get("XAI_API_KEY")),
|
|
||||||
"x_source": x_source_status["source"],
|
"x_source": x_source_status["source"],
|
||||||
"bird_installed": x_source_status["bird_installed"],
|
"x_method": x_source_status.get("method"),
|
||||||
"bird_authenticated": x_source_status["bird_authenticated"],
|
|
||||||
"bird_username": x_source_status.get("bird_username"),
|
|
||||||
"youtube": has_ytdlp,
|
"youtube": has_ytdlp,
|
||||||
"tiktok": has_tiktok,
|
"tiktok": has_tiktok,
|
||||||
"instagram": has_instagram,
|
"instagram": has_instagram,
|
||||||
"xiaohongshu": has_xiaohongshu,
|
|
||||||
"hackernews": True,
|
"hackernews": True,
|
||||||
"bluesky": True,
|
|
||||||
"truthsocial": has_truthsocial,
|
|
||||||
"polymarket": True,
|
"polymarket": True,
|
||||||
|
"bluesky": has_bluesky,
|
||||||
|
"truthsocial": has_truthsocial,
|
||||||
|
"xiaohongshu": has_xiaohongshu,
|
||||||
|
"scrapecreators": bool(config.get("SCRAPECREATORS_API_KEY")),
|
||||||
"web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
|
"web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
|
||||||
}
|
}
|
||||||
ui.show_diagnostic_banner(diag)
|
ui.show_diagnostic_banner(diag)
|
||||||
|
|
||||||
# Check available sources (accounting for Bird auto-detection)
|
# Check available sources (now accounts for Bird/cookie auth automatically)
|
||||||
available = env.get_available_sources(config)
|
available = env.get_available_sources(config)
|
||||||
|
|
||||||
# Override available if Bird provides X
|
|
||||||
if x_source == 'bird':
|
|
||||||
if available == 'reddit':
|
|
||||||
available = 'both' # Now have both Reddit + X
|
|
||||||
elif available == 'reddit-web':
|
|
||||||
available = 'all' # Reddit + X + Web
|
|
||||||
elif available == 'web':
|
|
||||||
available = 'x-web' # X + Web
|
|
||||||
|
|
||||||
# Mock mode can work without keys
|
# Mock mode can work without keys
|
||||||
if args.mock:
|
if args.mock:
|
||||||
if args.sources == "auto":
|
if args.sources == "auto":
|
||||||
@@ -1709,6 +1735,18 @@ def main():
|
|||||||
search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type)
|
search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type)
|
||||||
search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type)
|
search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type)
|
||||||
search_run_xiaohongshu = has_xiaohongshu
|
search_run_xiaohongshu = has_xiaohongshu
|
||||||
|
|
||||||
|
# INCLUDE_SOURCES override: force specific sources on regardless of tier
|
||||||
|
_include_sources = {s.strip().lower() for s in config.get('INCLUDE_SOURCES', '').split(',') if s.strip()}
|
||||||
|
if _include_sources:
|
||||||
|
if 'tiktok' in _include_sources and has_tiktok:
|
||||||
|
if not search_run_tiktok:
|
||||||
|
sys.stderr.write("[Config] INCLUDE_SOURCES override: forcing tiktok\n")
|
||||||
|
search_run_tiktok = True
|
||||||
|
if 'instagram' in _include_sources and has_instagram:
|
||||||
|
if not search_run_instagram:
|
||||||
|
sys.stderr.write("[Config] INCLUDE_SOURCES override: forcing instagram\n")
|
||||||
|
search_run_instagram = True
|
||||||
if args.search:
|
if args.search:
|
||||||
search_sources = parse_search_flag(args.search)
|
search_sources = parse_search_flag(args.search)
|
||||||
has_reddit = "reddit" in search_sources
|
has_reddit = "reddit" in search_sources
|
||||||
@@ -1902,9 +1940,9 @@ def main():
|
|||||||
elif has_ytdlp and not report.youtube:
|
elif has_ytdlp and not report.youtube:
|
||||||
source_info["youtube_skip_reason"] = "0 results (query may be too specific)"
|
source_info["youtube_skip_reason"] = "0 results (query may be too specific)"
|
||||||
if not has_tiktok:
|
if not has_tiktok:
|
||||||
source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)"
|
source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free API calls, no credit card)"
|
||||||
if not has_instagram:
|
if not has_instagram:
|
||||||
source_info["instagram_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)"
|
source_info["instagram_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free API calls, no credit card)"
|
||||||
if not has_xiaohongshu:
|
if not has_xiaohongshu:
|
||||||
source_info["xiaohongshu_skip_reason"] = (
|
source_info["xiaohongshu_skip_reason"] = (
|
||||||
f"Xiaohongshu API unavailable or not logged in - start xiaohongshu-mcp and login "
|
f"Xiaohongshu API unavailable or not logged in - start xiaohongshu-mcp and login "
|
||||||
@@ -1913,8 +1951,16 @@ def main():
|
|||||||
if not web_source:
|
if not web_source:
|
||||||
source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)"
|
source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)"
|
||||||
|
|
||||||
|
# Compute quality score and upgrade nudge
|
||||||
|
research_results = {
|
||||||
|
"x_error": x_error,
|
||||||
|
"youtube_error": youtube_error,
|
||||||
|
"reddit_error": reddit_error,
|
||||||
|
}
|
||||||
|
quality = quality_nudge.compute_quality_score(config, research_results)
|
||||||
|
|
||||||
# Output result
|
# Output result
|
||||||
output_result(report, args.emit, web_needed, args.topic, from_date, to_date, missing_keys, args.days, source_info)
|
output_result(report, args.emit, web_needed, args.topic, from_date, to_date, missing_keys, args.days, source_info, first_run=first_run, quality=quality)
|
||||||
|
|
||||||
# Auto-save raw research to file if --save-dir is set
|
# Auto-save raw research to file if --save-dir is set
|
||||||
if args.save_dir:
|
if args.save_dir:
|
||||||
@@ -1926,6 +1972,8 @@ def main():
|
|||||||
if save_path.exists():
|
if save_path.exists():
|
||||||
save_path = save_dir / f"{slug}-raw-{datetime.now().strftime('%Y-%m-%d')}.md"
|
save_path = save_dir / f"{slug}-raw-{datetime.now().strftime('%Y-%m-%d')}.md"
|
||||||
content = render.render_compact(report, missing_keys=missing_keys)
|
content = render.render_compact(report, missing_keys=missing_keys)
|
||||||
|
if quality and quality.get("nudge_text"):
|
||||||
|
content += "\n" + render.render_quality_nudge(quality)
|
||||||
content += "\n" + render.render_source_status(report, source_info)
|
content += "\n" + render.render_source_status(report, source_info)
|
||||||
save_path.write_text(content, encoding="utf-8")
|
save_path.write_text(content, encoding="utf-8")
|
||||||
print(f"📎 {save_path}", file=sys.stderr)
|
print(f"📎 {save_path}", file=sys.stderr)
|
||||||
@@ -2043,10 +2091,19 @@ def output_result(
|
|||||||
missing_keys: str = "none",
|
missing_keys: str = "none",
|
||||||
days: int = 30,
|
days: int = 30,
|
||||||
source_info: dict = None,
|
source_info: dict = None,
|
||||||
|
first_run: bool = False,
|
||||||
|
quality: dict = None,
|
||||||
):
|
):
|
||||||
"""Output the result based on emit mode."""
|
"""Output the result based on emit mode."""
|
||||||
if emit_mode == "compact":
|
if emit_mode == "compact":
|
||||||
|
# Emit first-run flag before research output so SKILL.md can detect it
|
||||||
|
if first_run:
|
||||||
|
print("FIRST_RUN: true")
|
||||||
|
print("")
|
||||||
print(render.render_compact(report, missing_keys=missing_keys))
|
print(render.render_compact(report, missing_keys=missing_keys))
|
||||||
|
# Quality nudge (right before source status/stats block)
|
||||||
|
if quality and quality.get("nudge_text"):
|
||||||
|
print(render.render_quality_nudge(quality))
|
||||||
# Append source status footer
|
# Append source status footer
|
||||||
print(render.render_source_status(report, source_info))
|
print(render.render_source_status(report, source_info))
|
||||||
elif emit_mode == "json":
|
elif emit_mode == "json":
|
||||||
|
|||||||
@@ -0,0 +1,265 @@
|
|||||||
|
"""Chrome cookie extraction for macOS.
|
||||||
|
|
||||||
|
Extracts cookies from Chrome's encrypted SQLite database using only stdlib
|
||||||
|
modules and the system openssl CLI (ships with macOS). Zero pip dependencies.
|
||||||
|
|
||||||
|
Chrome on macOS uses v10 encryption (AES-128-CBC with Keychain-stored key).
|
||||||
|
This is NOT affected by Windows App-Bound Encryption (v20).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Chrome cookie DB location on macOS
|
||||||
|
CHROME_COOKIES_DB = Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "Cookies"
|
||||||
|
|
||||||
|
# Chrome v10 encryption constants
|
||||||
|
CHROME_SALT = b"saltysalt"
|
||||||
|
CHROME_PBKDF2_ITERATIONS = 1003
|
||||||
|
CHROME_KEY_LENGTH = 16
|
||||||
|
# IV is 16 space characters (0x20)
|
||||||
|
CHROME_IV_HEX = "20" * 16
|
||||||
|
|
||||||
|
|
||||||
|
def _get_chrome_encryption_key() -> Optional[bytes]:
|
||||||
|
"""Retrieve Chrome's encryption passphrase from macOS Keychain.
|
||||||
|
|
||||||
|
Calls `security find-generic-password` which may trigger a system dialog
|
||||||
|
on first access.
|
||||||
|
|
||||||
|
Returns the raw passphrase bytes, or None on failure.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.info("Chrome Keychain access denied or Chrome not installed: %s", result.stderr.strip())
|
||||||
|
return None
|
||||||
|
passphrase = result.stdout.strip()
|
||||||
|
if not passphrase:
|
||||||
|
logger.info("Chrome Keychain returned empty passphrase")
|
||||||
|
return None
|
||||||
|
return passphrase.encode("utf-8")
|
||||||
|
except FileNotFoundError:
|
||||||
|
logger.info("'security' command not found — not on macOS?")
|
||||||
|
return None
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
logger.info("Chrome Keychain access timed out")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger.info("Failed to get Chrome encryption key: %s", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _derive_aes_key(passphrase: bytes) -> bytes:
|
||||||
|
"""Derive 16-byte AES key from Chrome's Keychain passphrase via PBKDF2."""
|
||||||
|
return hashlib.pbkdf2_hmac(
|
||||||
|
"sha1",
|
||||||
|
passphrase,
|
||||||
|
CHROME_SALT,
|
||||||
|
CHROME_PBKDF2_ITERATIONS,
|
||||||
|
dklen=CHROME_KEY_LENGTH,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decrypt_v10_value(encrypted_value: bytes, aes_key: bytes, db_version: int) -> Optional[str]:
|
||||||
|
"""Decrypt a Chrome v10-encrypted cookie value.
|
||||||
|
|
||||||
|
Uses system openssl CLI for AES-128-CBC decryption (zero pip deps).
|
||||||
|
For Chrome 130+ (db_version >= 24), strips 32-byte SHA-256 prefix after decryption.
|
||||||
|
|
||||||
|
Returns decrypted string or None on failure.
|
||||||
|
"""
|
||||||
|
# Strip the 'v10' prefix
|
||||||
|
ciphertext = encrypted_value[3:]
|
||||||
|
if not ciphertext:
|
||||||
|
return None
|
||||||
|
|
||||||
|
hex_key = aes_key.hex()
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"openssl", "enc", "-aes-128-cbc", "-d",
|
||||||
|
"-K", hex_key,
|
||||||
|
"-iv", CHROME_IV_HEX,
|
||||||
|
"-nopad",
|
||||||
|
],
|
||||||
|
input=ciphertext,
|
||||||
|
capture_output=True,
|
||||||
|
timeout=5,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.debug("openssl decryption failed: %s", result.stderr.decode(errors="replace").strip())
|
||||||
|
return None
|
||||||
|
|
||||||
|
decrypted = result.stdout
|
||||||
|
if not decrypted:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Remove PKCS7 padding
|
||||||
|
decrypted = _remove_pkcs7_padding(decrypted)
|
||||||
|
if decrypted is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Chrome 130+ (db version >= 24): strip 32-byte SHA-256 prefix
|
||||||
|
if db_version >= 24 and len(decrypted) > 32:
|
||||||
|
decrypted = decrypted[32:]
|
||||||
|
|
||||||
|
return decrypted.decode("utf-8", errors="replace")
|
||||||
|
|
||||||
|
except FileNotFoundError:
|
||||||
|
logger.info("openssl not found — cannot decrypt Chrome cookies")
|
||||||
|
return None
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
logger.info("openssl decryption timed out")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("Chrome cookie decryption error: %s", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_pkcs7_padding(data: bytes) -> Optional[bytes]:
|
||||||
|
"""Remove PKCS7 padding from decrypted data.
|
||||||
|
|
||||||
|
The last byte indicates the number of padding bytes added.
|
||||||
|
All padding bytes must have the same value.
|
||||||
|
|
||||||
|
Returns unpadded data or None if padding is invalid.
|
||||||
|
"""
|
||||||
|
if not data:
|
||||||
|
return None
|
||||||
|
pad_len = data[-1]
|
||||||
|
if pad_len < 1 or pad_len > 16:
|
||||||
|
return None
|
||||||
|
# Verify all padding bytes match
|
||||||
|
if data[-pad_len:] != bytes([pad_len]) * pad_len:
|
||||||
|
return None
|
||||||
|
return data[:-pad_len]
|
||||||
|
|
||||||
|
|
||||||
|
def _get_db_version(cursor: sqlite3.Cursor) -> int:
|
||||||
|
"""Get Chrome cookie database version from the meta table.
|
||||||
|
|
||||||
|
Returns 0 if meta table doesn't exist or version can't be read.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
cursor.execute("SELECT value FROM meta WHERE key = 'version'")
|
||||||
|
row = cursor.fetchone()
|
||||||
|
if row:
|
||||||
|
return int(row[0])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def extract_chrome_cookies_macos(domain: str, cookie_names: list[str]) -> Optional[dict[str, str]]:
|
||||||
|
"""Extract cookies from Chrome on macOS.
|
||||||
|
|
||||||
|
Copies the locked Cookies database to a temp file, reads specified cookies,
|
||||||
|
and decrypts v10-encrypted values using the Keychain-stored key.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
domain: Cookie domain to match (e.g., ".twitter.com", ".x.com")
|
||||||
|
cookie_names: List of cookie names to extract
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict mapping cookie name to decrypted value, or None on failure.
|
||||||
|
Only includes cookies that were successfully found and decrypted.
|
||||||
|
"""
|
||||||
|
if not CHROME_COOKIES_DB.exists():
|
||||||
|
logger.info("Chrome cookies database not found at %s", CHROME_COOKIES_DB)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Get encryption key from Keychain
|
||||||
|
passphrase = _get_chrome_encryption_key()
|
||||||
|
aes_key = _derive_aes_key(passphrase) if passphrase else None
|
||||||
|
|
||||||
|
# Copy DB to temp file (Chrome locks the original)
|
||||||
|
tmp_fd = None
|
||||||
|
tmp_path = None
|
||||||
|
try:
|
||||||
|
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||||
|
shutil.copy2(str(CHROME_COOKIES_DB), tmp_path)
|
||||||
|
except Exception as e:
|
||||||
|
logger.info("Failed to copy Chrome cookies database: %s", e)
|
||||||
|
if tmp_path:
|
||||||
|
try:
|
||||||
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
if tmp_fd is not None:
|
||||||
|
import os
|
||||||
|
os.close(tmp_fd)
|
||||||
|
|
||||||
|
try:
|
||||||
|
conn = sqlite3.connect(tmp_path)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
|
db_version = _get_db_version(cursor)
|
||||||
|
logger.debug("Chrome cookie DB version: %d", db_version)
|
||||||
|
|
||||||
|
# Build query with placeholders for cookie names
|
||||||
|
placeholders = ",".join("?" for _ in cookie_names)
|
||||||
|
query = (
|
||||||
|
f"SELECT name, value, encrypted_value FROM cookies "
|
||||||
|
f"WHERE host_key LIKE ? AND name IN ({placeholders})"
|
||||||
|
)
|
||||||
|
# Use LIKE for domain matching (e.g., %.twitter.com matches .twitter.com)
|
||||||
|
params = [f"%{domain}"] + list(cookie_names)
|
||||||
|
cursor.execute(query, params)
|
||||||
|
|
||||||
|
results: dict[str, str] = {}
|
||||||
|
for name, value, encrypted_value in cursor.fetchall():
|
||||||
|
# Prefer unencrypted value if present
|
||||||
|
if value:
|
||||||
|
results[name] = value
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Handle encrypted value
|
||||||
|
if encrypted_value and encrypted_value[:3] == b"v10":
|
||||||
|
if aes_key is None:
|
||||||
|
logger.debug("Skipping encrypted cookie %s — no Keychain access", name)
|
||||||
|
continue
|
||||||
|
decrypted = _decrypt_v10_value(encrypted_value, aes_key, db_version)
|
||||||
|
if decrypted:
|
||||||
|
results[name] = decrypted
|
||||||
|
else:
|
||||||
|
logger.debug("Failed to decrypt cookie %s", name)
|
||||||
|
elif encrypted_value:
|
||||||
|
# Unknown encryption version
|
||||||
|
logger.debug("Unknown encryption for cookie %s (prefix: %r)", name, encrypted_value[:3])
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
if not results:
|
||||||
|
logger.info("No matching cookies found in Chrome for domain %s", domain)
|
||||||
|
return None
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
except sqlite3.Error as e:
|
||||||
|
logger.info("Failed to read Chrome cookies database: %s", e)
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger.info("Unexpected error reading Chrome cookies: %s", e)
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
@@ -0,0 +1,295 @@
|
|||||||
|
"""Browser cookie extraction for last30days.
|
||||||
|
|
||||||
|
Extracts cookies from local browser databases (Firefox, Chrome, Safari)
|
||||||
|
to enable zero-config authentication for services like X/Twitter.
|
||||||
|
|
||||||
|
Only uses Python stdlib — no external dependencies.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import configparser
|
||||||
|
import logging
|
||||||
|
import platform
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_firefox_profiles_dir() -> Optional[Path]:
|
||||||
|
"""Return the Firefox profiles directory for the current platform, or None."""
|
||||||
|
system = platform.system()
|
||||||
|
if system == "Darwin":
|
||||||
|
path = Path.home() / "Library" / "Application Support" / "Firefox"
|
||||||
|
elif system == "Linux":
|
||||||
|
path = Path.home() / ".mozilla" / "firefox"
|
||||||
|
else:
|
||||||
|
# Windows: %APPDATA%\Mozilla\Firefox — best-effort
|
||||||
|
appdata = Path.home() / "AppData" / "Roaming" / "Mozilla" / "Firefox"
|
||||||
|
path = appdata
|
||||||
|
return path if path.is_dir() else None
|
||||||
|
|
||||||
|
|
||||||
|
def _find_default_profile(profiles_dir: Path) -> Optional[Path]:
|
||||||
|
"""Parse profiles.ini to find the default profile directory.
|
||||||
|
|
||||||
|
Looks for a section with Default=1. Falls back to the first profile
|
||||||
|
directory found on disk if profiles.ini is missing or malformed.
|
||||||
|
"""
|
||||||
|
ini_path = profiles_dir / "profiles.ini"
|
||||||
|
|
||||||
|
if ini_path.is_file():
|
||||||
|
try:
|
||||||
|
config = configparser.ConfigParser()
|
||||||
|
config.read(str(ini_path), encoding="utf-8")
|
||||||
|
|
||||||
|
# First pass: look for Default=1
|
||||||
|
for section in config.sections():
|
||||||
|
if config.has_option(section, "Default") and config.get(section, "Default") == "1":
|
||||||
|
return _resolve_profile_path(profiles_dir, config, section)
|
||||||
|
|
||||||
|
# Second pass: first Install* section with Default key (Firefox >= 67 format)
|
||||||
|
for section in config.sections():
|
||||||
|
if section.startswith("Install") and config.has_option(section, "Default"):
|
||||||
|
raw = config.get(section, "Default")
|
||||||
|
candidate = profiles_dir / raw
|
||||||
|
if candidate.is_dir():
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
# Third pass: first Profile section that exists on disk
|
||||||
|
for section in config.sections():
|
||||||
|
if section.startswith("Profile"):
|
||||||
|
resolved = _resolve_profile_path(profiles_dir, config, section)
|
||||||
|
if resolved and resolved.is_dir():
|
||||||
|
return resolved
|
||||||
|
except (configparser.Error, OSError) as exc:
|
||||||
|
logger.debug("Failed to parse profiles.ini: %s", exc)
|
||||||
|
|
||||||
|
# Fallback: scan directory for anything that looks like a profile
|
||||||
|
return _fallback_find_profile(profiles_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_profile_path(
|
||||||
|
profiles_dir: Path, config: configparser.ConfigParser, section: str
|
||||||
|
) -> Optional[Path]:
|
||||||
|
"""Resolve a profile path from a ConfigParser section."""
|
||||||
|
if not config.has_option(section, "Path"):
|
||||||
|
return None
|
||||||
|
raw_path = config.get(section, "Path")
|
||||||
|
is_relative = config.has_option(section, "IsRelative") and config.get(section, "IsRelative") == "1"
|
||||||
|
if is_relative:
|
||||||
|
candidate = profiles_dir / raw_path
|
||||||
|
else:
|
||||||
|
candidate = Path(raw_path)
|
||||||
|
return candidate if candidate.is_dir() else None
|
||||||
|
|
||||||
|
|
||||||
|
def _fallback_find_profile(profiles_dir: Path) -> Optional[Path]:
|
||||||
|
"""Find the first directory that contains cookies.sqlite."""
|
||||||
|
try:
|
||||||
|
for child in sorted(profiles_dir.iterdir()):
|
||||||
|
if child.is_dir() and (child / "cookies.sqlite").is_file():
|
||||||
|
return child
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _query_cookies_db(
|
||||||
|
db_path: Path, domain: str, cookie_names: List[str]
|
||||||
|
) -> Optional[Dict[str, str]]:
|
||||||
|
"""Copy the cookies database to a temp file and query it.
|
||||||
|
|
||||||
|
Firefox locks cookies.sqlite while running, so we copy first.
|
||||||
|
Returns {name: value} dict or None if no matching cookies found.
|
||||||
|
"""
|
||||||
|
if not db_path.is_file():
|
||||||
|
return None
|
||||||
|
|
||||||
|
tmp_fd = None
|
||||||
|
tmp_path = None
|
||||||
|
try:
|
||||||
|
tmp_fd, tmp_path = tempfile.mkstemp(suffix=".sqlite")
|
||||||
|
shutil.copy2(str(db_path), tmp_path)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(tmp_path)
|
||||||
|
try:
|
||||||
|
# Build parameterized query — SQLite doesn't support array params,
|
||||||
|
# so we build the IN clause with individual placeholders.
|
||||||
|
placeholders = ",".join("?" for _ in cookie_names)
|
||||||
|
query = (
|
||||||
|
f"SELECT name, value FROM moz_cookies "
|
||||||
|
f"WHERE host LIKE ? AND name IN ({placeholders})"
|
||||||
|
)
|
||||||
|
# domain pattern: match .x.com, x.com, etc.
|
||||||
|
domain_pattern = f"%{domain}"
|
||||||
|
params = [domain_pattern] + list(cookie_names)
|
||||||
|
|
||||||
|
cursor = conn.execute(query, params)
|
||||||
|
rows = cursor.fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
return None
|
||||||
|
return {name: value for name, value in rows}
|
||||||
|
|
||||||
|
except (sqlite3.Error, OSError) as exc:
|
||||||
|
logger.debug("Failed to query cookies database %s: %s", db_path, exc)
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
if tmp_path:
|
||||||
|
try:
|
||||||
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
if tmp_fd is not None:
|
||||||
|
try:
|
||||||
|
import os
|
||||||
|
os.close(tmp_fd)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def extract_firefox_cookies(
|
||||||
|
domain: str, cookie_names: List[str]
|
||||||
|
) -> Optional[Dict[str, str]]:
|
||||||
|
"""Extract cookies from Firefox for the given domain and cookie names.
|
||||||
|
|
||||||
|
Finds the default Firefox profile, copies cookies.sqlite to a temp file
|
||||||
|
(to avoid lock conflicts), and queries for the requested cookies.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
domain: The cookie domain to match (e.g. ".x.com"). Matched with LIKE %domain.
|
||||||
|
cookie_names: List of cookie names to extract (e.g. ["auth_token", "ct0"]).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||||
|
"""
|
||||||
|
profiles_dir = _get_firefox_profiles_dir()
|
||||||
|
if profiles_dir is None:
|
||||||
|
logger.debug("Firefox profiles directory not found")
|
||||||
|
return None
|
||||||
|
|
||||||
|
profile_path = _find_default_profile(profiles_dir)
|
||||||
|
if profile_path is None:
|
||||||
|
logger.debug("No Firefox profile found in %s", profiles_dir)
|
||||||
|
return None
|
||||||
|
|
||||||
|
db_path = profile_path / "cookies.sqlite"
|
||||||
|
return _query_cookies_db(db_path, domain, cookie_names)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_chrome_cookies(
|
||||||
|
domain: str, cookie_names: List[str]
|
||||||
|
) -> Optional[Dict[str, str]]:
|
||||||
|
"""Extract cookies from Chrome for the given domain and cookie names.
|
||||||
|
|
||||||
|
macOS only — uses Keychain + system openssl for AES-128-CBC decryption.
|
||||||
|
Linux/Windows not supported (Chrome uses platform-specific encryption).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||||
|
"""
|
||||||
|
if platform.system() != "Darwin":
|
||||||
|
logger.debug("Chrome cookie extraction only supported on macOS")
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from .chrome_cookies import extract_chrome_cookies_macos
|
||||||
|
return extract_chrome_cookies_macos(domain, cookie_names)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Chrome cookie extraction failed: %s", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_safari_cookies(
|
||||||
|
domain: str, cookie_names: List[str]
|
||||||
|
) -> Optional[Dict[str, str]]:
|
||||||
|
"""Extract cookies from Safari for the given domain and cookie names.
|
||||||
|
|
||||||
|
macOS only — parses the unencrypted binary cookie file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||||
|
"""
|
||||||
|
if platform.system() != "Darwin":
|
||||||
|
logger.debug("Safari cookie extraction only supported on macOS")
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from .safari_cookies import extract_safari_cookies_macos
|
||||||
|
return extract_safari_cookies_macos(domain, cookie_names)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Safari cookie extraction failed: %s", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_cookies(
|
||||||
|
browser: str, domain: str, cookie_names: list[str]
|
||||||
|
) -> Optional[dict[str, str]]:
|
||||||
|
"""Extract cookies from the specified browser.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||||
|
'auto' tries browsers in platform-appropriate order:
|
||||||
|
- macOS: Chrome -> Firefox -> Safari
|
||||||
|
- Linux: Firefox only
|
||||||
|
domain: The cookie domain to match (e.g. ".x.com").
|
||||||
|
cookie_names: List of cookie names to extract.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict of {cookie_name: cookie_value} or None if extraction fails.
|
||||||
|
"""
|
||||||
|
result = extract_cookies_with_source(browser, domain, cookie_names)
|
||||||
|
if result is None:
|
||||||
|
return None
|
||||||
|
cookies, _browser_name = result
|
||||||
|
return cookies
|
||||||
|
|
||||||
|
|
||||||
|
def extract_cookies_with_source(
|
||||||
|
browser: str, domain: str, cookie_names: list[str]
|
||||||
|
) -> Optional[tuple[dict[str, str], str]]:
|
||||||
|
"""Extract cookies and report which browser they came from.
|
||||||
|
|
||||||
|
Same as extract_cookies() but returns a (cookies, browser_name) tuple
|
||||||
|
so callers can track the source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
browser: One of 'firefox', 'chrome', 'safari', or 'auto'.
|
||||||
|
domain: The cookie domain to match (e.g. ".x.com").
|
||||||
|
cookie_names: List of cookie names to extract.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of ({cookie_name: cookie_value}, browser_name) or None.
|
||||||
|
"""
|
||||||
|
extractors = {
|
||||||
|
"firefox": extract_firefox_cookies,
|
||||||
|
"chrome": extract_chrome_cookies,
|
||||||
|
"safari": extract_safari_cookies,
|
||||||
|
}
|
||||||
|
|
||||||
|
if browser != "auto":
|
||||||
|
extractor = extractors.get(browser)
|
||||||
|
if extractor is None:
|
||||||
|
logger.warning("Unknown browser: %s", browser)
|
||||||
|
return None
|
||||||
|
result = extractor(domain, cookie_names)
|
||||||
|
return (result, browser) if result is not None else None
|
||||||
|
|
||||||
|
# Auto mode: try browsers in platform-appropriate order
|
||||||
|
system = platform.system()
|
||||||
|
if system == "Darwin":
|
||||||
|
order = ["chrome", "firefox", "safari"]
|
||||||
|
elif system == "Linux":
|
||||||
|
order = ["firefox"]
|
||||||
|
else:
|
||||||
|
order = ["firefox"]
|
||||||
|
|
||||||
|
for name in order:
|
||||||
|
result = extractors[name](domain, cookie_names)
|
||||||
|
if result is not None:
|
||||||
|
return (result, name)
|
||||||
|
|
||||||
|
return None
|
||||||
+230
-25
@@ -2,11 +2,38 @@
|
|||||||
|
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional, Dict, Any, Literal
|
from typing import Optional, Dict, Any, List, Literal
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Cookie domain registry: maps source names to browser cookie extraction params.
|
||||||
|
# Each entry: (domain, cookie_names, config_key_mapping)
|
||||||
|
# config_key_mapping: {cookie_name: config_key} so we know which config key
|
||||||
|
# each extracted cookie should populate.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
COOKIE_DOMAINS: Dict[str, Dict[str, Any]] = {
|
||||||
|
"x": {
|
||||||
|
"domain": ".x.com",
|
||||||
|
"cookies": ["auth_token", "ct0"],
|
||||||
|
"mapping": {
|
||||||
|
"auth_token": "AUTH_TOKEN",
|
||||||
|
"ct0": "CT0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"truthsocial": {
|
||||||
|
"domain": ".truthsocial.com",
|
||||||
|
"cookies": ["_session_id"],
|
||||||
|
"mapping": {
|
||||||
|
"_session_id": "TRUTHSOCIAL_TOKEN",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
# Allow override via environment variable for testing
|
# Allow override via environment variable for testing
|
||||||
# Set LAST30DAYS_CONFIG_DIR="" for clean/no-config mode
|
# Set LAST30DAYS_CONFIG_DIR="" for clean/no-config mode
|
||||||
@@ -212,6 +239,87 @@ def _find_project_env() -> Optional[Path]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_browser_credentials(config: Dict[str, Any]) -> Dict[str, str]:
|
||||||
|
"""Extract credentials from browser cookies for sources that need them.
|
||||||
|
|
||||||
|
Checks the FROM_BROWSER config key to decide whether/how to extract:
|
||||||
|
- 'auto': try browsers in platform order
|
||||||
|
- 'firefox', 'chrome', 'safari': try only that browser
|
||||||
|
- 'off': skip extraction entirely
|
||||||
|
|
||||||
|
If SETUP_COMPLETE is not set AND FROM_BROWSER is not explicitly set,
|
||||||
|
defaults to 'off' (wizard hasn't run yet — no extraction without consent).
|
||||||
|
If SETUP_COMPLETE is set and FROM_BROWSER is not set, defaults to 'auto'.
|
||||||
|
|
||||||
|
Explicit env var/config values always take priority over extracted cookies.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict of {config_key: value} for credentials discovered from cookies.
|
||||||
|
"""
|
||||||
|
setup_complete = config.get("SETUP_COMPLETE")
|
||||||
|
from_browser = config.get("FROM_BROWSER")
|
||||||
|
|
||||||
|
# Determine effective browser setting
|
||||||
|
if from_browser is None:
|
||||||
|
if setup_complete:
|
||||||
|
from_browser = "auto"
|
||||||
|
else:
|
||||||
|
from_browser = "off"
|
||||||
|
|
||||||
|
from_browser = from_browser.lower().strip() if isinstance(from_browser, str) else "off"
|
||||||
|
|
||||||
|
if from_browser == "off":
|
||||||
|
return {}
|
||||||
|
|
||||||
|
# Lazy import to avoid loading cookie_extract at module level
|
||||||
|
try:
|
||||||
|
from . import cookie_extract
|
||||||
|
except Exception:
|
||||||
|
logger.debug("cookie_extract module not available")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
credentials: Dict[str, str] = {}
|
||||||
|
|
||||||
|
for source_name, spec in COOKIE_DOMAINS.items():
|
||||||
|
domain = spec["domain"]
|
||||||
|
cookie_names: List[str] = spec["cookies"]
|
||||||
|
mapping: Dict[str, str] = spec["mapping"]
|
||||||
|
|
||||||
|
# Skip if ALL mapped config keys already have values
|
||||||
|
all_present = all(config.get(config_key) for config_key in mapping.values())
|
||||||
|
if all_present:
|
||||||
|
logger.debug(
|
||||||
|
"Skipping cookie extraction for %s: credentials already set",
|
||||||
|
source_name,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = cookie_extract.extract_cookies_with_source(from_browser, domain, cookie_names)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug(
|
||||||
|
"Cookie extraction failed for %s: %s", source_name, exc
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if result is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
cookies, browser_name = result
|
||||||
|
filled_any = False
|
||||||
|
for cookie_name, config_key in mapping.items():
|
||||||
|
# Only fill in keys not already present
|
||||||
|
if not config.get(config_key) and cookie_name in cookies:
|
||||||
|
credentials[config_key] = cookies[cookie_name]
|
||||||
|
filled_any = True
|
||||||
|
|
||||||
|
# Track which browser provided the credentials for this source
|
||||||
|
if filled_any:
|
||||||
|
credentials[f"__{source_name.upper()}_BROWSER"] = browser_name
|
||||||
|
|
||||||
|
return credentials
|
||||||
|
|
||||||
|
|
||||||
def get_config() -> Dict[str, Any]:
|
def get_config() -> Dict[str, Any]:
|
||||||
"""Load configuration from multiple sources.
|
"""Load configuration from multiple sources.
|
||||||
|
|
||||||
@@ -219,6 +327,7 @@ def get_config() -> Dict[str, Any]:
|
|||||||
1. Environment variables (os.environ)
|
1. Environment variables (os.environ)
|
||||||
2. .claude/last30days.env (per-project config)
|
2. .claude/last30days.env (per-project config)
|
||||||
3. ~/.config/last30days/.env (global config)
|
3. ~/.config/last30days/.env (global config)
|
||||||
|
4. Browser cookies (only fills in missing keys)
|
||||||
"""
|
"""
|
||||||
# Load from global config file
|
# Load from global config file
|
||||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||||
@@ -249,6 +358,7 @@ def get_config() -> Dict[str, Any]:
|
|||||||
('OPENROUTER_API_KEY', None),
|
('OPENROUTER_API_KEY', None),
|
||||||
('PARALLEL_API_KEY', None),
|
('PARALLEL_API_KEY', None),
|
||||||
('BRAVE_API_KEY', None),
|
('BRAVE_API_KEY', None),
|
||||||
|
('EXA_API_KEY', None),
|
||||||
('XIAOHONGSHU_API_BASE', None),
|
('XIAOHONGSHU_API_BASE', None),
|
||||||
('GEMINI_MODEL', None),
|
('GEMINI_MODEL', None),
|
||||||
('OPENAI_MODEL_POLICY', 'auto'),
|
('OPENAI_MODEL_POLICY', 'auto'),
|
||||||
@@ -262,11 +372,32 @@ def get_config() -> Dict[str, Any]:
|
|||||||
('BSKY_HANDLE', None),
|
('BSKY_HANDLE', None),
|
||||||
('BSKY_APP_PASSWORD', None),
|
('BSKY_APP_PASSWORD', None),
|
||||||
('TRUTHSOCIAL_TOKEN', None),
|
('TRUTHSOCIAL_TOKEN', None),
|
||||||
|
('FROM_BROWSER', None),
|
||||||
|
('SETUP_COMPLETE', None),
|
||||||
|
('INCLUDE_SOURCES', None),
|
||||||
]
|
]
|
||||||
|
|
||||||
for key, default in keys:
|
for key, default in keys:
|
||||||
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
config[key] = os.environ.get(key) or merged_env.get(key, default)
|
||||||
|
|
||||||
|
# Inject browser cookies for any credentials not already set
|
||||||
|
browser_creds = extract_browser_credentials(config)
|
||||||
|
for key, value in browser_creds.items():
|
||||||
|
if not config.get(key):
|
||||||
|
config[key] = value
|
||||||
|
|
||||||
|
# Track AUTH_TOKEN source for status reporting
|
||||||
|
if config.get('AUTH_TOKEN'):
|
||||||
|
if os.environ.get('AUTH_TOKEN') or merged_env.get('AUTH_TOKEN'):
|
||||||
|
config['_AUTH_TOKEN_SOURCE'] = 'env'
|
||||||
|
elif browser_creds.get('AUTH_TOKEN'):
|
||||||
|
browser_name = browser_creds.get('__X_BROWSER', 'unknown')
|
||||||
|
config['_AUTH_TOKEN_SOURCE'] = f'browser-{browser_name}'
|
||||||
|
else:
|
||||||
|
config['_AUTH_TOKEN_SOURCE'] = 'env' # fallback
|
||||||
|
else:
|
||||||
|
config['_AUTH_TOKEN_SOURCE'] = None
|
||||||
|
|
||||||
# Track which config source was used
|
# Track which config source was used
|
||||||
if project_env_path:
|
if project_env_path:
|
||||||
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
|
||||||
@@ -314,14 +445,19 @@ def get_reddit_source(config: Dict[str, Any]) -> Optional[str]:
|
|||||||
def get_available_sources(config: Dict[str, Any]) -> str:
|
def get_available_sources(config: Dict[str, Any]) -> str:
|
||||||
"""Determine which sources are available.
|
"""Determine which sources are available.
|
||||||
|
|
||||||
|
X is available if ANY auth method works: AUTH_TOKEN/CT0 (env or cookies),
|
||||||
|
XAI_API_KEY, or Bird installed+authenticated.
|
||||||
|
Reddit is always available (public JSON fallback).
|
||||||
|
HN and Polymarket are always available.
|
||||||
|
YouTube available if yt-dlp installed.
|
||||||
|
|
||||||
Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
|
Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
|
||||||
"""
|
"""
|
||||||
# Reddit is available via public JSON fallback even without OpenAI auth.
|
|
||||||
has_reddit = True
|
has_reddit = True
|
||||||
has_xai = bool(config.get('XAI_API_KEY'))
|
has_x = get_x_source(config) is not None
|
||||||
has_web = has_web_search_keys(config)
|
has_web = has_web_search_keys(config)
|
||||||
|
|
||||||
if has_reddit and has_xai:
|
if has_reddit and has_x:
|
||||||
return 'all' if has_web else 'both'
|
return 'all' if has_web else 'both'
|
||||||
elif has_reddit:
|
elif has_reddit:
|
||||||
return 'reddit-web' if has_web else 'reddit'
|
return 'reddit-web' if has_web else 'reddit'
|
||||||
@@ -330,16 +466,18 @@ def get_available_sources(config: Dict[str, Any]) -> str:
|
|||||||
|
|
||||||
def has_web_search_keys(config: Dict[str, Any]) -> bool:
|
def has_web_search_keys(config: Dict[str, Any]) -> bool:
|
||||||
"""Check if any web search API keys are configured."""
|
"""Check if any web search API keys are configured."""
|
||||||
return bool(config.get('OPENROUTER_API_KEY') or config.get('PARALLEL_API_KEY') or config.get('BRAVE_API_KEY'))
|
return bool(config.get('EXA_API_KEY') or config.get('OPENROUTER_API_KEY') or config.get('PARALLEL_API_KEY') or config.get('BRAVE_API_KEY'))
|
||||||
|
|
||||||
|
|
||||||
def get_web_search_source(config: Dict[str, Any]) -> Optional[str]:
|
def get_web_search_source(config: Dict[str, Any]) -> Optional[str]:
|
||||||
"""Determine the best available web search backend.
|
"""Determine the best available web search backend.
|
||||||
|
|
||||||
Priority: Parallel AI > Brave > OpenRouter/Sonar Pro
|
Priority: Exa (free) > Parallel AI > Brave > OpenRouter/Sonar Pro
|
||||||
|
|
||||||
Returns: 'parallel', 'brave', 'openrouter', or None
|
Returns: 'exa', 'parallel', 'brave', 'openrouter', or None
|
||||||
"""
|
"""
|
||||||
|
if config.get('EXA_API_KEY'):
|
||||||
|
return 'exa'
|
||||||
if config.get('PARALLEL_API_KEY'):
|
if config.get('PARALLEL_API_KEY'):
|
||||||
return 'parallel'
|
return 'parallel'
|
||||||
if config.get('BRAVE_API_KEY'):
|
if config.get('BRAVE_API_KEY'):
|
||||||
@@ -441,9 +579,13 @@ def validate_sources(requested: str, available: str, include_web: bool = False)
|
|||||||
def get_x_source(config: Dict[str, Any]) -> Optional[str]:
|
def get_x_source(config: Dict[str, Any]) -> Optional[str]:
|
||||||
"""Determine the best available X/Twitter source.
|
"""Determine the best available X/Twitter source.
|
||||||
|
|
||||||
Priority: Bird (free) → xAI (paid API)
|
Priority chain:
|
||||||
|
1. AUTH_TOKEN/CT0 from env var or .env file → Bird with method "env"
|
||||||
|
2. AUTH_TOKEN/CT0 from browser cookie extraction → Bird with method "browser-{browser}"
|
||||||
|
3. XAI_API_KEY → xAI with method "api"
|
||||||
|
4. None
|
||||||
|
|
||||||
Keep X selection limited to documented, verified search backends.
|
Use get_x_source_with_method() to also get the method string.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Configuration dict from get_config()
|
config: Configuration dict from get_config()
|
||||||
@@ -453,20 +595,58 @@ def get_x_source(config: Dict[str, Any]) -> Optional[str]:
|
|||||||
'xai' if XAI_API_KEY is configured,
|
'xai' if XAI_API_KEY is configured,
|
||||||
None if no X source available.
|
None if no X source available.
|
||||||
"""
|
"""
|
||||||
# Import here to avoid circular dependency
|
source, _method = get_x_source_with_method(config)
|
||||||
|
return source
|
||||||
|
|
||||||
|
|
||||||
|
def get_x_source_with_method(config: Dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
|
||||||
|
"""Determine the best available X/Twitter source and auth method.
|
||||||
|
|
||||||
|
Priority chain:
|
||||||
|
1. AUTH_TOKEN/CT0 (env var or .env) → Bird with method "env"
|
||||||
|
2. AUTH_TOKEN/CT0 (browser cookies) → Bird with method "browser-{browser}"
|
||||||
|
3. XAI_API_KEY → xAI with method "api"
|
||||||
|
4. None
|
||||||
|
|
||||||
|
Args:
|
||||||
|
config: Configuration dict from get_config()
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (source, method) where source is 'bird', 'xai', or None
|
||||||
|
and method is 'env', 'browser-chrome', 'browser-firefox', 'browser-safari', 'api', or None.
|
||||||
|
"""
|
||||||
from . import bird_x
|
from . import bird_x
|
||||||
|
|
||||||
# Check Bird first (free option)
|
setup_complete = config.get('SETUP_COMPLETE')
|
||||||
|
|
||||||
|
# Check Bird first (free option — uses AUTH_TOKEN/CT0 from any source)
|
||||||
if bird_x.is_bird_installed():
|
if bird_x.is_bird_installed():
|
||||||
username = bird_x.is_bird_authenticated()
|
auth_source = config.get('_AUTH_TOKEN_SOURCE')
|
||||||
if username:
|
|
||||||
return 'bird'
|
# If SETUP_COMPLETE is not set, only allow explicit env var credentials.
|
||||||
|
# Do NOT call is_bird_authenticated() for browser-cookie probing —
|
||||||
|
# that requires user consent via the setup wizard.
|
||||||
|
if not setup_complete:
|
||||||
|
# Explicit AUTH_TOKEN from env var / .env file is always allowed
|
||||||
|
if auth_source == 'env' and config.get('AUTH_TOKEN'):
|
||||||
|
username = bird_x.is_bird_authenticated()
|
||||||
|
if username:
|
||||||
|
return 'bird', 'env'
|
||||||
|
else:
|
||||||
|
# SETUP_COMPLETE is set — normal flow, probe cookies if needed
|
||||||
|
username = bird_x.is_bird_authenticated()
|
||||||
|
if username:
|
||||||
|
if auth_source and auth_source.startswith('browser-'):
|
||||||
|
method = auth_source # e.g. "browser-firefox"
|
||||||
|
else:
|
||||||
|
method = 'env'
|
||||||
|
return 'bird', method
|
||||||
|
|
||||||
# Fall back to xAI if key exists
|
# Fall back to xAI if key exists
|
||||||
if config.get('XAI_API_KEY'):
|
if config.get('XAI_API_KEY'):
|
||||||
return 'xai'
|
return 'xai', 'api'
|
||||||
|
|
||||||
return None
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
def is_ytdlp_available() -> bool:
|
def is_ytdlp_available() -> bool:
|
||||||
@@ -576,24 +756,49 @@ def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
|
|||||||
"""Get detailed X source status for UI decisions.
|
"""Get detailed X source status for UI decisions.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict with keys: source, bird_installed, bird_authenticated,
|
Dict with keys: source, method, bird_installed, bird_authenticated,
|
||||||
bird_username, xai_available, can_install_bird
|
bird_username, xai_available, can_install_bird
|
||||||
|
|
||||||
|
The ``method`` field indicates HOW the active source is authenticated:
|
||||||
|
- "env" — AUTH_TOKEN came from an env var or .env file
|
||||||
|
- "browser-chrome", "browser-firefox", "browser-safari" — from cookie extraction
|
||||||
|
- "api" — using xAI API key
|
||||||
|
- None — no X source available
|
||||||
"""
|
"""
|
||||||
from . import bird_x
|
from . import bird_x
|
||||||
|
|
||||||
bird_status = bird_x.get_bird_status()
|
setup_complete = config.get('SETUP_COMPLETE')
|
||||||
xai_available = bool(config.get('XAI_API_KEY'))
|
xai_available = bool(config.get('XAI_API_KEY'))
|
||||||
|
|
||||||
# Determine active source
|
if not setup_complete:
|
||||||
if bird_status["authenticated"]:
|
# Before consent: do NOT call get_bird_status() which probes cookies.
|
||||||
source = 'bird'
|
# Only check if Bird is installed (no cookie probing) and use the
|
||||||
elif xai_available:
|
# gated get_x_source_with_method() which blocks cookie detection.
|
||||||
source = 'xai'
|
bird_installed = bird_x.is_bird_installed()
|
||||||
else:
|
source, method = get_x_source_with_method(config)
|
||||||
source = None
|
|
||||||
|
# Bird "authenticated" only if get_x_source_with_method found explicit creds
|
||||||
|
bird_authenticated = (source == 'bird')
|
||||||
|
|
||||||
|
return {
|
||||||
|
"source": source,
|
||||||
|
"method": method,
|
||||||
|
"bird_installed": bird_installed,
|
||||||
|
"bird_authenticated": bird_authenticated,
|
||||||
|
"bird_username": None if not bird_authenticated else "env AUTH_TOKEN",
|
||||||
|
"xai_available": xai_available,
|
||||||
|
"can_install_bird": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
# SETUP_COMPLETE is set — normal flow
|
||||||
|
bird_status = bird_x.get_bird_status()
|
||||||
|
|
||||||
|
# Use the unified resolution function for source + method
|
||||||
|
source, method = get_x_source_with_method(config)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"source": source,
|
"source": source,
|
||||||
|
"method": method,
|
||||||
"bird_installed": bird_status["installed"],
|
"bird_installed": bird_status["installed"],
|
||||||
"bird_authenticated": bird_status["authenticated"],
|
"bird_authenticated": bird_status["authenticated"],
|
||||||
"bird_username": bird_status["username"],
|
"bird_username": bird_status["username"],
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
"""Exa AI web search for last30days skill.
|
||||||
|
|
||||||
|
Uses the Exa Search API as a free web search backend.
|
||||||
|
Free tier: 1,000 searches/month, semantic search, no credit card required.
|
||||||
|
|
||||||
|
API docs: https://docs.exa.ai/reference/search
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from . import http
|
||||||
|
|
||||||
|
ENDPOINT = "https://api.exa.ai/search"
|
||||||
|
|
||||||
|
# Domains to exclude (handled by Reddit/X search)
|
||||||
|
EXCLUDED_DOMAINS = {
|
||||||
|
"reddit.com", "www.reddit.com", "old.reddit.com",
|
||||||
|
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def search_web(
|
||||||
|
topic: str,
|
||||||
|
from_date: str,
|
||||||
|
to_date: str,
|
||||||
|
api_key: str,
|
||||||
|
depth: str = "default",
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Search the web via Exa AI Search API.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
topic: Search topic
|
||||||
|
from_date: Start date (YYYY-MM-DD)
|
||||||
|
to_date: End date (YYYY-MM-DD)
|
||||||
|
api_key: Exa API key
|
||||||
|
depth: 'quick', 'default', or 'deep'
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
|
||||||
|
"""
|
||||||
|
num_results = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
|
||||||
|
max_chars = {"quick": 1000, "default": 2000, "deep": 3000}.get(depth, 2000)
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"query": f"{topic} (from {from_date} to {to_date})",
|
||||||
|
"type": "auto",
|
||||||
|
"numResults": num_results,
|
||||||
|
"contents": {"text": {"maxCharacters": max_chars}},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Add date filtering if dates are provided
|
||||||
|
if from_date:
|
||||||
|
payload["startPublishedDate"] = f"{from_date}T00:00:00.000Z"
|
||||||
|
if to_date:
|
||||||
|
payload["endPublishedDate"] = f"{to_date}T23:59:59.999Z"
|
||||||
|
|
||||||
|
sys.stderr.write(f"[Web] Searching Exa for: {topic}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = http.post(
|
||||||
|
ENDPOINT,
|
||||||
|
json_data=payload,
|
||||||
|
headers={
|
||||||
|
"x-api-key": api_key,
|
||||||
|
},
|
||||||
|
timeout=20,
|
||||||
|
retries=2,
|
||||||
|
)
|
||||||
|
except http.HTTPError as e:
|
||||||
|
status = e.status_code
|
||||||
|
if status == 401:
|
||||||
|
sys.stderr.write("[Web] Exa: invalid API key (401)\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
return []
|
||||||
|
if status == 429:
|
||||||
|
sys.stderr.write("[Web] Exa: rate limited (429)\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
return []
|
||||||
|
sys.stderr.write(f"[Web] Exa: HTTP error {status}: {e}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
sys.stderr.write(f"[Web] Exa: request failed: {e}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
return []
|
||||||
|
|
||||||
|
return _normalize_results(response)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||||
|
"""Convert Exa API response to websearch item schema.
|
||||||
|
|
||||||
|
Exa results have: title, url, text, publishedDate, score, author.
|
||||||
|
"""
|
||||||
|
items = []
|
||||||
|
|
||||||
|
results = response.get("results", [])
|
||||||
|
if not isinstance(results, list):
|
||||||
|
return items
|
||||||
|
|
||||||
|
for i, result in enumerate(results):
|
||||||
|
if not isinstance(result, dict):
|
||||||
|
continue
|
||||||
|
|
||||||
|
url = result.get("url", "")
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Skip excluded domains
|
||||||
|
try:
|
||||||
|
domain = urlparse(url).netloc.lower()
|
||||||
|
if domain in EXCLUDED_DOMAINS:
|
||||||
|
continue
|
||||||
|
if domain.startswith("www."):
|
||||||
|
domain = domain[4:]
|
||||||
|
except Exception:
|
||||||
|
domain = ""
|
||||||
|
|
||||||
|
title = str(result.get("title", "")).strip()
|
||||||
|
# Exa returns page content in "text" field
|
||||||
|
snippet = str(result.get("text", "")).strip()
|
||||||
|
|
||||||
|
if not title and not snippet:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Parse publishedDate (ISO format from Exa: "2026-03-15T00:00:00.000Z")
|
||||||
|
date = _parse_exa_date(result.get("publishedDate"))
|
||||||
|
date_confidence = "med" if date else "low"
|
||||||
|
|
||||||
|
# Exa provides a relevance score
|
||||||
|
relevance = result.get("score", 0.6)
|
||||||
|
try:
|
||||||
|
relevance = min(1.0, max(0.0, float(relevance)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
relevance = 0.6
|
||||||
|
|
||||||
|
items.append({
|
||||||
|
"id": f"W{i+1}",
|
||||||
|
"title": title[:200],
|
||||||
|
"url": url,
|
||||||
|
"source_domain": domain,
|
||||||
|
"snippet": snippet[:500],
|
||||||
|
"date": date,
|
||||||
|
"date_confidence": date_confidence,
|
||||||
|
"relevance": relevance,
|
||||||
|
"why_relevant": "",
|
||||||
|
})
|
||||||
|
|
||||||
|
sys.stderr.write(f"[Web] Exa: {len(items)} results\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_exa_date(published_date: Optional[str]) -> Optional[str]:
|
||||||
|
"""Parse Exa's publishedDate to YYYY-MM-DD.
|
||||||
|
|
||||||
|
Exa returns ISO format like "2026-03-15T00:00:00.000Z".
|
||||||
|
"""
|
||||||
|
if not published_date:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Extract YYYY-MM-DD from ISO datetime
|
||||||
|
if "T" in published_date:
|
||||||
|
return published_date.split("T")[0]
|
||||||
|
# Already YYYY-MM-DD
|
||||||
|
if len(published_date) >= 10:
|
||||||
|
return published_date[:10]
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return None
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
Uses ScrapeCreators REST API to search Instagram Reels by keyword, extract
|
Uses ScrapeCreators REST API to search Instagram Reels by keyword, extract
|
||||||
engagement metrics (views, likes, comments), and fetch video transcripts.
|
engagement metrics (views, likes, comments), and fetch video transcripts.
|
||||||
|
|
||||||
Requires SCRAPECREATORS_API_KEY in config. 100 free credits, then PAYG.
|
Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
|
||||||
API docs: https://scrapecreators.com/docs
|
API docs: https://scrapecreators.com/docs
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -193,6 +193,7 @@ def normalize_youtube_items(
|
|||||||
date_confidence="high",
|
date_confidence="high",
|
||||||
engagement=engagement,
|
engagement=engagement,
|
||||||
transcript_snippet=item.get("transcript_snippet", ""),
|
transcript_snippet=item.get("transcript_snippet", ""),
|
||||||
|
transcript_highlights=item.get("transcript_highlights", []),
|
||||||
relevance=item.get("relevance", 0.7),
|
relevance=item.get("relevance", 0.7),
|
||||||
why_relevant=item.get("why_relevant", ""),
|
why_relevant=item.get("why_relevant", ""),
|
||||||
))
|
))
|
||||||
|
|||||||
@@ -0,0 +1,201 @@
|
|||||||
|
"""Post-research quality score and upgrade nudge.
|
||||||
|
|
||||||
|
Computes a quality score based on 5 core sources and builds
|
||||||
|
a nudge message describing what the user missed and how to fix it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
# The 5 core sources
|
||||||
|
CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit_comments"]
|
||||||
|
|
||||||
|
# Labels for display
|
||||||
|
SOURCE_LABELS = {
|
||||||
|
"hn": "Hacker News",
|
||||||
|
"polymarket": "Polymarket",
|
||||||
|
"x": "X/Twitter",
|
||||||
|
"youtube": "YouTube",
|
||||||
|
"reddit_comments": "Reddit with comments",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_x_active(config: dict, research_results: dict) -> bool:
|
||||||
|
"""Check if X source is active (has credentials AND didn't error)."""
|
||||||
|
has_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||||
|
if not has_creds:
|
||||||
|
return False
|
||||||
|
# If X errored this run, it's configured but broken
|
||||||
|
if research_results.get("x_error"):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _is_youtube_active(config: dict, research_results: dict) -> bool:
|
||||||
|
"""Check if YouTube source is active (yt-dlp installed)."""
|
||||||
|
try:
|
||||||
|
from . import youtube_yt
|
||||||
|
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||||
|
except Exception:
|
||||||
|
has_ytdlp = False
|
||||||
|
if not has_ytdlp:
|
||||||
|
return False
|
||||||
|
if research_results.get("youtube_error"):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _is_reddit_comments_active(config: dict, research_results: dict) -> bool:
|
||||||
|
"""Check if Reddit with comments is active (ScrapeCreators)."""
|
||||||
|
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||||
|
if not has_sc:
|
||||||
|
return False
|
||||||
|
if research_results.get("reddit_error"):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def compute_quality_score(config: dict, research_results: dict) -> dict:
|
||||||
|
"""Compute research quality score based on 5 core sources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
config: Configuration dict from env.get_config()
|
||||||
|
research_results: Dict with keys like x_error, youtube_error,
|
||||||
|
reddit_error reflecting what happened this run.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
"score_pct": 40-100,
|
||||||
|
"core_active": ["hn", "polymarket", ...],
|
||||||
|
"core_missing": ["x", "youtube", "reddit_comments"],
|
||||||
|
"core_errored": ["reddit_comments"], # configured but errored
|
||||||
|
"nudge_text": "..." or None if 100%
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
core_active: List[str] = []
|
||||||
|
core_missing: List[str] = []
|
||||||
|
core_errored: List[str] = []
|
||||||
|
|
||||||
|
# HN and Polymarket are always active
|
||||||
|
core_active.append("hn")
|
||||||
|
core_active.append("polymarket")
|
||||||
|
|
||||||
|
# X
|
||||||
|
has_x_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
|
||||||
|
if _is_x_active(config, research_results):
|
||||||
|
core_active.append("x")
|
||||||
|
else:
|
||||||
|
core_missing.append("x")
|
||||||
|
if has_x_creds and research_results.get("x_error"):
|
||||||
|
core_errored.append("x")
|
||||||
|
|
||||||
|
# YouTube
|
||||||
|
try:
|
||||||
|
from . import youtube_yt
|
||||||
|
has_ytdlp = youtube_yt.is_ytdlp_installed()
|
||||||
|
except Exception:
|
||||||
|
has_ytdlp = False
|
||||||
|
if _is_youtube_active(config, research_results):
|
||||||
|
core_active.append("youtube")
|
||||||
|
else:
|
||||||
|
core_missing.append("youtube")
|
||||||
|
if has_ytdlp and research_results.get("youtube_error"):
|
||||||
|
core_errored.append("youtube")
|
||||||
|
|
||||||
|
# Reddit with comments (ScrapeCreators)
|
||||||
|
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
|
||||||
|
if _is_reddit_comments_active(config, research_results):
|
||||||
|
core_active.append("reddit_comments")
|
||||||
|
else:
|
||||||
|
core_missing.append("reddit_comments")
|
||||||
|
if has_sc and research_results.get("reddit_error"):
|
||||||
|
core_errored.append("reddit_comments")
|
||||||
|
|
||||||
|
score_pct = int(len(core_active) / 5 * 100)
|
||||||
|
|
||||||
|
nudge_text = _build_nudge_text(core_missing, core_errored) if core_missing else None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"score_pct": score_pct,
|
||||||
|
"core_active": core_active,
|
||||||
|
"core_missing": core_missing,
|
||||||
|
"core_errored": core_errored,
|
||||||
|
"nudge_text": nudge_text,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_nudge_text(core_missing: List[str], core_errored: List[str]) -> str:
|
||||||
|
"""Build human-readable nudge text describing what was missed.
|
||||||
|
|
||||||
|
Prioritizes free suggestions before paid ones.
|
||||||
|
"""
|
||||||
|
lines: List[str] = []
|
||||||
|
|
||||||
|
# Describe what was missed
|
||||||
|
missed_parts: List[str] = []
|
||||||
|
for src in core_missing:
|
||||||
|
label = SOURCE_LABELS[src]
|
||||||
|
if src in core_errored:
|
||||||
|
missed_parts.append(f"{label} (errored this run)")
|
||||||
|
else:
|
||||||
|
missed_parts.append(label)
|
||||||
|
|
||||||
|
active_count = 5 - len(core_missing)
|
||||||
|
lines.append(f"Research quality: {active_count}/5 core sources.")
|
||||||
|
lines.append(f"Missing: {', '.join(missed_parts)}.")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# Free suggestions first
|
||||||
|
free_suggestions: List[str] = []
|
||||||
|
paid_suggestions: List[str] = []
|
||||||
|
|
||||||
|
if "x" in core_missing:
|
||||||
|
if "x" in core_errored:
|
||||||
|
free_suggestions.append(
|
||||||
|
"X errored — try refreshing your browser cookies "
|
||||||
|
"(log into x.com, then re-run)."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
free_suggestions.append(
|
||||||
|
"X/Twitter: scan browser cookies automatically — "
|
||||||
|
"just log into x.com in any browser and re-run."
|
||||||
|
)
|
||||||
|
|
||||||
|
if "youtube" in core_missing:
|
||||||
|
if "youtube" in core_errored:
|
||||||
|
free_suggestions.append(
|
||||||
|
"YouTube errored — check that yt-dlp is up to date: "
|
||||||
|
"brew upgrade yt-dlp"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
free_suggestions.append(
|
||||||
|
"YouTube: install yt-dlp — brew install yt-dlp"
|
||||||
|
)
|
||||||
|
|
||||||
|
if "reddit_comments" in core_missing:
|
||||||
|
if "reddit_comments" in core_errored:
|
||||||
|
paid_suggestions.append(
|
||||||
|
"Reddit comments errored — check your ScrapeCreators API key "
|
||||||
|
"at scrapecreators.com."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
paid_suggestions.append(
|
||||||
|
"Reddit with comments: add SCRAPECREATORS_API_KEY — "
|
||||||
|
"100 free API calls, no credit card — scrapecreators.com"
|
||||||
|
)
|
||||||
|
|
||||||
|
if free_suggestions:
|
||||||
|
lines.append("Free fixes:")
|
||||||
|
for s in free_suggestions:
|
||||||
|
lines.append(f" - {s}")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
if paid_suggestions:
|
||||||
|
lines.append("Paid options:")
|
||||||
|
for s in paid_suggestions:
|
||||||
|
lines.append(f" - {s}")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
lines.append("last30days has no affiliation with any API provider.")
|
||||||
|
|
||||||
|
return "\n".join(lines)
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
"""Standalone Reddit public JSON search module.
|
||||||
|
|
||||||
|
Searches Reddit using the free public JSON endpoints (no API key required).
|
||||||
|
Promoted from last-resort fallback to robust primary free path.
|
||||||
|
|
||||||
|
Endpoints:
|
||||||
|
- Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit}
|
||||||
|
- Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month
|
||||||
|
|
||||||
|
Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
|
||||||
|
network timeouts, and missing subreddits.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
USER_AGENT = "last30days/3.0 (research tool)"
|
||||||
|
|
||||||
|
# Depth-aware limits for thread counts
|
||||||
|
DEPTH_LIMITS = {
|
||||||
|
"quick": 10,
|
||||||
|
"default": 25,
|
||||||
|
"deep": 50,
|
||||||
|
}
|
||||||
|
|
||||||
|
MAX_RETRIES = 3
|
||||||
|
BASE_BACKOFF = 2.0 # seconds
|
||||||
|
|
||||||
|
|
||||||
|
def _log(msg: str):
|
||||||
|
"""Log to stderr."""
|
||||||
|
sys.stderr.write(f"[RedditPublic] {msg}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _url_encode(text: str) -> str:
|
||||||
|
"""URL-encode a query string."""
|
||||||
|
return urllib.parse.quote_plus(text)
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Fetch JSON from a URL with retry on 429 and error handling.
|
||||||
|
|
||||||
|
Returns parsed JSON dict, or None on unrecoverable failure.
|
||||||
|
"""
|
||||||
|
headers = {
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
|
"Accept": "application/json",
|
||||||
|
}
|
||||||
|
req = urllib.request.Request(url, headers=headers)
|
||||||
|
|
||||||
|
for attempt in range(MAX_RETRIES):
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
content_type = resp.headers.get("Content-Type", "")
|
||||||
|
if "json" not in content_type and "text/html" in content_type:
|
||||||
|
_log(f"Anti-bot HTML response (Content-Type: {content_type})")
|
||||||
|
return None
|
||||||
|
|
||||||
|
body = resp.read().decode("utf-8")
|
||||||
|
return json.loads(body)
|
||||||
|
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code == 429:
|
||||||
|
delay = BASE_BACKOFF * (2 ** attempt)
|
||||||
|
retry_after = None
|
||||||
|
if hasattr(e, "headers"):
|
||||||
|
retry_after = e.headers.get("Retry-After")
|
||||||
|
if retry_after:
|
||||||
|
try:
|
||||||
|
delay = float(retry_after)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
_log(f"429 rate limited, retry {attempt + 1}/{MAX_RETRIES} after {delay:.1f}s")
|
||||||
|
if attempt < MAX_RETRIES - 1:
|
||||||
|
time.sleep(delay)
|
||||||
|
continue
|
||||||
|
# Last attempt exhausted
|
||||||
|
_log("429 retries exhausted")
|
||||||
|
return None
|
||||||
|
elif e.code == 404:
|
||||||
|
_log(f"404 not found: {url}")
|
||||||
|
return None
|
||||||
|
elif e.code == 403:
|
||||||
|
_log(f"403 forbidden: {url}")
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
_log(f"HTTP {e.code}: {e.reason}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except (urllib.error.URLError, OSError, TimeoutError) as e:
|
||||||
|
_log(f"Network error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
_log(f"JSON decode error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_posts(data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||||
|
"""Parse Reddit listing JSON into normalized post dicts."""
|
||||||
|
if not data:
|
||||||
|
return []
|
||||||
|
|
||||||
|
children = data.get("data", {}).get("children", [])
|
||||||
|
posts = []
|
||||||
|
|
||||||
|
for child in children:
|
||||||
|
if child.get("kind") != "t3":
|
||||||
|
continue
|
||||||
|
post = child.get("data", {})
|
||||||
|
permalink = str(post.get("permalink", "")).strip()
|
||||||
|
if not permalink or "/comments/" not in permalink:
|
||||||
|
continue
|
||||||
|
|
||||||
|
score = int(post.get("score", 0) or 0)
|
||||||
|
num_comments = int(post.get("num_comments", 0) or 0)
|
||||||
|
selftext = str(post.get("selftext", ""))
|
||||||
|
author = str(post.get("author", "[deleted]"))
|
||||||
|
created_utc = post.get("created_utc")
|
||||||
|
|
||||||
|
# Parse date
|
||||||
|
date_str = None
|
||||||
|
if created_utc:
|
||||||
|
try:
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
dt = datetime.fromtimestamp(float(created_utc), tz=timezone.utc)
|
||||||
|
date_str = dt.strftime("%Y-%m-%d")
|
||||||
|
except (ValueError, TypeError, OSError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
posts.append({
|
||||||
|
"id": "", # Will be assigned after dedup
|
||||||
|
"title": str(post.get("title", "")).strip(),
|
||||||
|
"url": f"https://www.reddit.com{permalink}",
|
||||||
|
"score": score,
|
||||||
|
"num_comments": num_comments,
|
||||||
|
"subreddit": str(post.get("subreddit", "")).strip(),
|
||||||
|
"created_utc": float(created_utc) if created_utc else None,
|
||||||
|
"author": author if author not in ("[deleted]", "[removed]") else "[deleted]",
|
||||||
|
"selftext": selftext[:500] if selftext else "",
|
||||||
|
# Normalized fields matching ScrapeCreators output
|
||||||
|
"date": date_str,
|
||||||
|
"engagement": {
|
||||||
|
"score": score,
|
||||||
|
"num_comments": num_comments,
|
||||||
|
"upvote_ratio": post.get("upvote_ratio"),
|
||||||
|
},
|
||||||
|
"relevance": _compute_relevance(score, num_comments),
|
||||||
|
"why_relevant": "Reddit public search",
|
||||||
|
})
|
||||||
|
|
||||||
|
return posts
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_relevance(score: int, num_comments: int) -> float:
|
||||||
|
"""Estimate relevance from engagement signals."""
|
||||||
|
score_component = min(1.0, max(0.0, score / 500.0))
|
||||||
|
comments_component = min(1.0, max(0.0, num_comments / 200.0))
|
||||||
|
return round((score_component * 0.6) + (comments_component * 0.4), 3)
|
||||||
|
|
||||||
|
|
||||||
|
def search(
|
||||||
|
query: str,
|
||||||
|
depth: str = "default",
|
||||||
|
subreddit: Optional[str] = None,
|
||||||
|
timeout: int = 15,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Search Reddit via the public JSON endpoint.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query string
|
||||||
|
depth: 'quick', 'default', or 'deep' — controls result limit
|
||||||
|
subreddit: Optional subreddit name (without r/) for scoped search
|
||||||
|
timeout: HTTP timeout in seconds
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of normalized post dicts. Empty list on any failure.
|
||||||
|
"""
|
||||||
|
limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
|
||||||
|
encoded_query = _url_encode(query)
|
||||||
|
|
||||||
|
if subreddit:
|
||||||
|
sub = subreddit.lstrip("r/").strip()
|
||||||
|
url = (
|
||||||
|
f"https://www.reddit.com/r/{sub}/search.json"
|
||||||
|
f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
url = (
|
||||||
|
f"https://www.reddit.com/search.json"
|
||||||
|
f"?q={encoded_query}&sort=relevance&t=month&limit={limit}&raw_json=1"
|
||||||
|
)
|
||||||
|
|
||||||
|
data = _fetch_json(url, timeout=timeout)
|
||||||
|
posts = _parse_posts(data)
|
||||||
|
|
||||||
|
# Dedupe by URL and assign IDs
|
||||||
|
seen_urls = set()
|
||||||
|
unique = []
|
||||||
|
for post in posts:
|
||||||
|
if post["url"] not in seen_urls:
|
||||||
|
seen_urls.add(post["url"])
|
||||||
|
unique.append(post)
|
||||||
|
|
||||||
|
for i, post in enumerate(unique):
|
||||||
|
post["id"] = f"R{i + 1}"
|
||||||
|
|
||||||
|
return unique[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def search_reddit_public(
|
||||||
|
topic: str,
|
||||||
|
from_date: str,
|
||||||
|
to_date: str,
|
||||||
|
depth: str = "default",
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""High-level Reddit public search matching the openai_reddit interface.
|
||||||
|
|
||||||
|
Runs global search, deduplicates, filters by date range, and sorts
|
||||||
|
by engagement. Compatible as a drop-in replacement in the fallback chain.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
topic: Search topic
|
||||||
|
from_date: Start date (YYYY-MM-DD)
|
||||||
|
to_date: End date (YYYY-MM-DD)
|
||||||
|
depth: 'quick', 'default', or 'deep'
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of normalized item dicts matching ScrapeCreators output format.
|
||||||
|
"""
|
||||||
|
results = search(topic, depth=depth)
|
||||||
|
|
||||||
|
# Date filter: keep posts in range or with unknown dates
|
||||||
|
filtered = []
|
||||||
|
for item in results:
|
||||||
|
d = item.get("date")
|
||||||
|
if d is None or (from_date <= d <= to_date):
|
||||||
|
filtered.append(item)
|
||||||
|
|
||||||
|
# Sort by engagement (score desc)
|
||||||
|
filtered.sort(
|
||||||
|
key=lambda x: x.get("engagement", {}).get("score", 0),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Re-index IDs
|
||||||
|
for i, item in enumerate(filtered):
|
||||||
|
item["id"] = f"R{i + 1}"
|
||||||
|
|
||||||
|
return filtered
|
||||||
+32
-6
@@ -115,7 +115,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
|||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append("---")
|
lines.append("---")
|
||||||
lines.append("**⚡ Want better results?** Add API keys to unlock Reddit, TikTok, Instagram & X data:")
|
lines.append("**⚡ Want better results?** Add API keys to unlock Reddit, TikTok, Instagram & X data:")
|
||||||
lines.append("- `SCRAPECREATORS_API_KEY` → Reddit + TikTok + Instagram (one key, all three!) — real upvotes, comments, views")
|
lines.append("- `SCRAPECREATORS_API_KEY` → Reddit + TikTok + Instagram (one key, all three!) — 100 free calls, no CC — scrapecreators.com (no affiliation)")
|
||||||
lines.append("- `XAI_API_KEY` → X posts with real likes & reposts")
|
lines.append("- `XAI_API_KEY` → X posts with real likes & reposts")
|
||||||
lines.append("- `OPENAI_API_KEY` (legacy) → Reddit threads (slower, higher cost)")
|
lines.append("- `OPENAI_API_KEY` (legacy) → Reddit threads (slower, higher cost)")
|
||||||
lines.append("- Edit `~/.config/last30days/.env` to add keys")
|
lines.append("- Edit `~/.config/last30days/.env` to add keys")
|
||||||
@@ -143,7 +143,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
|||||||
lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*")
|
lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
elif report.mode == "x-only" and missing_keys in ("reddit", "none"):
|
elif report.mode == "x-only" and missing_keys in ("reddit", "none"):
|
||||||
lines.append("*💡 Tip: Add `SCRAPECREATORS_API_KEY` for Reddit + TikTok + Instagram data (one key, all three) and better triangulation.*")
|
lines.append("*💡 Tip: Add `SCRAPECREATORS_API_KEY` for Reddit + TikTok + Instagram data (one key, all three) — 100 free calls, no CC — scrapecreators.com (no affiliation)*")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# Reddit items
|
# Reddit items
|
||||||
@@ -257,11 +257,15 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
|||||||
lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}{_xref_tag(item)}")
|
lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}{_xref_tag(item)}")
|
||||||
lines.append(f" {item.title}")
|
lines.append(f" {item.title}")
|
||||||
lines.append(f" {item.url}")
|
lines.append(f" {item.url}")
|
||||||
|
if item.transcript_highlights:
|
||||||
|
lines.append(" Highlights:")
|
||||||
|
for hl in item.transcript_highlights[:5]:
|
||||||
|
lines.append(f' - "{hl}"')
|
||||||
if item.transcript_snippet:
|
if item.transcript_snippet:
|
||||||
snippet = item.transcript_snippet[:200]
|
word_count = len(item.transcript_snippet.split())
|
||||||
if len(item.transcript_snippet) > 200:
|
lines.append(f" <details><summary>Full transcript ({word_count} words)</summary>")
|
||||||
snippet += "..."
|
lines.append(f" {item.transcript_snippet}")
|
||||||
lines.append(f" Transcript: {snippet}")
|
lines.append(" </details>")
|
||||||
lines.append(f" *{item.why_relevant}*")
|
lines.append(f" *{item.why_relevant}*")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
@@ -521,6 +525,28 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def render_quality_nudge(quality: dict) -> str:
|
||||||
|
"""Render the quality score nudge block.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
quality: Dict from quality_nudge.compute_quality_score()
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Markdown string with quality nudge, or empty string if no nudge.
|
||||||
|
"""
|
||||||
|
nudge_text = quality.get("nudge_text")
|
||||||
|
if not nudge_text:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
lines = []
|
||||||
|
lines.append("---")
|
||||||
|
lines.append(f"**🔍 Research Coverage: {quality['score_pct']}%**")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(nudge_text)
|
||||||
|
lines.append("")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
def render_source_status(report: schema.Report, source_info: dict = None) -> str:
|
def render_source_status(report: schema.Report, source_info: dict = None) -> str:
|
||||||
"""Render source status footer showing what was used/skipped and why.
|
"""Render source status footer showing what was used/skipped and why.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
"""
|
||||||
|
Safari binary cookie extractor for macOS.
|
||||||
|
|
||||||
|
Parses ~/Library/Cookies/Cookies.binarycookies (unencrypted binary format)
|
||||||
|
using only stdlib. Zero pip dependencies.
|
||||||
|
|
||||||
|
Reference: github.com/mdegrazia/Safari-Binary-Cookie-Parser
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Mac epoch: 2001-01-01 00:00:00 UTC (not used for filtering, but documented)
|
||||||
|
_MAC_EPOCH_OFFSET = 978307200 # seconds between Unix epoch and Mac epoch
|
||||||
|
|
||||||
|
_MAGIC = b"cook"
|
||||||
|
|
||||||
|
|
||||||
|
def _read_null_terminated(data: bytes, offset: int) -> str:
|
||||||
|
"""Read a null-terminated string from data starting at offset."""
|
||||||
|
end = data.find(b"\x00", offset)
|
||||||
|
if end == -1:
|
||||||
|
end = len(data)
|
||||||
|
return data[offset:end].decode("utf-8", errors="replace")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_cookie_record(data: bytes) -> dict | None:
|
||||||
|
"""Parse a single cookie record. Returns dict with url, name, value, path or None."""
|
||||||
|
if len(data) < 44:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
(size,) = struct.unpack("<I", data[0:4])
|
||||||
|
# flags at offset 4 (4 bytes, little-endian) — not needed for extraction
|
||||||
|
(url_offset,) = struct.unpack("<I", data[16:20])
|
||||||
|
(name_offset,) = struct.unpack("<I", data[20:24])
|
||||||
|
(path_offset,) = struct.unpack("<I", data[24:28])
|
||||||
|
(value_offset,) = struct.unpack("<I", data[28:32])
|
||||||
|
# expiry at offset 40 (8-byte double, little-endian) — not needed for filtering
|
||||||
|
# creation at offset 48 (8-byte double, little-endian) — not needed
|
||||||
|
|
||||||
|
url = _read_null_terminated(data, url_offset)
|
||||||
|
name = _read_null_terminated(data, name_offset)
|
||||||
|
path = _read_null_terminated(data, path_offset)
|
||||||
|
value = _read_null_terminated(data, value_offset)
|
||||||
|
|
||||||
|
return {"url": url, "name": name, "value": value, "path": path}
|
||||||
|
except (struct.error, IndexError, UnicodeDecodeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_page(page_data: bytes) -> list[dict]:
|
||||||
|
"""Parse a single page of cookies. Returns list of cookie dicts."""
|
||||||
|
cookies = []
|
||||||
|
if len(page_data) < 8:
|
||||||
|
return cookies
|
||||||
|
|
||||||
|
# Page header: 4 bytes (always 00 00 01 00), then 4-byte LE cookie count
|
||||||
|
try:
|
||||||
|
(num_cookies,) = struct.unpack("<I", page_data[4:8])
|
||||||
|
except struct.error:
|
||||||
|
return cookies
|
||||||
|
|
||||||
|
# Sanity check
|
||||||
|
if num_cookies > 10000:
|
||||||
|
return cookies
|
||||||
|
|
||||||
|
# Cookie offsets: array of 4-byte LE uint32 starting at offset 8
|
||||||
|
offsets_end = 8 + num_cookies * 4
|
||||||
|
if offsets_end > len(page_data):
|
||||||
|
return cookies
|
||||||
|
|
||||||
|
for i in range(num_cookies):
|
||||||
|
off_start = 8 + i * 4
|
||||||
|
try:
|
||||||
|
(cookie_offset,) = struct.unpack("<I", page_data[off_start : off_start + 4])
|
||||||
|
except struct.error:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if cookie_offset >= len(page_data):
|
||||||
|
continue
|
||||||
|
|
||||||
|
cookie_data = page_data[cookie_offset:]
|
||||||
|
record = _parse_cookie_record(cookie_data)
|
||||||
|
if record:
|
||||||
|
cookies.append(record)
|
||||||
|
|
||||||
|
return cookies
|
||||||
|
|
||||||
|
|
||||||
|
def extract_safari_cookies_macos(
|
||||||
|
domain: str, cookie_names: list[str]
|
||||||
|
) -> dict[str, str] | None:
|
||||||
|
"""
|
||||||
|
Extract cookies from Safari on macOS.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
domain: Domain to match (substring match, e.g. "x.com")
|
||||||
|
cookie_names: List of cookie names to extract (e.g. ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict mapping cookie name to value for found cookies, or None on failure.
|
||||||
|
"""
|
||||||
|
if sys.platform != "darwin":
|
||||||
|
return None
|
||||||
|
|
||||||
|
cookie_path = Path.home() / "Library" / "Cookies" / "Cookies.binarycookies"
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw = cookie_path.read_bytes()
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
except PermissionError:
|
||||||
|
print(
|
||||||
|
"[safari] Permission denied reading Cookies.binarycookies. "
|
||||||
|
"Enable Full Disk Access for Terminal in System Settings > "
|
||||||
|
"Privacy & Security > Full Disk Access.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return _parse_binary_cookies(raw, domain, cookie_names)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_binary_cookies(
|
||||||
|
raw: bytes, domain: str, cookie_names: list[str]
|
||||||
|
) -> dict[str, str] | None:
|
||||||
|
"""Parse raw binary cookie data. Separated for testability."""
|
||||||
|
if len(raw) < 8:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Validate magic
|
||||||
|
if raw[:4] != _MAGIC:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
(num_pages,) = struct.unpack(">I", raw[4:8])
|
||||||
|
except struct.error:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if num_pages > 100000:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Read page sizes (big-endian uint32 array)
|
||||||
|
page_sizes_end = 8 + num_pages * 4
|
||||||
|
if page_sizes_end > len(raw):
|
||||||
|
return None
|
||||||
|
|
||||||
|
page_sizes = []
|
||||||
|
for i in range(num_pages):
|
||||||
|
off = 8 + i * 4
|
||||||
|
try:
|
||||||
|
(ps,) = struct.unpack(">I", raw[off : off + 4])
|
||||||
|
page_sizes.append(ps)
|
||||||
|
except struct.error:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Parse each page
|
||||||
|
names_set = set(cookie_names)
|
||||||
|
result: dict[str, str] = {}
|
||||||
|
offset = page_sizes_end
|
||||||
|
|
||||||
|
for ps in page_sizes:
|
||||||
|
if offset + ps > len(raw):
|
||||||
|
break
|
||||||
|
page_data = raw[offset : offset + ps]
|
||||||
|
cookies = _parse_page(page_data)
|
||||||
|
for c in cookies:
|
||||||
|
# Substring match on domain (handles leading dots like ".x.com")
|
||||||
|
if domain in c["url"] and c["name"] in names_set:
|
||||||
|
result[c["name"]] = c["value"]
|
||||||
|
offset += ps
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return result
|
||||||
@@ -210,6 +210,7 @@ class YouTubeItem:
|
|||||||
date_confidence: str = "high" # YouTube dates are always reliable
|
date_confidence: str = "high" # YouTube dates are always reliable
|
||||||
engagement: Optional[Engagement] = None
|
engagement: Optional[Engagement] = None
|
||||||
transcript_snippet: str = ""
|
transcript_snippet: str = ""
|
||||||
|
transcript_highlights: List[str] = field(default_factory=list)
|
||||||
relevance: float = 0.7
|
relevance: float = 0.7
|
||||||
why_relevant: str = ""
|
why_relevant: str = ""
|
||||||
subs: SubScores = field(default_factory=SubScores)
|
subs: SubScores = field(default_factory=SubScores)
|
||||||
@@ -226,6 +227,7 @@ class YouTubeItem:
|
|||||||
'date_confidence': self.date_confidence,
|
'date_confidence': self.date_confidence,
|
||||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||||
'transcript_snippet': self.transcript_snippet,
|
'transcript_snippet': self.transcript_snippet,
|
||||||
|
'transcript_highlights': self.transcript_highlights,
|
||||||
'relevance': self.relevance,
|
'relevance': self.relevance,
|
||||||
'why_relevant': self.why_relevant,
|
'why_relevant': self.why_relevant,
|
||||||
'subs': self.subs.to_dict(),
|
'subs': self.subs.to_dict(),
|
||||||
@@ -655,6 +657,7 @@ class Report:
|
|||||||
date_confidence=y.get('date_confidence', 'high'),
|
date_confidence=y.get('date_confidence', 'high'),
|
||||||
engagement=eng,
|
engagement=eng,
|
||||||
transcript_snippet=y.get('transcript_snippet', ''),
|
transcript_snippet=y.get('transcript_snippet', ''),
|
||||||
|
transcript_highlights=y.get('transcript_highlights', []),
|
||||||
relevance=y.get('relevance', 0.7),
|
relevance=y.get('relevance', 0.7),
|
||||||
why_relevant=y.get('why_relevant', ''),
|
why_relevant=y.get('why_relevant', ''),
|
||||||
subs=subs,
|
subs=subs,
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
"""First-run setup wizard for last30days.
|
||||||
|
|
||||||
|
Detects first run, performs auto-setup (cookie extraction + yt-dlp check),
|
||||||
|
and writes configuration. The actual wizard UI is SKILL.md-driven (the LLM
|
||||||
|
presents it), but this module provides the detection and setup actions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def is_first_run(config: Dict[str, Any]) -> bool:
|
||||||
|
"""Return True if the setup wizard has not been completed.
|
||||||
|
|
||||||
|
Checks for SETUP_COMPLETE in the config dict. If it's not set
|
||||||
|
(None or empty string), the user hasn't gone through setup yet.
|
||||||
|
"""
|
||||||
|
return not config.get("SETUP_COMPLETE")
|
||||||
|
|
||||||
|
|
||||||
|
def run_auto_setup(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Perform the auto-setup actions.
|
||||||
|
|
||||||
|
- Runs cookie extraction in auto mode for all registered domains
|
||||||
|
- Checks if yt-dlp is installed
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with keys:
|
||||||
|
cookies_found: {source_name: browser_name} for each source where cookies were found
|
||||||
|
ytdlp_installed: bool
|
||||||
|
env_written: bool (always False here — caller writes config separately)
|
||||||
|
"""
|
||||||
|
from . import cookie_extract
|
||||||
|
from .env import COOKIE_DOMAINS
|
||||||
|
|
||||||
|
cookies_found: Dict[str, str] = {}
|
||||||
|
|
||||||
|
for source_name, spec in COOKIE_DOMAINS.items():
|
||||||
|
domain = spec["domain"]
|
||||||
|
cookie_names = spec["cookies"]
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = cookie_extract.extract_cookies_with_source("auto", domain, cookie_names)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Cookie extraction failed for %s: %s", source_name, exc)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if result is not None:
|
||||||
|
_cookies, browser_name = result
|
||||||
|
cookies_found[source_name] = browser_name
|
||||||
|
|
||||||
|
# Check yt-dlp availability and install via Homebrew if missing
|
||||||
|
ytdlp_action: str
|
||||||
|
if shutil.which("yt-dlp") is not None:
|
||||||
|
ytdlp_installed = True
|
||||||
|
ytdlp_action = "already_installed"
|
||||||
|
elif shutil.which("brew") is not None:
|
||||||
|
brew_stderr = ""
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
["brew", "install", "yt-dlp"],
|
||||||
|
capture_output=True, text=True, timeout=120,
|
||||||
|
)
|
||||||
|
if proc.returncode == 0:
|
||||||
|
ytdlp_installed = True
|
||||||
|
ytdlp_action = "installed"
|
||||||
|
else:
|
||||||
|
ytdlp_installed = False
|
||||||
|
ytdlp_action = "install_failed"
|
||||||
|
brew_stderr = proc.stderr
|
||||||
|
logger.warning("brew install yt-dlp failed: %s", proc.stderr)
|
||||||
|
except Exception as exc:
|
||||||
|
ytdlp_installed = False
|
||||||
|
ytdlp_action = "install_failed"
|
||||||
|
brew_stderr = str(exc)
|
||||||
|
logger.warning("brew install yt-dlp exception: %s", exc)
|
||||||
|
else:
|
||||||
|
ytdlp_installed = False
|
||||||
|
ytdlp_action = "no_homebrew"
|
||||||
|
|
||||||
|
results: Dict[str, Any] = {
|
||||||
|
"cookies_found": cookies_found,
|
||||||
|
"ytdlp_installed": ytdlp_installed,
|
||||||
|
"ytdlp_action": ytdlp_action,
|
||||||
|
"env_written": False,
|
||||||
|
}
|
||||||
|
if ytdlp_action == "install_failed":
|
||||||
|
results["ytdlp_stderr"] = brew_stderr
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def write_setup_config(env_path: Path, from_browser: str = "auto") -> bool:
|
||||||
|
"""Write SETUP_COMPLETE and FROM_BROWSER to the .env file.
|
||||||
|
|
||||||
|
Creates the file and parent directories if needed.
|
||||||
|
Appends to existing file without overwriting existing keys.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
env_path: Path to the .env file (e.g. ~/.config/last30days/.env)
|
||||||
|
from_browser: Browser extraction mode to write (default: "auto")
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if config was written successfully, False on error.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
env_path = Path(env_path)
|
||||||
|
env_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Read existing content to avoid overwriting keys
|
||||||
|
existing_keys: set = set()
|
||||||
|
existing_content = ""
|
||||||
|
if env_path.exists():
|
||||||
|
existing_content = env_path.read_text(encoding="utf-8")
|
||||||
|
for line in existing_content.splitlines():
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped and not stripped.startswith("#") and "=" in stripped:
|
||||||
|
key = stripped.split("=", 1)[0].strip()
|
||||||
|
existing_keys.add(key)
|
||||||
|
|
||||||
|
lines_to_add = []
|
||||||
|
if "SETUP_COMPLETE" not in existing_keys:
|
||||||
|
lines_to_add.append("SETUP_COMPLETE=true")
|
||||||
|
if "FROM_BROWSER" not in existing_keys:
|
||||||
|
lines_to_add.append(f"FROM_BROWSER={from_browser}")
|
||||||
|
|
||||||
|
if not lines_to_add:
|
||||||
|
return True # Nothing to write, already configured
|
||||||
|
|
||||||
|
# Ensure trailing newline before appending
|
||||||
|
with open(env_path, "a", encoding="utf-8") as f:
|
||||||
|
if existing_content and not existing_content.endswith("\n"):
|
||||||
|
f.write("\n")
|
||||||
|
f.write("\n".join(lines_to_add) + "\n")
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
except OSError as exc:
|
||||||
|
logger.error("Failed to write setup config to %s: %s", env_path, exc)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_setup_status_text(results: Dict[str, Any]) -> str:
|
||||||
|
"""Return a human-readable summary of auto-setup results.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
results: Dict from run_auto_setup()
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Multi-line status text.
|
||||||
|
"""
|
||||||
|
lines = []
|
||||||
|
lines.append("Setup complete! Here's what I found:")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
cookies_found = results.get("cookies_found", {})
|
||||||
|
if cookies_found:
|
||||||
|
for source, browser in cookies_found.items():
|
||||||
|
lines.append(f" - {source.upper()} cookies found in {browser}")
|
||||||
|
else:
|
||||||
|
lines.append(" - No browser cookies found for X/Twitter")
|
||||||
|
|
||||||
|
ytdlp_action = results.get("ytdlp_action", "")
|
||||||
|
if ytdlp_action == "installed":
|
||||||
|
lines.append(" - Installed yt-dlp via Homebrew")
|
||||||
|
elif ytdlp_action == "install_failed":
|
||||||
|
lines.append(" - yt-dlp install failed \u2014 run `brew install yt-dlp` manually")
|
||||||
|
elif ytdlp_action == "no_homebrew":
|
||||||
|
lines.append(" - yt-dlp not found. Install Homebrew first, then: brew install yt-dlp")
|
||||||
|
elif ytdlp_action == "already_installed":
|
||||||
|
lines.append(" - yt-dlp already installed")
|
||||||
|
elif results.get("ytdlp_installed", False):
|
||||||
|
lines.append(" - yt-dlp is installed (YouTube search ready)")
|
||||||
|
else:
|
||||||
|
lines.append(" - yt-dlp not found (install with: brew install yt-dlp)")
|
||||||
|
|
||||||
|
env_written = results.get("env_written", False)
|
||||||
|
if env_written:
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Configuration saved. Future runs will auto-detect your browsers.")
|
||||||
|
|
||||||
|
return "\n".join(lines)
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
Uses ScrapeCreators REST API to search TikTok by keyword, extract engagement
|
Uses ScrapeCreators REST API to search TikTok by keyword, extract engagement
|
||||||
metrics (views, likes, comments, shares), and fetch video transcripts.
|
metrics (views, likes, comments, shares), and fetch video transcripts.
|
||||||
|
|
||||||
Requires SCRAPECREATORS_API_KEY in config. 100 free credits, then PAYG.
|
Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
|
||||||
API docs: https://scrapecreators.com/docs
|
API docs: https://scrapecreators.com/docs
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
+177
-107
@@ -417,120 +417,190 @@ class ProgressDisplay:
|
|||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
def show_diagnostic_banner(diag: dict):
|
def _build_status_banner(diag: dict) -> list[str]:
|
||||||
"""Show pre-flight source status banner when sources are missing.
|
"""Build the status banner lines (plain text, no ANSI).
|
||||||
|
|
||||||
|
Returns a list of strings, each being a line of the banner box.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
diag: Dict from env diagnostics with keys:
|
diag: Dict with keys:
|
||||||
openai, xai, x_source, bird_installed, bird_authenticated,
|
setup_complete, reddit_source, x_source, x_method,
|
||||||
bird_username, youtube, web_search_backend
|
youtube, tiktok, instagram, hackernews, polymarket,
|
||||||
|
bluesky, truthsocial, xiaohongshu, scrapecreators,
|
||||||
|
web_search_backend
|
||||||
"""
|
"""
|
||||||
has_openai = diag.get("openai", False)
|
setup_complete = diag.get("setup_complete", False)
|
||||||
has_reddit_public = diag.get("reddit_public", False)
|
has_sc = diag.get("scrapecreators", False)
|
||||||
has_reddit = has_openai or has_reddit_public
|
|
||||||
has_x = diag.get("x_source") is not None
|
|
||||||
has_youtube = diag.get("youtube", False)
|
|
||||||
has_xiaohongshu = diag.get("xiaohongshu", False)
|
|
||||||
has_web = diag.get("web_search_backend") is not None
|
|
||||||
|
|
||||||
# If everything is available, no banner needed
|
# --- Build active sources list: (label, method_label) ---
|
||||||
if has_reddit and has_x and has_youtube and has_web:
|
active: list[str] = []
|
||||||
return
|
|
||||||
|
|
||||||
lines = []
|
# Reddit — always available; what matters to users is comments or not
|
||||||
|
reddit_src = diag.get("reddit_source")
|
||||||
|
if reddit_src == "scrapecreators":
|
||||||
|
active.append("Reddit (with comments)")
|
||||||
|
else:
|
||||||
|
active.append("Reddit (threads only)")
|
||||||
|
|
||||||
|
# X/Twitter
|
||||||
|
x_source = diag.get("x_source")
|
||||||
|
x_method = diag.get("x_method")
|
||||||
|
if x_source:
|
||||||
|
if x_method and x_method.startswith("browser-"):
|
||||||
|
browser = x_method.split("-", 1)[1].capitalize()
|
||||||
|
active.append(f"X ({browser})")
|
||||||
|
elif x_method == "env":
|
||||||
|
active.append("X (env)")
|
||||||
|
elif x_method == "api":
|
||||||
|
active.append("X (xAI)")
|
||||||
|
else:
|
||||||
|
active.append("X")
|
||||||
|
|
||||||
|
# YouTube
|
||||||
|
if diag.get("youtube"):
|
||||||
|
active.append("YouTube")
|
||||||
|
|
||||||
|
# HN — always available
|
||||||
|
if diag.get("hackernews"):
|
||||||
|
active.append("HN")
|
||||||
|
|
||||||
|
# Polymarket — always available
|
||||||
|
if diag.get("polymarket"):
|
||||||
|
active.append("Polymarket")
|
||||||
|
|
||||||
|
# TikTok (requires SC or Apify)
|
||||||
|
if diag.get("tiktok"):
|
||||||
|
active.append("TikTok")
|
||||||
|
|
||||||
|
# Instagram (requires SC)
|
||||||
|
if diag.get("instagram"):
|
||||||
|
active.append("Instagram")
|
||||||
|
|
||||||
|
# Bluesky
|
||||||
|
if diag.get("bluesky"):
|
||||||
|
active.append("Bluesky")
|
||||||
|
|
||||||
|
# Truth Social
|
||||||
|
if diag.get("truthsocial"):
|
||||||
|
active.append("Truth Social")
|
||||||
|
|
||||||
|
# Xiaohongshu
|
||||||
|
if diag.get("xiaohongshu"):
|
||||||
|
active.append("Xiaohongshu")
|
||||||
|
|
||||||
|
# --- Format active sources into wrapped lines ---
|
||||||
|
BOX_INNER = 53 # characters inside the box (between │ and │)
|
||||||
|
PREFIX = " " # 2-space indent inside box
|
||||||
|
|
||||||
|
def _wrap_sources(sources: list[str]) -> list[str]:
|
||||||
|
"""Wrap source labels into lines that fit the box width."""
|
||||||
|
result_lines: list[str] = []
|
||||||
|
current = PREFIX
|
||||||
|
for i, s in enumerate(sources):
|
||||||
|
token = f"✅ {s}"
|
||||||
|
sep = " " if current != PREFIX else ""
|
||||||
|
if len(current) + len(sep) + len(token) > BOX_INNER:
|
||||||
|
result_lines.append(current)
|
||||||
|
current = PREFIX + token
|
||||||
|
else:
|
||||||
|
current += sep + token
|
||||||
|
if current.strip():
|
||||||
|
result_lines.append(current)
|
||||||
|
return result_lines
|
||||||
|
|
||||||
|
source_lines = _wrap_sources(active)
|
||||||
|
|
||||||
|
# --- Title ---
|
||||||
|
if not setup_complete:
|
||||||
|
title = "/last30days v3.0 — First Run"
|
||||||
|
else:
|
||||||
|
title = "/last30days v3.0 — Source Status"
|
||||||
|
|
||||||
|
# --- Build upgrade suggestions ---
|
||||||
|
suggestions: list[str] = []
|
||||||
|
|
||||||
|
if not setup_complete:
|
||||||
|
suggestions.append("Run /last30days setup to unlock more sources")
|
||||||
|
else:
|
||||||
|
# Recommend ScrapeCreators if missing
|
||||||
|
if not has_sc:
|
||||||
|
suggestions.append("⭐ Add SCRAPECREATORS_API_KEY for Reddit comments")
|
||||||
|
suggestions.append(" + TikTok + Instagram")
|
||||||
|
suggestions.append(" 100 free calls, no CC — scrapecreators.com (no affiliation)")
|
||||||
|
|
||||||
|
# --- Assemble box lines ---
|
||||||
|
# Collect all content lines, then determine box width dynamically.
|
||||||
|
content: list[str] = []
|
||||||
|
content.append(f" {title}")
|
||||||
|
content.append("") # blank line
|
||||||
|
|
||||||
|
for sl in source_lines:
|
||||||
|
content.append(sl)
|
||||||
|
|
||||||
|
if suggestions:
|
||||||
|
content.append("") # blank line
|
||||||
|
for sg in suggestions:
|
||||||
|
content.append(f" {sg}")
|
||||||
|
|
||||||
|
content.append("") # blank line
|
||||||
|
content.append(" Config: ~/.config/last30days/.env")
|
||||||
|
|
||||||
|
# Width = widest content line + 1 for right margin
|
||||||
|
width = max(len(line) for line in content) + 1
|
||||||
|
if width < 53:
|
||||||
|
width = 53
|
||||||
|
|
||||||
|
lines: list[str] = []
|
||||||
|
lines.append("\u250c" + "\u2500" * width + "\u2510")
|
||||||
|
for c in content:
|
||||||
|
lines.append("\u2502" + c.ljust(width) + "\u2502")
|
||||||
|
lines.append("\u2514" + "\u2500" * width + "\u2518")
|
||||||
|
|
||||||
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
def _colorize_banner(lines: list[str]) -> list[str]:
|
||||||
|
"""Apply ANSI colors to plain-text banner lines for TTY output."""
|
||||||
|
colored: list[str] = []
|
||||||
|
for line in lines:
|
||||||
|
if line.startswith("\u250c") or line.startswith("\u2514"):
|
||||||
|
colored.append(f"{Colors.DIM}{line}{Colors.RESET}")
|
||||||
|
elif line.startswith("\u2502"):
|
||||||
|
inner = line[1:-1] # strip box chars on both sides
|
||||||
|
inner_width = len(inner)
|
||||||
|
# Colorize check marks green, star yellow
|
||||||
|
inner = inner.replace("\u2705", f"{Colors.GREEN}\u2705{Colors.RESET}")
|
||||||
|
inner = inner.replace("\u2b50", f"{Colors.YELLOW}\u2b50{Colors.RESET}")
|
||||||
|
# Bold the title line
|
||||||
|
if "/last30days v3.0" in inner:
|
||||||
|
stripped = inner.strip()
|
||||||
|
inner = f" {Colors.BOLD}{stripped}{Colors.RESET}"
|
||||||
|
# Re-pad to original width (ANSI codes are zero-width)
|
||||||
|
visible_len = 1 + len(stripped)
|
||||||
|
inner = inner + " " * max(0, inner_width - visible_len)
|
||||||
|
colored.append(f"{Colors.DIM}\u2502{Colors.RESET}{inner}{Colors.DIM}\u2502{Colors.RESET}")
|
||||||
|
else:
|
||||||
|
colored.append(line)
|
||||||
|
return colored
|
||||||
|
|
||||||
|
|
||||||
|
def show_diagnostic_banner(diag: dict):
|
||||||
|
"""Show pre-flight source status banner.
|
||||||
|
|
||||||
|
Free-first design: leads with what's working (✅), not what's broken.
|
||||||
|
Shows upgrade suggestions only when relevant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
diag: Dict with keys:
|
||||||
|
setup_complete, reddit_source, x_source, x_method,
|
||||||
|
youtube, tiktok, instagram, hackernews, polymarket,
|
||||||
|
bluesky, truthsocial, xiaohongshu, scrapecreators,
|
||||||
|
web_search_backend
|
||||||
|
"""
|
||||||
|
lines = _build_status_banner(diag)
|
||||||
|
|
||||||
if IS_TTY:
|
if IS_TTY:
|
||||||
lines.append(f"{Colors.DIM}┌─────────────────────────────────────────────────────┐{Colors.RESET}")
|
lines = _colorize_banner(lines)
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.BOLD}/last30days v2.1 — Source Status{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
|
||||||
|
|
||||||
# Reddit
|
|
||||||
if has_openai:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Reddit{Colors.RESET} — OpenAI/Codex auth found {Colors.DIM}│{Colors.RESET}")
|
|
||||||
elif has_reddit_public:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Reddit{Colors.RESET} — Public Reddit search (no key) {Colors.DIM}│{Colors.RESET}")
|
|
||||||
else:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ Reddit{Colors.RESET} — No OPENAI_API_KEY {Colors.DIM}│{Colors.RESET}")
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Add to ~/.config/last30days/.env {Colors.DIM}│{Colors.RESET}")
|
|
||||||
|
|
||||||
# X/Twitter
|
|
||||||
if has_x:
|
|
||||||
source = diag.get("x_source", "")
|
|
||||||
username = diag.get("bird_username", "")
|
|
||||||
label = f"Bird ({username})" if source == "bird" and username else source.upper()
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ X/Twitter{Colors.RESET} — {label} {Colors.DIM}│{Colors.RESET}")
|
|
||||||
else:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ X/Twitter{Colors.RESET} — No X auth or fallback key {Colors.DIM}│{Colors.RESET}")
|
|
||||||
if diag.get("bird_installed"):
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY {Colors.DIM}│{Colors.RESET}")
|
|
||||||
else:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Needs Node.js 22+ (Bird is bundled) {Colors.DIM}│{Colors.RESET}")
|
|
||||||
|
|
||||||
# YouTube
|
|
||||||
if has_youtube:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ YouTube{Colors.RESET} — yt-dlp found {Colors.DIM}│{Colors.RESET}")
|
|
||||||
else:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ YouTube{Colors.RESET} — yt-dlp not installed {Colors.DIM}│{Colors.RESET}")
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Fix: brew install yt-dlp (free) {Colors.DIM}│{Colors.RESET}")
|
|
||||||
|
|
||||||
# Xiaohongshu
|
|
||||||
if has_xiaohongshu:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Xiaohongshu{Colors.RESET} — API connected + logged in {Colors.DIM}│{Colors.RESET}")
|
|
||||||
else:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.YELLOW}⚡ Xiaohongshu{Colors.RESET} — API not connected/logged in {Colors.DIM}│{Colors.RESET}")
|
|
||||||
|
|
||||||
# Web
|
|
||||||
if has_web:
|
|
||||||
backend = diag.get("web_search_backend", "")
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Web{Colors.RESET} — {backend} API {Colors.DIM}│{Colors.RESET}")
|
|
||||||
else:
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.YELLOW}⚡ Web{Colors.RESET} — Using assistant's search tool {Colors.DIM}│{Colors.RESET}")
|
|
||||||
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
|
||||||
lines.append(f"{Colors.DIM}│{Colors.RESET} Config: {Colors.BOLD}~/.config/last30days/.env{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
|
||||||
lines.append(f"{Colors.DIM}└─────────────────────────────────────────────────────┘{Colors.RESET}")
|
|
||||||
else:
|
|
||||||
# Plain text for non-TTY (Claude Code / Codex)
|
|
||||||
lines.append("┌─────────────────────────────────────────────────────┐")
|
|
||||||
lines.append("│ /last30days v2.1 — Source Status │")
|
|
||||||
lines.append("│ │")
|
|
||||||
|
|
||||||
if has_openai:
|
|
||||||
lines.append("│ ✅ Reddit — OpenAI/Codex auth found │")
|
|
||||||
elif has_reddit_public:
|
|
||||||
lines.append("│ ✅ Reddit — Public Reddit search (no key) │")
|
|
||||||
else:
|
|
||||||
lines.append("│ ❌ Reddit — No OPENAI_API_KEY │")
|
|
||||||
lines.append("│ └─ Add to ~/.config/last30days/.env │")
|
|
||||||
|
|
||||||
if has_x:
|
|
||||||
lines.append("│ ✅ X/Twitter — available │")
|
|
||||||
else:
|
|
||||||
lines.append("│ ❌ X/Twitter — No X auth or fallback key │")
|
|
||||||
if diag.get("bird_installed"):
|
|
||||||
lines.append("│ └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY │")
|
|
||||||
else:
|
|
||||||
lines.append("│ └─ Needs Node.js 22+ (Bird is bundled) │")
|
|
||||||
|
|
||||||
if has_youtube:
|
|
||||||
lines.append("│ ✅ YouTube — yt-dlp found │")
|
|
||||||
else:
|
|
||||||
lines.append("│ ❌ YouTube — yt-dlp not installed │")
|
|
||||||
lines.append("│ └─ Fix: brew install yt-dlp (free) │")
|
|
||||||
|
|
||||||
if has_xiaohongshu:
|
|
||||||
lines.append("│ ✅ Xiaohongshu — API connected + logged in │")
|
|
||||||
else:
|
|
||||||
lines.append("│ ⚡ Xiaohongshu — API not connected/logged in │")
|
|
||||||
|
|
||||||
if has_web:
|
|
||||||
lines.append("│ ✅ Web — API search available │")
|
|
||||||
else:
|
|
||||||
lines.append("│ ⚡ Web — Using assistant's search tool │")
|
|
||||||
|
|
||||||
lines.append("│ │")
|
|
||||||
lines.append("│ Config: ~/.config/last30days/.env │")
|
|
||||||
lines.append("└─────────────────────────────────────────────────────┘")
|
|
||||||
|
|
||||||
sys.stderr.write("\n".join(lines) + "\n\n")
|
sys.stderr.write("\n".join(lines) + "\n\n")
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|||||||
+182
-7
@@ -15,6 +15,8 @@ import shutil
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||||
@@ -33,11 +35,57 @@ TRANSCRIPT_LIMITS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Max words to keep from each transcript
|
# Max words to keep from each transcript
|
||||||
TRANSCRIPT_MAX_WORDS = 500
|
TRANSCRIPT_MAX_WORDS = 5000
|
||||||
|
|
||||||
from .relevance import token_overlap_relevance as _compute_relevance
|
from .relevance import token_overlap_relevance as _compute_relevance
|
||||||
|
|
||||||
|
|
||||||
|
def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -> List[str]:
|
||||||
|
"""Extract quotable highlights from a YouTube transcript.
|
||||||
|
|
||||||
|
Similar to reddit_enrich.extract_comment_insights() but for
|
||||||
|
continuous speech-to-text rather than threaded comments.
|
||||||
|
"""
|
||||||
|
if not transcript:
|
||||||
|
return []
|
||||||
|
|
||||||
|
sentences = re.split(r'(?<=[.!?])\s+', transcript)
|
||||||
|
|
||||||
|
filler = [
|
||||||
|
r"^(hey |hi |what's up|welcome back|in today's video|don't forget to)",
|
||||||
|
r"(subscribe|like and comment|hit the bell|check out the link|down below)",
|
||||||
|
r"^(so |and |but |okay |alright |um |uh )",
|
||||||
|
r"(thanks for watching|see you (next|in the)|bye)",
|
||||||
|
]
|
||||||
|
|
||||||
|
topic_words = [w.lower() for w in topic.lower().split() if len(w) > 2]
|
||||||
|
|
||||||
|
candidates = []
|
||||||
|
for sent in sentences:
|
||||||
|
sent = sent.strip()
|
||||||
|
words = sent.split()
|
||||||
|
if len(words) < 8 or len(words) > 50:
|
||||||
|
continue
|
||||||
|
if any(re.search(p, sent, re.IGNORECASE) for p in filler):
|
||||||
|
continue
|
||||||
|
|
||||||
|
score = 0
|
||||||
|
if re.search(r'\d', sent):
|
||||||
|
score += 2
|
||||||
|
if re.search(r'[A-Z][a-z]+', sent):
|
||||||
|
score += 1
|
||||||
|
if '?' in sent:
|
||||||
|
score += 1
|
||||||
|
sent_lower = sent.lower()
|
||||||
|
if any(w in sent_lower for w in topic_words):
|
||||||
|
score += 2
|
||||||
|
|
||||||
|
candidates.append((score, sent))
|
||||||
|
|
||||||
|
candidates.sort(key=lambda x: -x[0])
|
||||||
|
return [sent for _, sent in candidates[:limit]]
|
||||||
|
|
||||||
|
|
||||||
def _log(msg: str):
|
def _log(msg: str):
|
||||||
"""Log to stderr."""
|
"""Log to stderr."""
|
||||||
sys.stderr.write(f"[YouTube] {msg}\n")
|
sys.stderr.write(f"[YouTube] {msg}\n")
|
||||||
@@ -209,15 +257,114 @@ def _clean_vtt(vtt_text: str) -> str:
|
|||||||
return re.sub(r'\s+', ' ', ' '.join(unique)).strip()
|
return re.sub(r'\s+', ' ', ' '.join(unique)).strip()
|
||||||
|
|
||||||
|
|
||||||
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
_YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||||
"""Fetch auto-generated transcript for a YouTube video.
|
|
||||||
|
|
||||||
|
def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]:
|
||||||
|
"""Fetch YouTube transcript via direct HTTP without yt-dlp.
|
||||||
|
|
||||||
|
Scrapes the watch page HTML for the captions track URL in
|
||||||
|
ytInitialPlayerResponse, then fetches the VTT subtitle file.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
video_id: YouTube video ID
|
||||||
|
timeout: HTTP request timeout in seconds
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Raw VTT text, or None if captions are unavailable.
|
||||||
|
"""
|
||||||
|
watch_url = f"https://www.youtube.com/watch?v={video_id}"
|
||||||
|
headers = {
|
||||||
|
"User-Agent": _YT_USER_AGENT,
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Step 1: Fetch the watch page HTML
|
||||||
|
req = urllib.request.Request(watch_url, headers=headers)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
html = resp.read().decode("utf-8", errors="replace")
|
||||||
|
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
|
||||||
|
_log(f"Direct transcript: failed to fetch watch page for {video_id}: {exc}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Step 2: Extract captions URL from ytInitialPlayerResponse
|
||||||
|
# YouTube embeds this as a JS variable in the page HTML
|
||||||
|
match = re.search(
|
||||||
|
r'ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;(?:\s*var\s|\s*<\/script>)',
|
||||||
|
html,
|
||||||
|
)
|
||||||
|
if not match:
|
||||||
|
# Fallback: try the JSON embedded in the script tag
|
||||||
|
match = re.search(
|
||||||
|
r'var\s+ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;',
|
||||||
|
html,
|
||||||
|
)
|
||||||
|
if not match:
|
||||||
|
_log(f"Direct transcript: no ytInitialPlayerResponse found for {video_id}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
player_response = json.loads(match.group(1))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
_log(f"Direct transcript: failed to parse ytInitialPlayerResponse for {video_id}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Navigate to caption tracks
|
||||||
|
captions = player_response.get("captions", {})
|
||||||
|
renderer = captions.get("playerCaptionsTracklistRenderer", {})
|
||||||
|
caption_tracks = renderer.get("captionTracks", [])
|
||||||
|
|
||||||
|
if not caption_tracks:
|
||||||
|
_log(f"Direct transcript: no caption tracks for {video_id}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Find English track (prefer exact 'en', then any en variant, then first track)
|
||||||
|
base_url = None
|
||||||
|
for track in caption_tracks:
|
||||||
|
lang = track.get("languageCode", "")
|
||||||
|
if lang == "en":
|
||||||
|
base_url = track.get("baseUrl")
|
||||||
|
break
|
||||||
|
if not base_url:
|
||||||
|
for track in caption_tracks:
|
||||||
|
lang = track.get("languageCode", "")
|
||||||
|
if lang.startswith("en"):
|
||||||
|
base_url = track.get("baseUrl")
|
||||||
|
break
|
||||||
|
if not base_url:
|
||||||
|
# Fall back to first available track
|
||||||
|
base_url = caption_tracks[0].get("baseUrl")
|
||||||
|
if not base_url:
|
||||||
|
_log(f"Direct transcript: no baseUrl in caption tracks for {video_id}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Step 3: Fetch the VTT subtitle file
|
||||||
|
sep = "&" if "?" in base_url else "?"
|
||||||
|
vtt_url = f"{base_url}{sep}fmt=vtt"
|
||||||
|
vtt_req = urllib.request.Request(vtt_url, headers=headers)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(vtt_req, timeout=timeout) as resp:
|
||||||
|
vtt_text = resp.read().decode("utf-8", errors="replace")
|
||||||
|
except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
|
||||||
|
_log(f"Direct transcript: failed to fetch VTT for {video_id}: {exc}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not vtt_text or not vtt_text.strip():
|
||||||
|
return None
|
||||||
|
|
||||||
|
return vtt_text
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:
|
||||||
|
"""Fetch transcript using yt-dlp (original implementation).
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
video_id: YouTube video ID
|
video_id: YouTube video ID
|
||||||
temp_dir: Temporary directory for subtitle files
|
temp_dir: Temporary directory for subtitle files
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Plaintext transcript string, or None if no captions available.
|
Raw VTT text, or None if no captions available.
|
||||||
"""
|
"""
|
||||||
cmd = [
|
cmd = [
|
||||||
"yt-dlp",
|
"yt-dlp",
|
||||||
@@ -265,11 +412,35 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
raw = vtt_path.read_text(encoding="utf-8", errors="replace")
|
return vtt_path.read_text(encoding="utf-8", errors="replace")
|
||||||
except OSError:
|
except OSError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
transcript = _clean_vtt(raw)
|
|
||||||
|
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
||||||
|
"""Fetch auto-generated transcript for a YouTube video.
|
||||||
|
|
||||||
|
Uses yt-dlp when available (preferred, more robust). Falls back to
|
||||||
|
direct HTTP transcript fetching when yt-dlp is not installed.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
video_id: YouTube video ID
|
||||||
|
temp_dir: Temporary directory for subtitle files
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Plaintext transcript string, or None if no captions available.
|
||||||
|
"""
|
||||||
|
raw_vtt = None
|
||||||
|
if is_ytdlp_installed():
|
||||||
|
raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
|
||||||
|
else:
|
||||||
|
_log("yt-dlp not installed, using direct HTTP transcript fetch")
|
||||||
|
raw_vtt = _fetch_transcript_direct(video_id)
|
||||||
|
|
||||||
|
if not raw_vtt:
|
||||||
|
return None
|
||||||
|
|
||||||
|
transcript = _clean_vtt(raw_vtt)
|
||||||
|
|
||||||
# Truncate to max words
|
# Truncate to max words
|
||||||
words = transcript.split()
|
words = transcript.split()
|
||||||
@@ -345,11 +516,15 @@ def search_and_transcribe(
|
|||||||
top_ids = [item["video_id"] for item in items[:transcript_limit]]
|
top_ids = [item["video_id"] for item in items[:transcript_limit]]
|
||||||
transcripts = fetch_transcripts_parallel(top_ids)
|
transcripts = fetch_transcripts_parallel(top_ids)
|
||||||
|
|
||||||
# Step 3: Attach transcripts to items
|
# Step 3: Attach transcripts and extract highlights
|
||||||
|
core_topic = _extract_core_subject(topic)
|
||||||
for item in items:
|
for item in items:
|
||||||
vid = item["video_id"]
|
vid = item["video_id"]
|
||||||
transcript = transcripts.get(vid)
|
transcript = transcripts.get(vid)
|
||||||
item["transcript_snippet"] = transcript or ""
|
item["transcript_snippet"] = transcript or ""
|
||||||
|
item["transcript_highlights"] = extract_transcript_highlights(
|
||||||
|
transcript or "", core_topic,
|
||||||
|
)
|
||||||
|
|
||||||
return {"items": items}
|
return {"items": items}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ set -euo pipefail
|
|||||||
# using `claude --print` to capture real end-to-end output.
|
# using `claude --print` to capture real end-to-end output.
|
||||||
|
|
||||||
SKILL_DIR="$HOME/.claude/skills/last30days"
|
SKILL_DIR="$HOME/.claude/skills/last30days"
|
||||||
REPO_DIR="/Users/mvanhorn/last30days-skill-private"
|
REPO_DIR="/Users/mvanhorn/last30days-skill"
|
||||||
|
|
||||||
# Safety: always restore V2 SKILL.md on exit/crash
|
# Safety: always restore V2 SKILL.md on exit/crash
|
||||||
cleanup() {
|
cleanup() {
|
||||||
|
|||||||
@@ -0,0 +1,350 @@
|
|||||||
|
"""Tests for Chrome cookie extraction on macOS."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scripts.lib.chrome_cookies import (
|
||||||
|
CHROME_COOKIES_DB,
|
||||||
|
CHROME_IV_HEX,
|
||||||
|
CHROME_KEY_LENGTH,
|
||||||
|
CHROME_PBKDF2_ITERATIONS,
|
||||||
|
CHROME_SALT,
|
||||||
|
_derive_aes_key,
|
||||||
|
_get_chrome_encryption_key,
|
||||||
|
_get_db_version,
|
||||||
|
_remove_pkcs7_padding,
|
||||||
|
_decrypt_v10_value,
|
||||||
|
extract_chrome_cookies_macos,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers — create real encrypted cookie values using known key + system openssl
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
KNOWN_PASSPHRASE = b"test_passphrase_for_unit_tests"
|
||||||
|
KNOWN_AES_KEY = _derive_aes_key(KNOWN_PASSPHRASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _encrypt_value_v10(plaintext: str, aes_key: bytes) -> bytes:
|
||||||
|
"""Encrypt a value the same way Chrome v10 does, using system openssl.
|
||||||
|
|
||||||
|
Returns b'v10' + AES-128-CBC ciphertext with PKCS7 padding.
|
||||||
|
"""
|
||||||
|
hex_key = aes_key.hex()
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"openssl", "enc", "-aes-128-cbc", "-e",
|
||||||
|
"-K", hex_key,
|
||||||
|
"-iv", CHROME_IV_HEX,
|
||||||
|
],
|
||||||
|
input=plaintext.encode("utf-8"),
|
||||||
|
capture_output=True,
|
||||||
|
timeout=5,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"openssl encrypt failed: {result.stderr}"
|
||||||
|
return b"v10" + result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def _encrypt_value_v10_with_sha_prefix(plaintext: str, aes_key: bytes) -> bytes:
|
||||||
|
"""Encrypt with a 32-byte SHA-256 prefix (Chrome 130+ style)."""
|
||||||
|
raw = b"\x00" * 32 + plaintext.encode("utf-8")
|
||||||
|
hex_key = aes_key.hex()
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"openssl", "enc", "-aes-128-cbc", "-e",
|
||||||
|
"-K", hex_key,
|
||||||
|
"-iv", CHROME_IV_HEX,
|
||||||
|
],
|
||||||
|
input=raw,
|
||||||
|
capture_output=True,
|
||||||
|
timeout=5,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"openssl encrypt failed: {result.stderr}"
|
||||||
|
return b"v10" + result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def _create_chrome_cookies_db(path: str, cookies: list[tuple], db_version: int = 20) -> None:
|
||||||
|
"""Create a minimal Chrome Cookies SQLite database.
|
||||||
|
|
||||||
|
cookies: list of (host_key, name, value, encrypted_value) tuples
|
||||||
|
"""
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
c = conn.cursor()
|
||||||
|
c.execute("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)")
|
||||||
|
c.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('version', ?)", (str(db_version),))
|
||||||
|
c.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS cookies ("
|
||||||
|
" host_key TEXT NOT NULL,"
|
||||||
|
" name TEXT NOT NULL,"
|
||||||
|
" value TEXT NOT NULL DEFAULT '',"
|
||||||
|
" encrypted_value BLOB NOT NULL DEFAULT x'',"
|
||||||
|
" path TEXT NOT NULL DEFAULT '/',"
|
||||||
|
" expires_utc INTEGER NOT NULL DEFAULT 0,"
|
||||||
|
" is_secure INTEGER NOT NULL DEFAULT 1,"
|
||||||
|
" is_httponly INTEGER NOT NULL DEFAULT 1,"
|
||||||
|
" creation_utc INTEGER NOT NULL DEFAULT 0,"
|
||||||
|
" last_access_utc INTEGER NOT NULL DEFAULT 0,"
|
||||||
|
" has_expires INTEGER NOT NULL DEFAULT 1,"
|
||||||
|
" is_persistent INTEGER NOT NULL DEFAULT 1,"
|
||||||
|
" priority INTEGER NOT NULL DEFAULT 1,"
|
||||||
|
" samesite INTEGER NOT NULL DEFAULT 0,"
|
||||||
|
" source_scheme INTEGER NOT NULL DEFAULT 2,"
|
||||||
|
" source_port INTEGER NOT NULL DEFAULT 443,"
|
||||||
|
" last_update_utc INTEGER NOT NULL DEFAULT 0"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
for host_key, name, value, encrypted_value in cookies:
|
||||||
|
c.execute(
|
||||||
|
"INSERT INTO cookies (host_key, name, value, encrypted_value) VALUES (?, ?, ?, ?)",
|
||||||
|
(host_key, name, value, encrypted_value),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# PKCS7 padding tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestPkcs7Padding:
|
||||||
|
def test_valid_padding_1(self):
|
||||||
|
# 1 byte of padding
|
||||||
|
data = b"hello world!!!!!" + b"\x01"
|
||||||
|
assert _remove_pkcs7_padding(data) == b"hello world!!!!!"
|
||||||
|
|
||||||
|
def test_valid_padding_5(self):
|
||||||
|
data = b"hello world" + b"\x05\x05\x05\x05\x05"
|
||||||
|
assert _remove_pkcs7_padding(data) == b"hello world"
|
||||||
|
|
||||||
|
def test_valid_padding_16(self):
|
||||||
|
# Full block of padding
|
||||||
|
data = b"\x10" * 16
|
||||||
|
assert _remove_pkcs7_padding(data) == b""
|
||||||
|
|
||||||
|
def test_invalid_padding_zero(self):
|
||||||
|
data = b"hello\x00"
|
||||||
|
assert _remove_pkcs7_padding(data) is None
|
||||||
|
|
||||||
|
def test_invalid_padding_mismatch(self):
|
||||||
|
data = b"hello\x03\x03\x02"
|
||||||
|
assert _remove_pkcs7_padding(data) is None
|
||||||
|
|
||||||
|
def test_empty_data(self):
|
||||||
|
assert _remove_pkcs7_padding(b"") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Key derivation test
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestKeyDerivation:
|
||||||
|
def test_derive_aes_key_deterministic(self):
|
||||||
|
key1 = _derive_aes_key(b"my_passphrase")
|
||||||
|
key2 = _derive_aes_key(b"my_passphrase")
|
||||||
|
assert key1 == key2
|
||||||
|
assert len(key1) == 16
|
||||||
|
|
||||||
|
def test_derive_aes_key_different_passphrases(self):
|
||||||
|
key1 = _derive_aes_key(b"passphrase_a")
|
||||||
|
key2 = _derive_aes_key(b"passphrase_b")
|
||||||
|
assert key1 != key2
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Decryption test (real openssl, known key)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestDecryption:
|
||||||
|
def test_decrypt_v10_roundtrip(self):
|
||||||
|
"""Encrypt then decrypt — verifies the full pipeline works."""
|
||||||
|
original = "my_secret_cookie_value_12345"
|
||||||
|
encrypted = _encrypt_value_v10(original, KNOWN_AES_KEY)
|
||||||
|
assert encrypted[:3] == b"v10"
|
||||||
|
|
||||||
|
decrypted = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=20)
|
||||||
|
assert decrypted == original
|
||||||
|
|
||||||
|
def test_decrypt_v10_chrome130_with_sha_prefix(self):
|
||||||
|
"""Chrome 130+ (db_version >= 24) strips 32-byte SHA-256 prefix."""
|
||||||
|
original = "session_token_abc"
|
||||||
|
encrypted = _encrypt_value_v10_with_sha_prefix(original, KNOWN_AES_KEY)
|
||||||
|
|
||||||
|
decrypted = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=24)
|
||||||
|
assert decrypted == original
|
||||||
|
|
||||||
|
def test_decrypt_wrong_key_returns_none_or_garbage(self):
|
||||||
|
"""Wrong key should either fail decryption or produce garbage."""
|
||||||
|
original = "secret"
|
||||||
|
encrypted = _encrypt_value_v10(original, KNOWN_AES_KEY)
|
||||||
|
wrong_key = _derive_aes_key(b"wrong_passphrase")
|
||||||
|
|
||||||
|
result = _decrypt_v10_value(encrypted, wrong_key, db_version=20)
|
||||||
|
# Either None (padding check fails) or garbage (not the original)
|
||||||
|
assert result is None or result != original
|
||||||
|
|
||||||
|
def test_decrypt_empty_ciphertext(self):
|
||||||
|
"""v10 prefix with no ciphertext should return None."""
|
||||||
|
assert _decrypt_v10_value(b"v10", KNOWN_AES_KEY, db_version=20) is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Chrome not installed → returns None
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestChromeNotInstalled:
|
||||||
|
def test_db_not_found(self):
|
||||||
|
with mock.patch(
|
||||||
|
"scripts.lib.chrome_cookies.CHROME_COOKIES_DB",
|
||||||
|
Path("/nonexistent/path/Cookies"),
|
||||||
|
):
|
||||||
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Keychain access denied → returns None
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestKeychainDenied:
|
||||||
|
def test_security_command_fails(self):
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies.subprocess.run") as mock_run:
|
||||||
|
mock_run.return_value = subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=44, stdout="", stderr="security: SecKeychainSearchCopyNext: The specified item could not be found in the keychain."
|
||||||
|
)
|
||||||
|
result = _get_chrome_encryption_key()
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_security_command_not_found(self):
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
|
||||||
|
result = _get_chrome_encryption_key()
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# openssl not found → returns None
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestOpensslNotFound:
|
||||||
|
def test_openssl_missing(self):
|
||||||
|
encrypted = _encrypt_value_v10("test", KNOWN_AES_KEY)
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies.subprocess.run", side_effect=FileNotFoundError):
|
||||||
|
result = _decrypt_v10_value(encrypted, KNOWN_AES_KEY, db_version=20)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Unencrypted cookie values → returned as-is
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestUnencryptedCookies:
|
||||||
|
def test_plain_value_returned(self, tmp_path):
|
||||||
|
"""Unencrypted cookies (value column populated) returned without decryption."""
|
||||||
|
db_path = str(tmp_path / "Cookies")
|
||||||
|
_create_chrome_cookies_db(db_path, [
|
||||||
|
(".x.com", "auth_token", "plain_token_value", b""),
|
||||||
|
(".x.com", "ct0", "plain_ct0_value", b""),
|
||||||
|
])
|
||||||
|
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
|
# No keychain needed for unencrypted values
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
|
||||||
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
assert result == {"auth_token": "plain_token_value", "ct0": "plain_ct0_value"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Full integration: mock DB with real v10 encryption, mock Keychain
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestFullExtraction:
|
||||||
|
def test_encrypted_cookies_extracted(self, tmp_path):
|
||||||
|
"""End-to-end: create DB with real v10-encrypted values, extract them."""
|
||||||
|
auth_val = "my_auth_token_123"
|
||||||
|
ct0_val = "my_ct0_csrf_456"
|
||||||
|
|
||||||
|
encrypted_auth = _encrypt_value_v10(auth_val, KNOWN_AES_KEY)
|
||||||
|
encrypted_ct0 = _encrypt_value_v10(ct0_val, KNOWN_AES_KEY)
|
||||||
|
|
||||||
|
db_path = str(tmp_path / "Cookies")
|
||||||
|
_create_chrome_cookies_db(db_path, [
|
||||||
|
(".x.com", "auth_token", "", encrypted_auth),
|
||||||
|
(".x.com", "ct0", "", encrypted_ct0),
|
||||||
|
(".other.com", "other", "", b""), # unrelated cookie
|
||||||
|
])
|
||||||
|
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
|
with mock.patch(
|
||||||
|
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
|
||||||
|
return_value=KNOWN_PASSPHRASE,
|
||||||
|
):
|
||||||
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == auth_val
|
||||||
|
assert result["ct0"] == ct0_val
|
||||||
|
|
||||||
|
def test_no_matching_cookies_returns_none(self, tmp_path):
|
||||||
|
db_path = str(tmp_path / "Cookies")
|
||||||
|
_create_chrome_cookies_db(db_path, [
|
||||||
|
(".other.com", "session", "val", b""),
|
||||||
|
])
|
||||||
|
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies._get_chrome_encryption_key", return_value=None):
|
||||||
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_chrome130_db_version_24(self, tmp_path):
|
||||||
|
"""Chrome 130+ with db_version >= 24 strips SHA-256 prefix."""
|
||||||
|
auth_val = "token_for_chrome130"
|
||||||
|
encrypted_auth = _encrypt_value_v10_with_sha_prefix(auth_val, KNOWN_AES_KEY)
|
||||||
|
|
||||||
|
db_path = str(tmp_path / "Cookies")
|
||||||
|
_create_chrome_cookies_db(db_path, [
|
||||||
|
(".x.com", "auth_token", "", encrypted_auth),
|
||||||
|
], db_version=24)
|
||||||
|
|
||||||
|
with mock.patch("scripts.lib.chrome_cookies.CHROME_COOKIES_DB", Path(db_path)):
|
||||||
|
with mock.patch(
|
||||||
|
"scripts.lib.chrome_cookies._get_chrome_encryption_key",
|
||||||
|
return_value=KNOWN_PASSPHRASE,
|
||||||
|
):
|
||||||
|
result = extract_chrome_cookies_macos(".x.com", ["auth_token"])
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == auth_val
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# DB version detection
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestDbVersion:
|
||||||
|
def test_reads_version_from_meta(self, tmp_path):
|
||||||
|
db_path = str(tmp_path / "test.db")
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
c = conn.cursor()
|
||||||
|
c.execute("CREATE TABLE meta (key TEXT, value TEXT)")
|
||||||
|
c.execute("INSERT INTO meta VALUES ('version', '24')")
|
||||||
|
conn.commit()
|
||||||
|
assert _get_db_version(c) == 24
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_no_meta_table_returns_zero(self, tmp_path):
|
||||||
|
db_path = str(tmp_path / "test.db")
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
c = conn.cursor()
|
||||||
|
c.execute("CREATE TABLE dummy (x TEXT)")
|
||||||
|
conn.commit()
|
||||||
|
assert _get_db_version(c) == 0
|
||||||
|
conn.close()
|
||||||
@@ -124,7 +124,8 @@ class TestLoadCodexAuth(unittest.TestCase):
|
|||||||
|
|
||||||
class TestGetAvailableSourcesWithAuth(unittest.TestCase):
|
class TestGetAvailableSourcesWithAuth(unittest.TestCase):
|
||||||
|
|
||||||
def test_codex_auth_ok_counts_as_openai(self):
|
@patch("lib.bird_x.is_bird_installed", return_value=False)
|
||||||
|
def test_codex_auth_ok_counts_as_openai(self, _mock_bird):
|
||||||
config = {
|
config = {
|
||||||
"OPENAI_API_KEY": "codex-token",
|
"OPENAI_API_KEY": "codex-token",
|
||||||
"OPENAI_AUTH_STATUS": "ok",
|
"OPENAI_AUTH_STATUS": "ok",
|
||||||
@@ -133,7 +134,8 @@ class TestGetAvailableSourcesWithAuth(unittest.TestCase):
|
|||||||
result = env.get_available_sources(config)
|
result = env.get_available_sources(config)
|
||||||
self.assertIn("reddit", result)
|
self.assertIn("reddit", result)
|
||||||
|
|
||||||
def test_codex_auth_expired_not_counted(self):
|
@patch("lib.bird_x.is_bird_installed", return_value=False)
|
||||||
|
def test_codex_auth_expired_not_counted(self, _mock_bird):
|
||||||
config = {
|
config = {
|
||||||
"OPENAI_API_KEY": None,
|
"OPENAI_API_KEY": None,
|
||||||
"OPENAI_AUTH_STATUS": "expired",
|
"OPENAI_AUTH_STATUS": "expired",
|
||||||
|
|||||||
@@ -0,0 +1,293 @@
|
|||||||
|
"""Tests for browser cookie extraction module."""
|
||||||
|
|
||||||
|
import configparser
|
||||||
|
import sqlite3
|
||||||
|
import textwrap
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Optional, Tuple
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scripts.lib.cookie_extract import (
|
||||||
|
extract_cookies,
|
||||||
|
extract_firefox_cookies,
|
||||||
|
_find_default_profile,
|
||||||
|
_get_firefox_profiles_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_firefox_env(tmp_path):
|
||||||
|
"""Create a mock Firefox profiles directory with cookies.sqlite.
|
||||||
|
|
||||||
|
Returns (profiles_dir, profile_dir) for patching.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _make(
|
||||||
|
*,
|
||||||
|
profiles_ini=None, # type: Optional[str]
|
||||||
|
profiles=None, # type: Optional[Dict[str, List[Tuple[str, str, str]]]]
|
||||||
|
default_profile="abc123.default-release", # type: str
|
||||||
|
):
|
||||||
|
profiles_dir = tmp_path / "Firefox"
|
||||||
|
profiles_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Default: one profile with X cookies
|
||||||
|
if profiles is None:
|
||||||
|
profiles = {
|
||||||
|
default_profile: [
|
||||||
|
(".x.com", "auth_token", "tok_abc123"),
|
||||||
|
(".x.com", "ct0", "ct0_xyz789"),
|
||||||
|
(".example.com", "session", "sess_other"),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create profile directories with cookies databases
|
||||||
|
for profile_name, cookies in profiles.items():
|
||||||
|
profile_dir = profiles_dir / profile_name
|
||||||
|
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
db_path = profile_dir / "cookies.sqlite"
|
||||||
|
conn = sqlite3.connect(str(db_path))
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE moz_cookies ("
|
||||||
|
" id INTEGER PRIMARY KEY,"
|
||||||
|
" name TEXT NOT NULL,"
|
||||||
|
" value TEXT NOT NULL,"
|
||||||
|
" host TEXT NOT NULL,"
|
||||||
|
" path TEXT DEFAULT '/',"
|
||||||
|
" expiry INTEGER DEFAULT 0,"
|
||||||
|
" isSecure INTEGER DEFAULT 1,"
|
||||||
|
" isHttpOnly INTEGER DEFAULT 1,"
|
||||||
|
" sameSite INTEGER DEFAULT 0,"
|
||||||
|
" schemeMap INTEGER DEFAULT 0"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
for host, name, value in cookies:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO moz_cookies (name, value, host) VALUES (?, ?, ?)",
|
||||||
|
(name, value, host),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
# Write profiles.ini
|
||||||
|
if profiles_ini is None:
|
||||||
|
profiles_ini = textwrap.dedent(f"""\
|
||||||
|
[General]
|
||||||
|
StartWithLastProfile=1
|
||||||
|
|
||||||
|
[Profile0]
|
||||||
|
Name=default-release
|
||||||
|
IsRelative=1
|
||||||
|
Path={default_profile}
|
||||||
|
Default=1
|
||||||
|
""")
|
||||||
|
|
||||||
|
(profiles_dir / "profiles.ini").write_text(profiles_ini)
|
||||||
|
|
||||||
|
return profiles_dir
|
||||||
|
|
||||||
|
return _make
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractFirefoxCookies:
|
||||||
|
"""Tests for extract_firefox_cookies."""
|
||||||
|
|
||||||
|
def test_valid_cookies_extracted(self, mock_firefox_env):
|
||||||
|
"""Cookies for the target domain are returned correctly."""
|
||||||
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
):
|
||||||
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "tok_abc123"
|
||||||
|
assert result["ct0"] == "ct0_xyz789"
|
||||||
|
assert "session" not in result # different domain cookie not included
|
||||||
|
|
||||||
|
def test_multiple_profiles_selects_default(self, mock_firefox_env):
|
||||||
|
"""When multiple profiles exist, the one with Default=1 is used."""
|
||||||
|
profiles_dir = mock_firefox_env(
|
||||||
|
profiles={
|
||||||
|
"aaa111.other": [
|
||||||
|
(".x.com", "auth_token", "wrong_token"),
|
||||||
|
],
|
||||||
|
"bbb222.default-release": [
|
||||||
|
(".x.com", "auth_token", "correct_token"),
|
||||||
|
(".x.com", "ct0", "correct_ct0"),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
profiles_ini=textwrap.dedent("""\
|
||||||
|
[General]
|
||||||
|
StartWithLastProfile=1
|
||||||
|
|
||||||
|
[Profile0]
|
||||||
|
Name=other
|
||||||
|
IsRelative=1
|
||||||
|
Path=aaa111.other
|
||||||
|
|
||||||
|
[Profile1]
|
||||||
|
Name=default-release
|
||||||
|
IsRelative=1
|
||||||
|
Path=bbb222.default-release
|
||||||
|
Default=1
|
||||||
|
"""),
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
):
|
||||||
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "correct_token"
|
||||||
|
assert result["ct0"] == "correct_ct0"
|
||||||
|
|
||||||
|
def test_firefox_not_installed(self):
|
||||||
|
"""Returns None when Firefox profiles directory doesn't exist."""
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=None,
|
||||||
|
):
|
||||||
|
result = extract_firefox_cookies(".x.com", ["auth_token"])
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_cookies_sqlite_empty(self, mock_firefox_env):
|
||||||
|
"""Returns None when cookies.sqlite has no rows."""
|
||||||
|
profiles_dir = mock_firefox_env(
|
||||||
|
profiles={"abc123.default-release": []}, # no cookies
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
):
|
||||||
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_domain_has_no_cookies(self, mock_firefox_env):
|
||||||
|
"""Returns None when cookies exist but not for the target domain."""
|
||||||
|
profiles_dir = mock_firefox_env(
|
||||||
|
profiles={
|
||||||
|
"abc123.default-release": [
|
||||||
|
(".example.com", "session", "sess_123"),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
):
|
||||||
|
result = extract_firefox_cookies(".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_malformed_profiles_ini_falls_back(self, mock_firefox_env):
|
||||||
|
"""Falls back to first profile on disk when profiles.ini is garbage."""
|
||||||
|
profiles_dir = mock_firefox_env(
|
||||||
|
profiles={
|
||||||
|
"zzz999.fallback": [
|
||||||
|
(".x.com", "auth_token", "fallback_token"),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
profiles_ini="this is not valid ini content\n[[[broken",
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
):
|
||||||
|
result = extract_firefox_cookies(".x.com", ["auth_token"])
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "fallback_token"
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractCookiesAuto:
|
||||||
|
"""Tests for extract_cookies with browser='auto'."""
|
||||||
|
|
||||||
|
def test_auto_macos_tries_chrome_then_firefox(self, mock_firefox_env):
|
||||||
|
"""On macOS, auto tries Chrome first, falls back to Firefox if Chrome fails."""
|
||||||
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("scripts.lib.cookie_extract.platform.system", return_value="Darwin"),
|
||||||
|
patch(
|
||||||
|
"scripts.lib.cookie_extract.extract_chrome_cookies",
|
||||||
|
return_value=None,
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"scripts.lib.cookie_extract.extract_safari_cookies",
|
||||||
|
return_value=None,
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
result = extract_cookies("auto", ".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
# Chrome and Safari return None, Firefox succeeds
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "tok_abc123"
|
||||||
|
assert result["ct0"] == "ct0_xyz789"
|
||||||
|
|
||||||
|
def test_auto_linux_tries_firefox_only(self, mock_firefox_env):
|
||||||
|
"""On Linux, auto only tries Firefox."""
|
||||||
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("scripts.lib.cookie_extract.platform.system", return_value="Linux"),
|
||||||
|
patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
result = extract_cookies("auto", ".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "tok_abc123"
|
||||||
|
|
||||||
|
def test_explicit_firefox(self, mock_firefox_env):
|
||||||
|
"""Explicit browser='firefox' goes directly to Firefox."""
|
||||||
|
profiles_dir = mock_firefox_env()
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract._get_firefox_profiles_dir",
|
||||||
|
return_value=profiles_dir,
|
||||||
|
):
|
||||||
|
result = extract_cookies("firefox", ".x.com", ["auth_token"])
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "tok_abc123"
|
||||||
|
|
||||||
|
def test_unknown_browser_returns_none(self):
|
||||||
|
"""Unknown browser name returns None."""
|
||||||
|
result = extract_cookies("netscape", ".x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_chrome_delegates_to_chrome_module(self):
|
||||||
|
"""Chrome extraction delegates to chrome_cookies module."""
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract.extract_chrome_cookies",
|
||||||
|
return_value={"auth_token": "chrome_tok"},
|
||||||
|
):
|
||||||
|
result = extract_cookies("chrome", ".x.com", ["auth_token"])
|
||||||
|
assert result == {"auth_token": "chrome_tok"}
|
||||||
|
|
||||||
|
def test_safari_delegates_to_safari_module(self):
|
||||||
|
"""Safari extraction delegates to safari_cookies module."""
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.cookie_extract.extract_safari_cookies",
|
||||||
|
return_value={"auth_token": "safari_tok"},
|
||||||
|
):
|
||||||
|
result = extract_cookies("safari", ".x.com", ["auth_token"])
|
||||||
|
assert result == {"auth_token": "safari_tok"}
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
"""Tests for browser cookie extraction integration in env.py."""
|
||||||
|
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scripts.lib.env import extract_browser_credentials, COOKIE_DOMAINS
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _base_config(**overrides):
|
||||||
|
"""Return a minimal config dict with common defaults."""
|
||||||
|
cfg = {
|
||||||
|
"AUTH_TOKEN": None,
|
||||||
|
"CT0": None,
|
||||||
|
"TRUTHSOCIAL_TOKEN": None,
|
||||||
|
"FROM_BROWSER": None,
|
||||||
|
"SETUP_COMPLETE": None,
|
||||||
|
}
|
||||||
|
cfg.update(overrides)
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestExtractBrowserCredentials:
|
||||||
|
"""Unit tests for extract_browser_credentials()."""
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_auto_with_setup_complete_populates_credentials(self, mock_extract):
|
||||||
|
"""FROM_BROWSER=auto, SETUP_COMPLETE=true, mock returns valid cookies
|
||||||
|
-> config now contains AUTH_TOKEN and CT0."""
|
||||||
|
mock_extract.return_value = ({"auth_token": "tok123", "ct0": "ct0val"}, "chrome")
|
||||||
|
|
||||||
|
config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert result["AUTH_TOKEN"] == "tok123"
|
||||||
|
assert result["CT0"] == "ct0val"
|
||||||
|
# Should have been called for x domain
|
||||||
|
mock_extract.assert_any_call("auto", ".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_explicit_auth_token_skips_x_extraction(self, mock_extract):
|
||||||
|
"""Config already has AUTH_TOKEN and CT0 from env var
|
||||||
|
-> cookie extraction skipped for X (explicit takes priority)."""
|
||||||
|
# Return None for any non-X domains that still get checked (e.g. Truth Social)
|
||||||
|
mock_extract.return_value = None
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="explicit_token",
|
||||||
|
CT0="explicit_ct0",
|
||||||
|
FROM_BROWSER="auto",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
)
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
# X cookies should not appear in result (already set)
|
||||||
|
assert "AUTH_TOKEN" not in result
|
||||||
|
assert "CT0" not in result
|
||||||
|
# extract_cookies should NOT have been called for .x.com
|
||||||
|
for call in mock_extract.call_args_list:
|
||||||
|
assert call[0][1] != ".x.com", "Should not extract cookies for X when credentials already set"
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_from_browser_off_skips_all(self, mock_extract):
|
||||||
|
"""FROM_BROWSER=off -> no cookie extraction attempted."""
|
||||||
|
config = _base_config(FROM_BROWSER="off", SETUP_COMPLETE="true")
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert result == {}
|
||||||
|
mock_extract.assert_not_called()
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_no_setup_complete_no_from_browser_defaults_off(self, mock_extract):
|
||||||
|
"""FROM_BROWSER not set and SETUP_COMPLETE not set
|
||||||
|
-> no extraction (wizard hasn't run)."""
|
||||||
|
config = _base_config() # both None
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert result == {}
|
||||||
|
mock_extract.assert_not_called()
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_setup_complete_no_from_browser_defaults_auto(self, mock_extract):
|
||||||
|
"""SETUP_COMPLETE is set but FROM_BROWSER is not
|
||||||
|
-> defaults to 'auto'."""
|
||||||
|
mock_extract.return_value = ({"auth_token": "found", "ct0": "found_ct0"}, "firefox")
|
||||||
|
|
||||||
|
config = _base_config(SETUP_COMPLETE="true") # FROM_BROWSER=None
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert result["AUTH_TOKEN"] == "found"
|
||||||
|
# extract_cookies should have been called with 'auto'
|
||||||
|
mock_extract.assert_any_call("auto", ".x.com", ["auth_token", "ct0"])
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_from_browser_firefox_only(self, mock_extract):
|
||||||
|
"""FROM_BROWSER=firefox -> only Firefox extraction attempted."""
|
||||||
|
mock_extract.return_value = ({"auth_token": "ff_tok", "ct0": "ff_ct0"}, "firefox")
|
||||||
|
|
||||||
|
config = _base_config(FROM_BROWSER="firefox", SETUP_COMPLETE="true")
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert result["AUTH_TOKEN"] == "ff_tok"
|
||||||
|
# All calls should use 'firefox' as the browser arg
|
||||||
|
for call in mock_extract.call_args_list:
|
||||||
|
assert call[0][0] == "firefox"
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_extraction_returns_none_config_unchanged(self, mock_extract):
|
||||||
|
"""Cookie extraction returns None for X -> config unchanged for AUTH_TOKEN."""
|
||||||
|
mock_extract.return_value = None
|
||||||
|
|
||||||
|
config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert "AUTH_TOKEN" not in result
|
||||||
|
assert "CT0" not in result
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_extraction_raises_exception_config_unchanged(self, mock_extract):
|
||||||
|
"""Cookie extraction raises exception -> caught, config unchanged."""
|
||||||
|
mock_extract.side_effect = RuntimeError("database locked")
|
||||||
|
|
||||||
|
config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
# Should not raise, and no credentials populated
|
||||||
|
assert "AUTH_TOKEN" not in result
|
||||||
|
assert "CT0" not in result
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_partial_credentials_only_fills_missing(self, mock_extract):
|
||||||
|
"""If AUTH_TOKEN is set but CT0 is not, only CT0 gets filled."""
|
||||||
|
mock_extract.return_value = ({"auth_token": "cookie_tok", "ct0": "cookie_ct0"}, "chrome")
|
||||||
|
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="explicit",
|
||||||
|
CT0=None,
|
||||||
|
FROM_BROWSER="auto",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
)
|
||||||
|
result = extract_browser_credentials(config)
|
||||||
|
|
||||||
|
# AUTH_TOKEN already set, should not be overridden
|
||||||
|
assert "AUTH_TOKEN" not in result
|
||||||
|
# CT0 was missing, should be filled
|
||||||
|
assert result["CT0"] == "cookie_ct0"
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetConfigCookieIntegration:
|
||||||
|
"""Integration test: get_config() calls extract_browser_credentials."""
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("scripts.lib.env._find_project_env", return_value=None)
|
||||||
|
@patch("scripts.lib.env.load_env_file", return_value={})
|
||||||
|
@patch("scripts.lib.env.get_openai_auth")
|
||||||
|
def test_get_config_injects_cookies(
|
||||||
|
self, mock_openai, mock_load, mock_proj, mock_extract
|
||||||
|
):
|
||||||
|
"""get_config merges browser cookies into the returned config."""
|
||||||
|
from scripts.lib.env import get_config, OpenAIAuth
|
||||||
|
|
||||||
|
mock_openai.return_value = OpenAIAuth(
|
||||||
|
token=None, source="none", status="missing",
|
||||||
|
account_id=None, codex_auth_file="/fake",
|
||||||
|
)
|
||||||
|
mock_extract.return_value = ({"auth_token": "browser_tok", "ct0": "browser_ct0"}, "firefox")
|
||||||
|
|
||||||
|
import os
|
||||||
|
env_patch = {
|
||||||
|
"SETUP_COMPLETE": "true",
|
||||||
|
"FROM_BROWSER": "auto",
|
||||||
|
"LAST30DAYS_CONFIG_DIR": "",
|
||||||
|
}
|
||||||
|
with patch.dict(os.environ, env_patch, clear=False):
|
||||||
|
config = get_config()
|
||||||
|
|
||||||
|
assert config["AUTH_TOKEN"] == "browser_tok"
|
||||||
|
assert config["CT0"] == "browser_ct0"
|
||||||
@@ -223,7 +223,9 @@ class TestXSourceSelection(unittest.TestCase):
|
|||||||
'can_install': False,
|
'can_install': False,
|
||||||
}
|
}
|
||||||
|
|
||||||
with patch('lib.bird_x.get_bird_status', return_value=bird_status):
|
with patch('lib.bird_x.get_bird_status', return_value=bird_status), \
|
||||||
|
patch('lib.bird_x.is_bird_installed', return_value=True), \
|
||||||
|
patch('lib.bird_x.is_bird_authenticated', return_value=None):
|
||||||
status = env.get_x_source_status(config)
|
status = env.get_x_source_status(config)
|
||||||
|
|
||||||
self.assertIsNone(status['source'])
|
self.assertIsNone(status['source'])
|
||||||
|
|||||||
@@ -0,0 +1,249 @@
|
|||||||
|
"""Tests for Exa Search module."""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
|
||||||
|
# Ensure scripts/ is on path
|
||||||
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts'))
|
||||||
|
|
||||||
|
# Force clean config mode for tests
|
||||||
|
os.environ['LAST30DAYS_CONFIG_DIR'] = ''
|
||||||
|
|
||||||
|
from lib.exa_search import search_web, _normalize_results, _parse_exa_date, EXCLUDED_DOMAINS
|
||||||
|
from lib import env, http
|
||||||
|
|
||||||
|
|
||||||
|
class TestNormalizeResults(unittest.TestCase):
|
||||||
|
"""Test result normalization from Exa API responses."""
|
||||||
|
|
||||||
|
def test_valid_results(self):
|
||||||
|
response = {
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"title": "AI Trends in 2026",
|
||||||
|
"url": "https://blog.example.com/ai-trends",
|
||||||
|
"text": "This article covers the latest AI trends...",
|
||||||
|
"publishedDate": "2026-03-15T00:00:00.000Z",
|
||||||
|
"score": 0.85,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Machine Learning Updates",
|
||||||
|
"url": "https://ml.example.com/updates",
|
||||||
|
"text": "New ML frameworks released this month...",
|
||||||
|
"publishedDate": "2026-03-10T12:30:00.000Z",
|
||||||
|
"score": 0.72,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
items = _normalize_results(response)
|
||||||
|
self.assertEqual(len(items), 2)
|
||||||
|
|
||||||
|
# Check first item structure
|
||||||
|
item = items[0]
|
||||||
|
self.assertEqual(item["title"], "AI Trends in 2026")
|
||||||
|
self.assertEqual(item["url"], "https://blog.example.com/ai-trends")
|
||||||
|
self.assertIn("AI trends", item["snippet"])
|
||||||
|
self.assertEqual(item["date"], "2026-03-15")
|
||||||
|
self.assertEqual(item["date_confidence"], "med")
|
||||||
|
self.assertAlmostEqual(item["relevance"], 0.85)
|
||||||
|
self.assertEqual(item["id"], "W1")
|
||||||
|
self.assertEqual(item["source_domain"], "blog.example.com")
|
||||||
|
|
||||||
|
def test_excludes_reddit_and_x(self):
|
||||||
|
response = {
|
||||||
|
"results": [
|
||||||
|
{"title": "Reddit post", "url": "https://www.reddit.com/r/test/123", "text": "Post"},
|
||||||
|
{"title": "X post", "url": "https://x.com/user/status/456", "text": "Tweet"},
|
||||||
|
{"title": "Good result", "url": "https://example.com/article", "text": "Content"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
items = _normalize_results(response)
|
||||||
|
self.assertEqual(len(items), 1)
|
||||||
|
self.assertEqual(items[0]["title"], "Good result")
|
||||||
|
|
||||||
|
def test_empty_results(self):
|
||||||
|
items = _normalize_results({"results": []})
|
||||||
|
self.assertEqual(items, [])
|
||||||
|
|
||||||
|
def test_missing_results_key(self):
|
||||||
|
items = _normalize_results({})
|
||||||
|
self.assertEqual(items, [])
|
||||||
|
|
||||||
|
def test_skips_items_without_url(self):
|
||||||
|
response = {
|
||||||
|
"results": [
|
||||||
|
{"title": "No URL", "text": "Content"},
|
||||||
|
{"title": "Has URL", "url": "https://example.com/a", "text": "Content"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
items = _normalize_results(response)
|
||||||
|
self.assertEqual(len(items), 1)
|
||||||
|
|
||||||
|
def test_skips_items_without_title_and_snippet(self):
|
||||||
|
response = {
|
||||||
|
"results": [
|
||||||
|
{"url": "https://example.com/a", "title": "", "text": ""},
|
||||||
|
{"url": "https://example.com/b", "title": "Valid", "text": "Content"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
items = _normalize_results(response)
|
||||||
|
self.assertEqual(len(items), 1)
|
||||||
|
|
||||||
|
def test_no_date_gives_low_confidence(self):
|
||||||
|
response = {
|
||||||
|
"results": [
|
||||||
|
{"title": "No date", "url": "https://example.com/a", "text": "Content"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
items = _normalize_results(response)
|
||||||
|
self.assertEqual(items[0]["date"], None)
|
||||||
|
self.assertEqual(items[0]["date_confidence"], "low")
|
||||||
|
|
||||||
|
def test_truncates_long_fields(self):
|
||||||
|
response = {
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"title": "T" * 300,
|
||||||
|
"url": "https://example.com/a",
|
||||||
|
"text": "S" * 1000,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
items = _normalize_results(response)
|
||||||
|
self.assertLessEqual(len(items[0]["title"]), 200)
|
||||||
|
self.assertLessEqual(len(items[0]["snippet"]), 500)
|
||||||
|
|
||||||
|
def test_relevance_clamped(self):
|
||||||
|
response = {
|
||||||
|
"results": [
|
||||||
|
{"title": "High", "url": "https://example.com/a", "text": "C", "score": 1.5},
|
||||||
|
{"title": "Low", "url": "https://example.com/b", "text": "C", "score": -0.5},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
items = _normalize_results(response)
|
||||||
|
self.assertEqual(items[0]["relevance"], 1.0)
|
||||||
|
self.assertEqual(items[1]["relevance"], 0.0)
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseExaDate(unittest.TestCase):
|
||||||
|
def test_iso_datetime(self):
|
||||||
|
self.assertEqual(_parse_exa_date("2026-03-15T00:00:00.000Z"), "2026-03-15")
|
||||||
|
|
||||||
|
def test_date_only(self):
|
||||||
|
self.assertEqual(_parse_exa_date("2026-03-15"), "2026-03-15")
|
||||||
|
|
||||||
|
def test_none(self):
|
||||||
|
self.assertIsNone(_parse_exa_date(None))
|
||||||
|
|
||||||
|
def test_empty_string(self):
|
||||||
|
self.assertIsNone(_parse_exa_date(""))
|
||||||
|
|
||||||
|
|
||||||
|
class TestSearchWebIntegration(unittest.TestCase):
|
||||||
|
"""Test search_web function with mocked HTTP calls."""
|
||||||
|
|
||||||
|
@patch("lib.exa_search.http.post")
|
||||||
|
def test_valid_search(self, mock_post):
|
||||||
|
mock_post.return_value = {
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"title": "Test Result",
|
||||||
|
"url": "https://example.com/test",
|
||||||
|
"text": "Test content here",
|
||||||
|
"publishedDate": "2026-03-20T00:00:00.000Z",
|
||||||
|
"score": 0.9,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-api-key")
|
||||||
|
self.assertEqual(len(results), 1)
|
||||||
|
self.assertEqual(results[0]["title"], "Test Result")
|
||||||
|
self.assertEqual(results[0]["url"], "https://example.com/test")
|
||||||
|
self.assertEqual(results[0]["snippet"], "Test content here")
|
||||||
|
|
||||||
|
# Verify API call
|
||||||
|
mock_post.assert_called_once()
|
||||||
|
call_args = mock_post.call_args
|
||||||
|
self.assertEqual(call_args[0][0], "https://api.exa.ai/search")
|
||||||
|
self.assertEqual(call_args[1]["headers"]["x-api-key"], "test-api-key")
|
||||||
|
|
||||||
|
@patch("lib.exa_search.http.post")
|
||||||
|
def test_401_invalid_key(self, mock_post):
|
||||||
|
mock_post.side_effect = http.HTTPError("HTTP 401: Unauthorized", status_code=401)
|
||||||
|
results = search_web("AI news", "2026-03-01", "2026-03-29", "bad-key")
|
||||||
|
self.assertEqual(results, [])
|
||||||
|
|
||||||
|
@patch("lib.exa_search.http.post")
|
||||||
|
def test_429_rate_limit(self, mock_post):
|
||||||
|
mock_post.side_effect = http.HTTPError("HTTP 429: Too Many Requests", status_code=429)
|
||||||
|
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-key")
|
||||||
|
self.assertEqual(results, [])
|
||||||
|
|
||||||
|
@patch("lib.exa_search.http.post")
|
||||||
|
def test_empty_results(self, mock_post):
|
||||||
|
mock_post.return_value = {"results": []}
|
||||||
|
results = search_web("obscure query", "2026-03-01", "2026-03-29", "test-key")
|
||||||
|
self.assertEqual(results, [])
|
||||||
|
|
||||||
|
@patch("lib.exa_search.http.post")
|
||||||
|
def test_network_timeout(self, mock_post):
|
||||||
|
mock_post.side_effect = http.HTTPError("Connection error: TimeoutError: timed out")
|
||||||
|
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-key")
|
||||||
|
self.assertEqual(results, [])
|
||||||
|
|
||||||
|
@patch("lib.exa_search.http.post")
|
||||||
|
def test_generic_exception(self, mock_post):
|
||||||
|
mock_post.side_effect = Exception("Something unexpected")
|
||||||
|
results = search_web("AI news", "2026-03-01", "2026-03-29", "test-key")
|
||||||
|
self.assertEqual(results, [])
|
||||||
|
|
||||||
|
|
||||||
|
class TestEnvExaPriority(unittest.TestCase):
|
||||||
|
"""Test that Exa is prioritized correctly in env.py."""
|
||||||
|
|
||||||
|
def test_no_exa_key_not_selected(self):
|
||||||
|
config = {}
|
||||||
|
self.assertIsNone(env.get_web_search_source(config))
|
||||||
|
|
||||||
|
def test_exa_key_selected(self):
|
||||||
|
config = {"EXA_API_KEY": "exa-test-key"}
|
||||||
|
self.assertEqual(env.get_web_search_source(config), "exa")
|
||||||
|
|
||||||
|
def test_exa_takes_priority_over_brave(self):
|
||||||
|
config = {"EXA_API_KEY": "exa-key", "BRAVE_API_KEY": "brave-key"}
|
||||||
|
self.assertEqual(env.get_web_search_source(config), "exa")
|
||||||
|
|
||||||
|
def test_exa_takes_priority_over_parallel(self):
|
||||||
|
config = {"EXA_API_KEY": "exa-key", "PARALLEL_API_KEY": "parallel-key"}
|
||||||
|
self.assertEqual(env.get_web_search_source(config), "exa")
|
||||||
|
|
||||||
|
def test_exa_takes_priority_over_openrouter(self):
|
||||||
|
config = {"EXA_API_KEY": "exa-key", "OPENROUTER_API_KEY": "or-key"}
|
||||||
|
self.assertEqual(env.get_web_search_source(config), "exa")
|
||||||
|
|
||||||
|
def test_exa_takes_priority_over_all(self):
|
||||||
|
config = {
|
||||||
|
"EXA_API_KEY": "exa-key",
|
||||||
|
"PARALLEL_API_KEY": "parallel-key",
|
||||||
|
"BRAVE_API_KEY": "brave-key",
|
||||||
|
"OPENROUTER_API_KEY": "or-key",
|
||||||
|
}
|
||||||
|
self.assertEqual(env.get_web_search_source(config), "exa")
|
||||||
|
|
||||||
|
def test_fallback_to_parallel_without_exa(self):
|
||||||
|
config = {"PARALLEL_API_KEY": "parallel-key", "BRAVE_API_KEY": "brave-key"}
|
||||||
|
self.assertEqual(env.get_web_search_source(config), "parallel")
|
||||||
|
|
||||||
|
def test_has_web_search_keys_with_exa(self):
|
||||||
|
config = {"EXA_API_KEY": "exa-key"}
|
||||||
|
self.assertTrue(env.has_web_search_keys(config))
|
||||||
|
|
||||||
|
def test_has_web_search_keys_without_any(self):
|
||||||
|
config = {}
|
||||||
|
self.assertFalse(env.has_web_search_keys(config))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,411 @@
|
|||||||
|
"""End-to-end NUX integration tests.
|
||||||
|
|
||||||
|
Verifies that setup wizard, status banner, quality nudge, and SKILL.md
|
||||||
|
first-run flow work together coherently:
|
||||||
|
- first_run flag emitted / not emitted based on SETUP_COMPLETE
|
||||||
|
- setup subcommand runs auto_setup and writes config
|
||||||
|
- quality score is consistent with source configuration
|
||||||
|
- quality nudge disappears at 100%
|
||||||
|
- banner and quality nudge don't contradict each other
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# Add scripts dir to path
|
||||||
|
SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
|
||||||
|
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||||
|
|
||||||
|
from lib import setup_wizard, quality_nudge, ui
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _base_config(**overrides):
|
||||||
|
"""Return a minimal config dict."""
|
||||||
|
config = {
|
||||||
|
"AUTH_TOKEN": None,
|
||||||
|
"CT0": None,
|
||||||
|
"XAI_API_KEY": None,
|
||||||
|
"SCRAPECREATORS_API_KEY": None,
|
||||||
|
}
|
||||||
|
config.update(overrides)
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def _base_results(**overrides):
|
||||||
|
"""Return a minimal research_results dict with no errors."""
|
||||||
|
results = {
|
||||||
|
"x_error": None,
|
||||||
|
"youtube_error": None,
|
||||||
|
"reddit_error": None,
|
||||||
|
}
|
||||||
|
results.update(overrides)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _base_diag(**overrides):
|
||||||
|
"""Return a minimal diag dict for banner testing."""
|
||||||
|
diag = {
|
||||||
|
"setup_complete": False,
|
||||||
|
"reddit_source": None,
|
||||||
|
"x_source": None,
|
||||||
|
"x_method": None,
|
||||||
|
"youtube": False,
|
||||||
|
"tiktok": False,
|
||||||
|
"instagram": False,
|
||||||
|
"hackernews": True,
|
||||||
|
"polymarket": True,
|
||||||
|
"bluesky": False,
|
||||||
|
"truthsocial": False,
|
||||||
|
"xiaohongshu": False,
|
||||||
|
"scrapecreators": False,
|
||||||
|
"web_search_backend": None,
|
||||||
|
}
|
||||||
|
diag.update(overrides)
|
||||||
|
return diag
|
||||||
|
|
||||||
|
|
||||||
|
def _compute(config_overrides=None, result_overrides=None, ytdlp_installed=False):
|
||||||
|
"""Helper to call compute_quality_score with mocked yt-dlp check."""
|
||||||
|
from lib import youtube_yt
|
||||||
|
|
||||||
|
config = _base_config(**(config_overrides or {}))
|
||||||
|
results = _base_results(**(result_overrides or {}))
|
||||||
|
|
||||||
|
with patch.object(youtube_yt, "is_ytdlp_installed", return_value=ytdlp_installed):
|
||||||
|
return quality_nudge.compute_quality_score(config, results)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# First-run flag detection
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestFirstRunDetection:
|
||||||
|
"""first_run flag is emitted based on SETUP_COMPLETE in config."""
|
||||||
|
|
||||||
|
def test_first_run_when_setup_not_complete(self):
|
||||||
|
"""SETUP_COMPLETE missing -> is_first_run returns True."""
|
||||||
|
config = _base_config()
|
||||||
|
assert setup_wizard.is_first_run(config) is True
|
||||||
|
|
||||||
|
def test_first_run_when_setup_complete_empty(self):
|
||||||
|
"""SETUP_COMPLETE='' -> is_first_run returns True."""
|
||||||
|
config = _base_config(SETUP_COMPLETE="")
|
||||||
|
assert setup_wizard.is_first_run(config) is True
|
||||||
|
|
||||||
|
def test_not_first_run_when_setup_complete(self):
|
||||||
|
"""SETUP_COMPLETE=true -> is_first_run returns False."""
|
||||||
|
config = _base_config(SETUP_COMPLETE="true")
|
||||||
|
assert setup_wizard.is_first_run(config) is False
|
||||||
|
|
||||||
|
def test_not_first_run_any_truthy_value(self):
|
||||||
|
"""SETUP_COMPLETE=1 -> is_first_run returns False."""
|
||||||
|
config = _base_config(SETUP_COMPLETE="1")
|
||||||
|
assert setup_wizard.is_first_run(config) is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Setup subcommand writes config
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestSetupSubcommandWritesConfig:
|
||||||
|
"""setup subcommand runs auto_setup and writes SETUP_COMPLETE."""
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_auto_setup_and_write(self, mock_which, mock_extract):
|
||||||
|
"""run_auto_setup + write_setup_config creates valid config."""
|
||||||
|
mock_extract.return_value = ({"auth_token": "abc", "ct0": "xyz"}, "chrome")
|
||||||
|
mock_which.return_value = "/usr/local/bin/yt-dlp"
|
||||||
|
|
||||||
|
config = _base_config()
|
||||||
|
results = setup_wizard.run_auto_setup(config)
|
||||||
|
|
||||||
|
assert results["cookies_found"]["x"] == "chrome"
|
||||||
|
assert results["ytdlp_installed"] is True
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / ".env"
|
||||||
|
written = setup_wizard.write_setup_config(env_path)
|
||||||
|
assert written is True
|
||||||
|
|
||||||
|
content = env_path.read_text()
|
||||||
|
assert "SETUP_COMPLETE=true" in content
|
||||||
|
assert "FROM_BROWSER=auto" in content
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_after_setup_not_first_run(self, mock_which, mock_extract):
|
||||||
|
"""After write_setup_config, is_first_run should return False."""
|
||||||
|
mock_extract.return_value = None
|
||||||
|
mock_which.return_value = None
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / ".env"
|
||||||
|
setup_wizard.write_setup_config(env_path)
|
||||||
|
|
||||||
|
# Simulate reading the config back
|
||||||
|
config = {"SETUP_COMPLETE": "true"}
|
||||||
|
assert setup_wizard.is_first_run(config) is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Quality score consistency with source config
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestQualityScoreConsistency:
|
||||||
|
"""Quality score matches the source configuration state."""
|
||||||
|
|
||||||
|
def test_zero_config_is_40_pct(self):
|
||||||
|
"""No X, no yt-dlp, no SC -> 40% (HN + Polymarket only)."""
|
||||||
|
q = _compute()
|
||||||
|
assert q["score_pct"] == 40
|
||||||
|
assert set(q["core_active"]) == {"hn", "polymarket"}
|
||||||
|
|
||||||
|
def test_x_cookies_is_60_pct(self):
|
||||||
|
"""X cookies active -> 60%."""
|
||||||
|
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
|
||||||
|
assert q["score_pct"] == 60
|
||||||
|
assert "x" in q["core_active"]
|
||||||
|
|
||||||
|
def test_x_plus_ytdlp_is_80_pct(self):
|
||||||
|
"""X cookies + yt-dlp -> 80%."""
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["score_pct"] == 80
|
||||||
|
assert "x" in q["core_active"]
|
||||||
|
assert "youtube" in q["core_active"]
|
||||||
|
|
||||||
|
def test_full_config_is_100_pct(self):
|
||||||
|
"""X + yt-dlp + ScrapeCreators -> 100%."""
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["score_pct"] == 100
|
||||||
|
assert len(q["core_active"]) == 5
|
||||||
|
|
||||||
|
def test_xai_key_also_enables_x(self):
|
||||||
|
"""XAI_API_KEY activates X source same as cookies."""
|
||||||
|
q = _compute(config_overrides={"XAI_API_KEY": "xai_key"})
|
||||||
|
assert q["score_pct"] == 60
|
||||||
|
assert "x" in q["core_active"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Quality nudge disappears at 100%
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestQualityNudgeDisappears:
|
||||||
|
"""Quality nudge is None at 100% and present below 100%."""
|
||||||
|
|
||||||
|
def test_nudge_present_at_40(self):
|
||||||
|
q = _compute()
|
||||||
|
assert q["nudge_text"] is not None
|
||||||
|
assert len(q["nudge_text"]) > 0
|
||||||
|
|
||||||
|
def test_nudge_present_at_60(self):
|
||||||
|
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
|
||||||
|
assert q["nudge_text"] is not None
|
||||||
|
|
||||||
|
def test_nudge_present_at_80(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["nudge_text"] is not None
|
||||||
|
|
||||||
|
def test_nudge_none_at_100(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["nudge_text"] is None
|
||||||
|
assert q["core_missing"] == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Banner and quality nudge consistency
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestBannerQualityNudgeConsistency:
|
||||||
|
"""Banner source count and quality nudge percentage don't contradict."""
|
||||||
|
|
||||||
|
def test_zero_config_banner_3_sources_nudge_40(self):
|
||||||
|
"""Banner shows 3 sources (Reddit, HN, PM), nudge says 40%."""
|
||||||
|
diag = _base_diag()
|
||||||
|
banner = "\n".join(ui._build_status_banner(diag))
|
||||||
|
|
||||||
|
# Banner shows 3 active sources
|
||||||
|
assert "Reddit (threads only)" in banner
|
||||||
|
assert "HN" in banner
|
||||||
|
assert "Polymarket" in banner
|
||||||
|
# X and YouTube should NOT be in the banner
|
||||||
|
assert "X (" not in banner
|
||||||
|
assert "YouTube" not in banner
|
||||||
|
|
||||||
|
# Quality nudge is 40%
|
||||||
|
q = _compute()
|
||||||
|
assert q["score_pct"] == 40
|
||||||
|
|
||||||
|
def test_after_wizard_banner_5_sources_nudge_80(self):
|
||||||
|
"""After wizard (X cookies + yt-dlp): banner shows 5+ sources, nudge ~80%."""
|
||||||
|
diag = _base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-chrome",
|
||||||
|
youtube=True,
|
||||||
|
)
|
||||||
|
banner = "\n".join(ui._build_status_banner(diag))
|
||||||
|
|
||||||
|
# Banner shows X and YouTube
|
||||||
|
assert "X (Chrome)" in banner
|
||||||
|
assert "YouTube" in banner
|
||||||
|
assert "Reddit (threads only)" in banner
|
||||||
|
assert "HN" in banner
|
||||||
|
assert "Polymarket" in banner
|
||||||
|
|
||||||
|
# Quality nudge is 80%
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["score_pct"] == 80
|
||||||
|
|
||||||
|
def test_full_config_banner_all_nudge_gone(self):
|
||||||
|
"""Full config: banner shows all sources, nudge disappears (100%)."""
|
||||||
|
diag = _base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source="scrapecreators",
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-chrome",
|
||||||
|
youtube=True,
|
||||||
|
tiktok=True,
|
||||||
|
instagram=True,
|
||||||
|
scrapecreators=True,
|
||||||
|
)
|
||||||
|
banner = "\n".join(ui._build_status_banner(diag))
|
||||||
|
|
||||||
|
assert "Reddit (with comments)" in banner
|
||||||
|
assert "X (Chrome)" in banner
|
||||||
|
assert "YouTube" in banner
|
||||||
|
assert "TikTok" in banner
|
||||||
|
assert "Instagram" in banner
|
||||||
|
|
||||||
|
# Quality nudge is 100% / gone
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["score_pct"] == 100
|
||||||
|
assert q["nudge_text"] is None
|
||||||
|
|
||||||
|
def test_banner_first_run_suggests_setup(self):
|
||||||
|
"""First-run banner suggests running setup wizard."""
|
||||||
|
diag = _base_diag(setup_complete=False)
|
||||||
|
banner = "\n".join(ui._build_status_banner(diag))
|
||||||
|
assert "First Run" in banner
|
||||||
|
assert "/last30days setup" in banner
|
||||||
|
|
||||||
|
def test_banner_partial_suggests_scrapecreators(self):
|
||||||
|
"""After setup, missing SC is the only recommendation."""
|
||||||
|
diag = _base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-chrome",
|
||||||
|
youtube=True,
|
||||||
|
scrapecreators=False,
|
||||||
|
)
|
||||||
|
banner = "\n".join(ui._build_status_banner(diag))
|
||||||
|
assert "SCRAPECREATORS_API_KEY" in banner
|
||||||
|
assert "100 free calls, no CC" in banner
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Progressive narrowing of nudge suggestions
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestProgressiveNarrowing:
|
||||||
|
"""As users configure more, nudge suggestions narrow."""
|
||||||
|
|
||||||
|
def test_baseline_mentions_all_three_missing(self):
|
||||||
|
q = _compute()
|
||||||
|
assert "X/Twitter" in q["nudge_text"]
|
||||||
|
assert "YouTube" in q["nudge_text"]
|
||||||
|
assert "Reddit with comments" in q["nudge_text"]
|
||||||
|
|
||||||
|
def test_with_x_mentions_yt_and_sc(self):
|
||||||
|
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
|
||||||
|
assert "X/Twitter" not in q["nudge_text"]
|
||||||
|
assert "YouTube" in q["nudge_text"]
|
||||||
|
assert "Reddit with comments" in q["nudge_text"]
|
||||||
|
|
||||||
|
def test_with_x_and_yt_mentions_sc_only(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert "X/Twitter" not in q["nudge_text"]
|
||||||
|
assert "YouTube" not in q["nudge_text"]
|
||||||
|
# SC nudge is present
|
||||||
|
assert "Reddit" in q["nudge_text"] or "ScrapeCreators" in q["nudge_text"].lower() or "scrapecreators" in q["nudge_text"]
|
||||||
|
|
||||||
|
def test_full_coverage_no_nudge(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["nudge_text"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Render quality nudge integration
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestRenderQualityNudge:
|
||||||
|
"""render_quality_nudge produces correct output."""
|
||||||
|
|
||||||
|
def test_render_at_80_pct(self):
|
||||||
|
from lib.render import render_quality_nudge
|
||||||
|
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
rendered = render_quality_nudge(q)
|
||||||
|
assert "80%" in rendered
|
||||||
|
assert "Research Coverage" in rendered
|
||||||
|
|
||||||
|
def test_render_at_100_pct_is_empty(self):
|
||||||
|
from lib.render import render_quality_nudge
|
||||||
|
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
rendered = render_quality_nudge(q)
|
||||||
|
assert rendered == ""
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
"""Tests for post-research quality score and upgrade nudge."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _base_config(**overrides):
|
||||||
|
"""Return a minimal config dict."""
|
||||||
|
config = {
|
||||||
|
"AUTH_TOKEN": None,
|
||||||
|
"CT0": None,
|
||||||
|
"XAI_API_KEY": None,
|
||||||
|
"SCRAPECREATORS_API_KEY": None,
|
||||||
|
}
|
||||||
|
config.update(overrides)
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def _base_results(**overrides):
|
||||||
|
"""Return a minimal research_results dict with no errors."""
|
||||||
|
results = {
|
||||||
|
"x_error": None,
|
||||||
|
"youtube_error": None,
|
||||||
|
"reddit_error": None,
|
||||||
|
}
|
||||||
|
results.update(overrides)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _compute(config_overrides=None, result_overrides=None, ytdlp_installed=False):
|
||||||
|
"""Helper to call compute_quality_score with mocked yt-dlp check."""
|
||||||
|
from scripts.lib.quality_nudge import compute_quality_score
|
||||||
|
from scripts.lib import youtube_yt
|
||||||
|
|
||||||
|
config = _base_config(**(config_overrides or {}))
|
||||||
|
results = _base_results(**(result_overrides or {}))
|
||||||
|
|
||||||
|
with patch.object(youtube_yt, "is_ytdlp_installed", return_value=ytdlp_installed):
|
||||||
|
return compute_quality_score(config, results)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestBaseline:
|
||||||
|
"""HN + Polymarket only (no X, no YT, no SC) -> 40%."""
|
||||||
|
|
||||||
|
def test_score_40(self):
|
||||||
|
q = _compute()
|
||||||
|
assert q["score_pct"] == 40
|
||||||
|
|
||||||
|
def test_active_sources(self):
|
||||||
|
q = _compute()
|
||||||
|
assert "hn" in q["core_active"]
|
||||||
|
assert "polymarket" in q["core_active"]
|
||||||
|
assert len(q["core_active"]) == 2
|
||||||
|
|
||||||
|
def test_missing_all_three(self):
|
||||||
|
q = _compute()
|
||||||
|
assert set(q["core_missing"]) == {"x", "youtube", "reddit_comments"}
|
||||||
|
|
||||||
|
def test_nudge_mentions_all_missing(self):
|
||||||
|
q = _compute()
|
||||||
|
assert q["nudge_text"] is not None
|
||||||
|
assert "X/Twitter" in q["nudge_text"]
|
||||||
|
assert "YouTube" in q["nudge_text"]
|
||||||
|
assert "Reddit with comments" in q["nudge_text"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestXCookies:
|
||||||
|
"""+X cookies -> 60%."""
|
||||||
|
|
||||||
|
def test_score_60(self):
|
||||||
|
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
|
||||||
|
assert q["score_pct"] == 60
|
||||||
|
|
||||||
|
def test_nudge_mentions_yt_and_sc(self):
|
||||||
|
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
|
||||||
|
assert "YouTube" in q["nudge_text"]
|
||||||
|
assert "Reddit with comments" in q["nudge_text"]
|
||||||
|
assert "X/Twitter" not in q["nudge_text"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestXPlusYtdlp:
|
||||||
|
"""+X + yt-dlp -> 80%."""
|
||||||
|
|
||||||
|
def test_score_80(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["score_pct"] == 80
|
||||||
|
|
||||||
|
def test_nudge_mentions_sc_only(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert "ScrapeCreators" in q["nudge_text"] or "scrapecreators" in q["nudge_text"]
|
||||||
|
assert "YouTube" not in q["nudge_text"]
|
||||||
|
assert "X/Twitter" not in q["nudge_text"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestFullCoverage:
|
||||||
|
"""+X + yt-dlp + SC -> 100%, no nudge."""
|
||||||
|
|
||||||
|
def test_score_100(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["score_pct"] == 100
|
||||||
|
|
||||||
|
def test_nudge_is_none(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["nudge_text"] is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestSCActiveNoX:
|
||||||
|
"""SC active but no X -> 80%, nudge suggests browser cookies (free)."""
|
||||||
|
|
||||||
|
def test_score_80(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"SCRAPECREATORS_API_KEY": "sc_key"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["score_pct"] == 80
|
||||||
|
|
||||||
|
def test_nudge_suggests_browser_cookies(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"SCRAPECREATORS_API_KEY": "sc_key"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["nudge_text"] is not None
|
||||||
|
assert "browser" in q["nudge_text"].lower()
|
||||||
|
assert "x.com" in q["nudge_text"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
class TestRedditErrored:
|
||||||
|
"""SC is configured but Reddit errored this run."""
|
||||||
|
|
||||||
|
def test_nudge_mentions_error(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
result_overrides={"reddit_error": "ScrapeCreators: 500 Internal Server Error"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert "reddit_comments" in q["core_errored"]
|
||||||
|
assert "errored" in q["nudge_text"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
class TestDisclaimerAlwaysPresent:
|
||||||
|
"""Nudge always includes no-affiliate disclaimer when present."""
|
||||||
|
|
||||||
|
def test_disclaimer_baseline(self):
|
||||||
|
q = _compute()
|
||||||
|
assert "no affiliation" in q["nudge_text"]
|
||||||
|
|
||||||
|
def test_disclaimer_partial(self):
|
||||||
|
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
|
||||||
|
assert "no affiliation" in q["nudge_text"]
|
||||||
|
|
||||||
|
def test_disclaimer_not_present_at_100(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={
|
||||||
|
"AUTH_TOKEN": "tok123",
|
||||||
|
"SCRAPECREATORS_API_KEY": "sc_key",
|
||||||
|
},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert q["nudge_text"] is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestSCNudgeContent:
|
||||||
|
"""SC nudge always includes '100 free API calls, no credit card'."""
|
||||||
|
|
||||||
|
def test_sc_nudge_content(self):
|
||||||
|
q = _compute()
|
||||||
|
assert "100 free API calls, no credit card" in q["nudge_text"]
|
||||||
|
|
||||||
|
def test_sc_nudge_content_when_only_missing_sc(self):
|
||||||
|
q = _compute(
|
||||||
|
config_overrides={"AUTH_TOKEN": "tok123"},
|
||||||
|
ytdlp_installed=True,
|
||||||
|
)
|
||||||
|
assert "100 free API calls, no credit card" in q["nudge_text"]
|
||||||
@@ -0,0 +1,330 @@
|
|||||||
|
"""Tests for scripts/lib/reddit_public.py — standalone Reddit public JSON search."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import urllib.error
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
|
||||||
|
# Ensure lib is importable
|
||||||
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
||||||
|
|
||||||
|
from lib import reddit_public
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures / helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _make_reddit_listing(posts):
|
||||||
|
"""Build a Reddit listing JSON structure from a list of post dicts."""
|
||||||
|
children = []
|
||||||
|
for p in posts:
|
||||||
|
children.append({
|
||||||
|
"kind": "t3",
|
||||||
|
"data": {
|
||||||
|
"title": p.get("title", "Test Post"),
|
||||||
|
"permalink": p.get("permalink", "/r/test/comments/abc123/test_post/"),
|
||||||
|
"subreddit": p.get("subreddit", "test"),
|
||||||
|
"score": p.get("score", 42),
|
||||||
|
"num_comments": p.get("num_comments", 10),
|
||||||
|
"created_utc": p.get("created_utc", 1711670400), # 2024-03-29
|
||||||
|
"author": p.get("author", "testuser"),
|
||||||
|
"selftext": p.get("selftext", "Some body text"),
|
||||||
|
"upvote_ratio": p.get("upvote_ratio", 0.95),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return {"data": {"children": children}}
|
||||||
|
|
||||||
|
|
||||||
|
SAMPLE_LISTING = _make_reddit_listing([
|
||||||
|
{
|
||||||
|
"title": "Claude Code is amazing",
|
||||||
|
"permalink": "/r/ClaudeAI/comments/abc123/claude_code_is_amazing/",
|
||||||
|
"subreddit": "ClaudeAI",
|
||||||
|
"score": 250,
|
||||||
|
"num_comments": 45,
|
||||||
|
"created_utc": 1711670400,
|
||||||
|
"author": "ai_fan",
|
||||||
|
"selftext": "I've been using Claude Code for a week and it changed my workflow.",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Tips for Claude Code prompting",
|
||||||
|
"permalink": "/r/ClaudeAI/comments/def456/tips_for_claude_code/",
|
||||||
|
"subreddit": "ClaudeAI",
|
||||||
|
"score": 120,
|
||||||
|
"num_comments": 22,
|
||||||
|
"created_utc": 1711584000,
|
||||||
|
"author": "prompt_engineer",
|
||||||
|
"selftext": "Here are my top tips for getting the most out of Claude Code.",
|
||||||
|
},
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_urlopen_ok(listing_data):
|
||||||
|
"""Return a context-manager mock for urllib.request.urlopen that returns listing_data."""
|
||||||
|
resp = mock.MagicMock()
|
||||||
|
resp.read.return_value = json.dumps(listing_data).encode("utf-8")
|
||||||
|
resp.headers = {"Content-Type": "application/json"}
|
||||||
|
resp.__enter__ = mock.MagicMock(return_value=resp)
|
||||||
|
resp.__exit__ = mock.MagicMock(return_value=False)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestSearchReturnsCorrectFields:
|
||||||
|
"""Search query returns parsed results with correct fields."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_search_returns_parsed_results(self, mock_urlopen):
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
|
||||||
|
|
||||||
|
results = reddit_public.search("Claude Code", depth="quick")
|
||||||
|
|
||||||
|
assert len(results) == 2
|
||||||
|
first = results[0]
|
||||||
|
|
||||||
|
# Check all required fields exist
|
||||||
|
assert "id" in first
|
||||||
|
assert "title" in first
|
||||||
|
assert "url" in first
|
||||||
|
assert "score" in first
|
||||||
|
assert "num_comments" in first
|
||||||
|
assert "subreddit" in first
|
||||||
|
assert "created_utc" in first
|
||||||
|
assert "author" in first
|
||||||
|
assert "selftext" in first
|
||||||
|
|
||||||
|
# Check values
|
||||||
|
assert first["title"] == "Claude Code is amazing"
|
||||||
|
assert first["subreddit"] == "ClaudeAI"
|
||||||
|
assert first["score"] == 250
|
||||||
|
assert first["num_comments"] == 45
|
||||||
|
assert first["author"] == "ai_fan"
|
||||||
|
assert "/comments/" in first["url"]
|
||||||
|
assert first["id"] == "R1"
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_search_includes_normalized_fields(self, mock_urlopen):
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
|
||||||
|
|
||||||
|
results = reddit_public.search("Claude Code")
|
||||||
|
first = results[0]
|
||||||
|
|
||||||
|
# Normalized fields matching ScrapeCreators format
|
||||||
|
assert "date" in first
|
||||||
|
assert "engagement" in first
|
||||||
|
assert "relevance" in first
|
||||||
|
assert "why_relevant" in first
|
||||||
|
|
||||||
|
assert isinstance(first["engagement"], dict)
|
||||||
|
assert "score" in first["engagement"]
|
||||||
|
assert "num_comments" in first["engagement"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestSubredditScopedSearch:
|
||||||
|
"""Subreddit-scoped search works."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_subreddit_search_builds_correct_url(self, mock_urlopen):
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
|
||||||
|
|
||||||
|
reddit_public.search("Claude Code", subreddit="ClaudeAI")
|
||||||
|
|
||||||
|
# Check the URL passed to urlopen
|
||||||
|
call_args = mock_urlopen.call_args
|
||||||
|
req = call_args[0][0] # First positional arg is the Request object
|
||||||
|
assert "/r/ClaudeAI/search.json" in req.full_url
|
||||||
|
assert "restrict_sr=on" in req.full_url
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_subreddit_search_strips_prefix(self, mock_urlopen):
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
|
||||||
|
|
||||||
|
reddit_public.search("Claude Code", subreddit="r/ClaudeAI")
|
||||||
|
|
||||||
|
req = mock_urlopen.call_args[0][0]
|
||||||
|
# Should strip the r/ prefix, not double it
|
||||||
|
assert "/r/ClaudeAI/search.json" in req.full_url
|
||||||
|
assert "/r/r/" not in req.full_url
|
||||||
|
|
||||||
|
|
||||||
|
class TestRetryOn429:
|
||||||
|
"""429 response triggers retries, eventually returns partial results."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.time.sleep")
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_429_retries_then_returns_empty(self, mock_urlopen, mock_sleep):
|
||||||
|
error = urllib.error.HTTPError(
|
||||||
|
"https://reddit.com/search.json", 429, "Too Many Requests",
|
||||||
|
{"Retry-After": "1"}, None,
|
||||||
|
)
|
||||||
|
mock_urlopen.side_effect = error
|
||||||
|
|
||||||
|
results = reddit_public.search("test query")
|
||||||
|
|
||||||
|
assert results == []
|
||||||
|
# Should have retried (MAX_RETRIES = 3, sleeps happen between attempts)
|
||||||
|
assert mock_sleep.call_count == reddit_public.MAX_RETRIES - 1
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.time.sleep")
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_429_then_success(self, mock_urlopen, mock_sleep):
|
||||||
|
error = urllib.error.HTTPError(
|
||||||
|
"https://reddit.com/search.json", 429, "Too Many Requests",
|
||||||
|
{}, None,
|
||||||
|
)
|
||||||
|
success_resp = _mock_urlopen_ok(SAMPLE_LISTING)
|
||||||
|
|
||||||
|
mock_urlopen.side_effect = [error, success_resp]
|
||||||
|
|
||||||
|
results = reddit_public.search("test query")
|
||||||
|
|
||||||
|
assert len(results) == 2
|
||||||
|
assert mock_sleep.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestHtmlAntiBot:
|
||||||
|
"""HTML response (anti-bot) is detected, returns empty."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_html_response_returns_empty(self, mock_urlopen):
|
||||||
|
resp = mock.MagicMock()
|
||||||
|
resp.read.return_value = b"<html><body>Please verify you are human</body></html>"
|
||||||
|
resp.headers = {"Content-Type": "text/html; charset=utf-8"}
|
||||||
|
resp.__enter__ = mock.MagicMock(return_value=resp)
|
||||||
|
resp.__exit__ = mock.MagicMock(return_value=False)
|
||||||
|
mock_urlopen.return_value = resp
|
||||||
|
|
||||||
|
results = reddit_public.search("test query")
|
||||||
|
|
||||||
|
assert results == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestNetworkTimeout:
|
||||||
|
"""Network timeout returns empty."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_timeout_returns_empty(self, mock_urlopen):
|
||||||
|
mock_urlopen.side_effect = TimeoutError("Connection timed out")
|
||||||
|
|
||||||
|
results = reddit_public.search("test query")
|
||||||
|
|
||||||
|
assert results == []
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_url_error_returns_empty(self, mock_urlopen):
|
||||||
|
mock_urlopen.side_effect = urllib.error.URLError("Connection refused")
|
||||||
|
|
||||||
|
results = reddit_public.search("test query")
|
||||||
|
|
||||||
|
assert results == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestNormalizationMatchesScrapeCreators:
|
||||||
|
"""Results normalize to same schema as ScrapeCreators (field names match)."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_field_names_match_scrapecreators(self, mock_urlopen):
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
|
||||||
|
|
||||||
|
results = reddit_public.search("Claude Code")
|
||||||
|
assert len(results) > 0
|
||||||
|
item = results[0]
|
||||||
|
|
||||||
|
# These fields must exist to match ScrapeCreators _normalize_post output
|
||||||
|
sc_required_fields = {"id", "title", "url", "subreddit", "date",
|
||||||
|
"engagement", "relevance", "why_relevant"}
|
||||||
|
actual_fields = set(item.keys())
|
||||||
|
assert sc_required_fields.issubset(actual_fields), (
|
||||||
|
f"Missing fields: {sc_required_fields - actual_fields}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Engagement sub-fields
|
||||||
|
eng = item["engagement"]
|
||||||
|
assert "score" in eng
|
||||||
|
assert "num_comments" in eng
|
||||||
|
assert "upvote_ratio" in eng
|
||||||
|
|
||||||
|
|
||||||
|
class TestDepthLimits:
|
||||||
|
"""Depth-aware limits are respected."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_quick_limit(self, mock_urlopen):
|
||||||
|
# Create more posts than the quick limit
|
||||||
|
many_posts = _make_reddit_listing([
|
||||||
|
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/"}
|
||||||
|
for i in range(20)
|
||||||
|
])
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(many_posts)
|
||||||
|
|
||||||
|
results = reddit_public.search("test", depth="quick")
|
||||||
|
assert len(results) <= 10
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_default_limit(self, mock_urlopen):
|
||||||
|
many_posts = _make_reddit_listing([
|
||||||
|
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/"}
|
||||||
|
for i in range(50)
|
||||||
|
])
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(many_posts)
|
||||||
|
|
||||||
|
results = reddit_public.search("test", depth="default")
|
||||||
|
assert len(results) <= 25
|
||||||
|
|
||||||
|
|
||||||
|
class TestSearchRedditPublicHighLevel:
|
||||||
|
"""Test the high-level search_reddit_public interface."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_date_filtering(self, mock_urlopen):
|
||||||
|
listing = _make_reddit_listing([
|
||||||
|
{
|
||||||
|
"title": "In range",
|
||||||
|
"permalink": "/r/test/comments/aaa/in_range/",
|
||||||
|
"created_utc": 1711670400, # 2024-03-29
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Out of range",
|
||||||
|
"permalink": "/r/test/comments/bbb/out_of_range/",
|
||||||
|
"created_utc": 1609459200, # 2021-01-01
|
||||||
|
},
|
||||||
|
])
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(listing)
|
||||||
|
|
||||||
|
results = reddit_public.search_reddit_public(
|
||||||
|
"test", "2024-03-01", "2024-03-31"
|
||||||
|
)
|
||||||
|
|
||||||
|
titles = [r["title"] for r in results]
|
||||||
|
assert "In range" in titles
|
||||||
|
assert "Out of range" not in titles
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_user_agent_header(self, mock_urlopen):
|
||||||
|
mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING)
|
||||||
|
|
||||||
|
reddit_public.search("test")
|
||||||
|
|
||||||
|
req = mock_urlopen.call_args[0][0]
|
||||||
|
assert req.get_header("User-agent") == "last30days/3.0 (research tool)"
|
||||||
|
|
||||||
|
|
||||||
|
class TestMissingSubreddit:
|
||||||
|
"""Subreddit doesn't exist returns empty."""
|
||||||
|
|
||||||
|
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||||
|
def test_404_subreddit_returns_empty(self, mock_urlopen):
|
||||||
|
mock_urlopen.side_effect = urllib.error.HTTPError(
|
||||||
|
"https://reddit.com/r/nonexistent/search.json",
|
||||||
|
404, "Not Found", {}, None,
|
||||||
|
)
|
||||||
|
|
||||||
|
results = reddit_public.search("test", subreddit="nonexistent")
|
||||||
|
assert results == []
|
||||||
@@ -0,0 +1,219 @@
|
|||||||
|
"""Tests for Safari binary cookie extraction."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# Import the internal parser directly for testability (avoids platform check)
|
||||||
|
from scripts.lib.safari_cookies import (
|
||||||
|
_parse_binary_cookies,
|
||||||
|
extract_safari_cookies_macos,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_cookie_record(url: str, name: str, value: str, path: str = "/") -> bytes:
|
||||||
|
"""Build a single binary cookie record."""
|
||||||
|
# Fixed header: size(4) + flags(4) + padding(8) + url_off(4) + name_off(4) + path_off(4) + value_off(4) + comment(8) + expiry(8) + creation(8)
|
||||||
|
# Total fixed = 4 + 4 + 8 + 4 + 4 + 4 + 4 + 8 + 8 + 8 = 56 bytes
|
||||||
|
# String data starts at offset 56
|
||||||
|
|
||||||
|
url_b = url.encode("utf-8") + b"\x00"
|
||||||
|
name_b = name.encode("utf-8") + b"\x00"
|
||||||
|
path_b = path.encode("utf-8") + b"\x00"
|
||||||
|
value_b = value.encode("utf-8") + b"\x00"
|
||||||
|
|
||||||
|
str_offset_base = 56
|
||||||
|
url_offset = str_offset_base
|
||||||
|
name_offset = url_offset + len(url_b)
|
||||||
|
path_offset = name_offset + len(name_b)
|
||||||
|
value_offset = path_offset + len(path_b)
|
||||||
|
|
||||||
|
total_size = value_offset + len(value_b)
|
||||||
|
|
||||||
|
record = struct.pack("<I", total_size) # size
|
||||||
|
record += struct.pack("<I", 0) # flags
|
||||||
|
record += b"\x00" * 8 # padding/unknown
|
||||||
|
record += struct.pack("<I", url_offset) # url offset
|
||||||
|
record += struct.pack("<I", name_offset) # name offset
|
||||||
|
record += struct.pack("<I", path_offset) # path offset
|
||||||
|
record += struct.pack("<I", value_offset) # value offset
|
||||||
|
record += b"\x00" * 8 # comment (unused)
|
||||||
|
record += struct.pack("<d", 700000000.0) # expiry (Mac epoch)
|
||||||
|
record += struct.pack("<d", 690000000.0) # creation (Mac epoch)
|
||||||
|
record += url_b + name_b + path_b + value_b
|
||||||
|
|
||||||
|
return record
|
||||||
|
|
||||||
|
|
||||||
|
def _build_page(cookie_records: list[bytes]) -> bytes:
|
||||||
|
"""Build a binary cookies page from a list of cookie records."""
|
||||||
|
num_cookies = len(cookie_records)
|
||||||
|
|
||||||
|
# Page header: 4-byte marker + 4-byte cookie count + offset array
|
||||||
|
header_size = 4 + 4 + num_cookies * 4
|
||||||
|
# Also add 4 bytes for end-of-page marker
|
||||||
|
offsets_start = header_size
|
||||||
|
|
||||||
|
# Calculate offsets for each cookie record
|
||||||
|
offsets = []
|
||||||
|
current_offset = offsets_start
|
||||||
|
for rec in cookie_records:
|
||||||
|
offsets.append(current_offset)
|
||||||
|
current_offset += len(rec)
|
||||||
|
|
||||||
|
page = b"\x00\x00\x01\x00" # page header marker
|
||||||
|
page += struct.pack("<I", num_cookies)
|
||||||
|
for off in offsets:
|
||||||
|
page += struct.pack("<I", off)
|
||||||
|
for rec in cookie_records:
|
||||||
|
page += rec
|
||||||
|
|
||||||
|
return page
|
||||||
|
|
||||||
|
|
||||||
|
def _build_binary_cookies_file(pages: list[bytes]) -> bytes:
|
||||||
|
"""Build a complete Cookies.binarycookies file from pages."""
|
||||||
|
num_pages = len(pages)
|
||||||
|
|
||||||
|
data = b"cook" # magic
|
||||||
|
data += struct.pack(">I", num_pages) # page count (big-endian)
|
||||||
|
|
||||||
|
# Page sizes (big-endian)
|
||||||
|
for page in pages:
|
||||||
|
data += struct.pack(">I", len(page))
|
||||||
|
|
||||||
|
# Page data
|
||||||
|
for page in pages:
|
||||||
|
data += page
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def x_cookies_file() -> bytes:
|
||||||
|
"""Build a minimal valid binary cookies file with .x.com cookies."""
|
||||||
|
rec1 = _build_cookie_record(".x.com", "auth_token", "test_auth_abc123")
|
||||||
|
rec2 = _build_cookie_record(".x.com", "ct0", "test_ct0_xyz789")
|
||||||
|
rec3 = _build_cookie_record(".google.com", "NID", "google_nid_value")
|
||||||
|
page = _build_page([rec1, rec2, rec3])
|
||||||
|
return _build_binary_cookies_file([page])
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseValidCookies:
|
||||||
|
def test_extracts_matching_cookies(self, x_cookies_file: bytes):
|
||||||
|
result = _parse_binary_cookies(x_cookies_file, "x.com", ["auth_token", "ct0"])
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "test_auth_abc123"
|
||||||
|
assert result["ct0"] == "test_ct0_xyz789"
|
||||||
|
|
||||||
|
def test_ignores_other_domains(self, x_cookies_file: bytes):
|
||||||
|
result = _parse_binary_cookies(x_cookies_file, "google.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_partial_match_returns_found_only(self, x_cookies_file: bytes):
|
||||||
|
result = _parse_binary_cookies(
|
||||||
|
x_cookies_file, "x.com", ["auth_token", "nonexistent"]
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "test_auth_abc123"
|
||||||
|
assert "nonexistent" not in result
|
||||||
|
|
||||||
|
def test_no_matching_cookie_names(self, x_cookies_file: bytes):
|
||||||
|
result = _parse_binary_cookies(x_cookies_file, "x.com", ["bogus"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_domain_substring_match_with_leading_dot(self, x_cookies_file: bytes):
|
||||||
|
"""Domain '.x.com' in cookie should match search for 'x.com'."""
|
||||||
|
result = _parse_binary_cookies(x_cookies_file, "x.com", ["auth_token"])
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "test_auth_abc123"
|
||||||
|
|
||||||
|
|
||||||
|
class TestMultiplePages:
|
||||||
|
def test_cookies_across_pages(self):
|
||||||
|
rec1 = _build_cookie_record(".x.com", "auth_token", "page1_auth")
|
||||||
|
rec2 = _build_cookie_record(".x.com", "ct0", "page2_ct0")
|
||||||
|
page1 = _build_page([rec1])
|
||||||
|
page2 = _build_page([rec2])
|
||||||
|
data = _build_binary_cookies_file([page1, page2])
|
||||||
|
|
||||||
|
result = _parse_binary_cookies(data, "x.com", ["auth_token", "ct0"])
|
||||||
|
assert result is not None
|
||||||
|
assert result["auth_token"] == "page1_auth"
|
||||||
|
assert result["ct0"] == "page2_ct0"
|
||||||
|
|
||||||
|
|
||||||
|
class TestErrorPaths:
|
||||||
|
def test_file_not_found(self, tmp_path: Path):
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.safari_cookies.Path.home", return_value=tmp_path
|
||||||
|
), patch("scripts.lib.safari_cookies.sys") as mock_sys:
|
||||||
|
mock_sys.platform = "darwin"
|
||||||
|
mock_sys.stderr = sys.stderr
|
||||||
|
result = extract_safari_cookies_macos("x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_permission_denied(self, tmp_path: Path, capsys):
|
||||||
|
cookie_dir = tmp_path / "Library" / "Cookies"
|
||||||
|
cookie_dir.mkdir(parents=True)
|
||||||
|
cookie_file = cookie_dir / "Cookies.binarycookies"
|
||||||
|
cookie_file.write_bytes(b"cook")
|
||||||
|
cookie_file.chmod(0o000)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with patch(
|
||||||
|
"scripts.lib.safari_cookies.Path.home", return_value=tmp_path
|
||||||
|
), patch("scripts.lib.safari_cookies.sys") as mock_sys:
|
||||||
|
mock_sys.platform = "darwin"
|
||||||
|
mock_sys.stderr = sys.stderr
|
||||||
|
result = extract_safari_cookies_macos("x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
captured = capsys.readouterr()
|
||||||
|
assert "Full Disk Access" in captured.err
|
||||||
|
finally:
|
||||||
|
cookie_file.chmod(0o644)
|
||||||
|
|
||||||
|
def test_truncated_magic_only(self):
|
||||||
|
result = _parse_binary_cookies(b"cook", "x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_empty_file(self):
|
||||||
|
result = _parse_binary_cookies(b"", "x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_wrong_magic(self):
|
||||||
|
result = _parse_binary_cookies(b"notcook!", "x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_truncated_page_sizes(self):
|
||||||
|
# Header says 5 pages but data is too short
|
||||||
|
data = b"cook" + struct.pack(">I", 5) + b"\x00" * 4
|
||||||
|
result = _parse_binary_cookies(data, "x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_truncated_page_data(self):
|
||||||
|
# Valid header with 1 page of size 1000, but no actual page data
|
||||||
|
data = b"cook" + struct.pack(">I", 1) + struct.pack(">I", 1000)
|
||||||
|
result = _parse_binary_cookies(data, "x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_non_darwin_returns_none(self):
|
||||||
|
with patch("scripts.lib.safari_cookies.sys") as mock_sys:
|
||||||
|
mock_sys.platform = "linux"
|
||||||
|
result = extract_safari_cookies_macos("x.com", ["auth_token"])
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_garbage_data_no_crash(self):
|
||||||
|
"""Random bytes after valid magic should not crash."""
|
||||||
|
import os
|
||||||
|
|
||||||
|
data = b"cook" + os.urandom(200)
|
||||||
|
# Should not raise — may return None or a dict
|
||||||
|
result = _parse_binary_cookies(data, "x.com", ["auth_token"])
|
||||||
|
# Just verify no exception; result is either None or dict
|
||||||
|
assert result is None or isinstance(result, dict)
|
||||||
@@ -0,0 +1,342 @@
|
|||||||
|
"""Tests for the first-run setup wizard module."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# Add scripts dir to path
|
||||||
|
SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
|
||||||
|
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||||
|
|
||||||
|
from lib import setup_wizard
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsFirstRun:
|
||||||
|
"""Tests for is_first_run()."""
|
||||||
|
|
||||||
|
def test_first_run_when_setup_complete_not_set(self):
|
||||||
|
"""SETUP_COMPLETE not in config -> first run."""
|
||||||
|
config = {"AUTH_TOKEN": "abc", "CT0": "xyz"}
|
||||||
|
assert setup_wizard.is_first_run(config) is True
|
||||||
|
|
||||||
|
def test_first_run_when_setup_complete_is_none(self):
|
||||||
|
"""SETUP_COMPLETE=None -> first run."""
|
||||||
|
config = {"SETUP_COMPLETE": None}
|
||||||
|
assert setup_wizard.is_first_run(config) is True
|
||||||
|
|
||||||
|
def test_first_run_when_setup_complete_is_empty(self):
|
||||||
|
"""SETUP_COMPLETE="" -> first run."""
|
||||||
|
config = {"SETUP_COMPLETE": ""}
|
||||||
|
assert setup_wizard.is_first_run(config) is True
|
||||||
|
|
||||||
|
def test_not_first_run_when_setup_complete_true(self):
|
||||||
|
"""SETUP_COMPLETE=true -> not first run."""
|
||||||
|
config = {"SETUP_COMPLETE": "true"}
|
||||||
|
assert setup_wizard.is_first_run(config) is False
|
||||||
|
|
||||||
|
def test_not_first_run_when_setup_complete_any_value(self):
|
||||||
|
"""SETUP_COMPLETE set to any truthy value -> not first run."""
|
||||||
|
config = {"SETUP_COMPLETE": "yes"}
|
||||||
|
assert setup_wizard.is_first_run(config) is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestRunAutoSetup:
|
||||||
|
"""Tests for run_auto_setup()."""
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_cookies_found(self, mock_which, mock_extract):
|
||||||
|
"""When cookies are found, results dict includes them."""
|
||||||
|
mock_extract.return_value = ({"auth_token": "abc", "ct0": "xyz"}, "chrome")
|
||||||
|
mock_which.return_value = "/usr/local/bin/yt-dlp"
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
results = setup_wizard.run_auto_setup(config)
|
||||||
|
|
||||||
|
assert "x" in results["cookies_found"]
|
||||||
|
assert results["cookies_found"]["x"] == "chrome"
|
||||||
|
assert results["ytdlp_installed"] is True
|
||||||
|
assert results["ytdlp_action"] == "already_installed"
|
||||||
|
assert results["env_written"] is False
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_no_cookies_found(self, mock_which, mock_extract):
|
||||||
|
"""When no cookies found, results dict has empty cookies_found."""
|
||||||
|
mock_extract.return_value = None
|
||||||
|
mock_which.return_value = None
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
results = setup_wizard.run_auto_setup(config)
|
||||||
|
|
||||||
|
assert results["cookies_found"] == {}
|
||||||
|
assert results["ytdlp_installed"] is False
|
||||||
|
assert results["ytdlp_action"] == "no_homebrew"
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_cookie_extraction_exception(self, mock_which, mock_extract):
|
||||||
|
"""Cookie extraction raising an exception is handled gracefully."""
|
||||||
|
mock_extract.side_effect = Exception("DB locked")
|
||||||
|
mock_which.return_value = None
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
results = setup_wizard.run_auto_setup(config)
|
||||||
|
|
||||||
|
assert results["cookies_found"] == {}
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_multiple_sources(self, mock_which, mock_extract):
|
||||||
|
"""Multiple cookie sources can be found."""
|
||||||
|
def side_effect(browser, domain, cookie_names):
|
||||||
|
if domain == ".x.com":
|
||||||
|
return ({"auth_token": "abc", "ct0": "xyz"}, "firefox")
|
||||||
|
elif domain == ".truthsocial.com":
|
||||||
|
return ({"_session_id": "sess123"}, "firefox")
|
||||||
|
return None
|
||||||
|
|
||||||
|
mock_extract.side_effect = side_effect
|
||||||
|
mock_which.return_value = None
|
||||||
|
|
||||||
|
config = {}
|
||||||
|
results = setup_wizard.run_auto_setup(config)
|
||||||
|
|
||||||
|
assert results["cookies_found"]["x"] == "firefox"
|
||||||
|
assert results["cookies_found"]["truthsocial"] == "firefox"
|
||||||
|
|
||||||
|
|
||||||
|
class TestYtdlpAutoInstall:
|
||||||
|
"""Tests for yt-dlp auto-install via Homebrew in run_auto_setup()."""
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
|
||||||
|
@patch("subprocess.run")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_ytdlp_missing_brew_available_installs(self, mock_which, mock_subproc, mock_extract):
|
||||||
|
"""yt-dlp missing + brew available -> installs via brew."""
|
||||||
|
def which_side_effect(cmd):
|
||||||
|
if cmd == "yt-dlp":
|
||||||
|
return None
|
||||||
|
if cmd == "brew":
|
||||||
|
return "/opt/homebrew/bin/brew"
|
||||||
|
return None
|
||||||
|
mock_which.side_effect = which_side_effect
|
||||||
|
mock_subproc.return_value = MagicMock(returncode=0, stderr="")
|
||||||
|
|
||||||
|
results = setup_wizard.run_auto_setup({})
|
||||||
|
|
||||||
|
mock_subproc.assert_called_once_with(
|
||||||
|
["brew", "install", "yt-dlp"],
|
||||||
|
capture_output=True, text=True, timeout=120,
|
||||||
|
)
|
||||||
|
assert results["ytdlp_installed"] is True
|
||||||
|
assert results["ytdlp_action"] == "installed"
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_ytdlp_missing_brew_missing(self, mock_which, mock_extract):
|
||||||
|
"""yt-dlp missing + brew missing -> no_homebrew."""
|
||||||
|
mock_which.return_value = None
|
||||||
|
|
||||||
|
results = setup_wizard.run_auto_setup({})
|
||||||
|
|
||||||
|
assert results["ytdlp_installed"] is False
|
||||||
|
assert results["ytdlp_action"] == "no_homebrew"
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_ytdlp_already_installed(self, mock_which, mock_extract):
|
||||||
|
"""yt-dlp already installed -> already_installed."""
|
||||||
|
mock_which.return_value = "/usr/local/bin/yt-dlp"
|
||||||
|
|
||||||
|
results = setup_wizard.run_auto_setup({})
|
||||||
|
|
||||||
|
assert results["ytdlp_installed"] is True
|
||||||
|
assert results["ytdlp_action"] == "already_installed"
|
||||||
|
|
||||||
|
@patch("lib.cookie_extract.extract_cookies_with_source", return_value=None)
|
||||||
|
@patch("subprocess.run")
|
||||||
|
@patch("shutil.which")
|
||||||
|
def test_brew_install_fails(self, mock_which, mock_subproc, mock_extract):
|
||||||
|
"""brew install yt-dlp fails -> install_failed with stderr."""
|
||||||
|
def which_side_effect(cmd):
|
||||||
|
if cmd == "yt-dlp":
|
||||||
|
return None
|
||||||
|
if cmd == "brew":
|
||||||
|
return "/opt/homebrew/bin/brew"
|
||||||
|
return None
|
||||||
|
mock_which.side_effect = which_side_effect
|
||||||
|
mock_subproc.return_value = MagicMock(returncode=1, stderr="Error: something broke")
|
||||||
|
|
||||||
|
results = setup_wizard.run_auto_setup({})
|
||||||
|
|
||||||
|
assert results["ytdlp_installed"] is False
|
||||||
|
assert results["ytdlp_action"] == "install_failed"
|
||||||
|
assert "something broke" in results["ytdlp_stderr"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestWriteSetupConfig:
|
||||||
|
"""Tests for write_setup_config()."""
|
||||||
|
|
||||||
|
def test_creates_new_env_file(self):
|
||||||
|
"""Creates .env file with SETUP_COMPLETE and FROM_BROWSER."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / "subdir" / ".env"
|
||||||
|
|
||||||
|
result = setup_wizard.write_setup_config(env_path)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
assert env_path.exists()
|
||||||
|
content = env_path.read_text()
|
||||||
|
assert "SETUP_COMPLETE=true" in content
|
||||||
|
assert "FROM_BROWSER=auto" in content
|
||||||
|
|
||||||
|
def test_appends_to_existing_file(self):
|
||||||
|
"""Appends to existing .env without overwriting keys."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / ".env"
|
||||||
|
env_path.write_text("XAI_API_KEY=my-key\nAUTH_TOKEN=tok123\n")
|
||||||
|
|
||||||
|
result = setup_wizard.write_setup_config(env_path)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
content = env_path.read_text()
|
||||||
|
# Original keys preserved
|
||||||
|
assert "XAI_API_KEY=my-key" in content
|
||||||
|
assert "AUTH_TOKEN=tok123" in content
|
||||||
|
# New keys appended
|
||||||
|
assert "SETUP_COMPLETE=true" in content
|
||||||
|
assert "FROM_BROWSER=auto" in content
|
||||||
|
|
||||||
|
def test_does_not_overwrite_existing_keys(self):
|
||||||
|
"""If SETUP_COMPLETE or FROM_BROWSER already exist, don't duplicate."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / ".env"
|
||||||
|
env_path.write_text("SETUP_COMPLETE=true\nFROM_BROWSER=firefox\n")
|
||||||
|
|
||||||
|
result = setup_wizard.write_setup_config(env_path)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
content = env_path.read_text()
|
||||||
|
# Should only appear once
|
||||||
|
assert content.count("SETUP_COMPLETE") == 1
|
||||||
|
assert content.count("FROM_BROWSER") == 1
|
||||||
|
# Original value preserved
|
||||||
|
assert "FROM_BROWSER=firefox" in content
|
||||||
|
|
||||||
|
def test_custom_from_browser_value(self):
|
||||||
|
"""Custom from_browser value is written."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / ".env"
|
||||||
|
|
||||||
|
result = setup_wizard.write_setup_config(env_path, from_browser="chrome")
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
content = env_path.read_text()
|
||||||
|
assert "FROM_BROWSER=chrome" in content
|
||||||
|
|
||||||
|
def test_creates_parent_directories(self):
|
||||||
|
"""Creates parent directories if they don't exist."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / "a" / "b" / "c" / ".env"
|
||||||
|
|
||||||
|
result = setup_wizard.write_setup_config(env_path)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
assert env_path.exists()
|
||||||
|
|
||||||
|
def test_handles_file_without_trailing_newline(self):
|
||||||
|
"""Appends correctly when existing file has no trailing newline."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
env_path = Path(tmpdir) / ".env"
|
||||||
|
env_path.write_text("EXISTING_KEY=value") # no trailing newline
|
||||||
|
|
||||||
|
result = setup_wizard.write_setup_config(env_path)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
content = env_path.read_text()
|
||||||
|
# Should have newline separator
|
||||||
|
lines = content.strip().split("\n")
|
||||||
|
assert len(lines) == 3
|
||||||
|
assert lines[0] == "EXISTING_KEY=value"
|
||||||
|
assert "SETUP_COMPLETE=true" in lines[1]
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetSetupStatusText:
|
||||||
|
"""Tests for get_setup_status_text()."""
|
||||||
|
|
||||||
|
def test_with_cookies_and_ytdlp(self):
|
||||||
|
"""Status text mentions found cookies and yt-dlp."""
|
||||||
|
results = {
|
||||||
|
"cookies_found": {"x": "chrome"},
|
||||||
|
"ytdlp_installed": True,
|
||||||
|
"ytdlp_action": "already_installed",
|
||||||
|
"env_written": True,
|
||||||
|
}
|
||||||
|
text = setup_wizard.get_setup_status_text(results)
|
||||||
|
assert "X cookies found in chrome" in text
|
||||||
|
assert "yt-dlp already installed" in text
|
||||||
|
assert "Configuration saved" in text
|
||||||
|
|
||||||
|
def test_with_no_cookies_no_ytdlp(self):
|
||||||
|
"""Status text shows no cookies and suggests yt-dlp install."""
|
||||||
|
results = {
|
||||||
|
"cookies_found": {},
|
||||||
|
"ytdlp_installed": False,
|
||||||
|
"ytdlp_action": "no_homebrew",
|
||||||
|
"env_written": False,
|
||||||
|
}
|
||||||
|
text = setup_wizard.get_setup_status_text(results)
|
||||||
|
assert "No browser cookies found" in text
|
||||||
|
assert "Install Homebrew first" in text
|
||||||
|
|
||||||
|
def test_status_text_installed(self):
|
||||||
|
"""Status text for freshly installed yt-dlp."""
|
||||||
|
results = {
|
||||||
|
"cookies_found": {},
|
||||||
|
"ytdlp_installed": True,
|
||||||
|
"ytdlp_action": "installed",
|
||||||
|
"env_written": False,
|
||||||
|
}
|
||||||
|
text = setup_wizard.get_setup_status_text(results)
|
||||||
|
assert "Installed yt-dlp via Homebrew" in text
|
||||||
|
|
||||||
|
def test_status_text_install_failed(self):
|
||||||
|
"""Status text for failed yt-dlp install."""
|
||||||
|
results = {
|
||||||
|
"cookies_found": {},
|
||||||
|
"ytdlp_installed": False,
|
||||||
|
"ytdlp_action": "install_failed",
|
||||||
|
"env_written": False,
|
||||||
|
}
|
||||||
|
text = setup_wizard.get_setup_status_text(results)
|
||||||
|
assert "yt-dlp install failed" in text
|
||||||
|
assert "manually" in text
|
||||||
|
|
||||||
|
|
||||||
|
class TestSetupSubcommand:
|
||||||
|
"""Tests for setup subcommand detection in argument parsing."""
|
||||||
|
|
||||||
|
def test_setup_detected_as_topic(self):
|
||||||
|
"""The word 'setup' is treated as the setup subcommand."""
|
||||||
|
# Simulate what argparse produces
|
||||||
|
import argparse
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("topic", nargs="*")
|
||||||
|
args = parser.parse_args(["setup"])
|
||||||
|
topic = " ".join(args.topic) if args.topic else None
|
||||||
|
assert topic is not None
|
||||||
|
assert topic.strip().lower() == "setup"
|
||||||
|
|
||||||
|
def test_normal_topic_not_setup(self):
|
||||||
|
"""A normal topic is not confused with setup."""
|
||||||
|
import argparse
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("topic", nargs="*")
|
||||||
|
args = parser.parse_args(["AI", "video", "tools"])
|
||||||
|
topic = " ".join(args.topic) if args.topic else None
|
||||||
|
assert topic.strip().lower() != "setup"
|
||||||
@@ -0,0 +1,737 @@
|
|||||||
|
"""Tests for source resolution priority hierarchy (Unit 4).
|
||||||
|
|
||||||
|
Validates the free-first priority chain:
|
||||||
|
env AUTH_TOKEN/CT0 -> browser cookies -> XAI_API_KEY -> None
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scripts.lib import env
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _base_config(**overrides):
|
||||||
|
"""Return a minimal config dict with typical defaults."""
|
||||||
|
cfg = {
|
||||||
|
"AUTH_TOKEN": None,
|
||||||
|
"CT0": None,
|
||||||
|
"XAI_API_KEY": None,
|
||||||
|
"SCRAPECREATORS_API_KEY": None,
|
||||||
|
"OPENAI_API_KEY": None,
|
||||||
|
"OPENAI_AUTH_STATUS": "missing",
|
||||||
|
"OPENROUTER_API_KEY": None,
|
||||||
|
"PARALLEL_API_KEY": None,
|
||||||
|
"BRAVE_API_KEY": None,
|
||||||
|
"BSKY_HANDLE": None,
|
||||||
|
"BSKY_APP_PASSWORD": None,
|
||||||
|
"TRUTHSOCIAL_TOKEN": None,
|
||||||
|
"FROM_BROWSER": None,
|
||||||
|
"SETUP_COMPLETE": None,
|
||||||
|
"_AUTH_TOKEN_SOURCE": None,
|
||||||
|
}
|
||||||
|
cfg.update(overrides)
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_bird_installed(installed=True):
|
||||||
|
"""Patch bird_x.is_bird_installed to return the given value."""
|
||||||
|
return patch("scripts.lib.bird_x.is_bird_installed", return_value=installed)
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_bird_authenticated(username=None):
|
||||||
|
"""Patch bird_x.is_bird_authenticated to return the given value."""
|
||||||
|
return patch("scripts.lib.bird_x.is_bird_authenticated", return_value=username)
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_bird_status(installed=True, authenticated=True, username="env AUTH_TOKEN"):
|
||||||
|
"""Patch bird_x.get_bird_status to return a status dict."""
|
||||||
|
return patch("scripts.lib.bird_x.get_bird_status", return_value={
|
||||||
|
"installed": installed,
|
||||||
|
"authenticated": authenticated,
|
||||||
|
"username": username,
|
||||||
|
"can_install": True,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: X source resolution priority
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestXSourcePriority:
|
||||||
|
"""Test the X source priority chain: env -> browser cookies -> xAI -> None."""
|
||||||
|
|
||||||
|
def test_no_env_cookies_found_resolves_bird_browser(self):
|
||||||
|
"""No env vars, SETUP_COMPLETE=true, cookies found -> Bird with browser method."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="cookie_tok",
|
||||||
|
CT0="cookie_ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-firefox",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "bird"
|
||||||
|
assert method == "browser-firefox"
|
||||||
|
|
||||||
|
def test_env_auth_token_plus_cookies_env_wins(self):
|
||||||
|
"""AUTH_TOKEN in .env + cookies available -> env wins (method='env')."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="explicit_token",
|
||||||
|
CT0="explicit_ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="env",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "bird"
|
||||||
|
assert method == "env"
|
||||||
|
|
||||||
|
def test_no_env_no_cookies_xai_key_resolves_xai(self):
|
||||||
|
"""No env vars, no cookies, XAI_API_KEY set -> xAI with method 'api'."""
|
||||||
|
config = _base_config(XAI_API_KEY="xai-key-123")
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated(None):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "xai"
|
||||||
|
assert method == "api"
|
||||||
|
|
||||||
|
def test_no_env_no_cookies_no_api_keys_none(self):
|
||||||
|
"""No env vars, no cookies, no API keys -> X not available."""
|
||||||
|
config = _base_config()
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated(None):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source is None
|
||||||
|
assert method is None
|
||||||
|
|
||||||
|
def test_bird_not_installed_falls_to_xai(self):
|
||||||
|
"""Bird not installed, XAI_API_KEY set -> xAI."""
|
||||||
|
config = _base_config(XAI_API_KEY="xai-key")
|
||||||
|
|
||||||
|
with _mock_bird_installed(False), _mock_bird_authenticated(None):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "xai"
|
||||||
|
assert method == "api"
|
||||||
|
|
||||||
|
def test_bird_not_installed_no_xai_none(self):
|
||||||
|
"""Bird not installed, no XAI_API_KEY -> None."""
|
||||||
|
config = _base_config()
|
||||||
|
|
||||||
|
with _mock_bird_installed(False), _mock_bird_authenticated(None):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source is None
|
||||||
|
assert method is None
|
||||||
|
|
||||||
|
def test_browser_chrome_method_tracked(self):
|
||||||
|
"""Cookies from Chrome -> method is 'browser-chrome'."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="chrome_tok",
|
||||||
|
CT0="chrome_ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-chrome",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "bird"
|
||||||
|
assert method == "browser-chrome"
|
||||||
|
|
||||||
|
def test_browser_safari_method_tracked(self):
|
||||||
|
"""Cookies from Safari -> method is 'browser-safari'."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="safari_tok",
|
||||||
|
CT0="safari_ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-safari",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "bird"
|
||||||
|
assert method == "browser-safari"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: get_x_source() backward compat
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestGetXSourceBackwardCompat:
|
||||||
|
"""Ensure get_x_source() returns the same string as before."""
|
||||||
|
|
||||||
|
def test_bird_returns_bird(self):
|
||||||
|
config = _base_config(AUTH_TOKEN="tok", CT0="ct0", _AUTH_TOKEN_SOURCE="env")
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
assert env.get_x_source(config) == "bird"
|
||||||
|
|
||||||
|
def test_xai_returns_xai(self):
|
||||||
|
config = _base_config(XAI_API_KEY="key")
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated(None):
|
||||||
|
assert env.get_x_source(config) == "xai"
|
||||||
|
|
||||||
|
def test_none_returns_none(self):
|
||||||
|
config = _base_config()
|
||||||
|
with _mock_bird_installed(False):
|
||||||
|
assert env.get_x_source(config) is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: get_x_source_status() method field
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestGetXSourceStatusMethod:
|
||||||
|
"""Test that get_x_source_status() includes the method field."""
|
||||||
|
|
||||||
|
def test_bird_env_method(self):
|
||||||
|
config = _base_config(AUTH_TOKEN="tok", CT0="ct0", _AUTH_TOKEN_SOURCE="env")
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"), \
|
||||||
|
_mock_bird_status(installed=True, authenticated=True, username="env AUTH_TOKEN"):
|
||||||
|
status = env.get_x_source_status(config)
|
||||||
|
|
||||||
|
assert status["source"] == "bird"
|
||||||
|
assert status["method"] == "env"
|
||||||
|
assert "bird_installed" in status
|
||||||
|
assert "xai_available" in status
|
||||||
|
|
||||||
|
def test_bird_browser_method(self):
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="tok", CT0="ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-firefox",
|
||||||
|
)
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"), \
|
||||||
|
_mock_bird_status(installed=True, authenticated=True):
|
||||||
|
status = env.get_x_source_status(config)
|
||||||
|
|
||||||
|
assert status["source"] == "bird"
|
||||||
|
assert status["method"] == "browser-firefox"
|
||||||
|
|
||||||
|
def test_xai_api_method(self):
|
||||||
|
config = _base_config(XAI_API_KEY="key")
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated(None), \
|
||||||
|
_mock_bird_status(installed=True, authenticated=False, username=None):
|
||||||
|
status = env.get_x_source_status(config)
|
||||||
|
|
||||||
|
assert status["source"] == "xai"
|
||||||
|
assert status["method"] == "api"
|
||||||
|
|
||||||
|
def test_no_source_method_none(self):
|
||||||
|
config = _base_config()
|
||||||
|
with _mock_bird_installed(False), \
|
||||||
|
_mock_bird_status(installed=False, authenticated=False, username=None):
|
||||||
|
status = env.get_x_source_status(config)
|
||||||
|
|
||||||
|
assert status["source"] is None
|
||||||
|
assert status["method"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: get_available_sources() with various configs
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestGetAvailableSources:
|
||||||
|
"""Test get_available_sources() returns correct strings."""
|
||||||
|
|
||||||
|
def test_no_x_no_web_reddit_only(self):
|
||||||
|
"""No X source, no web keys -> 'reddit' (Reddit always available)."""
|
||||||
|
config = _base_config()
|
||||||
|
with _mock_bird_installed(False):
|
||||||
|
result = env.get_available_sources(config)
|
||||||
|
assert result == "reddit"
|
||||||
|
|
||||||
|
def test_xai_key_no_web(self):
|
||||||
|
"""XAI_API_KEY set, no web keys -> 'both'."""
|
||||||
|
config = _base_config(XAI_API_KEY="key")
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated(None):
|
||||||
|
result = env.get_available_sources(config)
|
||||||
|
assert result == "both"
|
||||||
|
|
||||||
|
def test_bird_auth_no_web(self):
|
||||||
|
"""Bird authenticated (cookies), SETUP_COMPLETE=true, no web keys -> 'both'."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="tok", CT0="ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-firefox",
|
||||||
|
)
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
result = env.get_available_sources(config)
|
||||||
|
assert result == "both"
|
||||||
|
|
||||||
|
def test_bird_auth_with_web(self):
|
||||||
|
"""Bird authenticated + web keys -> 'all'."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="tok", CT0="ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-firefox",
|
||||||
|
BRAVE_API_KEY="brave-key",
|
||||||
|
)
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
result = env.get_available_sources(config)
|
||||||
|
assert result == "all"
|
||||||
|
|
||||||
|
def test_no_x_with_web(self):
|
||||||
|
"""No X source, web keys -> 'reddit-web'."""
|
||||||
|
config = _base_config(BRAVE_API_KEY="brave-key")
|
||||||
|
with _mock_bird_installed(False):
|
||||||
|
result = env.get_available_sources(config)
|
||||||
|
assert result == "reddit-web"
|
||||||
|
|
||||||
|
def test_reddit_hn_polymarket_always_available(self):
|
||||||
|
"""Reddit, HN, and Polymarket are always available regardless of config."""
|
||||||
|
config = _base_config()
|
||||||
|
# These functions don't depend on config
|
||||||
|
assert env.is_hackernews_available() is True
|
||||||
|
assert env.is_polymarket_available() is True
|
||||||
|
# Reddit: get_available_sources always includes it
|
||||||
|
with _mock_bird_installed(False):
|
||||||
|
result = env.get_available_sources(config)
|
||||||
|
assert result in ("reddit", "reddit-web") # never 'none' when Reddit is always True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: Full resolution with mixed config
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestFullResolution:
|
||||||
|
"""Test that each source resolves independently with mixed config."""
|
||||||
|
|
||||||
|
def test_mixed_config_all_sources(self):
|
||||||
|
"""Bird for X, public Reddit, web keys, YouTube/TikTok available."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="tok", CT0="ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-chrome",
|
||||||
|
BRAVE_API_KEY="brave-key",
|
||||||
|
SCRAPECREATORS_API_KEY="sc-key",
|
||||||
|
BSKY_HANDLE="user.bsky.social",
|
||||||
|
BSKY_APP_PASSWORD="app-pw",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
x_source, x_method = env.get_x_source_with_method(config)
|
||||||
|
available = env.get_available_sources(config)
|
||||||
|
|
||||||
|
assert x_source == "bird"
|
||||||
|
assert x_method == "browser-chrome"
|
||||||
|
assert available == "all" # reddit + x + web
|
||||||
|
assert env.is_bluesky_available(config) is True
|
||||||
|
assert env.is_tiktok_available(config) is True
|
||||||
|
assert env.is_hackernews_available() is True
|
||||||
|
assert env.is_polymarket_available() is True
|
||||||
|
|
||||||
|
def test_no_config_minimal_sources(self):
|
||||||
|
"""No API keys, no cookies -> Reddit + HN + Polymarket only."""
|
||||||
|
config = _base_config()
|
||||||
|
|
||||||
|
with _mock_bird_installed(False):
|
||||||
|
x_source = env.get_x_source(config)
|
||||||
|
available = env.get_available_sources(config)
|
||||||
|
|
||||||
|
assert x_source is None
|
||||||
|
assert available == "reddit"
|
||||||
|
assert env.is_hackernews_available() is True
|
||||||
|
assert env.is_polymarket_available() is True
|
||||||
|
assert env.is_bluesky_available(config) is False
|
||||||
|
assert env.is_tiktok_available(config) is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: extract_browser_credentials tracks browser source
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestExtractBrowserCredentialsSource:
|
||||||
|
"""Test that extract_browser_credentials tracks __X_BROWSER."""
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_tracks_firefox_source(self, mock_extract):
|
||||||
|
"""Cookies from Firefox -> __X_BROWSER set to 'firefox'."""
|
||||||
|
mock_extract.return_value = (
|
||||||
|
{"auth_token": "tok", "ct0": "ct0val"},
|
||||||
|
"firefox",
|
||||||
|
)
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"AUTH_TOKEN": None,
|
||||||
|
"CT0": None,
|
||||||
|
"TRUTHSOCIAL_TOKEN": None,
|
||||||
|
"FROM_BROWSER": "auto",
|
||||||
|
"SETUP_COMPLETE": "true",
|
||||||
|
}
|
||||||
|
result = env.extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert result["AUTH_TOKEN"] == "tok"
|
||||||
|
assert result["CT0"] == "ct0val"
|
||||||
|
assert result["__X_BROWSER"] == "firefox"
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_tracks_chrome_source(self, mock_extract):
|
||||||
|
"""Cookies from Chrome -> __X_BROWSER set to 'chrome'."""
|
||||||
|
mock_extract.return_value = (
|
||||||
|
{"auth_token": "tok", "ct0": "ct0val"},
|
||||||
|
"chrome",
|
||||||
|
)
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"AUTH_TOKEN": None,
|
||||||
|
"CT0": None,
|
||||||
|
"TRUTHSOCIAL_TOKEN": None,
|
||||||
|
"FROM_BROWSER": "chrome",
|
||||||
|
"SETUP_COMPLETE": "true",
|
||||||
|
}
|
||||||
|
result = env.extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert result["__X_BROWSER"] == "chrome"
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
def test_no_cookies_no_browser_key(self, mock_extract):
|
||||||
|
"""No cookies found -> no __X_BROWSER key."""
|
||||||
|
mock_extract.return_value = None
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"AUTH_TOKEN": None,
|
||||||
|
"CT0": None,
|
||||||
|
"TRUTHSOCIAL_TOKEN": None,
|
||||||
|
"FROM_BROWSER": "auto",
|
||||||
|
"SETUP_COMPLETE": "true",
|
||||||
|
}
|
||||||
|
result = env.extract_browser_credentials(config)
|
||||||
|
|
||||||
|
assert "__X_BROWSER" not in result
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: get_config() _AUTH_TOKEN_SOURCE tracking
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestGetConfigAuthTokenSource:
|
||||||
|
"""Test that get_config() sets _AUTH_TOKEN_SOURCE correctly."""
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("scripts.lib.env._find_project_env", return_value=None)
|
||||||
|
@patch("scripts.lib.env.load_env_file", return_value={})
|
||||||
|
@patch("scripts.lib.env.get_openai_auth")
|
||||||
|
def test_env_var_auth_token_source_env(
|
||||||
|
self, mock_openai, mock_load, mock_proj, mock_extract
|
||||||
|
):
|
||||||
|
"""AUTH_TOKEN from env var -> _AUTH_TOKEN_SOURCE='env'."""
|
||||||
|
from scripts.lib.env import get_config, OpenAIAuth
|
||||||
|
|
||||||
|
mock_openai.return_value = OpenAIAuth(
|
||||||
|
token=None, source="none", status="missing",
|
||||||
|
account_id=None, codex_auth_file="/fake",
|
||||||
|
)
|
||||||
|
mock_extract.return_value = None
|
||||||
|
|
||||||
|
env_patch = {
|
||||||
|
"AUTH_TOKEN": "env_token",
|
||||||
|
"CT0": "env_ct0",
|
||||||
|
"LAST30DAYS_CONFIG_DIR": "",
|
||||||
|
}
|
||||||
|
with patch.dict(os.environ, env_patch, clear=False):
|
||||||
|
config = get_config()
|
||||||
|
|
||||||
|
assert config["_AUTH_TOKEN_SOURCE"] == "env"
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("scripts.lib.env._find_project_env", return_value=None)
|
||||||
|
@patch("scripts.lib.env.load_env_file", return_value={})
|
||||||
|
@patch("scripts.lib.env.get_openai_auth")
|
||||||
|
def test_browser_cookies_auth_token_source_browser(
|
||||||
|
self, mock_openai, mock_load, mock_proj, mock_extract
|
||||||
|
):
|
||||||
|
"""AUTH_TOKEN from cookies -> _AUTH_TOKEN_SOURCE='browser-firefox'."""
|
||||||
|
from scripts.lib.env import get_config, OpenAIAuth
|
||||||
|
|
||||||
|
mock_openai.return_value = OpenAIAuth(
|
||||||
|
token=None, source="none", status="missing",
|
||||||
|
account_id=None, codex_auth_file="/fake",
|
||||||
|
)
|
||||||
|
mock_extract.return_value = (
|
||||||
|
{"auth_token": "cookie_tok", "ct0": "cookie_ct0"},
|
||||||
|
"firefox",
|
||||||
|
)
|
||||||
|
|
||||||
|
env_patch = {
|
||||||
|
"SETUP_COMPLETE": "true",
|
||||||
|
"FROM_BROWSER": "auto",
|
||||||
|
"LAST30DAYS_CONFIG_DIR": "",
|
||||||
|
}
|
||||||
|
# Ensure AUTH_TOKEN is NOT in env
|
||||||
|
clean_env = {k: v for k, v in os.environ.items()
|
||||||
|
if k not in ("AUTH_TOKEN", "CT0")}
|
||||||
|
clean_env.update(env_patch)
|
||||||
|
with patch.dict(os.environ, clean_env, clear=True):
|
||||||
|
config = get_config()
|
||||||
|
|
||||||
|
assert config["AUTH_TOKEN"] == "cookie_tok"
|
||||||
|
assert config["_AUTH_TOKEN_SOURCE"] == "browser-firefox"
|
||||||
|
|
||||||
|
@patch("scripts.lib.cookie_extract.extract_cookies_with_source")
|
||||||
|
@patch("scripts.lib.env._find_project_env", return_value=None)
|
||||||
|
@patch("scripts.lib.env.load_env_file", return_value={})
|
||||||
|
@patch("scripts.lib.env.get_openai_auth")
|
||||||
|
def test_no_auth_token_source_none(
|
||||||
|
self, mock_openai, mock_load, mock_proj, mock_extract
|
||||||
|
):
|
||||||
|
"""No AUTH_TOKEN at all -> _AUTH_TOKEN_SOURCE=None."""
|
||||||
|
from scripts.lib.env import get_config, OpenAIAuth
|
||||||
|
|
||||||
|
mock_openai.return_value = OpenAIAuth(
|
||||||
|
token=None, source="none", status="missing",
|
||||||
|
account_id=None, codex_auth_file="/fake",
|
||||||
|
)
|
||||||
|
mock_extract.return_value = None
|
||||||
|
|
||||||
|
clean_env = {k: v for k, v in os.environ.items()
|
||||||
|
if k not in ("AUTH_TOKEN", "CT0")}
|
||||||
|
clean_env["LAST30DAYS_CONFIG_DIR"] = ""
|
||||||
|
with patch.dict(os.environ, clean_env, clear=True):
|
||||||
|
config = get_config()
|
||||||
|
|
||||||
|
assert config["_AUTH_TOKEN_SOURCE"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: SETUP_COMPLETE gate — Bird cookie probing blocked before consent
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestSetupCompleteGate:
|
||||||
|
"""Test that Bird cookie probing is gated behind SETUP_COMPLETE consent."""
|
||||||
|
|
||||||
|
def test_no_setup_complete_bird_has_cookies_returns_none(self):
|
||||||
|
"""SETUP_COMPLETE not set, Bird has browser cookies -> returns None (not 'bird').
|
||||||
|
|
||||||
|
Bird's is_bird_authenticated() should NOT be called at all because
|
||||||
|
there is no consent yet. Cookie-sourced AUTH_TOKEN must be ignored.
|
||||||
|
"""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="cookie_tok",
|
||||||
|
CT0="cookie_ct0",
|
||||||
|
SETUP_COMPLETE=None, # not set — first run
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-chrome",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), \
|
||||||
|
_mock_bird_authenticated("Chrome") as mock_auth:
|
||||||
|
source = env.get_x_source(config)
|
||||||
|
|
||||||
|
assert source is None
|
||||||
|
# is_bird_authenticated should NOT have been called (no cookie probing)
|
||||||
|
mock_auth.assert_not_called()
|
||||||
|
|
||||||
|
def test_setup_complete_bird_has_cookies_returns_bird(self):
|
||||||
|
"""SETUP_COMPLETE=true, Bird has browser cookies -> returns 'bird'.
|
||||||
|
|
||||||
|
After user consent, cookie probing should work normally.
|
||||||
|
"""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="cookie_tok",
|
||||||
|
CT0="cookie_ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-chrome",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("Chrome"):
|
||||||
|
source = env.get_x_source(config)
|
||||||
|
|
||||||
|
assert source == "bird"
|
||||||
|
|
||||||
|
def test_no_setup_complete_explicit_auth_token_returns_bird(self):
|
||||||
|
"""SETUP_COMPLETE not set, AUTH_TOKEN from env var -> returns 'bird' with method 'env'.
|
||||||
|
|
||||||
|
Explicit env var credentials must ALWAYS work regardless of SETUP_COMPLETE.
|
||||||
|
"""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="explicit_token",
|
||||||
|
CT0="explicit_ct0",
|
||||||
|
SETUP_COMPLETE=None, # not set
|
||||||
|
_AUTH_TOKEN_SOURCE="env",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "bird"
|
||||||
|
assert method == "env"
|
||||||
|
|
||||||
|
def test_no_setup_complete_xai_key_returns_xai(self):
|
||||||
|
"""SETUP_COMPLETE not set, XAI_API_KEY configured -> returns 'xai'.
|
||||||
|
|
||||||
|
API keys always work without setup consent.
|
||||||
|
"""
|
||||||
|
config = _base_config(
|
||||||
|
XAI_API_KEY="xai-key-123",
|
||||||
|
SETUP_COMPLETE=None, # not set
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated(None):
|
||||||
|
source, method = env.get_x_source_with_method(config)
|
||||||
|
|
||||||
|
assert source == "xai"
|
||||||
|
assert method == "api"
|
||||||
|
|
||||||
|
def test_no_setup_complete_status_reports_not_configured(self):
|
||||||
|
"""First-run status banner should show X as not configured when only cookies exist."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="cookie_tok",
|
||||||
|
CT0="cookie_ct0",
|
||||||
|
SETUP_COMPLETE=None,
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-firefox",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True):
|
||||||
|
status = env.get_x_source_status(config)
|
||||||
|
|
||||||
|
assert status["source"] is None
|
||||||
|
assert status["method"] is None
|
||||||
|
assert status["bird_authenticated"] is False
|
||||||
|
|
||||||
|
def test_setup_complete_status_reports_bird(self):
|
||||||
|
"""After consent, status banner should show Bird as configured."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="cookie_tok",
|
||||||
|
CT0="cookie_ct0",
|
||||||
|
SETUP_COMPLETE="true",
|
||||||
|
_AUTH_TOKEN_SOURCE="browser-firefox",
|
||||||
|
)
|
||||||
|
|
||||||
|
with _mock_bird_installed(True), _mock_bird_authenticated("Firefox"), \
|
||||||
|
_mock_bird_status(installed=True, authenticated=True, username="Firefox"):
|
||||||
|
status = env.get_x_source_status(config)
|
||||||
|
|
||||||
|
assert status["source"] == "bird"
|
||||||
|
assert status["method"] == "browser-firefox"
|
||||||
|
assert status["bird_authenticated"] is True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: BIRD_DISABLE_BROWSER_COOKIES env var on first run
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestBirdDisableBrowserCookiesEnvVar:
|
||||||
|
"""Test that BIRD_DISABLE_BROWSER_COOKIES is set to block Bird's Node.js
|
||||||
|
sweet-cookie scanner on first run before user consent."""
|
||||||
|
|
||||||
|
def _run_main_flow_env_setup(self, config, first_run):
|
||||||
|
"""Simulate the env-var-setting logic from last30days.py main flow.
|
||||||
|
|
||||||
|
Mirrors the block right after first_run detection in main().
|
||||||
|
"""
|
||||||
|
if first_run and config.get('_AUTH_TOKEN_SOURCE') != 'env':
|
||||||
|
os.environ['BIRD_DISABLE_BROWSER_COOKIES'] = '1'
|
||||||
|
else:
|
||||||
|
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
|
||||||
|
|
||||||
|
def test_first_run_no_auth_token_sets_env_var(self):
|
||||||
|
"""first_run=True, no AUTH_TOKEN -> BIRD_DISABLE_BROWSER_COOKIES is set."""
|
||||||
|
config = _base_config(SETUP_COMPLETE=None, _AUTH_TOKEN_SOURCE=None)
|
||||||
|
try:
|
||||||
|
self._run_main_flow_env_setup(config, first_run=True)
|
||||||
|
assert os.environ.get('BIRD_DISABLE_BROWSER_COOKIES') == '1'
|
||||||
|
finally:
|
||||||
|
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
|
||||||
|
|
||||||
|
def test_not_first_run_no_env_var(self):
|
||||||
|
"""first_run=False -> BIRD_DISABLE_BROWSER_COOKIES is NOT set."""
|
||||||
|
config = _base_config(SETUP_COMPLETE="true", _AUTH_TOKEN_SOURCE="browser-firefox")
|
||||||
|
try:
|
||||||
|
self._run_main_flow_env_setup(config, first_run=False)
|
||||||
|
assert 'BIRD_DISABLE_BROWSER_COOKIES' not in os.environ
|
||||||
|
finally:
|
||||||
|
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
|
||||||
|
|
||||||
|
def test_first_run_explicit_auth_token_no_env_var(self):
|
||||||
|
"""first_run=True, AUTH_TOKEN explicitly set -> BIRD_DISABLE_BROWSER_COOKIES is NOT set."""
|
||||||
|
config = _base_config(
|
||||||
|
AUTH_TOKEN="explicit_token",
|
||||||
|
CT0="explicit_ct0",
|
||||||
|
SETUP_COMPLETE=None,
|
||||||
|
_AUTH_TOKEN_SOURCE="env",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
self._run_main_flow_env_setup(config, first_run=True)
|
||||||
|
assert 'BIRD_DISABLE_BROWSER_COOKIES' not in os.environ
|
||||||
|
finally:
|
||||||
|
os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests: INCLUDE_SOURCES config override
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestIncludeSourcesOverride:
|
||||||
|
"""Test that INCLUDE_SOURCES forces sources on regardless of tier."""
|
||||||
|
|
||||||
|
def _simulate_source_decisions(self, config, query_type="breaking_news"):
|
||||||
|
"""Simulate the source decision logic from last30days.py main flow.
|
||||||
|
|
||||||
|
Returns (search_run_tiktok, search_run_instagram) after tier + override.
|
||||||
|
"""
|
||||||
|
from scripts.lib import query_type as qt
|
||||||
|
|
||||||
|
has_tiktok = env.is_tiktok_available(config)
|
||||||
|
has_instagram = env.is_instagram_available(config)
|
||||||
|
|
||||||
|
# Tier system decision
|
||||||
|
search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type)
|
||||||
|
search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type)
|
||||||
|
|
||||||
|
# INCLUDE_SOURCES override (mirrors last30days.py logic)
|
||||||
|
_include_sources = {
|
||||||
|
s.strip().lower()
|
||||||
|
for s in config.get('INCLUDE_SOURCES', '').split(',')
|
||||||
|
if s.strip()
|
||||||
|
}
|
||||||
|
if _include_sources:
|
||||||
|
if 'tiktok' in _include_sources and has_tiktok:
|
||||||
|
if not search_run_tiktok:
|
||||||
|
search_run_tiktok = True
|
||||||
|
if 'instagram' in _include_sources and has_instagram:
|
||||||
|
if not search_run_instagram:
|
||||||
|
search_run_instagram = True
|
||||||
|
|
||||||
|
return search_run_tiktok, search_run_instagram
|
||||||
|
|
||||||
|
def test_include_sources_forces_tiktok_and_instagram_on(self):
|
||||||
|
"""INCLUDE_SOURCES=tiktok,instagram + SC key + GENERAL query -> both forced on."""
|
||||||
|
config = _base_config(
|
||||||
|
SCRAPECREATORS_API_KEY="sc-key",
|
||||||
|
INCLUDE_SOURCES="tiktok,instagram",
|
||||||
|
)
|
||||||
|
run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
|
||||||
|
assert run_tiktok is True
|
||||||
|
assert run_instagram is True
|
||||||
|
|
||||||
|
def test_no_include_sources_tier_controls(self):
|
||||||
|
"""No INCLUDE_SOURCES + SC key + GENERAL query -> tier system controls (both off)."""
|
||||||
|
config = _base_config(
|
||||||
|
SCRAPECREATORS_API_KEY="sc-key",
|
||||||
|
)
|
||||||
|
run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
|
||||||
|
# breaking_news tier doesn't include tiktok or instagram
|
||||||
|
assert run_tiktok is False
|
||||||
|
assert run_instagram is False
|
||||||
|
|
||||||
|
def test_include_sources_no_sc_key_still_off(self):
|
||||||
|
"""INCLUDE_SOURCES=tiktok but no SC key -> TikTok still off (no backend)."""
|
||||||
|
config = _base_config(
|
||||||
|
INCLUDE_SOURCES="tiktok",
|
||||||
|
)
|
||||||
|
run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
|
||||||
|
assert run_tiktok is False
|
||||||
|
assert run_instagram is False
|
||||||
@@ -0,0 +1,283 @@
|
|||||||
|
"""Tests for the redesigned status banner (free-first design)."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scripts.lib.ui import _build_status_banner
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _base_diag(**overrides):
|
||||||
|
"""Return a minimal diag dict with common defaults."""
|
||||||
|
diag = {
|
||||||
|
"setup_complete": False,
|
||||||
|
"reddit_source": None, # None = public fallback
|
||||||
|
"x_source": None,
|
||||||
|
"x_method": None,
|
||||||
|
"youtube": False,
|
||||||
|
"tiktok": False,
|
||||||
|
"instagram": False,
|
||||||
|
"hackernews": True,
|
||||||
|
"polymarket": True,
|
||||||
|
"bluesky": False,
|
||||||
|
"truthsocial": False,
|
||||||
|
"xiaohongshu": False,
|
||||||
|
"scrapecreators": False,
|
||||||
|
"web_search_backend": None,
|
||||||
|
}
|
||||||
|
diag.update(overrides)
|
||||||
|
return diag
|
||||||
|
|
||||||
|
|
||||||
|
def _banner_text(diag):
|
||||||
|
"""Return full banner as a single string."""
|
||||||
|
return "\n".join(_build_status_banner(diag))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestZeroConfig:
|
||||||
|
"""Zero-config state: SETUP_COMPLETE not set, wizard hasn't run."""
|
||||||
|
|
||||||
|
def test_shows_first_run_title(self):
|
||||||
|
banner = _banner_text(_base_diag())
|
||||||
|
assert "First Run" in banner
|
||||||
|
|
||||||
|
def test_shows_three_free_sources(self):
|
||||||
|
banner = _banner_text(_base_diag())
|
||||||
|
assert "Reddit (threads only)" in banner
|
||||||
|
assert "HN" in banner
|
||||||
|
assert "Polymarket" in banner
|
||||||
|
|
||||||
|
def test_shows_setup_prompt(self):
|
||||||
|
banner = _banner_text(_base_diag())
|
||||||
|
assert "/last30days setup" in banner
|
||||||
|
|
||||||
|
def test_does_not_show_source_status_title(self):
|
||||||
|
banner = _banner_text(_base_diag())
|
||||||
|
assert "Source Status" not in banner
|
||||||
|
|
||||||
|
|
||||||
|
class TestFullConfig:
|
||||||
|
"""Fully configured: X + yt-dlp + ScrapeCreators = everything active."""
|
||||||
|
|
||||||
|
def _full_diag(self):
|
||||||
|
return _base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source="scrapecreators",
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-chrome",
|
||||||
|
youtube=True,
|
||||||
|
tiktok=True,
|
||||||
|
instagram=True,
|
||||||
|
hackernews=True,
|
||||||
|
polymarket=True,
|
||||||
|
bluesky=True,
|
||||||
|
truthsocial=True,
|
||||||
|
xiaohongshu=True,
|
||||||
|
scrapecreators=True,
|
||||||
|
web_search_backend="parallel",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_shows_source_status_title(self):
|
||||||
|
banner = _banner_text(self._full_diag())
|
||||||
|
assert "Source Status" in banner
|
||||||
|
|
||||||
|
def test_shows_all_sources(self):
|
||||||
|
banner = _banner_text(self._full_diag())
|
||||||
|
assert "Reddit (with comments)" in banner
|
||||||
|
assert "X (Chrome)" in banner
|
||||||
|
assert "YouTube" in banner
|
||||||
|
assert "HN" in banner
|
||||||
|
assert "Polymarket" in banner
|
||||||
|
assert "TikTok" in banner
|
||||||
|
assert "Instagram" in banner
|
||||||
|
assert "Bluesky" in banner
|
||||||
|
assert "Truth Social" in banner
|
||||||
|
assert "Xiaohongshu" in banner
|
||||||
|
|
||||||
|
def test_no_recommendations(self):
|
||||||
|
banner = _banner_text(self._full_diag())
|
||||||
|
assert "⭐" not in banner
|
||||||
|
assert "scrapecreators.com" not in banner
|
||||||
|
assert "/last30days setup" not in banner
|
||||||
|
|
||||||
|
|
||||||
|
class TestPartialConfig:
|
||||||
|
"""Partially configured: X + yt-dlp but no ScrapeCreators."""
|
||||||
|
|
||||||
|
def _partial_diag(self):
|
||||||
|
return _base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source=None, # public fallback
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-chrome",
|
||||||
|
youtube=True,
|
||||||
|
scrapecreators=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_shows_active_sources(self):
|
||||||
|
banner = _banner_text(self._partial_diag())
|
||||||
|
assert "Reddit (threads only)" in banner
|
||||||
|
assert "X (Chrome)" in banner
|
||||||
|
assert "YouTube" in banner
|
||||||
|
assert "HN" in banner
|
||||||
|
assert "Polymarket" in banner
|
||||||
|
|
||||||
|
def test_recommends_scrapecreators(self):
|
||||||
|
banner = _banner_text(self._partial_diag())
|
||||||
|
assert "SCRAPECREATORS_API_KEY" in banner
|
||||||
|
|
||||||
|
def test_scrapecreators_free_calls_copy(self):
|
||||||
|
banner = _banner_text(self._partial_diag())
|
||||||
|
assert "100 free calls, no CC" in banner
|
||||||
|
assert "scrapecreators.com" in banner
|
||||||
|
|
||||||
|
def test_shows_tiktok_instagram_unlock(self):
|
||||||
|
banner = _banner_text(self._partial_diag())
|
||||||
|
assert "TikTok" in banner
|
||||||
|
assert "Instagram" in banner
|
||||||
|
|
||||||
|
|
||||||
|
class TestScrapeCreatorsRecommendation:
|
||||||
|
"""ScrapeCreators recommendation always includes key copy."""
|
||||||
|
|
||||||
|
def test_always_includes_free_calls_no_cc(self):
|
||||||
|
"""Any config missing SC should show the free-calls copy."""
|
||||||
|
# Zero config
|
||||||
|
banner_zero = _banner_text(_base_diag())
|
||||||
|
# Zero config doesn't recommend SC directly (recommends setup wizard)
|
||||||
|
# But after setup, missing SC should always include the copy
|
||||||
|
banner_partial = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
scrapecreators=False,
|
||||||
|
))
|
||||||
|
assert "100 free calls, no CC" in banner_partial
|
||||||
|
|
||||||
|
def test_present_when_x_available_but_no_sc(self):
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
x_source="xai",
|
||||||
|
x_method="api",
|
||||||
|
scrapecreators=False,
|
||||||
|
))
|
||||||
|
assert "100 free calls, no CC" in banner
|
||||||
|
assert "scrapecreators.com" in banner
|
||||||
|
|
||||||
|
|
||||||
|
class TestRedditLabelDisplay:
|
||||||
|
"""Reddit label reflects comment availability, not implementation details."""
|
||||||
|
|
||||||
|
def test_no_scrapecreators_shows_threads_only(self):
|
||||||
|
"""Without SC, Reddit label should say 'threads only' regardless of OpenAI auth."""
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source="openai",
|
||||||
|
scrapecreators=False,
|
||||||
|
))
|
||||||
|
assert "Reddit (threads only)" in banner
|
||||||
|
assert "OpenAI" not in banner
|
||||||
|
assert "Codex" not in banner
|
||||||
|
|
||||||
|
def test_no_scrapecreators_public_shows_threads_only(self):
|
||||||
|
"""Public fallback also shows 'threads only'."""
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source=None,
|
||||||
|
scrapecreators=False,
|
||||||
|
))
|
||||||
|
assert "Reddit (threads only)" in banner
|
||||||
|
|
||||||
|
def test_scrapecreators_shows_with_comments(self):
|
||||||
|
"""With SC configured, Reddit label should say 'with comments'."""
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source="scrapecreators",
|
||||||
|
scrapecreators=True,
|
||||||
|
))
|
||||||
|
assert "Reddit (with comments)" in banner
|
||||||
|
|
||||||
|
def test_no_openai_or_codex_in_banner(self):
|
||||||
|
"""Banner should never mention OpenAI or Codex — those are implementation details."""
|
||||||
|
for source in [None, "openai", "scrapecreators"]:
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source=source,
|
||||||
|
scrapecreators=(source == "scrapecreators"),
|
||||||
|
))
|
||||||
|
assert "OpenAI" not in banner, f"Found 'OpenAI' with reddit_source={source}"
|
||||||
|
assert "Codex" not in banner, f"Found 'Codex' with reddit_source={source}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestXMethodDisplay:
|
||||||
|
"""X source shows auth method in parens."""
|
||||||
|
|
||||||
|
def test_browser_chrome(self):
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-chrome",
|
||||||
|
))
|
||||||
|
assert "X (Chrome)" in banner
|
||||||
|
|
||||||
|
def test_browser_firefox(self):
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-firefox",
|
||||||
|
))
|
||||||
|
assert "X (Firefox)" in banner
|
||||||
|
|
||||||
|
def test_env_method(self):
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
x_source="bird",
|
||||||
|
x_method="env",
|
||||||
|
))
|
||||||
|
assert "X (env)" in banner
|
||||||
|
|
||||||
|
def test_xai_api(self):
|
||||||
|
banner = _banner_text(_base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
x_source="xai",
|
||||||
|
x_method="api",
|
||||||
|
))
|
||||||
|
assert "X (xAI)" in banner
|
||||||
|
|
||||||
|
|
||||||
|
class TestBannerStructure:
|
||||||
|
"""Banner formatting and structure tests."""
|
||||||
|
|
||||||
|
def test_has_box_drawing(self):
|
||||||
|
lines = _build_status_banner(_base_diag())
|
||||||
|
assert lines[0].startswith("┌")
|
||||||
|
assert lines[-1].startswith("└")
|
||||||
|
|
||||||
|
def test_shows_config_path(self):
|
||||||
|
banner = _banner_text(_base_diag())
|
||||||
|
assert "~/.config/last30days/.env" in banner
|
||||||
|
|
||||||
|
def test_max_10_inner_lines(self):
|
||||||
|
"""Banner should be compact — max 10 lines inside the box."""
|
||||||
|
# Full config (most lines)
|
||||||
|
diag = _base_diag(
|
||||||
|
setup_complete=True,
|
||||||
|
reddit_source="scrapecreators",
|
||||||
|
x_source="bird",
|
||||||
|
x_method="browser-chrome",
|
||||||
|
youtube=True,
|
||||||
|
tiktok=True,
|
||||||
|
instagram=True,
|
||||||
|
bluesky=True,
|
||||||
|
truthsocial=True,
|
||||||
|
xiaohongshu=True,
|
||||||
|
scrapecreators=True,
|
||||||
|
)
|
||||||
|
lines = _build_status_banner(diag)
|
||||||
|
# Subtract top and bottom border
|
||||||
|
inner_lines = [l for l in lines if l.startswith("│")]
|
||||||
|
assert len(inner_lines) <= 10, f"Banner has {len(inner_lines)} inner lines, max is 10"
|
||||||
@@ -1,8 +1,10 @@
|
|||||||
"""Tests for yt-dlp invocation safety flags."""
|
"""Tests for yt-dlp invocation safety flags."""
|
||||||
|
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
import urllib.error
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
@@ -37,6 +39,7 @@ class TestYtDlpFlags(unittest.TestCase):
|
|||||||
def test_transcript_fetch_ignores_global_config_and_browser_cookies(self):
|
def test_transcript_fetch_ignores_global_config_and_browser_cookies(self):
|
||||||
proc = _DummyProc()
|
proc = _DummyProc()
|
||||||
with tempfile.TemporaryDirectory() as temp_dir, \
|
with tempfile.TemporaryDirectory() as temp_dir, \
|
||||||
|
mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
|
||||||
mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock:
|
mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock:
|
||||||
youtube_yt.fetch_transcript("abc123", temp_dir)
|
youtube_yt.fetch_transcript("abc123", temp_dir)
|
||||||
|
|
||||||
@@ -45,5 +48,171 @@ class TestYtDlpFlags(unittest.TestCase):
|
|||||||
self.assertIn("--no-cookies-from-browser", cmd)
|
self.assertIn("--no-cookies-from-browser", cmd)
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractTranscriptHighlights(unittest.TestCase):
|
||||||
|
def test_extracts_specific_sentences(self):
|
||||||
|
transcript = (
|
||||||
|
"Hey guys welcome back to the channel. "
|
||||||
|
"In today's video we're looking at something special. "
|
||||||
|
"The Lego Bugatti Chiron took 13,438 hours to build with over 1 million pieces. "
|
||||||
|
"Don't forget to subscribe and hit the bell. "
|
||||||
|
"The tolerance on each brick is 0.002 millimeters which is insane for injection molding. "
|
||||||
|
"So yeah that's pretty cool. "
|
||||||
|
"Thanks for watching see you next time."
|
||||||
|
)
|
||||||
|
highlights = youtube_yt.extract_transcript_highlights(transcript, "Lego")
|
||||||
|
self.assertTrue(len(highlights) > 0)
|
||||||
|
# Should pick the sentences with numbers and topic relevance, not filler
|
||||||
|
joined = " ".join(highlights)
|
||||||
|
self.assertIn("13,438", joined)
|
||||||
|
self.assertNotIn("subscribe", joined)
|
||||||
|
self.assertNotIn("welcome back", joined)
|
||||||
|
|
||||||
|
def test_empty_transcript(self):
|
||||||
|
self.assertEqual(youtube_yt.extract_transcript_highlights("", "test"), [])
|
||||||
|
|
||||||
|
def test_respects_limit(self):
|
||||||
|
sentences = ". ".join(
|
||||||
|
f"The model {i} has {i * 100} parameters and runs at {i * 10} tokens per second"
|
||||||
|
for i in range(20)
|
||||||
|
) + "."
|
||||||
|
highlights = youtube_yt.extract_transcript_highlights(sentences, "model", limit=3)
|
||||||
|
self.assertEqual(len(highlights), 3)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetchTranscriptDirect(unittest.TestCase):
|
||||||
|
"""Tests for _fetch_transcript_direct() — direct HTTP transcript fetching."""
|
||||||
|
|
||||||
|
# Minimal ytInitialPlayerResponse JSON with a caption track
|
||||||
|
_PLAYER_RESPONSE = json.dumps({
|
||||||
|
"captions": {
|
||||||
|
"playerCaptionsTracklistRenderer": {
|
||||||
|
"captionTracks": [
|
||||||
|
{
|
||||||
|
"baseUrl": "https://www.youtube.com/api/timedtext?v=abc123&lang=en",
|
||||||
|
"languageCode": "en",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
_WATCH_HTML = (
|
||||||
|
'<html><script>var ytInitialPlayerResponse = '
|
||||||
|
+ _PLAYER_RESPONSE
|
||||||
|
+ ';</script></html>'
|
||||||
|
)
|
||||||
|
|
||||||
|
_SAMPLE_VTT = (
|
||||||
|
"WEBVTT\n\n"
|
||||||
|
"00:00:00.000 --> 00:00:02.000\n"
|
||||||
|
"Hello world this is a test sentence with enough words to pass.\n\n"
|
||||||
|
"00:00:02.000 --> 00:00:04.000\n"
|
||||||
|
"Another line of transcript text here for testing purposes.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _mock_urlopen(self, url_or_req, *, timeout=None):
|
||||||
|
"""Return watch HTML or VTT depending on URL."""
|
||||||
|
url = url_or_req.full_url if hasattr(url_or_req, 'full_url') else url_or_req
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
def __init__(self, data):
|
||||||
|
self._data = data.encode("utf-8")
|
||||||
|
def read(self):
|
||||||
|
return self._data
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
def __exit__(self, *a):
|
||||||
|
pass
|
||||||
|
|
||||||
|
if "watch?" in url:
|
||||||
|
return _Resp(self._WATCH_HTML)
|
||||||
|
elif "timedtext" in url:
|
||||||
|
return _Resp(self._SAMPLE_VTT)
|
||||||
|
raise urllib.error.URLError("unexpected URL")
|
||||||
|
|
||||||
|
def test_extracts_vtt_from_mock_page(self):
|
||||||
|
"""Happy path: extracts VTT text from a page with captions."""
|
||||||
|
with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=self._mock_urlopen):
|
||||||
|
result = youtube_yt._fetch_transcript_direct("abc123")
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
self.assertIn("WEBVTT", result)
|
||||||
|
self.assertIn("Hello world", result)
|
||||||
|
|
||||||
|
def test_no_captions_returns_none(self):
|
||||||
|
"""Video with no caption tracks returns None."""
|
||||||
|
no_captions_response = json.dumps({"captions": {"playerCaptionsTracklistRenderer": {"captionTracks": []}}})
|
||||||
|
html = f'<html><script>var ytInitialPlayerResponse = {no_captions_response};</script></html>'
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
def __init__(self, data):
|
||||||
|
self._data = data.encode("utf-8")
|
||||||
|
def read(self):
|
||||||
|
return self._data
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
def __exit__(self, *a):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def mock_open(req, *, timeout=None):
|
||||||
|
return _Resp(html)
|
||||||
|
|
||||||
|
with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=mock_open):
|
||||||
|
result = youtube_yt._fetch_transcript_direct("nocaps")
|
||||||
|
self.assertIsNone(result)
|
||||||
|
|
||||||
|
def test_http_timeout_returns_none(self):
|
||||||
|
"""HTTP timeout on watch page returns None."""
|
||||||
|
def timeout_open(req, *, timeout=None):
|
||||||
|
raise TimeoutError("timed out")
|
||||||
|
|
||||||
|
with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=timeout_open):
|
||||||
|
result = youtube_yt._fetch_transcript_direct("timeout_vid")
|
||||||
|
self.assertIsNone(result)
|
||||||
|
|
||||||
|
def test_direct_vtt_feeds_into_clean_vtt(self):
|
||||||
|
"""VTT from direct fetch produces clean plaintext via _clean_vtt()."""
|
||||||
|
cleaned = youtube_yt._clean_vtt(self._SAMPLE_VTT)
|
||||||
|
self.assertNotIn("WEBVTT", cleaned)
|
||||||
|
self.assertNotIn("-->", cleaned)
|
||||||
|
self.assertIn("Hello world", cleaned)
|
||||||
|
self.assertIn("Another line", cleaned)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetchTranscriptFallback(unittest.TestCase):
|
||||||
|
"""Tests that fetch_transcript picks yt-dlp or direct path correctly."""
|
||||||
|
|
||||||
|
def test_uses_ytdlp_when_installed(self):
|
||||||
|
"""When yt-dlp is installed, uses _fetch_transcript_ytdlp."""
|
||||||
|
with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
|
||||||
|
mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", return_value="WEBVTT\n\nfake") as yt_mock, \
|
||||||
|
mock.patch.object(youtube_yt, "_fetch_transcript_direct") as direct_mock:
|
||||||
|
result = youtube_yt.fetch_transcript("vid1", "/tmp/test")
|
||||||
|
yt_mock.assert_called_once_with("vid1", "/tmp/test")
|
||||||
|
direct_mock.assert_not_called()
|
||||||
|
|
||||||
|
def test_uses_direct_when_ytdlp_missing(self):
|
||||||
|
"""When yt-dlp is NOT installed, falls back to _fetch_transcript_direct."""
|
||||||
|
sample_vtt = (
|
||||||
|
"WEBVTT\n\n"
|
||||||
|
"00:00:00.000 --> 00:00:02.000\n"
|
||||||
|
"Direct transcript content with enough words for testing.\n"
|
||||||
|
)
|
||||||
|
with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \
|
||||||
|
mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp") as yt_mock, \
|
||||||
|
mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=sample_vtt) as direct_mock:
|
||||||
|
result = youtube_yt.fetch_transcript("vid2", "/tmp/test")
|
||||||
|
yt_mock.assert_not_called()
|
||||||
|
direct_mock.assert_called_once_with("vid2")
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
self.assertIn("Direct transcript content", result)
|
||||||
|
|
||||||
|
def test_returns_none_when_both_fail(self):
|
||||||
|
"""Returns None when the chosen path returns None."""
|
||||||
|
with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \
|
||||||
|
mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=None):
|
||||||
|
result = youtube_yt.fetch_transcript("novid", "/tmp/test")
|
||||||
|
self.assertIsNone(result)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user