From 5ca4829be43eb2a4db5ff5fa4a3f65d6b9a93c40 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Fri, 23 Jan 2026 12:37:31 -0800 Subject: [PATCH] Initial commit: last30days skill Research topics across Reddit + X from the last 30 days using OpenAI and xAI APIs. Features: - Auto model selection (GPT-5.x, Grok-3) - Popularity-aware scoring (relevance + recency + engagement) - Reddit thread enrichment with real metrics - Near-duplicate detection - Multiple emit modes (compact, json, context, path) - 24h caching with --refresh bypass - NUX for API key setup - 87 passing unit tests Co-Authored-By: Claude Opus 4.5 --- SKILL.md | 145 +++++++++++++++ SPEC.md | 75 ++++++++ TASKS.md | 47 +++++ fixtures/models_openai_sample.json | 41 +++++ fixtures/models_xai_sample.json | 23 +++ fixtures/openai_sample.json | 22 +++ fixtures/reddit_thread_sample.json | 108 +++++++++++ fixtures/xai_sample.json | 22 +++ scripts/last30days.py | 271 ++++++++++++++++++++++++++++ scripts/lib/__init__.py | 1 + scripts/lib/cache.py | 119 +++++++++++++ scripts/lib/dates.py | 124 +++++++++++++ scripts/lib/dedupe.py | 120 +++++++++++++ scripts/lib/env.py | 102 +++++++++++ scripts/lib/http.py | 126 +++++++++++++ scripts/lib/models.py | 175 ++++++++++++++++++ scripts/lib/normalize.py | 118 ++++++++++++ scripts/lib/openai_reddit.py | 158 ++++++++++++++++ scripts/lib/reddit_enrich.py | 232 ++++++++++++++++++++++++ scripts/lib/render.py | 277 +++++++++++++++++++++++++++++ scripts/lib/schema.py | 193 ++++++++++++++++++++ scripts/lib/score.py | 240 +++++++++++++++++++++++++ scripts/lib/xai_x.py | 192 ++++++++++++++++++++ tests/__init__.py | 1 + tests/test_cache.py | 59 ++++++ tests/test_dates.py | 114 ++++++++++++ tests/test_dedupe.py | 111 ++++++++++++ tests/test_models.py | 135 ++++++++++++++ tests/test_normalize.py | 138 ++++++++++++++ tests/test_render.py | 116 ++++++++++++ tests/test_score.py | 168 +++++++++++++++++ 31 files changed, 3773 insertions(+) create mode 100644 SKILL.md create mode 100644 SPEC.md create mode 100644 TASKS.md create mode 100644 fixtures/models_openai_sample.json create mode 100644 fixtures/models_xai_sample.json create mode 100644 fixtures/openai_sample.json create mode 100644 fixtures/reddit_thread_sample.json create mode 100644 fixtures/xai_sample.json create mode 100644 scripts/last30days.py create mode 100644 scripts/lib/__init__.py create mode 100644 scripts/lib/cache.py create mode 100644 scripts/lib/dates.py create mode 100644 scripts/lib/dedupe.py create mode 100644 scripts/lib/env.py create mode 100644 scripts/lib/http.py create mode 100644 scripts/lib/models.py create mode 100644 scripts/lib/normalize.py create mode 100644 scripts/lib/openai_reddit.py create mode 100644 scripts/lib/reddit_enrich.py create mode 100644 scripts/lib/render.py create mode 100644 scripts/lib/schema.py create mode 100644 scripts/lib/score.py create mode 100644 scripts/lib/xai_x.py create mode 100644 tests/__init__.py create mode 100644 tests/test_cache.py create mode 100644 tests/test_dates.py create mode 100644 tests/test_dedupe.py create mode 100644 tests/test_models.py create mode 100644 tests/test_normalize.py create mode 100644 tests/test_render.py create mode 100644 tests/test_score.py diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..b2ac908 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,145 @@ +--- +name: last30days +description: Research a topic from the last 30 days on Reddit + X; judge/summarize into best practices, a prompt pack, and a reusable context snippet. +argument-hint: "[topic]" +context: fork +agent: Explore +disable-model-invocation: true +allowed-tools: Bash, Read, Write +--- + +# last30days: 30-Day Research Synthesis + +Research a topic across Reddit and X from the last 30 days, then synthesize findings into actionable best practices, prompts, and reusable context. + +## Setup Check + +First, verify API key configuration exists: + +```bash +if [ ! -f ~/.config/last30days/.env ]; then + echo "SETUP_NEEDED" +else + echo "CONFIGURED" +fi +``` + +### If SETUP_NEEDED + +Run the NUX flow to configure API keys. Use the AskUserQuestion tool to collect: + +1. **OpenAI API Key** (optional but recommended for Reddit research) +2. **xAI API Key** (optional but recommended for X research) +3. **Model policies** (optional, defaults are usually fine) + +Then create the config: + +```bash +mkdir -p ~/.config/last30days +cat > ~/.config/last30days/.env << 'ENVEOF' +# last30days API Configuration +# At least one key is required + +OPENAI_API_KEY= +XAI_API_KEY= + +# Model selection (optional) +# OPENAI_MODEL_POLICY=auto|pinned (default: auto) +# OPENAI_MODEL_PIN=gpt-5.2 (only if pinned) +# XAI_MODEL_POLICY=latest|stable|pinned (default: latest) +# XAI_MODEL_PIN=grok-4 (only if pinned) +ENVEOF + +chmod 600 ~/.config/last30days/.env +echo "Config created at ~/.config/last30days/.env" +echo "Please edit it to add your API keys, then run the skill again." +``` + +After creating the file, instruct the user to edit `~/.config/last30days/.env` and add their keys. + +**STOP HERE if setup was needed. Do not proceed until keys are configured.** + +--- + +## Research Execution + +If configured, run the research orchestrator: + +```bash +python3 ~/.claude/skills/last30days/scripts/last30days.py "$ARGUMENTS" --emit=compact 2>&1 +``` + +The script will: +- Auto-detect which keys are available +- Auto-select the best models (or use pinned versions) +- Search Reddit via OpenAI Responses API (if OpenAI key present) +- Search X via xAI Responses API (if xAI key present) +- Enrich Reddit threads with real engagement metrics +- Score, rank, and dedupe results +- Output files to `~/.local/share/last30days/out/` + +--- + +## RESEARCH DATA + +The output above contains the research data. Now synthesize it. + +--- + +## Your Role: Judge and Synthesizer + +You are now the expert judge. Using the research data above, produce: + +### A) Best Practices (Grouped & Actionable) + +Group findings into 3-7 thematic categories. For each best practice: +- State the practice clearly and actionably +- Cite supporting item IDs (e.g., "supported by R3, R7, X2") +- Note if it's **strongly supported** (multiple high-score sources) or **niche** (single source or low engagement) + +### B) Prompt Pack (3-7 Copy/Paste Prompts) + +Create ready-to-use prompts tailored to the topic. Each prompt should: +- Be immediately usable (copy/paste ready) +- Target a specific use case discovered in the research +- Include any relevant context or constraints from the findings + +### C) Reusable Context Snippet + +Create a compact (~200-400 words) context block that other skills/tools can import. Include: +- Core concepts and terminology +- Key techniques or patterns +- Common pitfalls to avoid +- Brief source attribution + +### D) Sources Appendix + +List all source URLs organized by platform: +- **Reddit**: Title, subreddit, URL, score +- **X**: Author, text excerpt, URL, engagement + +### E) Confidence Assessment + +Explicitly state: +- **Strongly Supported**: Practices backed by multiple high-engagement sources +- **Emerging/Niche**: Practices from single sources or low engagement (still valuable but use with awareness) +- **Gaps**: What the research didn't cover well + +--- + +## Final Output + +After completing your synthesis: + +1. Display the full report to the user +2. Confirm the files were written: + - `~/.local/share/last30days/out/report.md` + - `~/.local/share/last30days/out/report.json` + - `~/.local/share/last30days/out/last30days.context.md` + +3. Show the header summary: + ``` + Models used: OpenAI={model} xAI={model} + Mode: {reddit-only|x-only|both} + Coverage: {note about triangulation if single-source} + ``` diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..a464c0b --- /dev/null +++ b/SPEC.md @@ -0,0 +1,75 @@ +# last30days Skill Specification + +## Overview + +`last30days` is a Claude Code skill that researches a given topic across Reddit and X (Twitter) using the OpenAI Responses API and xAI Responses API respectively. It enforces a strict 30-day recency window, popularity-aware ranking, and produces actionable outputs including best practices, a prompt pack, and a reusable context snippet. + +The skill operates in three modes depending on available API keys: **reddit-only** (OpenAI key), **x-only** (xAI key), or **both** (full cross-validation). It uses automatic model selection to stay current with the latest models from both providers, with optional pinning for stability. + +## Architecture + +The orchestrator (`last30days.py`) coordinates discovery, enrichment, normalization, scoring, deduplication, and rendering. Each concern is isolated in `scripts/lib/`: + +- **env.py**: Load and validate API keys from `~/.config/last30days/.env` +- **dates.py**: Date range calculation and confidence scoring +- **cache.py**: 24-hour TTL caching keyed by topic + date range +- **http.py**: stdlib-only HTTP client with retry logic +- **models.py**: Auto-selection of OpenAI/xAI models with 7-day caching +- **openai_reddit.py**: OpenAI Responses API + web_search for Reddit +- **xai_x.py**: xAI Responses API + x_search for X +- **reddit_enrich.py**: Fetch Reddit thread JSON for real engagement metrics +- **normalize.py**: Convert raw API responses to canonical schema +- **score.py**: Compute popularity-aware scores (relevance + recency + engagement) +- **dedupe.py**: Near-duplicate detection via text similarity +- **render.py**: Generate markdown and JSON outputs +- **schema.py**: Type definitions and validation + +## Embedding in Other Skills + +Other skills can import the research context in several ways: + +### Inline Context Injection +```markdown +## Recent Research Context +!python3 ~/.claude/skills/last30days/scripts/last30days.py "your topic" --emit=context +``` + +### Read from File +```markdown +## Research Context +!cat ~/.local/share/last30days/out/last30days.context.md +``` + +### Get Path for Dynamic Loading +```bash +CONTEXT_PATH=$(python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=path) +cat "$CONTEXT_PATH" +``` + +### JSON for Programmatic Use +```bash +python3 ~/.claude/skills/last30days/scripts/last30days.py "topic" --emit=json > research.json +``` + +## CLI Reference + +``` +python3 ~/.claude/skills/last30days/scripts/last30days.py [options] + +Options: + --refresh Bypass cache and fetch fresh data + --mock Use fixtures instead of real API calls + --emit=MODE Output mode: compact|json|md|context|path (default: compact) + --sources=MODE Source selection: auto|reddit|x|both (default: auto) +``` + +## Output Files + +All outputs are written to `~/.local/share/last30days/out/`: + +- `report.md` - Human-readable full report +- `report.json` - Normalized data with scores +- `last30days.context.md` - Compact reusable snippet for other skills +- `raw_openai.json` - Raw OpenAI API response +- `raw_xai.json` - Raw xAI API response +- `raw_reddit_threads_enriched.json` - Enriched Reddit thread data diff --git a/TASKS.md b/TASKS.md new file mode 100644 index 0000000..8f9272d --- /dev/null +++ b/TASKS.md @@ -0,0 +1,47 @@ +# last30days Implementation Tasks + +## Setup & Configuration +- [x] Create directory structure +- [x] Write SPEC.md +- [x] Write TASKS.md +- [x] Write SKILL.md with proper frontmatter + +## Core Library Modules +- [x] scripts/lib/env.py - Environment and API key loading +- [x] scripts/lib/dates.py - Date range and confidence utilities +- [x] scripts/lib/cache.py - TTL-based caching +- [x] scripts/lib/http.py - HTTP client with retry +- [x] scripts/lib/models.py - Auto model selection +- [x] scripts/lib/schema.py - Data structures +- [x] scripts/lib/openai_reddit.py - OpenAI Responses API +- [x] scripts/lib/xai_x.py - xAI Responses API +- [x] scripts/lib/reddit_enrich.py - Reddit thread JSON fetcher +- [x] scripts/lib/normalize.py - Schema normalization +- [x] scripts/lib/score.py - Popularity scoring +- [x] scripts/lib/dedupe.py - Near-duplicate detection +- [x] scripts/lib/render.py - Output rendering + +## Main Script +- [x] scripts/last30days.py - CLI orchestrator + +## Fixtures +- [x] fixtures/openai_sample.json +- [x] fixtures/xai_sample.json +- [x] fixtures/reddit_thread_sample.json +- [x] fixtures/models_openai_sample.json +- [x] fixtures/models_xai_sample.json + +## Tests +- [x] tests/test_dates.py +- [x] tests/test_cache.py +- [x] tests/test_models.py +- [x] tests/test_score.py +- [x] tests/test_dedupe.py +- [x] tests/test_normalize.py +- [x] tests/test_render.py + +## Validation +- [x] Run tests in mock mode +- [x] Demo --emit=compact +- [x] Demo --emit=context +- [x] Verify file tree diff --git a/fixtures/models_openai_sample.json b/fixtures/models_openai_sample.json new file mode 100644 index 0000000..e972479 --- /dev/null +++ b/fixtures/models_openai_sample.json @@ -0,0 +1,41 @@ +{ + "object": "list", + "data": [ + { + "id": "gpt-5.2", + "object": "model", + "created": 1704067200, + "owned_by": "openai" + }, + { + "id": "gpt-5.1", + "object": "model", + "created": 1701388800, + "owned_by": "openai" + }, + { + "id": "gpt-5", + "object": "model", + "created": 1698710400, + "owned_by": "openai" + }, + { + "id": "gpt-5-mini", + "object": "model", + "created": 1704067200, + "owned_by": "openai" + }, + { + "id": "gpt-4o", + "object": "model", + "created": 1683158400, + "owned_by": "openai" + }, + { + "id": "gpt-4-turbo", + "object": "model", + "created": 1680566400, + "owned_by": "openai" + } + ] +} diff --git a/fixtures/models_xai_sample.json b/fixtures/models_xai_sample.json new file mode 100644 index 0000000..5e571ed --- /dev/null +++ b/fixtures/models_xai_sample.json @@ -0,0 +1,23 @@ +{ + "object": "list", + "data": [ + { + "id": "grok-4-latest", + "object": "model", + "created": 1704067200, + "owned_by": "xai" + }, + { + "id": "grok-4", + "object": "model", + "created": 1701388800, + "owned_by": "xai" + }, + { + "id": "grok-3", + "object": "model", + "created": 1698710400, + "owned_by": "xai" + } + ] +} diff --git a/fixtures/openai_sample.json b/fixtures/openai_sample.json new file mode 100644 index 0000000..ce0d023 --- /dev/null +++ b/fixtures/openai_sample.json @@ -0,0 +1,22 @@ +{ + "id": "resp_mock123", + "object": "response", + "created": 1706140800, + "model": "gpt-5.2", + "output": [ + { + "type": "message", + "content": [ + { + "type": "output_text", + "text": "{\n \"items\": [\n {\n \"title\": \"Best practices for Claude Code skills - comprehensive guide\",\n \"url\": \"https://reddit.com/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills\",\n \"subreddit\": \"ClaudeAI\",\n \"date\": \"2026-01-15\",\n \"why_relevant\": \"Detailed discussion of skill creation patterns and best practices\",\n \"relevance\": 0.95\n },\n {\n \"title\": \"How I built a research skill for Claude Code\",\n \"url\": \"https://reddit.com/r/ClaudeAI/comments/def456/how_i_built_a_research_skill\",\n \"subreddit\": \"ClaudeAI\",\n \"date\": \"2026-01-10\",\n \"why_relevant\": \"Real-world example of building a Claude Code skill with API integrations\",\n \"relevance\": 0.90\n },\n {\n \"title\": \"Claude Code vs Cursor vs Windsurf - January 2026 comparison\",\n \"url\": \"https://reddit.com/r/LocalLLaMA/comments/ghi789/claude_code_vs_cursor_vs_windsurf\",\n \"subreddit\": \"LocalLLaMA\",\n \"date\": \"2026-01-08\",\n \"why_relevant\": \"Compares Claude Code features including skills system\",\n \"relevance\": 0.85\n },\n {\n \"title\": \"Tips for effective prompt engineering in Claude Code\",\n \"url\": \"https://reddit.com/r/PromptEngineering/comments/jkl012/tips_for_claude_code_prompts\",\n \"subreddit\": \"PromptEngineering\",\n \"date\": \"2026-01-05\",\n \"why_relevant\": \"Discusses prompt patterns that work well with Claude Code skills\",\n \"relevance\": 0.80\n },\n {\n \"title\": \"New Claude Code update: improved skill loading\",\n \"url\": \"https://reddit.com/r/ClaudeAI/comments/mno345/new_claude_code_update_improved_skill_loading\",\n \"subreddit\": \"ClaudeAI\",\n \"date\": \"2026-01-03\",\n \"why_relevant\": \"Announcement of new skill features in Claude Code\",\n \"relevance\": 0.75\n }\n ]\n}" + } + ] + } + ], + "usage": { + "prompt_tokens": 150, + "completion_tokens": 500, + "total_tokens": 650 + } +} diff --git a/fixtures/reddit_thread_sample.json b/fixtures/reddit_thread_sample.json new file mode 100644 index 0000000..502d560 --- /dev/null +++ b/fixtures/reddit_thread_sample.json @@ -0,0 +1,108 @@ +[ + { + "kind": "Listing", + "data": { + "children": [ + { + "kind": "t3", + "data": { + "title": "Best practices for Claude Code skills - comprehensive guide", + "score": 847, + "num_comments": 156, + "upvote_ratio": 0.94, + "created_utc": 1705363200, + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/", + "selftext": "After building 20+ skills for Claude Code, here are my key learnings..." + } + } + ] + } + }, + { + "kind": "Listing", + "data": { + "children": [ + { + "kind": "t1", + "data": { + "score": 234, + "created_utc": 1705366800, + "author": "skill_expert", + "body": "Great guide! One thing I'd add: always use explicit tool permissions in your SKILL.md. Don't default to allowing everything.", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment1/" + } + }, + { + "kind": "t1", + "data": { + "score": 189, + "created_utc": 1705370400, + "author": "claude_dev", + "body": "The context: fork tip is gold. I was wondering why my heavy research skill was slow - it was blocking the main thread!", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment2/" + } + }, + { + "kind": "t1", + "data": { + "score": 145, + "created_utc": 1705374000, + "author": "ai_builder", + "body": "For anyone starting out: begin with a simple skill that just runs one bash command. Once that works, build up complexity gradually.", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment3/" + } + }, + { + "kind": "t1", + "data": { + "score": 98, + "created_utc": 1705377600, + "author": "dev_tips", + "body": "The --mock flag pattern for testing without API calls is essential. I always build that in from day one now.", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment4/" + } + }, + { + "kind": "t1", + "data": { + "score": 76, + "created_utc": 1705381200, + "author": "code_writer", + "body": "Thanks for sharing! Question: how do you handle API key storage securely in skills?", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment5/" + } + }, + { + "kind": "t1", + "data": { + "score": 65, + "created_utc": 1705384800, + "author": "security_minded", + "body": "I use ~/.config/skillname/.env with chmod 600. Never hardcode keys, and definitely don't commit them!", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment6/" + } + }, + { + "kind": "t1", + "data": { + "score": 52, + "created_utc": 1705388400, + "author": "helpful_user", + "body": "The caching pattern you described saved me so much on API costs. 24h TTL is perfect for most research skills.", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment7/" + } + }, + { + "kind": "t1", + "data": { + "score": 34, + "created_utc": 1705392000, + "author": "newbie_coder", + "body": "This is exactly what I needed. Starting my first skill this weekend!", + "permalink": "/r/ClaudeAI/comments/abc123/best_practices_for_claude_code_skills/comment8/" + } + } + ] + } + } +] diff --git a/fixtures/xai_sample.json b/fixtures/xai_sample.json new file mode 100644 index 0000000..fd035cb --- /dev/null +++ b/fixtures/xai_sample.json @@ -0,0 +1,22 @@ +{ + "id": "resp_xai_mock456", + "object": "response", + "created": 1706140800, + "model": "grok-4-latest", + "output": [ + { + "type": "message", + "content": [ + { + "type": "output_text", + "text": "{\n \"items\": [\n {\n \"text\": \"Just shipped my first Claude Code skill! The SKILL.md format is incredibly intuitive. Pro tip: use context: fork for resource-intensive operations.\",\n \"url\": \"https://x.com/devuser1/status/1234567890\",\n \"author_handle\": \"devuser1\",\n \"date\": \"2026-01-18\",\n \"engagement\": {\n \"likes\": 542,\n \"reposts\": 87,\n \"replies\": 34,\n \"quotes\": 12\n },\n \"why_relevant\": \"First-hand experience building Claude Code skills with practical tips\",\n \"relevance\": 0.92\n },\n {\n \"text\": \"Thread: Everything I learned building 10 Claude Code skills in 30 days. 1/ Start simple. Your first skill should be < 50 lines of markdown.\",\n \"url\": \"https://x.com/aibuilder/status/1234567891\",\n \"author_handle\": \"aibuilder\",\n \"date\": \"2026-01-12\",\n \"engagement\": {\n \"likes\": 1203,\n \"reposts\": 245,\n \"replies\": 89,\n \"quotes\": 56\n },\n \"why_relevant\": \"Comprehensive thread on skill building best practices\",\n \"relevance\": 0.95\n },\n {\n \"text\": \"The allowed-tools field in SKILL.md is crucial for security. Don't give skills more permissions than they need.\",\n \"url\": \"https://x.com/securitydev/status/1234567892\",\n \"author_handle\": \"securitydev\",\n \"date\": \"2026-01-08\",\n \"engagement\": {\n \"likes\": 328,\n \"reposts\": 67,\n \"replies\": 23,\n \"quotes\": 8\n },\n \"why_relevant\": \"Security best practices for Claude Code skills\",\n \"relevance\": 0.85\n },\n {\n \"text\": \"Loving the new /skill command in Claude Code. Makes testing skills so much easier during development.\",\n \"url\": \"https://x.com/codeenthusiast/status/1234567893\",\n \"author_handle\": \"codeenthusiast\",\n \"date\": \"2026-01-05\",\n \"engagement\": {\n \"likes\": 156,\n \"reposts\": 23,\n \"replies\": 12,\n \"quotes\": 4\n },\n \"why_relevant\": \"Discusses skill development workflow\",\n \"relevance\": 0.78\n }\n ]\n}" + } + ] + } + ], + "usage": { + "prompt_tokens": 180, + "completion_tokens": 450, + "total_tokens": 630 + } +} diff --git a/scripts/last30days.py b/scripts/last30days.py new file mode 100644 index 0000000..2c1e2f9 --- /dev/null +++ b/scripts/last30days.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +""" +last30days - Research a topic from the last 30 days on Reddit + X. + +Usage: + python3 last30days.py [options] + +Options: + --refresh Bypass cache and fetch fresh data + --mock Use fixtures instead of real API calls + --emit=MODE Output mode: compact|json|md|context|path (default: compact) + --sources=MODE Source selection: auto|reddit|x|both (default: auto) +""" + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +# Add lib to path +SCRIPT_DIR = Path(__file__).parent.resolve() +sys.path.insert(0, str(SCRIPT_DIR)) + +from lib import ( + cache, + dates, + dedupe, + env, + models, + normalize, + openai_reddit, + reddit_enrich, + render, + schema, + score, + xai_x, +) + + +def load_fixture(name: str) -> dict: + """Load a fixture file.""" + fixture_path = SCRIPT_DIR.parent / "fixtures" / name + if fixture_path.exists(): + with open(fixture_path) as f: + return json.load(f) + return {} + + +def run_research( + topic: str, + sources: str, + config: dict, + selected_models: dict, + from_date: str, + to_date: str, + mock: bool = False, +) -> tuple: + """Run the research pipeline. + + Returns: + Tuple of (reddit_items, x_items, raw_openai, raw_xai, raw_reddit_enriched) + """ + reddit_items = [] + x_items = [] + raw_openai = None + raw_xai = None + raw_reddit_enriched = [] + + # Reddit search via OpenAI + if sources in ("both", "reddit"): + if mock: + raw_openai = load_fixture("openai_sample.json") + else: + raw_openai = openai_reddit.search_reddit( + config["OPENAI_API_KEY"], + selected_models["openai"], + topic, + ) + + # Parse response + reddit_items = openai_reddit.parse_reddit_response(raw_openai) + + # Enrich with real Reddit data + for i, item in enumerate(reddit_items): + if mock: + mock_thread = load_fixture("reddit_thread_sample.json") + reddit_items[i] = reddit_enrich.enrich_reddit_item(item, mock_thread) + else: + reddit_items[i] = reddit_enrich.enrich_reddit_item(item) + + raw_reddit_enriched.append(reddit_items[i]) + + # X search via xAI + if sources in ("both", "x"): + if mock: + raw_xai = load_fixture("xai_sample.json") + else: + raw_xai = xai_x.search_x( + config["XAI_API_KEY"], + selected_models["xai"], + topic, + from_date, + to_date, + ) + + # Parse response + x_items = xai_x.parse_x_response(raw_xai) + + return reddit_items, x_items, raw_openai, raw_xai, raw_reddit_enriched + + +def main(): + parser = argparse.ArgumentParser( + description="Research a topic from the last 30 days on Reddit + X" + ) + parser.add_argument("topic", nargs="?", help="Topic to research") + parser.add_argument("--refresh", action="store_true", help="Bypass cache") + parser.add_argument("--mock", action="store_true", help="Use fixtures") + parser.add_argument( + "--emit", + choices=["compact", "json", "md", "context", "path"], + default="compact", + help="Output mode", + ) + parser.add_argument( + "--sources", + choices=["auto", "reddit", "x", "both"], + default="auto", + help="Source selection", + ) + + args = parser.parse_args() + + if not args.topic: + print("Error: Please provide a topic to research.", file=sys.stderr) + print("Usage: python3 last30days.py [options]", file=sys.stderr) + sys.exit(1) + + # Load config + config = env.get_config() + + # Check available sources + available = env.get_available_sources(config) + if available == "none" and not args.mock: + print("Error: No API keys configured.", file=sys.stderr) + print("Please add at least one key to ~/.config/last30days/.env:", file=sys.stderr) + print(" OPENAI_API_KEY=sk-...", file=sys.stderr) + print(" XAI_API_KEY=xai-...", file=sys.stderr) + sys.exit(1) + + # Mock mode can work without keys + if args.mock: + if args.sources == "auto": + sources = "both" + else: + sources = args.sources + else: + # Validate requested sources against available + sources, error = env.validate_sources(args.sources, available) + if error: + print(f"Error: {error}", file=sys.stderr) + sys.exit(1) + + # Get date range + from_date, to_date = dates.get_date_range(30) + + # Check cache (unless refresh or mock) + cache_key = cache.get_cache_key(args.topic, from_date, to_date, sources) + if not args.refresh and not args.mock: + cached = cache.load_cache(cache_key) + if cached: + # Use cached data + report = schema.Report(**cached) + output_result(report, args.emit) + return + + # Select models + if args.mock: + # Use mock models + mock_openai_models = load_fixture("models_openai_sample.json").get("data", []) + mock_xai_models = load_fixture("models_xai_sample.json").get("data", []) + selected_models = models.get_models( + { + "OPENAI_API_KEY": "mock", + "XAI_API_KEY": "mock", + **config, + }, + mock_openai_models, + mock_xai_models, + ) + else: + selected_models = models.get_models(config) + + # Determine mode string + if sources == "both": + mode = "both" + elif sources == "reddit": + mode = "reddit-only" + else: + mode = "x-only" + + # Run research + reddit_items, x_items, raw_openai, raw_xai, raw_reddit_enriched = run_research( + args.topic, + sources, + config, + selected_models, + from_date, + to_date, + args.mock, + ) + + # Normalize items + normalized_reddit = normalize.normalize_reddit_items(reddit_items, from_date, to_date) + normalized_x = normalize.normalize_x_items(x_items, from_date, to_date) + + # Score items + scored_reddit = score.score_reddit_items(normalized_reddit) + scored_x = score.score_x_items(normalized_x) + + # Sort items + sorted_reddit = score.sort_items(scored_reddit) + sorted_x = score.sort_items(scored_x) + + # Dedupe items + deduped_reddit = dedupe.dedupe_reddit(sorted_reddit) + deduped_x = dedupe.dedupe_x(sorted_x) + + # Create report + report = schema.create_report( + args.topic, + from_date, + to_date, + mode, + selected_models.get("openai"), + selected_models.get("xai"), + ) + report.reddit = deduped_reddit + report.x = deduped_x + + # Generate context snippet + report.context_snippet_md = render.render_context_snippet(report) + + # Write outputs + render.write_outputs(report, raw_openai, raw_xai, raw_reddit_enriched) + + # Cache the result (if not mock) + if not args.mock: + cache.save_cache(cache_key, report.to_dict()) + + # Output result + output_result(report, args.emit) + + +def output_result(report: schema.Report, emit_mode: str): + """Output the result based on emit mode.""" + if emit_mode == "compact": + print(render.render_compact(report)) + elif emit_mode == "json": + print(json.dumps(report.to_dict(), indent=2)) + elif emit_mode == "md": + print(render.render_full_report(report)) + elif emit_mode == "context": + print(report.context_snippet_md) + elif emit_mode == "path": + print(render.get_context_path()) + + +if __name__ == "__main__": + main() diff --git a/scripts/lib/__init__.py b/scripts/lib/__init__.py new file mode 100644 index 0000000..2297618 --- /dev/null +++ b/scripts/lib/__init__.py @@ -0,0 +1 @@ +# last30days library modules diff --git a/scripts/lib/cache.py b/scripts/lib/cache.py new file mode 100644 index 0000000..0545ba6 --- /dev/null +++ b/scripts/lib/cache.py @@ -0,0 +1,119 @@ +"""Caching utilities for last30days skill.""" + +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +CACHE_DIR = Path.home() / ".cache" / "last30days" +DEFAULT_TTL_HOURS = 24 +MODEL_CACHE_TTL_DAYS = 7 + + +def ensure_cache_dir(): + """Ensure cache directory exists.""" + CACHE_DIR.mkdir(parents=True, exist_ok=True) + + +def get_cache_key(topic: str, from_date: str, to_date: str, sources: str) -> str: + """Generate a cache key from query parameters.""" + key_data = f"{topic}|{from_date}|{to_date}|{sources}" + return hashlib.sha256(key_data.encode()).hexdigest()[:16] + + +def get_cache_path(cache_key: str) -> Path: + """Get path to cache file.""" + return CACHE_DIR / f"{cache_key}.json" + + +def is_cache_valid(cache_path: Path, ttl_hours: int = DEFAULT_TTL_HOURS) -> bool: + """Check if cache file exists and is within TTL.""" + if not cache_path.exists(): + return False + + try: + stat = cache_path.stat() + mtime = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc) + now = datetime.now(timezone.utc) + age_hours = (now - mtime).total_seconds() / 3600 + return age_hours < ttl_hours + except OSError: + return False + + +def load_cache(cache_key: str, ttl_hours: int = DEFAULT_TTL_HOURS) -> Optional[dict]: + """Load data from cache if valid.""" + cache_path = get_cache_path(cache_key) + + if not is_cache_valid(cache_path, ttl_hours): + return None + + try: + with open(cache_path, 'r') as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return None + + +def save_cache(cache_key: str, data: dict): + """Save data to cache.""" + ensure_cache_dir() + cache_path = get_cache_path(cache_key) + + try: + with open(cache_path, 'w') as f: + json.dump(data, f) + except OSError: + pass # Silently fail on cache write errors + + +def clear_cache(): + """Clear all cache files.""" + if CACHE_DIR.exists(): + for f in CACHE_DIR.glob("*.json"): + try: + f.unlink() + except OSError: + pass + + +# Model selection cache (longer TTL) +MODEL_CACHE_FILE = CACHE_DIR / "model_selection.json" + + +def load_model_cache() -> dict: + """Load model selection cache.""" + if not is_cache_valid(MODEL_CACHE_FILE, MODEL_CACHE_TTL_DAYS * 24): + return {} + + try: + with open(MODEL_CACHE_FILE, 'r') as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return {} + + +def save_model_cache(data: dict): + """Save model selection cache.""" + ensure_cache_dir() + try: + with open(MODEL_CACHE_FILE, 'w') as f: + json.dump(data, f) + except OSError: + pass + + +def get_cached_model(provider: str) -> Optional[str]: + """Get cached model selection for a provider.""" + cache = load_model_cache() + return cache.get(provider) + + +def set_cached_model(provider: str, model: str): + """Cache model selection for a provider.""" + cache = load_model_cache() + cache[provider] = model + cache['updated_at'] = datetime.now(timezone.utc).isoformat() + save_model_cache(cache) diff --git a/scripts/lib/dates.py b/scripts/lib/dates.py new file mode 100644 index 0000000..fd6c2d7 --- /dev/null +++ b/scripts/lib/dates.py @@ -0,0 +1,124 @@ +"""Date utilities for last30days skill.""" + +from datetime import datetime, timedelta, timezone +from typing import Optional, Tuple + + +def get_date_range(days: int = 30) -> Tuple[str, str]: + """Get the date range for the last N days. + + Returns: + Tuple of (from_date, to_date) as YYYY-MM-DD strings + """ + today = datetime.now(timezone.utc).date() + from_date = today - timedelta(days=days) + return from_date.isoformat(), today.isoformat() + + +def parse_date(date_str: Optional[str]) -> Optional[datetime]: + """Parse a date string in various formats. + + Supports: YYYY-MM-DD, ISO 8601, Unix timestamp + """ + if not date_str: + return None + + # Try Unix timestamp (from Reddit) + try: + ts = float(date_str) + return datetime.fromtimestamp(ts, tz=timezone.utc) + except (ValueError, TypeError): + pass + + # Try ISO formats + formats = [ + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S.%f%z", + ] + + for fmt in formats: + try: + return datetime.strptime(date_str, fmt).replace(tzinfo=timezone.utc) + except ValueError: + continue + + return None + + +def timestamp_to_date(ts: Optional[float]) -> Optional[str]: + """Convert Unix timestamp to YYYY-MM-DD string.""" + if ts is None: + return None + try: + dt = datetime.fromtimestamp(ts, tz=timezone.utc) + return dt.date().isoformat() + except (ValueError, TypeError, OSError): + return None + + +def get_date_confidence(date_str: Optional[str], from_date: str, to_date: str) -> str: + """Determine confidence level for a date. + + Args: + date_str: The date to check (YYYY-MM-DD or None) + from_date: Start of valid range (YYYY-MM-DD) + to_date: End of valid range (YYYY-MM-DD) + + Returns: + 'high', 'med', or 'low' + """ + if not date_str: + return 'low' + + try: + dt = datetime.strptime(date_str, "%Y-%m-%d").date() + start = datetime.strptime(from_date, "%Y-%m-%d").date() + end = datetime.strptime(to_date, "%Y-%m-%d").date() + + if start <= dt <= end: + return 'high' + elif dt < start: + # Older than range + return 'low' + else: + # Future date (suspicious) + return 'low' + except ValueError: + return 'low' + + +def days_ago(date_str: Optional[str]) -> Optional[int]: + """Calculate how many days ago a date is. + + Returns None if date is invalid or missing. + """ + if not date_str: + return None + + try: + dt = datetime.strptime(date_str, "%Y-%m-%d").date() + today = datetime.now(timezone.utc).date() + delta = today - dt + return delta.days + except ValueError: + return None + + +def recency_score(date_str: Optional[str], max_days: int = 30) -> int: + """Calculate recency score (0-100). + + 0 days ago = 100, max_days ago = 0, clamped. + """ + age = days_ago(date_str) + if age is None: + return 0 # Unknown date gets worst score + + if age < 0: + return 100 # Future date (treat as today) + if age >= max_days: + return 0 + + return int(100 * (1 - age / max_days)) diff --git a/scripts/lib/dedupe.py b/scripts/lib/dedupe.py new file mode 100644 index 0000000..a42024f --- /dev/null +++ b/scripts/lib/dedupe.py @@ -0,0 +1,120 @@ +"""Near-duplicate detection for last30days skill.""" + +import re +from typing import List, Set, Tuple, Union + +from . import schema + + +def normalize_text(text: str) -> str: + """Normalize text for comparison. + + - Lowercase + - Remove punctuation + - Collapse whitespace + """ + text = text.lower() + text = re.sub(r'[^\w\s]', ' ', text) + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +def get_ngrams(text: str, n: int = 3) -> Set[str]: + """Get character n-grams from text.""" + text = normalize_text(text) + if len(text) < n: + return {text} + return {text[i:i+n] for i in range(len(text) - n + 1)} + + +def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float: + """Compute Jaccard similarity between two sets.""" + if not set1 or not set2: + return 0.0 + intersection = len(set1 & set2) + union = len(set1 | set2) + return intersection / union if union > 0 else 0.0 + + +def get_item_text(item: Union[schema.RedditItem, schema.XItem]) -> str: + """Get comparable text from an item.""" + if isinstance(item, schema.RedditItem): + return item.title + else: + return item.text + + +def find_duplicates( + items: List[Union[schema.RedditItem, schema.XItem]], + threshold: float = 0.7, +) -> List[Tuple[int, int]]: + """Find near-duplicate pairs in items. + + Args: + items: List of items to check + threshold: Similarity threshold (0-1) + + Returns: + List of (i, j) index pairs where i < j and items are similar + """ + duplicates = [] + + # Pre-compute n-grams + ngrams = [get_ngrams(get_item_text(item)) for item in items] + + for i in range(len(items)): + for j in range(i + 1, len(items)): + similarity = jaccard_similarity(ngrams[i], ngrams[j]) + if similarity >= threshold: + duplicates.append((i, j)) + + return duplicates + + +def dedupe_items( + items: List[Union[schema.RedditItem, schema.XItem]], + threshold: float = 0.7, +) -> List[Union[schema.RedditItem, schema.XItem]]: + """Remove near-duplicates, keeping highest-scored item. + + Args: + items: List of items (should be pre-sorted by score descending) + threshold: Similarity threshold + + Returns: + Deduplicated items + """ + if len(items) <= 1: + return items + + # Find duplicate pairs + dup_pairs = find_duplicates(items, threshold) + + # Mark indices to remove (always remove the lower-scored one) + # Since items are pre-sorted by score, the second index is always lower + to_remove = set() + for i, j in dup_pairs: + # Keep the higher-scored one (lower index in sorted list) + if items[i].score >= items[j].score: + to_remove.add(j) + else: + to_remove.add(i) + + # Return items not marked for removal + return [item for idx, item in enumerate(items) if idx not in to_remove] + + +def dedupe_reddit( + items: List[schema.RedditItem], + threshold: float = 0.7, +) -> List[schema.RedditItem]: + """Dedupe Reddit items.""" + return dedupe_items(items, threshold) + + +def dedupe_x( + items: List[schema.XItem], + threshold: float = 0.7, +) -> List[schema.XItem]: + """Dedupe X items.""" + return dedupe_items(items, threshold) diff --git a/scripts/lib/env.py b/scripts/lib/env.py new file mode 100644 index 0000000..191dc56 --- /dev/null +++ b/scripts/lib/env.py @@ -0,0 +1,102 @@ +"""Environment and API key management for last30days skill.""" + +import os +from pathlib import Path +from typing import Optional, Dict, Any + +CONFIG_DIR = Path.home() / ".config" / "last30days" +CONFIG_FILE = CONFIG_DIR / ".env" + + +def load_env_file(path: Path) -> Dict[str, str]: + """Load environment variables from a file.""" + env = {} + if not path.exists(): + return env + + with open(path, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + if '=' in line: + key, _, value = line.partition('=') + key = key.strip() + value = value.strip() + # Remove quotes if present + if value and value[0] in ('"', "'") and value[-1] == value[0]: + value = value[1:-1] + if key and value: + env[key] = value + return env + + +def get_config() -> Dict[str, Any]: + """Load configuration from ~/.config/last30days/.env and environment.""" + # Load from config file first + file_env = load_env_file(CONFIG_FILE) + + # Environment variables override file + config = { + 'OPENAI_API_KEY': os.environ.get('OPENAI_API_KEY') or file_env.get('OPENAI_API_KEY'), + 'XAI_API_KEY': os.environ.get('XAI_API_KEY') or file_env.get('XAI_API_KEY'), + 'OPENAI_MODEL_POLICY': os.environ.get('OPENAI_MODEL_POLICY') or file_env.get('OPENAI_MODEL_POLICY', 'auto'), + 'OPENAI_MODEL_PIN': os.environ.get('OPENAI_MODEL_PIN') or file_env.get('OPENAI_MODEL_PIN'), + 'XAI_MODEL_POLICY': os.environ.get('XAI_MODEL_POLICY') or file_env.get('XAI_MODEL_POLICY', 'latest'), + 'XAI_MODEL_PIN': os.environ.get('XAI_MODEL_PIN') or file_env.get('XAI_MODEL_PIN'), + } + + return config + + +def config_exists() -> bool: + """Check if configuration file exists.""" + return CONFIG_FILE.exists() + + +def get_available_sources(config: Dict[str, Any]) -> str: + """Determine which sources are available based on API keys. + + Returns: 'both', 'reddit', 'x', or 'none' + """ + has_openai = bool(config.get('OPENAI_API_KEY')) + has_xai = bool(config.get('XAI_API_KEY')) + + if has_openai and has_xai: + return 'both' + elif has_openai: + return 'reddit' + elif has_xai: + return 'x' + else: + return 'none' + + +def validate_sources(requested: str, available: str) -> tuple[str, Optional[str]]: + """Validate requested sources against available keys. + + Args: + requested: 'auto', 'reddit', 'x', or 'both' + available: Result from get_available_sources() + + Returns: + Tuple of (effective_sources, error_message) + """ + if available == 'none': + return 'none', "No API keys configured. Please add at least one key to ~/.config/last30days/.env" + + if requested == 'auto': + return available, None + + if requested == 'both': + if available != 'both': + missing = 'xAI' if available == 'reddit' else 'OpenAI' + return 'none', f"Requested both sources but {missing} key is missing. Use --sources=auto to use available keys." + + if requested == 'reddit' and available == 'x': + return 'none', "Requested Reddit but only xAI key is available." + + if requested == 'x' and available == 'reddit': + return 'none', "Requested X but only OpenAI key is available." + + return requested, None diff --git a/scripts/lib/http.py b/scripts/lib/http.py new file mode 100644 index 0000000..3af767f --- /dev/null +++ b/scripts/lib/http.py @@ -0,0 +1,126 @@ +"""HTTP utilities for last30days skill (stdlib only).""" + +import json +import time +import urllib.error +import urllib.request +from typing import Any, Dict, Optional +from urllib.parse import urlencode + +DEFAULT_TIMEOUT = 30 +MAX_RETRIES = 3 +RETRY_DELAY = 1.0 +USER_AGENT = "last30days-skill/1.0 (Claude Code Skill)" + + +class HTTPError(Exception): + """HTTP request error with status code.""" + def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None): + super().__init__(message) + self.status_code = status_code + self.body = body + + +def request( + method: str, + url: str, + headers: Optional[Dict[str, str]] = None, + json_data: Optional[Dict[str, Any]] = None, + timeout: int = DEFAULT_TIMEOUT, + retries: int = MAX_RETRIES, +) -> Dict[str, Any]: + """Make an HTTP request and return JSON response. + + Args: + method: HTTP method (GET, POST, etc.) + url: Request URL + headers: Optional headers dict + json_data: Optional JSON body (for POST) + timeout: Request timeout in seconds + retries: Number of retries on failure + + Returns: + Parsed JSON response + + Raises: + HTTPError: On request failure + """ + headers = headers or {} + headers.setdefault("User-Agent", USER_AGENT) + + data = None + if json_data is not None: + data = json.dumps(json_data).encode('utf-8') + headers.setdefault("Content-Type", "application/json") + + req = urllib.request.Request(url, data=data, headers=headers, method=method) + + last_error = None + for attempt in range(retries): + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + body = response.read().decode('utf-8') + return json.loads(body) if body else {} + except urllib.error.HTTPError as e: + body = None + try: + body = e.read().decode('utf-8') + except: + pass + last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body) + + # Don't retry client errors (4xx) except rate limits + if 400 <= e.code < 500 and e.code != 429: + raise last_error + + if attempt < retries - 1: + time.sleep(RETRY_DELAY * (attempt + 1)) + except urllib.error.URLError as e: + last_error = HTTPError(f"URL Error: {e.reason}") + if attempt < retries - 1: + time.sleep(RETRY_DELAY * (attempt + 1)) + except json.JSONDecodeError as e: + last_error = HTTPError(f"Invalid JSON response: {e}") + raise last_error + + if last_error: + raise last_error + raise HTTPError("Request failed with no error details") + + +def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: + """Make a GET request.""" + return request("GET", url, headers=headers, **kwargs) + + +def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: + """Make a POST request with JSON body.""" + return request("POST", url, headers=headers, json_data=json_data, **kwargs) + + +def get_reddit_json(path: str) -> Dict[str, Any]: + """Fetch Reddit thread JSON. + + Args: + path: Reddit path (e.g., /r/subreddit/comments/id/title) + + Returns: + Parsed JSON response + """ + # Ensure path starts with / + if not path.startswith('/'): + path = '/' + path + + # Remove trailing slash and add .json + path = path.rstrip('/') + if not path.endswith('.json'): + path = path + '.json' + + url = f"https://www.reddit.com{path}?raw_json=1" + + headers = { + "User-Agent": USER_AGENT, + "Accept": "application/json", + } + + return get(url, headers=headers) diff --git a/scripts/lib/models.py b/scripts/lib/models.py new file mode 100644 index 0000000..0a74fe1 --- /dev/null +++ b/scripts/lib/models.py @@ -0,0 +1,175 @@ +"""Model auto-selection for last30days skill.""" + +import re +from typing import Dict, List, Optional, Tuple + +from . import cache, http + +# OpenAI API +OPENAI_MODELS_URL = "https://api.openai.com/v1/models" +OPENAI_FALLBACK_MODELS = ["gpt-5.2", "gpt-5.1", "gpt-5", "gpt-4o"] + +# xAI API +XAI_MODELS_URL = "https://api.x.ai/v1/models" +XAI_ALIASES = { + "latest": "grok-3", + "stable": "grok-3", +} + + +def parse_version(model_id: str) -> Optional[Tuple[int, ...]]: + """Parse semantic version from model ID. + + Examples: + gpt-5 -> (5,) + gpt-5.2 -> (5, 2) + gpt-5.2.1 -> (5, 2, 1) + """ + match = re.search(r'(\d+(?:\.\d+)*)', model_id) + if match: + return tuple(int(x) for x in match.group(1).split('.')) + return None + + +def is_mainline_openai_model(model_id: str) -> bool: + """Check if model is a mainline GPT model (not mini/nano/chat/codex/pro).""" + model_lower = model_id.lower() + + # Must be gpt-5 series + if not re.match(r'^gpt-5(\.\d+)*$', model_lower): + return False + + # Exclude variants + excludes = ['mini', 'nano', 'chat', 'codex', 'pro', 'preview', 'turbo'] + for exc in excludes: + if exc in model_lower: + return False + + return True + + +def select_openai_model( + api_key: str, + policy: str = "auto", + pin: Optional[str] = None, + mock_models: Optional[List[Dict]] = None, +) -> str: + """Select the best OpenAI model based on policy. + + Args: + api_key: OpenAI API key + policy: 'auto' or 'pinned' + pin: Model to use if policy is 'pinned' + mock_models: Mock model list for testing + + Returns: + Selected model ID + """ + if policy == "pinned" and pin: + return pin + + # Check cache first + cached = cache.get_cached_model("openai") + if cached: + return cached + + # Fetch model list + if mock_models is not None: + models = mock_models + else: + try: + headers = {"Authorization": f"Bearer {api_key}"} + response = http.get(OPENAI_MODELS_URL, headers=headers) + models = response.get("data", []) + except http.HTTPError: + # Fall back to known models + return OPENAI_FALLBACK_MODELS[0] + + # Filter to mainline models + candidates = [m for m in models if is_mainline_openai_model(m.get("id", ""))] + + if not candidates: + # No gpt-5 models found, use fallback + return OPENAI_FALLBACK_MODELS[0] + + # Sort by version (descending), then by created timestamp + def sort_key(m): + version = parse_version(m.get("id", "")) or (0,) + created = m.get("created", 0) + return (version, created) + + candidates.sort(key=sort_key, reverse=True) + selected = candidates[0]["id"] + + # Cache the selection + cache.set_cached_model("openai", selected) + + return selected + + +def select_xai_model( + api_key: str, + policy: str = "latest", + pin: Optional[str] = None, + mock_models: Optional[List[Dict]] = None, +) -> str: + """Select the best xAI model based on policy. + + Args: + api_key: xAI API key + policy: 'latest', 'stable', or 'pinned' + pin: Model to use if policy is 'pinned' + mock_models: Mock model list for testing + + Returns: + Selected model ID + """ + if policy == "pinned" and pin: + return pin + + # Use alias system + if policy in XAI_ALIASES: + alias = XAI_ALIASES[policy] + + # Check cache first + cached = cache.get_cached_model("xai") + if cached: + return cached + + # Cache the alias + cache.set_cached_model("xai", alias) + return alias + + # Default to latest + return XAI_ALIASES["latest"] + + +def get_models( + config: Dict, + mock_openai_models: Optional[List[Dict]] = None, + mock_xai_models: Optional[List[Dict]] = None, +) -> Dict[str, Optional[str]]: + """Get selected models for both providers. + + Returns: + Dict with 'openai' and 'xai' keys + """ + result = {"openai": None, "xai": None} + + if config.get("OPENAI_API_KEY"): + result["openai"] = select_openai_model( + config["OPENAI_API_KEY"], + config.get("OPENAI_MODEL_POLICY", "auto"), + config.get("OPENAI_MODEL_PIN"), + mock_openai_models, + ) + + if config.get("XAI_API_KEY"): + result["xai"] = select_xai_model( + config["XAI_API_KEY"], + config.get("XAI_MODEL_POLICY", "latest"), + config.get("XAI_MODEL_PIN"), + mock_xai_models, + ) + + return result diff --git a/scripts/lib/normalize.py b/scripts/lib/normalize.py new file mode 100644 index 0000000..62c216a --- /dev/null +++ b/scripts/lib/normalize.py @@ -0,0 +1,118 @@ +"""Normalization of raw API data to canonical schema.""" + +from typing import Any, Dict, List + +from . import dates, schema + + +def normalize_reddit_items( + items: List[Dict[str, Any]], + from_date: str, + to_date: str, +) -> List[schema.RedditItem]: + """Normalize raw Reddit items to schema. + + Args: + items: Raw Reddit items from API + from_date: Start of date range + to_date: End of date range + + Returns: + List of RedditItem objects + """ + normalized = [] + + for item in items: + # Parse engagement + engagement = None + eng_raw = item.get("engagement") + if isinstance(eng_raw, dict): + engagement = schema.Engagement( + score=eng_raw.get("score"), + num_comments=eng_raw.get("num_comments"), + upvote_ratio=eng_raw.get("upvote_ratio"), + ) + + # Parse comments + top_comments = [] + for c in item.get("top_comments", []): + top_comments.append(schema.Comment( + score=c.get("score", 0), + date=c.get("date"), + author=c.get("author", ""), + excerpt=c.get("excerpt", ""), + url=c.get("url", ""), + )) + + # Determine date confidence + date_str = item.get("date") + date_confidence = dates.get_date_confidence(date_str, from_date, to_date) + + normalized.append(schema.RedditItem( + id=item.get("id", ""), + title=item.get("title", ""), + url=item.get("url", ""), + subreddit=item.get("subreddit", ""), + date=date_str, + date_confidence=date_confidence, + engagement=engagement, + top_comments=top_comments, + comment_insights=item.get("comment_insights", []), + relevance=item.get("relevance", 0.5), + why_relevant=item.get("why_relevant", ""), + )) + + return normalized + + +def normalize_x_items( + items: List[Dict[str, Any]], + from_date: str, + to_date: str, +) -> List[schema.XItem]: + """Normalize raw X items to schema. + + Args: + items: Raw X items from API + from_date: Start of date range + to_date: End of date range + + Returns: + List of XItem objects + """ + normalized = [] + + for item in items: + # Parse engagement + engagement = None + eng_raw = item.get("engagement") + if isinstance(eng_raw, dict): + engagement = schema.Engagement( + likes=eng_raw.get("likes"), + reposts=eng_raw.get("reposts"), + replies=eng_raw.get("replies"), + quotes=eng_raw.get("quotes"), + ) + + # Determine date confidence + date_str = item.get("date") + date_confidence = dates.get_date_confidence(date_str, from_date, to_date) + + normalized.append(schema.XItem( + id=item.get("id", ""), + text=item.get("text", ""), + url=item.get("url", ""), + author_handle=item.get("author_handle", ""), + date=date_str, + date_confidence=date_confidence, + engagement=engagement, + relevance=item.get("relevance", 0.5), + why_relevant=item.get("why_relevant", ""), + )) + + return normalized + + +def items_to_dicts(items: List) -> List[Dict[str, Any]]: + """Convert schema items to dicts for JSON serialization.""" + return [item.to_dict() for item in items] diff --git a/scripts/lib/openai_reddit.py b/scripts/lib/openai_reddit.py new file mode 100644 index 0000000..82fbf6b --- /dev/null +++ b/scripts/lib/openai_reddit.py @@ -0,0 +1,158 @@ +"""OpenAI Responses API client for Reddit discovery.""" + +import json +import re +from typing import Any, Dict, List, Optional + +from . import http + +OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses" + +REDDIT_SEARCH_PROMPT = """Search Reddit for discussions about: {topic} + +Focus on threads from the last 30 days. Find 15-30 high-quality, relevant threads. + +IMPORTANT: Return ONLY valid JSON in this exact format, no other text: +{{ + "items": [ + {{ + "title": "Thread title", + "url": "https://reddit.com/r/...", + "subreddit": "subreddit_name", + "date": "YYYY-MM-DD or null if unknown", + "why_relevant": "Brief explanation of relevance", + "relevance": 0.85 + }} + ] +}} + +Rules: +- relevance is 0.0 to 1.0 (1.0 = highly relevant) +- date must be YYYY-MM-DD format or null +- Include diverse subreddits if applicable +- Prefer threads with substantive discussions +- Do NOT include engagement metrics (upvotes, comments) - those will be fetched separately""" + + +def search_reddit( + api_key: str, + model: str, + topic: str, + mock_response: Optional[Dict] = None, +) -> Dict[str, Any]: + """Search Reddit for relevant threads using OpenAI Responses API. + + Args: + api_key: OpenAI API key + model: Model to use + topic: Search topic + mock_response: Mock response for testing + + Returns: + Raw API response + """ + if mock_response is not None: + return mock_response + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + payload = { + "model": model, + "tools": [ + { + "type": "web_search", + "filters": { + "allowed_domains": ["reddit.com"] + } + } + ], + "include": ["web_search_call.action.sources"], + "input": REDDIT_SEARCH_PROMPT.format(topic=topic), + } + + return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=60) + + +def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: + """Parse OpenAI response to extract Reddit items. + + Args: + response: Raw API response + + Returns: + List of item dicts + """ + items = [] + + # Try to find the output text + output_text = "" + if "output" in response: + output = response["output"] + if isinstance(output, str): + output_text = output + elif isinstance(output, list): + for item in output: + if isinstance(item, dict): + if item.get("type") == "message": + content = item.get("content", []) + for c in content: + if isinstance(c, dict) and c.get("type") == "output_text": + output_text = c.get("text", "") + break + elif "text" in item: + output_text = item["text"] + elif isinstance(item, str): + output_text = item + if output_text: + break + + # Also check for choices (older format) + if not output_text and "choices" in response: + for choice in response["choices"]: + if "message" in choice: + output_text = choice["message"].get("content", "") + break + + if not output_text: + return items + + # Extract JSON from the response + json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text) + if json_match: + try: + data = json.loads(json_match.group()) + items = data.get("items", []) + except json.JSONDecodeError: + pass + + # Validate and clean items + clean_items = [] + for i, item in enumerate(items): + if not isinstance(item, dict): + continue + + url = item.get("url", "") + if not url or "reddit.com" not in url: + continue + + clean_item = { + "id": f"R{i+1}", + "title": str(item.get("title", "")).strip(), + "url": url, + "subreddit": str(item.get("subreddit", "")).strip().lstrip("r/"), + "date": item.get("date"), + "why_relevant": str(item.get("why_relevant", "")).strip(), + "relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))), + } + + # Validate date format + if clean_item["date"]: + if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])): + clean_item["date"] = None + + clean_items.append(clean_item) + + return clean_items diff --git a/scripts/lib/reddit_enrich.py b/scripts/lib/reddit_enrich.py new file mode 100644 index 0000000..589cc63 --- /dev/null +++ b/scripts/lib/reddit_enrich.py @@ -0,0 +1,232 @@ +"""Reddit thread enrichment with real engagement metrics.""" + +import re +from typing import Any, Dict, List, Optional +from urllib.parse import urlparse + +from . import http, dates + + +def extract_reddit_path(url: str) -> Optional[str]: + """Extract the path from a Reddit URL. + + Args: + url: Reddit URL + + Returns: + Path component or None + """ + try: + parsed = urlparse(url) + if "reddit.com" not in parsed.netloc: + return None + return parsed.path + except: + return None + + +def fetch_thread_data(url: str, mock_data: Optional[Dict] = None) -> Optional[Dict[str, Any]]: + """Fetch Reddit thread JSON data. + + Args: + url: Reddit thread URL + mock_data: Mock data for testing + + Returns: + Thread data dict or None on failure + """ + if mock_data is not None: + return mock_data + + path = extract_reddit_path(url) + if not path: + return None + + try: + data = http.get_reddit_json(path) + return data + except http.HTTPError: + return None + + +def parse_thread_data(data: Any) -> Dict[str, Any]: + """Parse Reddit thread JSON into structured data. + + Args: + data: Raw Reddit JSON response + + Returns: + Dict with submission and comments data + """ + result = { + "submission": None, + "comments": [], + } + + if not isinstance(data, list) or len(data) < 1: + return result + + # First element is submission listing + submission_listing = data[0] + if isinstance(submission_listing, dict): + children = submission_listing.get("data", {}).get("children", []) + if children: + sub_data = children[0].get("data", {}) + result["submission"] = { + "score": sub_data.get("score"), + "num_comments": sub_data.get("num_comments"), + "upvote_ratio": sub_data.get("upvote_ratio"), + "created_utc": sub_data.get("created_utc"), + "permalink": sub_data.get("permalink"), + "title": sub_data.get("title"), + "selftext": sub_data.get("selftext", "")[:500], # Truncate + } + + # Second element is comments listing + if len(data) >= 2: + comments_listing = data[1] + if isinstance(comments_listing, dict): + children = comments_listing.get("data", {}).get("children", []) + for child in children: + if child.get("kind") != "t1": # t1 = comment + continue + c_data = child.get("data", {}) + if not c_data.get("body"): + continue + + comment = { + "score": c_data.get("score", 0), + "created_utc": c_data.get("created_utc"), + "author": c_data.get("author", "[deleted]"), + "body": c_data.get("body", "")[:300], # Truncate + "permalink": c_data.get("permalink"), + } + result["comments"].append(comment) + + return result + + +def get_top_comments(comments: List[Dict], limit: int = 10) -> List[Dict[str, Any]]: + """Get top comments sorted by score. + + Args: + comments: List of comment dicts + limit: Maximum number to return + + Returns: + Top comments sorted by score + """ + # Filter out deleted/removed + valid = [c for c in comments if c.get("author") not in ("[deleted]", "[removed]")] + + # Sort by score descending + sorted_comments = sorted(valid, key=lambda c: c.get("score", 0), reverse=True) + + return sorted_comments[:limit] + + +def extract_comment_insights(comments: List[Dict], limit: int = 7) -> List[str]: + """Extract key insights from top comments. + + Uses simple heuristics to identify valuable comments: + - Has substantive text + - Contains actionable information + - Not just agreement/disagreement + + Args: + comments: Top comments + limit: Max insights to extract + + Returns: + List of insight strings + """ + insights = [] + + for comment in comments[:limit * 2]: # Look at more comments than we need + body = comment.get("body", "").strip() + if not body or len(body) < 30: + continue + + # Skip low-value patterns + skip_patterns = [ + r'^(this|same|agreed|exactly|yep|nope|yes|no|thanks|thank you)\.?$', + r'^lol|lmao|haha', + r'^\[deleted\]', + r'^\[removed\]', + ] + if any(re.match(p, body.lower()) for p in skip_patterns): + continue + + # Truncate to first meaningful sentence or ~150 chars + insight = body[:150] + if len(body) > 150: + # Try to find a sentence boundary + for i, char in enumerate(insight): + if char in '.!?' and i > 50: + insight = insight[:i+1] + break + else: + insight = insight.rstrip() + "..." + + insights.append(insight) + if len(insights) >= limit: + break + + return insights + + +def enrich_reddit_item( + item: Dict[str, Any], + mock_thread_data: Optional[Dict] = None, +) -> Dict[str, Any]: + """Enrich a Reddit item with real engagement data. + + Args: + item: Reddit item dict + mock_thread_data: Mock data for testing + + Returns: + Enriched item dict + """ + url = item.get("url", "") + + # Fetch thread data + thread_data = fetch_thread_data(url, mock_thread_data) + if not thread_data: + return item + + parsed = parse_thread_data(thread_data) + submission = parsed.get("submission") + comments = parsed.get("comments", []) + + # Update engagement metrics + if submission: + item["engagement"] = { + "score": submission.get("score"), + "num_comments": submission.get("num_comments"), + "upvote_ratio": submission.get("upvote_ratio"), + } + + # Update date from actual data + created_utc = submission.get("created_utc") + if created_utc: + item["date"] = dates.timestamp_to_date(created_utc) + + # Get top comments + top_comments = get_top_comments(comments) + item["top_comments"] = [] + for c in top_comments: + permalink = c.get("permalink", "") + comment_url = f"https://reddit.com{permalink}" if permalink else "" + item["top_comments"].append({ + "score": c.get("score", 0), + "date": dates.timestamp_to_date(c.get("created_utc")), + "author": c.get("author", ""), + "excerpt": c.get("body", "")[:200], + "url": comment_url, + }) + + # Extract insights + item["comment_insights"] = extract_comment_insights(top_comments) + + return item diff --git a/scripts/lib/render.py b/scripts/lib/render.py new file mode 100644 index 0000000..af59ccf --- /dev/null +++ b/scripts/lib/render.py @@ -0,0 +1,277 @@ +"""Output rendering for last30days skill.""" + +import json +from pathlib import Path +from typing import List, Optional + +from . import schema + +OUTPUT_DIR = Path.home() / ".local" / "share" / "last30days" / "out" + + +def ensure_output_dir(): + """Ensure output directory exists.""" + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + +def render_compact(report: schema.Report, limit: int = 15) -> str: + """Render compact output for Claude to synthesize. + + Args: + report: Report data + limit: Max items per source + + Returns: + Compact markdown string + """ + lines = [] + + # Header + lines.append(f"## Research Results: {report.topic}") + lines.append("") + lines.append(f"**Date Range:** {report.range_from} to {report.range_to}") + lines.append(f"**Mode:** {report.mode}") + if report.openai_model_used: + lines.append(f"**OpenAI Model:** {report.openai_model_used}") + if report.xai_model_used: + lines.append(f"**xAI Model:** {report.xai_model_used}") + lines.append("") + + # Coverage note + if report.mode == "reddit-only": + lines.append("*Tip: Add xAI key for X coverage and better triangulation.*") + lines.append("") + elif report.mode == "x-only": + lines.append("*Tip: Add OpenAI key for Reddit coverage and better triangulation.*") + lines.append("") + + # Reddit items + if report.reddit: + lines.append("### Reddit Threads") + lines.append("") + for item in report.reddit[:limit]: + eng_str = "" + if item.engagement: + eng = item.engagement + parts = [] + if eng.score is not None: + parts.append(f"{eng.score}pts") + if eng.num_comments is not None: + parts.append(f"{eng.num_comments}cmt") + if parts: + eng_str = f" [{', '.join(parts)}]" + + date_str = f" ({item.date})" if item.date else " (date unknown)" + conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else "" + + lines.append(f"**{item.id}** (score:{item.score}) r/{item.subreddit}{date_str}{conf_str}{eng_str}") + lines.append(f" {item.title}") + lines.append(f" {item.url}") + lines.append(f" *{item.why_relevant}*") + + # Top comment insights + if item.comment_insights: + lines.append(f" Insights:") + for insight in item.comment_insights[:3]: + lines.append(f" - {insight}") + + lines.append("") + + # X items + if report.x: + lines.append("### X Posts") + lines.append("") + for item in report.x[:limit]: + eng_str = "" + if item.engagement: + eng = item.engagement + parts = [] + if eng.likes is not None: + parts.append(f"{eng.likes}likes") + if eng.reposts is not None: + parts.append(f"{eng.reposts}rt") + if parts: + eng_str = f" [{', '.join(parts)}]" + + date_str = f" ({item.date})" if item.date else " (date unknown)" + conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else "" + + lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{conf_str}{eng_str}") + lines.append(f" {item.text[:200]}...") + lines.append(f" {item.url}") + lines.append(f" *{item.why_relevant}*") + lines.append("") + + return "\n".join(lines) + + +def render_context_snippet(report: schema.Report) -> str: + """Render reusable context snippet. + + Args: + report: Report data + + Returns: + Context markdown string + """ + lines = [] + lines.append(f"# Context: {report.topic} (Last 30 Days)") + lines.append("") + lines.append(f"*Generated: {report.generated_at[:10]} | Sources: {report.mode}*") + lines.append("") + + # Key sources summary + lines.append("## Key Sources") + lines.append("") + + all_items = [] + for item in report.reddit[:5]: + all_items.append((item.score, "Reddit", item.title, item.url)) + for item in report.x[:5]: + all_items.append((item.score, "X", item.text[:50] + "...", item.url)) + + all_items.sort(key=lambda x: -x[0]) + for score, source, text, url in all_items[:7]: + lines.append(f"- [{source}] {text}") + + lines.append("") + lines.append("## Summary") + lines.append("") + lines.append("*See full report for best practices, prompt pack, and detailed sources.*") + lines.append("") + + return "\n".join(lines) + + +def render_full_report(report: schema.Report) -> str: + """Render full markdown report. + + Args: + report: Report data + + Returns: + Full report markdown + """ + lines = [] + + # Title + lines.append(f"# {report.topic} - Last 30 Days Research Report") + lines.append("") + lines.append(f"**Generated:** {report.generated_at}") + lines.append(f"**Date Range:** {report.range_from} to {report.range_to}") + lines.append(f"**Mode:** {report.mode}") + lines.append("") + + # Models + lines.append("## Models Used") + lines.append("") + if report.openai_model_used: + lines.append(f"- **OpenAI:** {report.openai_model_used}") + if report.xai_model_used: + lines.append(f"- **xAI:** {report.xai_model_used}") + lines.append("") + + # Reddit section + if report.reddit: + lines.append("## Reddit Threads") + lines.append("") + for item in report.reddit: + lines.append(f"### {item.id}: {item.title}") + lines.append("") + lines.append(f"- **Subreddit:** r/{item.subreddit}") + lines.append(f"- **URL:** {item.url}") + lines.append(f"- **Date:** {item.date or 'Unknown'} (confidence: {item.date_confidence})") + lines.append(f"- **Score:** {item.score}/100") + lines.append(f"- **Relevance:** {item.why_relevant}") + + if item.engagement: + eng = item.engagement + lines.append(f"- **Engagement:** {eng.score or '?'} points, {eng.num_comments or '?'} comments") + + if item.comment_insights: + lines.append("") + lines.append("**Key Insights from Comments:**") + for insight in item.comment_insights: + lines.append(f"- {insight}") + + lines.append("") + + # X section + if report.x: + lines.append("## X Posts") + lines.append("") + for item in report.x: + lines.append(f"### {item.id}: @{item.author_handle}") + lines.append("") + lines.append(f"- **URL:** {item.url}") + lines.append(f"- **Date:** {item.date or 'Unknown'} (confidence: {item.date_confidence})") + lines.append(f"- **Score:** {item.score}/100") + lines.append(f"- **Relevance:** {item.why_relevant}") + + if item.engagement: + eng = item.engagement + lines.append(f"- **Engagement:** {eng.likes or '?'} likes, {eng.reposts or '?'} reposts") + + lines.append("") + lines.append(f"> {item.text}") + lines.append("") + + # Placeholders for Claude synthesis + lines.append("## Best Practices") + lines.append("") + lines.append("*To be synthesized by Claude*") + lines.append("") + + lines.append("## Prompt Pack") + lines.append("") + lines.append("*To be synthesized by Claude*") + lines.append("") + + return "\n".join(lines) + + +def write_outputs( + report: schema.Report, + raw_openai: Optional[dict] = None, + raw_xai: Optional[dict] = None, + raw_reddit_enriched: Optional[list] = None, +): + """Write all output files. + + Args: + report: Report data + raw_openai: Raw OpenAI API response + raw_xai: Raw xAI API response + raw_reddit_enriched: Raw enriched Reddit thread data + """ + ensure_output_dir() + + # report.json + with open(OUTPUT_DIR / "report.json", 'w') as f: + json.dump(report.to_dict(), f, indent=2) + + # report.md + with open(OUTPUT_DIR / "report.md", 'w') as f: + f.write(render_full_report(report)) + + # last30days.context.md + with open(OUTPUT_DIR / "last30days.context.md", 'w') as f: + f.write(render_context_snippet(report)) + + # Raw responses + if raw_openai: + with open(OUTPUT_DIR / "raw_openai.json", 'w') as f: + json.dump(raw_openai, f, indent=2) + + if raw_xai: + with open(OUTPUT_DIR / "raw_xai.json", 'w') as f: + json.dump(raw_xai, f, indent=2) + + if raw_reddit_enriched: + with open(OUTPUT_DIR / "raw_reddit_threads_enriched.json", 'w') as f: + json.dump(raw_reddit_enriched, f, indent=2) + + +def get_context_path() -> str: + """Get path to context file.""" + return str(OUTPUT_DIR / "last30days.context.md") diff --git a/scripts/lib/schema.py b/scripts/lib/schema.py new file mode 100644 index 0000000..0069603 --- /dev/null +++ b/scripts/lib/schema.py @@ -0,0 +1,193 @@ +"""Data schemas for last30days skill.""" + +from dataclasses import dataclass, field, asdict +from typing import Any, Dict, List, Optional +from datetime import datetime, timezone + + +@dataclass +class Engagement: + """Engagement metrics.""" + # Reddit fields + score: Optional[int] = None + num_comments: Optional[int] = None + upvote_ratio: Optional[float] = None + + # X fields + likes: Optional[int] = None + reposts: Optional[int] = None + replies: Optional[int] = None + quotes: Optional[int] = None + + def to_dict(self) -> Dict[str, Any]: + d = {} + if self.score is not None: + d['score'] = self.score + if self.num_comments is not None: + d['num_comments'] = self.num_comments + if self.upvote_ratio is not None: + d['upvote_ratio'] = self.upvote_ratio + if self.likes is not None: + d['likes'] = self.likes + if self.reposts is not None: + d['reposts'] = self.reposts + if self.replies is not None: + d['replies'] = self.replies + if self.quotes is not None: + d['quotes'] = self.quotes + return d if d else None + + +@dataclass +class Comment: + """Reddit comment.""" + score: int + date: Optional[str] + author: str + excerpt: str + url: str + + def to_dict(self) -> Dict[str, Any]: + return { + 'score': self.score, + 'date': self.date, + 'author': self.author, + 'excerpt': self.excerpt, + 'url': self.url, + } + + +@dataclass +class SubScores: + """Component scores.""" + relevance: int = 0 + recency: int = 0 + engagement: int = 0 + + def to_dict(self) -> Dict[str, int]: + return { + 'relevance': self.relevance, + 'recency': self.recency, + 'engagement': self.engagement, + } + + +@dataclass +class RedditItem: + """Normalized Reddit item.""" + id: str + title: str + url: str + subreddit: str + date: Optional[str] = None + date_confidence: str = "low" + engagement: Optional[Engagement] = None + top_comments: List[Comment] = field(default_factory=list) + comment_insights: List[str] = field(default_factory=list) + relevance: float = 0.5 + why_relevant: str = "" + subs: SubScores = field(default_factory=SubScores) + score: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return { + 'id': self.id, + 'title': self.title, + 'url': self.url, + 'subreddit': self.subreddit, + 'date': self.date, + 'date_confidence': self.date_confidence, + 'engagement': self.engagement.to_dict() if self.engagement else None, + 'top_comments': [c.to_dict() for c in self.top_comments], + 'comment_insights': self.comment_insights, + 'relevance': self.relevance, + 'why_relevant': self.why_relevant, + 'subs': self.subs.to_dict(), + 'score': self.score, + } + + +@dataclass +class XItem: + """Normalized X item.""" + id: str + text: str + url: str + author_handle: str + date: Optional[str] = None + date_confidence: str = "low" + engagement: Optional[Engagement] = None + relevance: float = 0.5 + why_relevant: str = "" + subs: SubScores = field(default_factory=SubScores) + score: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return { + 'id': self.id, + 'text': self.text, + 'url': self.url, + 'author_handle': self.author_handle, + 'date': self.date, + 'date_confidence': self.date_confidence, + 'engagement': self.engagement.to_dict() if self.engagement else None, + 'relevance': self.relevance, + 'why_relevant': self.why_relevant, + 'subs': self.subs.to_dict(), + 'score': self.score, + } + + +@dataclass +class Report: + """Full research report.""" + topic: str + range_from: str + range_to: str + generated_at: str + mode: str # 'reddit-only', 'x-only', 'both' + openai_model_used: Optional[str] = None + xai_model_used: Optional[str] = None + reddit: List[RedditItem] = field(default_factory=list) + x: List[XItem] = field(default_factory=list) + best_practices: List[str] = field(default_factory=list) + prompt_pack: List[str] = field(default_factory=list) + context_snippet_md: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + 'topic': self.topic, + 'range': { + 'from': self.range_from, + 'to': self.range_to, + }, + 'generated_at': self.generated_at, + 'mode': self.mode, + 'openai_model_used': self.openai_model_used, + 'xai_model_used': self.xai_model_used, + 'reddit': [r.to_dict() for r in self.reddit], + 'x': [x.to_dict() for x in self.x], + 'best_practices': self.best_practices, + 'prompt_pack': self.prompt_pack, + 'context_snippet_md': self.context_snippet_md, + } + + +def create_report( + topic: str, + from_date: str, + to_date: str, + mode: str, + openai_model: Optional[str] = None, + xai_model: Optional[str] = None, +) -> Report: + """Create a new report with metadata.""" + return Report( + topic=topic, + range_from=from_date, + range_to=to_date, + generated_at=datetime.now(timezone.utc).isoformat(), + mode=mode, + openai_model_used=openai_model, + xai_model_used=xai_model, + ) diff --git a/scripts/lib/score.py b/scripts/lib/score.py new file mode 100644 index 0000000..8a96182 --- /dev/null +++ b/scripts/lib/score.py @@ -0,0 +1,240 @@ +"""Popularity-aware scoring for last30days skill.""" + +import math +from typing import List, Optional, Union + +from . import dates, schema + +# Score weights +WEIGHT_RELEVANCE = 0.45 +WEIGHT_RECENCY = 0.25 +WEIGHT_ENGAGEMENT = 0.30 + +# Default engagement score for unknown +DEFAULT_ENGAGEMENT = 35 +UNKNOWN_ENGAGEMENT_PENALTY = 10 + + +def log1p_safe(x: Optional[int]) -> float: + """Safe log1p that handles None and negative values.""" + if x is None or x < 0: + return 0.0 + return math.log1p(x) + + +def compute_reddit_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: + """Compute raw engagement score for Reddit item. + + Formula: 0.55*log1p(score) + 0.40*log1p(num_comments) + 0.05*(upvote_ratio*10) + """ + if engagement is None: + return None + + if engagement.score is None and engagement.num_comments is None: + return None + + score = log1p_safe(engagement.score) + comments = log1p_safe(engagement.num_comments) + ratio = (engagement.upvote_ratio or 0.5) * 10 + + return 0.55 * score + 0.40 * comments + 0.05 * ratio + + +def compute_x_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: + """Compute raw engagement score for X item. + + Formula: 0.55*log1p(likes) + 0.25*log1p(reposts) + 0.15*log1p(replies) + 0.05*log1p(quotes) + """ + if engagement is None: + return None + + if engagement.likes is None and engagement.reposts is None: + return None + + likes = log1p_safe(engagement.likes) + reposts = log1p_safe(engagement.reposts) + replies = log1p_safe(engagement.replies) + quotes = log1p_safe(engagement.quotes) + + return 0.55 * likes + 0.25 * reposts + 0.15 * replies + 0.05 * quotes + + +def normalize_to_100(values: List[float], default: float = 50) -> List[float]: + """Normalize a list of values to 0-100 scale. + + Args: + values: Raw values (None values are preserved) + default: Default value for None entries + + Returns: + Normalized values + """ + # Filter out None + valid = [v for v in values if v is not None] + if not valid: + return [default if v is None else 50 for v in values] + + min_val = min(valid) + max_val = max(valid) + range_val = max_val - min_val + + if range_val == 0: + return [50 if v is None else 50 for v in values] + + result = [] + for v in values: + if v is None: + result.append(None) + else: + normalized = ((v - min_val) / range_val) * 100 + result.append(normalized) + + return result + + +def score_reddit_items(items: List[schema.RedditItem]) -> List[schema.RedditItem]: + """Compute scores for Reddit items. + + Args: + items: List of Reddit items + + Returns: + Items with updated scores + """ + if not items: + return items + + # Compute raw engagement scores + eng_raw = [compute_reddit_engagement_raw(item.engagement) for item in items] + + # Normalize engagement to 0-100 + eng_normalized = normalize_to_100(eng_raw) + + for i, item in enumerate(items): + # Relevance subscore (model-provided, convert to 0-100) + rel_score = int(item.relevance * 100) + + # Recency subscore + rec_score = dates.recency_score(item.date) + + # Engagement subscore + if eng_normalized[i] is not None: + eng_score = int(eng_normalized[i]) + else: + eng_score = DEFAULT_ENGAGEMENT + + # Store subscores + item.subs = schema.SubScores( + relevance=rel_score, + recency=rec_score, + engagement=eng_score, + ) + + # Compute overall score + overall = ( + WEIGHT_RELEVANCE * rel_score + + WEIGHT_RECENCY * rec_score + + WEIGHT_ENGAGEMENT * eng_score + ) + + # Apply penalty for unknown engagement + if eng_raw[i] is None: + overall -= UNKNOWN_ENGAGEMENT_PENALTY + + # Apply penalty for low date confidence + if item.date_confidence == "low": + overall -= 10 + elif item.date_confidence == "med": + overall -= 5 + + item.score = max(0, min(100, int(overall))) + + return items + + +def score_x_items(items: List[schema.XItem]) -> List[schema.XItem]: + """Compute scores for X items. + + Args: + items: List of X items + + Returns: + Items with updated scores + """ + if not items: + return items + + # Compute raw engagement scores + eng_raw = [compute_x_engagement_raw(item.engagement) for item in items] + + # Normalize engagement to 0-100 + eng_normalized = normalize_to_100(eng_raw) + + for i, item in enumerate(items): + # Relevance subscore (model-provided, convert to 0-100) + rel_score = int(item.relevance * 100) + + # Recency subscore + rec_score = dates.recency_score(item.date) + + # Engagement subscore + if eng_normalized[i] is not None: + eng_score = int(eng_normalized[i]) + else: + eng_score = DEFAULT_ENGAGEMENT + + # Store subscores + item.subs = schema.SubScores( + relevance=rel_score, + recency=rec_score, + engagement=eng_score, + ) + + # Compute overall score + overall = ( + WEIGHT_RELEVANCE * rel_score + + WEIGHT_RECENCY * rec_score + + WEIGHT_ENGAGEMENT * eng_score + ) + + # Apply penalty for unknown engagement + if eng_raw[i] is None: + overall -= UNKNOWN_ENGAGEMENT_PENALTY + + # Apply penalty for low date confidence + if item.date_confidence == "low": + overall -= 10 + elif item.date_confidence == "med": + overall -= 5 + + item.score = max(0, min(100, int(overall))) + + return items + + +def sort_items(items: List[Union[schema.RedditItem, schema.XItem]]) -> List: + """Sort items by score (descending), then date, then source priority. + + Args: + items: List of items to sort + + Returns: + Sorted items + """ + def sort_key(item): + # Primary: score descending (negate for descending) + score = -item.score + + # Secondary: date descending (recent first) + date = item.date or "0000-00-00" + date_key = -int(date.replace("-", "")) + + # Tertiary: source priority (Reddit before X) + source_priority = 0 if isinstance(item, schema.RedditItem) else 1 + + # Quaternary: title/text for stability + text = getattr(item, "title", "") or getattr(item, "text", "") + + return (score, date_key, source_priority, text) + + return sorted(items, key=sort_key) diff --git a/scripts/lib/xai_x.py b/scripts/lib/xai_x.py new file mode 100644 index 0000000..331ea0f --- /dev/null +++ b/scripts/lib/xai_x.py @@ -0,0 +1,192 @@ +"""xAI API client for X (Twitter) discovery.""" + +import json +import re +from typing import Any, Dict, List, Optional + +from . import http + +# xAI uses chat completions endpoint +XAI_CHAT_URL = "https://api.x.ai/v1/chat/completions" + +X_SEARCH_PROMPT = """You have access to real-time X (Twitter) data. Search for posts about: {topic} + +Focus on posts from {from_date} to {to_date}. Find 15-30 high-quality, relevant posts. + +IMPORTANT: Return ONLY valid JSON in this exact format, no other text: +{{ + "items": [ + {{ + "text": "Post text content (truncated if long)", + "url": "https://x.com/user/status/...", + "author_handle": "username", + "date": "YYYY-MM-DD or null if unknown", + "engagement": {{ + "likes": 100, + "reposts": 25, + "replies": 15, + "quotes": 5 + }}, + "why_relevant": "Brief explanation of relevance", + "relevance": 0.85 + }} + ] +}} + +Rules: +- relevance is 0.0 to 1.0 (1.0 = highly relevant) +- date must be YYYY-MM-DD format or null +- engagement can be null if unknown +- Include diverse voices/accounts if applicable +- Prefer posts with substantive content, not just links""" + + +def search_x( + api_key: str, + model: str, + topic: str, + from_date: str, + to_date: str, + mock_response: Optional[Dict] = None, +) -> Dict[str, Any]: + """Search X for relevant posts using xAI API with live search. + + Args: + api_key: xAI API key + model: Model to use + topic: Search topic + from_date: Start date (YYYY-MM-DD) + to_date: End date (YYYY-MM-DD) + mock_response: Mock response for testing + + Returns: + Raw API response + """ + if mock_response is not None: + return mock_response + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + # Use chat completions format with search enabled + payload = { + "model": model, + "messages": [ + { + "role": "system", + "content": "You are a research assistant with access to real-time X (Twitter) data. Search X and return structured results." + }, + { + "role": "user", + "content": X_SEARCH_PROMPT.format( + topic=topic, + from_date=from_date, + to_date=to_date, + ), + } + ], + "search_parameters": { + "mode": "auto", + "sources": [{"type": "x"}], + "from_date": from_date, + "to_date": to_date, + }, + } + + return http.post(XAI_CHAT_URL, payload, headers=headers, timeout=90) + + +def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: + """Parse xAI response to extract X items. + + Args: + response: Raw API response + + Returns: + List of item dicts + """ + items = [] + + # Try to find the output text + output_text = "" + if "output" in response: + output = response["output"] + if isinstance(output, str): + output_text = output + elif isinstance(output, list): + for item in output: + if isinstance(item, dict): + if item.get("type") == "message": + content = item.get("content", []) + for c in content: + if isinstance(c, dict) and c.get("type") == "output_text": + output_text = c.get("text", "") + break + elif "text" in item: + output_text = item["text"] + elif isinstance(item, str): + output_text = item + if output_text: + break + + # Also check for choices (older format) + if not output_text and "choices" in response: + for choice in response["choices"]: + if "message" in choice: + output_text = choice["message"].get("content", "") + break + + if not output_text: + return items + + # Extract JSON from the response + json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text) + if json_match: + try: + data = json.loads(json_match.group()) + items = data.get("items", []) + except json.JSONDecodeError: + pass + + # Validate and clean items + clean_items = [] + for i, item in enumerate(items): + if not isinstance(item, dict): + continue + + url = item.get("url", "") + if not url: + continue + + # Parse engagement + engagement = None + eng_raw = item.get("engagement") + if isinstance(eng_raw, dict): + engagement = { + "likes": int(eng_raw.get("likes", 0)) if eng_raw.get("likes") else None, + "reposts": int(eng_raw.get("reposts", 0)) if eng_raw.get("reposts") else None, + "replies": int(eng_raw.get("replies", 0)) if eng_raw.get("replies") else None, + "quotes": int(eng_raw.get("quotes", 0)) if eng_raw.get("quotes") else None, + } + + clean_item = { + "id": f"X{i+1}", + "text": str(item.get("text", "")).strip()[:500], # Truncate long text + "url": url, + "author_handle": str(item.get("author_handle", "")).strip().lstrip("@"), + "date": item.get("date"), + "engagement": engagement, + "why_relevant": str(item.get("why_relevant", "")).strip(), + "relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))), + } + + # Validate date format + if clean_item["date"]: + if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])): + clean_item["date"] = None + + clean_items.append(clean_item) + + return clean_items diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..6bcb2af --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# last30days tests diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..fe1d9a0 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,59 @@ +"""Tests for cache module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import cache + + +class TestGetCacheKey(unittest.TestCase): + def test_returns_string(self): + result = cache.get_cache_key("test topic", "2026-01-01", "2026-01-31", "both") + self.assertIsInstance(result, str) + + def test_consistent_for_same_inputs(self): + key1 = cache.get_cache_key("test topic", "2026-01-01", "2026-01-31", "both") + key2 = cache.get_cache_key("test topic", "2026-01-01", "2026-01-31", "both") + self.assertEqual(key1, key2) + + def test_different_for_different_inputs(self): + key1 = cache.get_cache_key("topic a", "2026-01-01", "2026-01-31", "both") + key2 = cache.get_cache_key("topic b", "2026-01-01", "2026-01-31", "both") + self.assertNotEqual(key1, key2) + + def test_key_length(self): + key = cache.get_cache_key("test", "2026-01-01", "2026-01-31", "both") + self.assertEqual(len(key), 16) + + +class TestCachePath(unittest.TestCase): + def test_returns_path(self): + result = cache.get_cache_path("abc123") + self.assertIsInstance(result, Path) + + def test_has_json_extension(self): + result = cache.get_cache_path("abc123") + self.assertEqual(result.suffix, ".json") + + +class TestCacheValidity(unittest.TestCase): + def test_nonexistent_file_is_invalid(self): + fake_path = Path("/nonexistent/path/file.json") + result = cache.is_cache_valid(fake_path) + self.assertFalse(result) + + +class TestModelCache(unittest.TestCase): + def test_get_cached_model_returns_none_for_missing(self): + # Clear any existing cache first + result = cache.get_cached_model("nonexistent_provider") + # May be None or a cached value, but should not error + self.assertTrue(result is None or isinstance(result, str)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dates.py b/tests/test_dates.py new file mode 100644 index 0000000..6d932ec --- /dev/null +++ b/tests/test_dates.py @@ -0,0 +1,114 @@ +"""Tests for dates module.""" + +import sys +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import dates + + +class TestGetDateRange(unittest.TestCase): + def test_returns_tuple_of_two_strings(self): + from_date, to_date = dates.get_date_range(30) + self.assertIsInstance(from_date, str) + self.assertIsInstance(to_date, str) + + def test_date_format(self): + from_date, to_date = dates.get_date_range(30) + # Should be YYYY-MM-DD format + self.assertRegex(from_date, r'^\d{4}-\d{2}-\d{2}$') + self.assertRegex(to_date, r'^\d{4}-\d{2}-\d{2}$') + + def test_range_is_correct_days(self): + from_date, to_date = dates.get_date_range(30) + start = datetime.strptime(from_date, "%Y-%m-%d") + end = datetime.strptime(to_date, "%Y-%m-%d") + delta = end - start + self.assertEqual(delta.days, 30) + + +class TestParseDate(unittest.TestCase): + def test_parse_iso_date(self): + result = dates.parse_date("2026-01-15") + self.assertIsNotNone(result) + self.assertEqual(result.year, 2026) + self.assertEqual(result.month, 1) + self.assertEqual(result.day, 15) + + def test_parse_timestamp(self): + # Unix timestamp for 2026-01-15 00:00:00 UTC + result = dates.parse_date("1768435200") + self.assertIsNotNone(result) + + def test_parse_none(self): + result = dates.parse_date(None) + self.assertIsNone(result) + + def test_parse_empty_string(self): + result = dates.parse_date("") + self.assertIsNone(result) + + +class TestTimestampToDate(unittest.TestCase): + def test_valid_timestamp(self): + # 2026-01-15 00:00:00 UTC + result = dates.timestamp_to_date(1768435200) + self.assertEqual(result, "2026-01-15") + + def test_none_timestamp(self): + result = dates.timestamp_to_date(None) + self.assertIsNone(result) + + +class TestGetDateConfidence(unittest.TestCase): + def test_high_confidence_in_range(self): + result = dates.get_date_confidence("2026-01-15", "2026-01-01", "2026-01-31") + self.assertEqual(result, "high") + + def test_low_confidence_before_range(self): + result = dates.get_date_confidence("2025-12-15", "2026-01-01", "2026-01-31") + self.assertEqual(result, "low") + + def test_low_confidence_no_date(self): + result = dates.get_date_confidence(None, "2026-01-01", "2026-01-31") + self.assertEqual(result, "low") + + +class TestDaysAgo(unittest.TestCase): + def test_today(self): + today = datetime.now(timezone.utc).date().isoformat() + result = dates.days_ago(today) + self.assertEqual(result, 0) + + def test_none_date(self): + result = dates.days_ago(None) + self.assertIsNone(result) + + +class TestRecencyScore(unittest.TestCase): + def test_today_is_100(self): + today = datetime.now(timezone.utc).date().isoformat() + result = dates.recency_score(today) + self.assertEqual(result, 100) + + def test_30_days_ago_is_0(self): + old_date = (datetime.now(timezone.utc).date() - timedelta(days=30)).isoformat() + result = dates.recency_score(old_date) + self.assertEqual(result, 0) + + def test_15_days_ago_is_50(self): + mid_date = (datetime.now(timezone.utc).date() - timedelta(days=15)).isoformat() + result = dates.recency_score(mid_date) + self.assertEqual(result, 50) + + def test_none_date_is_0(self): + result = dates.recency_score(None) + self.assertEqual(result, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dedupe.py b/tests/test_dedupe.py new file mode 100644 index 0000000..a790db5 --- /dev/null +++ b/tests/test_dedupe.py @@ -0,0 +1,111 @@ +"""Tests for dedupe module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import dedupe, schema + + +class TestNormalizeText(unittest.TestCase): + def test_lowercase(self): + result = dedupe.normalize_text("HELLO World") + self.assertEqual(result, "hello world") + + def test_removes_punctuation(self): + result = dedupe.normalize_text("Hello, World!") + # Punctuation replaced with space, then whitespace collapsed + self.assertEqual(result, "hello world") + + def test_collapses_whitespace(self): + result = dedupe.normalize_text("hello world") + self.assertEqual(result, "hello world") + + +class TestGetNgrams(unittest.TestCase): + def test_short_text(self): + result = dedupe.get_ngrams("ab", n=3) + self.assertEqual(result, {"ab"}) + + def test_normal_text(self): + result = dedupe.get_ngrams("hello", n=3) + self.assertIn("hel", result) + self.assertIn("ell", result) + self.assertIn("llo", result) + + +class TestJaccardSimilarity(unittest.TestCase): + def test_identical_sets(self): + set1 = {"a", "b", "c"} + result = dedupe.jaccard_similarity(set1, set1) + self.assertEqual(result, 1.0) + + def test_disjoint_sets(self): + set1 = {"a", "b", "c"} + set2 = {"d", "e", "f"} + result = dedupe.jaccard_similarity(set1, set2) + self.assertEqual(result, 0.0) + + def test_partial_overlap(self): + set1 = {"a", "b", "c"} + set2 = {"b", "c", "d"} + result = dedupe.jaccard_similarity(set1, set2) + self.assertEqual(result, 0.5) # 2 overlap / 4 union + + def test_empty_sets(self): + result = dedupe.jaccard_similarity(set(), set()) + self.assertEqual(result, 0.0) + + +class TestFindDuplicates(unittest.TestCase): + def test_no_duplicates(self): + items = [ + schema.RedditItem(id="R1", title="Completely different topic A", url="", subreddit=""), + schema.RedditItem(id="R2", title="Another unrelated subject B", url="", subreddit=""), + ] + result = dedupe.find_duplicates(items) + self.assertEqual(result, []) + + def test_finds_duplicates(self): + items = [ + schema.RedditItem(id="R1", title="Best practices for Claude Code skills", url="", subreddit=""), + schema.RedditItem(id="R2", title="Best practices for Claude Code skills guide", url="", subreddit=""), + ] + result = dedupe.find_duplicates(items, threshold=0.7) + self.assertEqual(len(result), 1) + self.assertEqual(result[0], (0, 1)) + + +class TestDedupeItems(unittest.TestCase): + def test_keeps_higher_scored(self): + items = [ + schema.RedditItem(id="R1", title="Best practices for skills", url="", subreddit="", score=90), + schema.RedditItem(id="R2", title="Best practices for skills guide", url="", subreddit="", score=50), + ] + result = dedupe.dedupe_items(items, threshold=0.6) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].id, "R1") + + def test_keeps_all_unique(self): + items = [ + schema.RedditItem(id="R1", title="Topic about apples", url="", subreddit="", score=90), + schema.RedditItem(id="R2", title="Discussion of oranges", url="", subreddit="", score=50), + ] + result = dedupe.dedupe_items(items) + self.assertEqual(len(result), 2) + + def test_empty_list(self): + result = dedupe.dedupe_items([]) + self.assertEqual(result, []) + + def test_single_item(self): + items = [schema.RedditItem(id="R1", title="Test", url="", subreddit="")] + result = dedupe.dedupe_items(items) + self.assertEqual(len(result), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..0baa42b --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,135 @@ +"""Tests for models module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import models + + +class TestParseVersion(unittest.TestCase): + def test_simple_version(self): + result = models.parse_version("gpt-5") + self.assertEqual(result, (5,)) + + def test_minor_version(self): + result = models.parse_version("gpt-5.2") + self.assertEqual(result, (5, 2)) + + def test_patch_version(self): + result = models.parse_version("gpt-5.2.1") + self.assertEqual(result, (5, 2, 1)) + + def test_no_version(self): + result = models.parse_version("custom-model") + self.assertIsNone(result) + + +class TestIsMainlineOpenAIModel(unittest.TestCase): + def test_gpt5_is_mainline(self): + self.assertTrue(models.is_mainline_openai_model("gpt-5")) + + def test_gpt52_is_mainline(self): + self.assertTrue(models.is_mainline_openai_model("gpt-5.2")) + + def test_gpt5_mini_is_not_mainline(self): + self.assertFalse(models.is_mainline_openai_model("gpt-5-mini")) + + def test_gpt4_is_not_mainline(self): + self.assertFalse(models.is_mainline_openai_model("gpt-4")) + + +class TestSelectOpenAIModel(unittest.TestCase): + def test_pinned_policy(self): + result = models.select_openai_model( + "fake-key", + policy="pinned", + pin="gpt-5.1" + ) + self.assertEqual(result, "gpt-5.1") + + def test_auto_with_mock_models(self): + mock_models = [ + {"id": "gpt-5.2", "created": 1704067200}, + {"id": "gpt-5.1", "created": 1701388800}, + {"id": "gpt-5", "created": 1698710400}, + ] + result = models.select_openai_model( + "fake-key", + policy="auto", + mock_models=mock_models + ) + self.assertEqual(result, "gpt-5.2") + + def test_auto_filters_variants(self): + mock_models = [ + {"id": "gpt-5.2", "created": 1704067200}, + {"id": "gpt-5-mini", "created": 1704067200}, + {"id": "gpt-5.1", "created": 1701388800}, + ] + result = models.select_openai_model( + "fake-key", + policy="auto", + mock_models=mock_models + ) + self.assertEqual(result, "gpt-5.2") + + +class TestSelectXAIModel(unittest.TestCase): + def test_latest_policy(self): + result = models.select_xai_model( + "fake-key", + policy="latest" + ) + self.assertEqual(result, "grok-4-latest") + + def test_stable_policy(self): + # Clear cache first to avoid interference + from lib import cache + cache.MODEL_CACHE_FILE.unlink(missing_ok=True) + result = models.select_xai_model( + "fake-key", + policy="stable" + ) + self.assertEqual(result, "grok-4") + + def test_pinned_policy(self): + result = models.select_xai_model( + "fake-key", + policy="pinned", + pin="grok-3" + ) + self.assertEqual(result, "grok-3") + + +class TestGetModels(unittest.TestCase): + def test_no_keys_returns_none(self): + config = {} + result = models.get_models(config) + self.assertIsNone(result["openai"]) + self.assertIsNone(result["xai"]) + + def test_openai_key_only(self): + config = {"OPENAI_API_KEY": "sk-test"} + mock_models = [{"id": "gpt-5.2", "created": 1704067200}] + result = models.get_models(config, mock_openai_models=mock_models) + self.assertEqual(result["openai"], "gpt-5.2") + self.assertIsNone(result["xai"]) + + def test_both_keys(self): + config = { + "OPENAI_API_KEY": "sk-test", + "XAI_API_KEY": "xai-test", + } + mock_openai = [{"id": "gpt-5.2", "created": 1704067200}] + mock_xai = [{"id": "grok-4-latest", "created": 1704067200}] + result = models.get_models(config, mock_openai, mock_xai) + self.assertEqual(result["openai"], "gpt-5.2") + self.assertEqual(result["xai"], "grok-4-latest") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_normalize.py b/tests/test_normalize.py new file mode 100644 index 0000000..4ccdd67 --- /dev/null +++ b/tests/test_normalize.py @@ -0,0 +1,138 @@ +"""Tests for normalize module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import normalize, schema + + +class TestNormalizeRedditItems(unittest.TestCase): + def test_normalizes_basic_item(self): + items = [ + { + "id": "R1", + "title": "Test Thread", + "url": "https://reddit.com/r/test/1", + "subreddit": "test", + "date": "2026-01-15", + "why_relevant": "Relevant because...", + "relevance": 0.85, + } + ] + + result = normalize.normalize_reddit_items(items, "2026-01-01", "2026-01-31") + + self.assertEqual(len(result), 1) + self.assertIsInstance(result[0], schema.RedditItem) + self.assertEqual(result[0].id, "R1") + self.assertEqual(result[0].title, "Test Thread") + self.assertEqual(result[0].date_confidence, "high") + + def test_sets_low_confidence_for_old_date(self): + items = [ + { + "id": "R1", + "title": "Old Thread", + "url": "https://reddit.com/r/test/1", + "subreddit": "test", + "date": "2025-12-01", # Before range + "relevance": 0.5, + } + ] + + result = normalize.normalize_reddit_items(items, "2026-01-01", "2026-01-31") + + self.assertEqual(result[0].date_confidence, "low") + + def test_handles_engagement(self): + items = [ + { + "id": "R1", + "title": "Thread with engagement", + "url": "https://reddit.com/r/test/1", + "subreddit": "test", + "engagement": { + "score": 100, + "num_comments": 50, + "upvote_ratio": 0.9, + }, + "relevance": 0.5, + } + ] + + result = normalize.normalize_reddit_items(items, "2026-01-01", "2026-01-31") + + self.assertIsNotNone(result[0].engagement) + self.assertEqual(result[0].engagement.score, 100) + self.assertEqual(result[0].engagement.num_comments, 50) + + +class TestNormalizeXItems(unittest.TestCase): + def test_normalizes_basic_item(self): + items = [ + { + "id": "X1", + "text": "Test post content", + "url": "https://x.com/user/status/123", + "author_handle": "testuser", + "date": "2026-01-15", + "why_relevant": "Relevant because...", + "relevance": 0.9, + } + ] + + result = normalize.normalize_x_items(items, "2026-01-01", "2026-01-31") + + self.assertEqual(len(result), 1) + self.assertIsInstance(result[0], schema.XItem) + self.assertEqual(result[0].id, "X1") + self.assertEqual(result[0].author_handle, "testuser") + + def test_handles_x_engagement(self): + items = [ + { + "id": "X1", + "text": "Post with engagement", + "url": "https://x.com/user/status/123", + "author_handle": "user", + "engagement": { + "likes": 100, + "reposts": 25, + "replies": 15, + "quotes": 5, + }, + "relevance": 0.5, + } + ] + + result = normalize.normalize_x_items(items, "2026-01-01", "2026-01-31") + + self.assertIsNotNone(result[0].engagement) + self.assertEqual(result[0].engagement.likes, 100) + self.assertEqual(result[0].engagement.reposts, 25) + + +class TestItemsToDicts(unittest.TestCase): + def test_converts_items(self): + items = [ + schema.RedditItem( + id="R1", + title="Test", + url="https://reddit.com/r/test/1", + subreddit="test", + ) + ] + + result = normalize.items_to_dicts(items) + + self.assertEqual(len(result), 1) + self.assertIsInstance(result[0], dict) + self.assertEqual(result[0]["id"], "R1") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_render.py b/tests/test_render.py new file mode 100644 index 0000000..01a99bc --- /dev/null +++ b/tests/test_render.py @@ -0,0 +1,116 @@ +"""Tests for render module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import render, schema + + +class TestRenderCompact(unittest.TestCase): + def test_renders_basic_report(self): + report = schema.Report( + topic="test topic", + range_from="2026-01-01", + range_to="2026-01-31", + generated_at="2026-01-31T12:00:00Z", + mode="both", + openai_model_used="gpt-5.2", + xai_model_used="grok-4-latest", + ) + + result = render.render_compact(report) + + self.assertIn("test topic", result) + self.assertIn("2026-01-01", result) + self.assertIn("both", result) + self.assertIn("gpt-5.2", result) + + def test_renders_reddit_items(self): + report = schema.Report( + topic="test", + range_from="2026-01-01", + range_to="2026-01-31", + generated_at="2026-01-31T12:00:00Z", + mode="reddit-only", + reddit=[ + schema.RedditItem( + id="R1", + title="Test Thread", + url="https://reddit.com/r/test/1", + subreddit="test", + date="2026-01-15", + date_confidence="high", + score=85, + why_relevant="Very relevant", + ) + ], + ) + + result = render.render_compact(report) + + self.assertIn("R1", result) + self.assertIn("Test Thread", result) + self.assertIn("r/test", result) + + def test_shows_coverage_tip_for_reddit_only(self): + report = schema.Report( + topic="test", + range_from="2026-01-01", + range_to="2026-01-31", + generated_at="2026-01-31T12:00:00Z", + mode="reddit-only", + ) + + result = render.render_compact(report) + + self.assertIn("xAI key", result) + + +class TestRenderContextSnippet(unittest.TestCase): + def test_renders_snippet(self): + report = schema.Report( + topic="Claude Code Skills", + range_from="2026-01-01", + range_to="2026-01-31", + generated_at="2026-01-31T12:00:00Z", + mode="both", + ) + + result = render.render_context_snippet(report) + + self.assertIn("Claude Code Skills", result) + self.assertIn("Last 30 Days", result) + + +class TestRenderFullReport(unittest.TestCase): + def test_renders_full_report(self): + report = schema.Report( + topic="test topic", + range_from="2026-01-01", + range_to="2026-01-31", + generated_at="2026-01-31T12:00:00Z", + mode="both", + openai_model_used="gpt-5.2", + xai_model_used="grok-4-latest", + ) + + result = render.render_full_report(report) + + self.assertIn("# test topic", result) + self.assertIn("## Models Used", result) + self.assertIn("gpt-5.2", result) + + +class TestGetContextPath(unittest.TestCase): + def test_returns_path_string(self): + result = render.get_context_path() + self.assertIsInstance(result, str) + self.assertIn("last30days.context.md", result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_score.py b/tests/test_score.py new file mode 100644 index 0000000..b1183f2 --- /dev/null +++ b/tests/test_score.py @@ -0,0 +1,168 @@ +"""Tests for score module.""" + +import sys +import unittest +from datetime import datetime, timezone +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import schema, score + + +class TestLog1pSafe(unittest.TestCase): + def test_positive_value(self): + result = score.log1p_safe(100) + self.assertGreater(result, 0) + + def test_zero(self): + result = score.log1p_safe(0) + self.assertEqual(result, 0) + + def test_none(self): + result = score.log1p_safe(None) + self.assertEqual(result, 0) + + def test_negative(self): + result = score.log1p_safe(-5) + self.assertEqual(result, 0) + + +class TestComputeRedditEngagementRaw(unittest.TestCase): + def test_with_engagement(self): + eng = schema.Engagement(score=100, num_comments=50, upvote_ratio=0.9) + result = score.compute_reddit_engagement_raw(eng) + self.assertIsNotNone(result) + self.assertGreater(result, 0) + + def test_without_engagement(self): + result = score.compute_reddit_engagement_raw(None) + self.assertIsNone(result) + + def test_empty_engagement(self): + eng = schema.Engagement() + result = score.compute_reddit_engagement_raw(eng) + self.assertIsNone(result) + + +class TestComputeXEngagementRaw(unittest.TestCase): + def test_with_engagement(self): + eng = schema.Engagement(likes=100, reposts=25, replies=15, quotes=5) + result = score.compute_x_engagement_raw(eng) + self.assertIsNotNone(result) + self.assertGreater(result, 0) + + def test_without_engagement(self): + result = score.compute_x_engagement_raw(None) + self.assertIsNone(result) + + +class TestNormalizeTo100(unittest.TestCase): + def test_normalizes_values(self): + values = [0, 50, 100] + result = score.normalize_to_100(values) + self.assertEqual(result[0], 0) + self.assertEqual(result[1], 50) + self.assertEqual(result[2], 100) + + def test_handles_none(self): + values = [0, None, 100] + result = score.normalize_to_100(values) + self.assertIsNone(result[1]) + + def test_single_value(self): + values = [50] + result = score.normalize_to_100(values) + self.assertEqual(result[0], 50) + + +class TestScoreRedditItems(unittest.TestCase): + def test_scores_items(self): + today = datetime.now(timezone.utc).date().isoformat() + items = [ + schema.RedditItem( + id="R1", + title="Test", + url="https://reddit.com/r/test/1", + subreddit="test", + date=today, + date_confidence="high", + engagement=schema.Engagement(score=100, num_comments=50, upvote_ratio=0.9), + relevance=0.9, + ), + schema.RedditItem( + id="R2", + title="Test 2", + url="https://reddit.com/r/test/2", + subreddit="test", + date=today, + date_confidence="high", + engagement=schema.Engagement(score=10, num_comments=5, upvote_ratio=0.8), + relevance=0.5, + ), + ] + + result = score.score_reddit_items(items) + + self.assertEqual(len(result), 2) + self.assertGreater(result[0].score, 0) + self.assertGreater(result[1].score, 0) + # Higher relevance and engagement should score higher + self.assertGreater(result[0].score, result[1].score) + + def test_empty_list(self): + result = score.score_reddit_items([]) + self.assertEqual(result, []) + + +class TestScoreXItems(unittest.TestCase): + def test_scores_items(self): + today = datetime.now(timezone.utc).date().isoformat() + items = [ + schema.XItem( + id="X1", + text="Test post", + url="https://x.com/user/1", + author_handle="user1", + date=today, + date_confidence="high", + engagement=schema.Engagement(likes=100, reposts=25, replies=15, quotes=5), + relevance=0.9, + ), + ] + + result = score.score_x_items(items) + + self.assertEqual(len(result), 1) + self.assertGreater(result[0].score, 0) + + +class TestSortItems(unittest.TestCase): + def test_sorts_by_score_descending(self): + items = [ + schema.RedditItem(id="R1", title="Low", url="", subreddit="", score=30), + schema.RedditItem(id="R2", title="High", url="", subreddit="", score=90), + schema.RedditItem(id="R3", title="Mid", url="", subreddit="", score=60), + ] + + result = score.sort_items(items) + + self.assertEqual(result[0].id, "R2") + self.assertEqual(result[1].id, "R3") + self.assertEqual(result[2].id, "R1") + + def test_stable_sort(self): + items = [ + schema.RedditItem(id="R1", title="A", url="", subreddit="", score=50), + schema.RedditItem(id="R2", title="B", url="", subreddit="", score=50), + ] + + result = score.sort_items(items) + + # Both have same score, should maintain order by title + self.assertEqual(len(result), 2) + + +if __name__ == "__main__": + unittest.main()