;
-export {};
-//# sourceMappingURL=nodeSqlite.d.ts.map
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.d.ts.map b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.d.ts.map
deleted file mode 100644
index 45e57aa..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.d.ts.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"nodeSqlite.d.ts","sourceRoot":"","sources":["../../src/util/nodeSqlite.ts"],"names":[],"mappings":"AAAA,KAAK,gBAAgB,GAAG,cAAc,aAAa,CAAC,CAAC;AAIrD,wBAAgB,mBAAmB,IAAI,OAAO,CAQ7C;AA4BD,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAmBlE"}
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.js b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.js
deleted file mode 100644
index 68c344e..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.js
+++ /dev/null
@@ -1,58 +0,0 @@
-let cached = null;
-export function supportsReadBigInts() {
- const [majorRaw, minorRaw] = process.versions.node.split('.');
- const major = Number.parseInt(majorRaw ?? '', 10);
- const minor = Number.parseInt(minorRaw ?? '', 10);
- if (!Number.isFinite(major) || !Number.isFinite(minor))
- return false;
- if (major > 24)
- return true;
- if (major < 24)
- return false;
- return minor >= 4;
-}
-function shouldSuppressSqliteExperimentalWarning(warning, args) {
- const message = typeof warning === 'string'
- ? warning
- : warning instanceof Error
- ? warning.message
- : typeof warning?.message === 'string'
- ? warning.message
- : null;
- if (!message || !message.includes('SQLite is an experimental feature'))
- return false;
- // Best-effort: only swallow the one noisy warning that Node emits when loading `node:sqlite`.
- // We do *not* suppress arbitrary warnings, and we restore the original handler immediately.
- const firstArg = args[0];
- if (firstArg === 'ExperimentalWarning')
- return true;
- if (typeof firstArg === 'object' && firstArg) {
- const type = firstArg.type;
- if (type === 'ExperimentalWarning')
- return true;
- }
- if (warning instanceof Error && warning.name === 'ExperimentalWarning')
- return true;
- return false;
-}
-export async function importNodeSqlite() {
- if (cached)
- return cached;
- // Node currently emits an ExperimentalWarning when importing `node:sqlite`.
- // This is harmless noise for consumers of this library, so we silence only that specific warning.
- const originalEmitWarning = process.emitWarning.bind(process);
- process.emitWarning = ((warning, ...args) => {
- if (shouldSuppressSqliteExperimentalWarning(warning, args))
- return;
- // @ts-expect-error - Node's overloads are awkward; preserve runtime behavior.
- return originalEmitWarning(warning, ...args);
- });
- try {
- cached = await import('node:sqlite');
- return cached;
- }
- finally {
- process.emitWarning = originalEmitWarning;
- }
-}
-//# sourceMappingURL=nodeSqlite.js.map
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.js.map b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.js.map
deleted file mode 100644
index 627007d..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/nodeSqlite.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"nodeSqlite.js","sourceRoot":"","sources":["../../src/util/nodeSqlite.ts"],"names":[],"mappings":"AAEA,IAAI,MAAM,GAA4B,IAAI,CAAC;AAE3C,MAAM,UAAU,mBAAmB;IAClC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrE,IAAI,KAAK,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,KAAK,GAAG,EAAE;QAAE,OAAO,KAAK,CAAC;IAC7B,OAAO,KAAK,IAAI,CAAC,CAAC;AACnB,CAAC;AAED,SAAS,uCAAuC,CAAC,OAAgB,EAAE,IAAe;IACjF,MAAM,OAAO,GACZ,OAAO,OAAO,KAAK,QAAQ;QAC1B,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,OAAO,YAAY,KAAK;YACzB,CAAC,CAAC,OAAO,CAAC,OAAO;YACjB,CAAC,CAAC,OAAQ,OAAwC,EAAE,OAAO,KAAK,QAAQ;gBACvE,CAAC,CAAE,OAA+B,CAAC,OAAO;gBAC1C,CAAC,CAAC,IAAI,CAAC;IAEX,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,mCAAmC,CAAC;QAAE,OAAO,KAAK,CAAC;IAErF,8FAA8F;IAC9F,4FAA4F;IAC5F,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,QAAQ,KAAK,qBAAqB;QAAE,OAAO,IAAI,CAAC;IACpD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAI,QAA+B,CAAC,IAAI,CAAC;QACnD,IAAI,IAAI,KAAK,qBAAqB;YAAE,OAAO,IAAI,CAAC;IACjD,CAAC;IACD,IAAI,OAAO,YAAY,KAAK,IAAK,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC7F,OAAO,IAAI,CAAC;IAEb,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACrC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,4EAA4E;IAC5E,kGAAkG;IAClG,MAAM,mBAAmB,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAE9D,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,OAAgB,EAAE,GAAG,IAAe,EAAE,EAAE;QAC/D,IAAI,uCAAuC,CAAC,OAAO,EAAE,IAAI,CAAC;YAAE,OAAO;QACnE,8EAA8E;QAC9E,OAAO,mBAAmB,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC9C,CAAC,CAA+B,CAAC;IAEjC,IAAI,CAAC;QACJ,MAAM,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACrC,OAAO,MAAM,CAAC;IACf,CAAC;YAAS,CAAC;QACV,OAAO,CAAC,WAAW,GAAG,mBAAmB,CAAC;IAC3C,CAAC;AACF,CAAC"}
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.d.ts b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.d.ts
deleted file mode 100644
index 8a61f62..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.d.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export declare function normalizeOrigins(url: string, extraOrigins?: string[]): string[];
-//# sourceMappingURL=origins.d.ts.map
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.d.ts.map b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.d.ts.map
deleted file mode 100644
index a94554c..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.d.ts.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"origins.d.ts","sourceRoot":"","sources":["../../src/util/origins.ts"],"names":[],"mappings":"AAAA,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAqB/E"}
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.js b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.js
deleted file mode 100644
index f325dbe..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.js
+++ /dev/null
@@ -1,27 +0,0 @@
-export function normalizeOrigins(url, extraOrigins) {
- const origins = [];
- try {
- const parsed = new URL(url);
- origins.push(ensureTrailingSlash(parsed.origin));
- }
- catch {
- // ignore
- }
- for (const raw of extraOrigins ?? []) {
- const trimmed = raw.trim();
- if (!trimmed)
- continue;
- try {
- const parsed = new URL(trimmed);
- origins.push(ensureTrailingSlash(parsed.origin));
- }
- catch {
- // ignore malformed extras
- }
- }
- return Array.from(new Set(origins));
-}
-function ensureTrailingSlash(origin) {
- return origin.endsWith('/') ? origin : `${origin}/`;
-}
-//# sourceMappingURL=origins.js.map
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.js.map b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.js.map
deleted file mode 100644
index f30490d..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/origins.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"origins.js","sourceRoot":"","sources":["../../src/util/origins.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,YAAuB;IACpE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACR,SAAS;IACV,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACR,0BAA0B;QAC3B,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IAC1C,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC;AACrD,CAAC"}
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.d.ts b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.d.ts
deleted file mode 100644
index 0effb84..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.d.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export declare function isBunRuntime(): boolean;
-//# sourceMappingURL=runtime.d.ts.map
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.d.ts.map b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.d.ts.map
deleted file mode 100644
index e79fd7b..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.d.ts.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/util/runtime.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,IAAI,OAAO,CAKtC"}
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.js b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.js
deleted file mode 100644
index c0f48b4..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.js
+++ /dev/null
@@ -1,8 +0,0 @@
-export function isBunRuntime() {
- // Bun: https://bun.sh/docs/runtime/environment-variables#bun-specific
- if (typeof process === 'undefined')
- return false;
- const bunVersion = process.versions.bun;
- return Boolean(typeof process.versions === 'object' && typeof bunVersion === 'string');
-}
-//# sourceMappingURL=runtime.js.map
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.js.map b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.js.map
deleted file mode 100644
index 4c02d7a..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/util/runtime.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../src/util/runtime.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY;IAC3B,sEAAsE;IACtE,IAAI,OAAO,OAAO,KAAK,WAAW;QAAE,OAAO,KAAK,CAAC;IACjD,MAAM,UAAU,GAAI,OAAO,CAAC,QAAyC,CAAC,GAAG,CAAC;IAC1E,OAAO,OAAO,CAAC,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC;AACxF,CAAC"}
\ No newline at end of file
diff --git a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/package.json b/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/package.json
deleted file mode 100644
index ffef682..0000000
--- a/scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/package.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "name": "@steipete/sweet-cookie",
- "version": "0.1.0",
- "description": "Inline-first browser cookie extraction for local tooling (no native addons).",
- "license": "MIT",
- "author": "Peter Steinberger",
- "type": "module",
- "main": "dist/index.js",
- "types": "dist/index.d.ts",
- "files": [
- "dist",
- "README.md",
- "LICENSE"
- ],
- "repository": {
- "type": "git",
- "url": "git+https://github.com/steipete/sweet-cookie.git",
- "directory": "packages/core"
- },
- "bugs": {
- "url": "https://github.com/steipete/sweet-cookie/issues"
- },
- "homepage": "https://github.com/steipete/sweet-cookie#readme",
- "exports": {
- ".": {
- "types": "./dist/index.d.ts",
- "import": "./dist/index.js"
- }
- },
- "publishConfig": {
- "access": "public"
- },
- "scripts": {
- "build": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json",
- "typecheck": "tsc -p tsconfig.json --noEmit"
- },
- "engines": {
- "node": ">=22"
- }
-}
diff --git a/scripts/lib/websearch.py b/scripts/lib/websearch.py
deleted file mode 100644
index e7d779a..0000000
--- a/scripts/lib/websearch.py
+++ /dev/null
@@ -1,401 +0,0 @@
-"""WebSearch module for last30days skill.
-
-NOTE: WebSearch uses the assistant's built-in web search tool, which runs inside the host environment.
-Unlike Reddit/X which use external APIs, web search results are obtained by the assistant
-directly and passed to this module for normalization and scoring.
-
-The typical flow is:
-1. The assistant invokes its web search tool with the topic
-2. The assistant passes results to parse_websearch_results()
-3. Results are normalized into WebSearchItem objects
-"""
-
-import re
-from datetime import datetime, timedelta
-from typing import Any, Dict, List, Optional, Tuple
-from urllib.parse import urlparse
-
-from . import schema
-
-
-# Month name mappings for date parsing
-MONTH_MAP = {
- "jan": 1, "january": 1,
- "feb": 2, "february": 2,
- "mar": 3, "march": 3,
- "apr": 4, "april": 4,
- "may": 5,
- "jun": 6, "june": 6,
- "jul": 7, "july": 7,
- "aug": 8, "august": 8,
- "sep": 9, "sept": 9, "september": 9,
- "oct": 10, "october": 10,
- "nov": 11, "november": 11,
- "dec": 12, "december": 12,
-}
-
-
-def extract_date_from_url(url: str) -> Optional[str]:
- """Try to extract a date from URL path.
-
- Many sites embed dates in URLs like:
- - /2026/01/24/article-title
- - /2026-01-24/article
- - /blog/20260124/title
-
- Args:
- url: URL to parse
-
- Returns:
- Date string in YYYY-MM-DD format, or None
- """
- # Pattern 1: /YYYY/MM/DD/ (most common)
- match = re.search(r'/(\d{4})/(\d{2})/(\d{2})/', url)
- if match:
- year, month, day = match.groups()
- if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
- return f"{year}-{month}-{day}"
-
- # Pattern 2: /YYYY-MM-DD/ or /YYYY-MM-DD-
- match = re.search(r'/(\d{4})-(\d{2})-(\d{2})[-/]', url)
- if match:
- year, month, day = match.groups()
- if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
- return f"{year}-{month}-{day}"
-
- # Pattern 3: /YYYYMMDD/ (compact)
- match = re.search(r'/(\d{4})(\d{2})(\d{2})/', url)
- if match:
- year, month, day = match.groups()
- if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
- return f"{year}-{month}-{day}"
-
- return None
-
-
-def extract_date_from_snippet(text: str) -> Optional[str]:
- """Try to extract a date from text snippet or title.
-
- Looks for patterns like:
- - January 24, 2026 or Jan 24, 2026
- - 24 January 2026
- - 2026-01-24
- - "3 days ago", "yesterday", "last week"
-
- Args:
- text: Text to parse
-
- Returns:
- Date string in YYYY-MM-DD format, or None
- """
- if not text:
- return None
-
- text_lower = text.lower()
-
- # Pattern 1: Month DD, YYYY (e.g., "January 24, 2026")
- match = re.search(
- r'\b(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|'
- r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)'
- r'\s+(\d{1,2})(?:st|nd|rd|th)?,?\s*(\d{4})\b',
- text_lower
- )
- if match:
- month_str, day, year = match.groups()
- month = MONTH_MAP.get(month_str[:3])
- if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31:
- return f"{year}-{month:02d}-{int(day):02d}"
-
- # Pattern 2: DD Month YYYY (e.g., "24 January 2026")
- match = re.search(
- r'\b(\d{1,2})(?:st|nd|rd|th)?\s+'
- r'(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|'
- r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)'
- r'\s+(\d{4})\b',
- text_lower
- )
- if match:
- day, month_str, year = match.groups()
- month = MONTH_MAP.get(month_str[:3])
- if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31:
- return f"{year}-{month:02d}-{int(day):02d}"
-
- # Pattern 3: YYYY-MM-DD (ISO format)
- match = re.search(r'\b(\d{4})-(\d{2})-(\d{2})\b', text)
- if match:
- year, month, day = match.groups()
- if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
- return f"{year}-{month}-{day}"
-
- # Pattern 4: Relative dates ("3 days ago", "yesterday", etc.)
- today = datetime.now()
-
- if "yesterday" in text_lower:
- date = today - timedelta(days=1)
- return date.strftime("%Y-%m-%d")
-
- if "today" in text_lower:
- return today.strftime("%Y-%m-%d")
-
- # "N days ago"
- match = re.search(r'\b(\d+)\s*days?\s*ago\b', text_lower)
- if match:
- days = int(match.group(1))
- if days <= 60: # Reasonable range
- date = today - timedelta(days=days)
- return date.strftime("%Y-%m-%d")
-
- # "N hours ago" -> today
- match = re.search(r'\b(\d+)\s*hours?\s*ago\b', text_lower)
- if match:
- return today.strftime("%Y-%m-%d")
-
- # "last week" -> ~7 days ago
- if "last week" in text_lower:
- date = today - timedelta(days=7)
- return date.strftime("%Y-%m-%d")
-
- # "this week" -> ~3 days ago (middle of week)
- if "this week" in text_lower:
- date = today - timedelta(days=3)
- return date.strftime("%Y-%m-%d")
-
- return None
-
-
-def extract_date_signals(
- url: str,
- snippet: str,
- title: str,
-) -> Tuple[Optional[str], str]:
- """Extract date from any available signal.
-
- Tries URL first (most reliable), then snippet, then title.
-
- Args:
- url: Page URL
- snippet: Page snippet/description
- title: Page title
-
- Returns:
- Tuple of (date_string, confidence)
- - date from URL: 'high' confidence
- - date from snippet/title: 'med' confidence
- - no date found: None, 'low' confidence
- """
- # Try URL first (most reliable)
- url_date = extract_date_from_url(url)
- if url_date:
- return url_date, "high"
-
- # Try snippet
- snippet_date = extract_date_from_snippet(snippet)
- if snippet_date:
- return snippet_date, "med"
-
- # Try title
- title_date = extract_date_from_snippet(title)
- if title_date:
- return title_date, "med"
-
- return None, "low"
-
-
-# Domains to exclude (Reddit and X are handled separately)
-EXCLUDED_DOMAINS = {
- "reddit.com",
- "www.reddit.com",
- "old.reddit.com",
- "twitter.com",
- "www.twitter.com",
- "x.com",
- "www.x.com",
- "mobile.twitter.com",
-}
-
-
-def extract_domain(url: str) -> str:
- """Extract the domain from a URL.
-
- Args:
- url: Full URL
-
- Returns:
- Domain string (e.g., "medium.com")
- """
- try:
- parsed = urlparse(url)
- domain = parsed.netloc.lower()
- # Remove www. prefix for cleaner display
- if domain.startswith("www."):
- domain = domain[4:]
- return domain
- except Exception:
- return ""
-
-
-def is_excluded_domain(url: str) -> bool:
- """Check if URL is from an excluded domain (Reddit/X).
-
- Args:
- url: URL to check
-
- Returns:
- True if URL should be excluded
- """
- try:
- parsed = urlparse(url)
- domain = parsed.netloc.lower()
- return domain in EXCLUDED_DOMAINS
- except Exception:
- return False
-
-
-def parse_websearch_results(
- results: List[Dict[str, Any]],
- topic: str,
- from_date: str = "",
- to_date: str = "",
-) -> List[Dict[str, Any]]:
- """Parse WebSearch results into normalized format.
-
- This function expects results from Claude's WebSearch tool.
- Each result should have: title, url, snippet, and optionally date/relevance.
-
- Uses "Date Detective" approach:
- 1. Extract dates from URLs (high confidence)
- 2. Extract dates from snippets/titles (med confidence)
- 3. Hard filter: exclude items with verified old dates
- 4. Keep items with no date signals (with low confidence penalty)
-
- Args:
- results: List of WebSearch result dicts
- topic: Original search topic (for context)
- from_date: Start date for filtering (YYYY-MM-DD)
- to_date: End date for filtering (YYYY-MM-DD)
-
- Returns:
- List of normalized item dicts ready for WebSearchItem creation
- """
- items = []
-
- for i, result in enumerate(results):
- if not isinstance(result, dict):
- continue
-
- url = result.get("url", "")
- if not url:
- continue
-
- # Skip Reddit/X URLs (handled separately)
- if is_excluded_domain(url):
- continue
-
- title = str(result.get("title", "")).strip()
- snippet = str(result.get("snippet", result.get("description", ""))).strip()
-
- if not title and not snippet:
- continue
-
- # Use Date Detective to extract date signals
- date = result.get("date") # Use provided date if available
- date_confidence = "low"
-
- if date and re.match(r'^\d{4}-\d{2}-\d{2}$', str(date)):
- # Provided date is valid
- date_confidence = "med"
- else:
- # Try to extract date from URL/snippet/title
- extracted_date, confidence = extract_date_signals(url, snippet, title)
- if extracted_date:
- date = extracted_date
- date_confidence = confidence
-
- # Hard filter: if we found a date and it's too old, skip
- if date and from_date and date < from_date:
- continue # DROP - verified old content
-
- # Hard filter: if date is in the future, skip (parsing error)
- if date and to_date and date > to_date:
- continue # DROP - future date
-
- # Get relevance if provided, default to 0.5
- relevance = result.get("relevance", 0.5)
- try:
- relevance = min(1.0, max(0.0, float(relevance)))
- except (TypeError, ValueError):
- relevance = 0.5
-
- item = {
- "id": f"W{i+1}",
- "title": title[:200], # Truncate long titles
- "url": url,
- "source_domain": extract_domain(url),
- "snippet": snippet[:500], # Truncate long snippets
- "date": date,
- "date_confidence": date_confidence,
- "relevance": relevance,
- "why_relevant": str(result.get("why_relevant", "")).strip(),
- }
-
- items.append(item)
-
- return items
-
-
-def normalize_websearch_items(
- items: List[Dict[str, Any]],
- from_date: str,
- to_date: str,
-) -> List[schema.WebSearchItem]:
- """Convert parsed dicts to WebSearchItem objects.
-
- Args:
- items: List of parsed item dicts
- from_date: Start of date range (YYYY-MM-DD)
- to_date: End of date range (YYYY-MM-DD)
-
- Returns:
- List of WebSearchItem objects
- """
- result = []
-
- for item in items:
- web_item = schema.WebSearchItem(
- id=item["id"],
- title=item["title"],
- url=item["url"],
- source_domain=item["source_domain"],
- snippet=item["snippet"],
- date=item.get("date"),
- date_confidence=item.get("date_confidence", "low"),
- relevance=item.get("relevance", 0.5),
- why_relevant=item.get("why_relevant", ""),
- )
- result.append(web_item)
-
- return result
-
-
-def dedupe_websearch(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
- """Remove duplicate WebSearch items.
-
- Deduplication is based on URL.
-
- Args:
- items: List of WebSearchItem objects
-
- Returns:
- Deduplicated list
- """
- seen_urls = set()
- result = []
-
- for item in items:
- # Normalize URL for comparison
- url_key = item.url.lower().rstrip("/")
- if url_key not in seen_urls:
- seen_urls.add(url_key)
- result.append(item)
-
- return result
diff --git a/scripts/lib/xai_x.py b/scripts/lib/xai_x.py
index b00e215..fdb044b 100644
--- a/scripts/lib/xai_x.py
+++ b/scripts/lib/xai_x.py
@@ -5,13 +5,24 @@ import re
import sys
from typing import Any, Dict, List, Optional
-from . import http
+from . import http, log
+
+
+def _safe_text(val) -> str:
+ """Extract text from string or localized object."""
+ if isinstance(val, str):
+ return val
+ if isinstance(val, dict):
+ return str(val.get("text", val.get("en", "")))
+ return str(val) if val is not None else ""
+
+
+def _log(msg: str):
+ log.source_log("xAI", msg, tty_only=False)
def _log_error(msg: str):
- """Log error to stderr."""
- sys.stderr.write(f"[X ERROR] {msg}\n")
- sys.stderr.flush()
+ log.source_log("xAI ERROR", msg, tty_only=False)
# xAI uses responses endpoint with Agent Tools API
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
@@ -173,7 +184,7 @@ def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
data = json.loads(json_match.group())
items = data.get("items", [])
except json.JSONDecodeError:
- pass
+ _log(f"Failed to parse xAI response JSON: {output_text[:200]}")
# Validate and clean items
clean_items = []
@@ -190,20 +201,20 @@ def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
eng_raw = item.get("engagement")
if isinstance(eng_raw, dict):
engagement = {
- "likes": int(eng_raw.get("likes", 0)) if eng_raw.get("likes") else None,
- "reposts": int(eng_raw.get("reposts", 0)) if eng_raw.get("reposts") else None,
- "replies": int(eng_raw.get("replies", 0)) if eng_raw.get("replies") else None,
- "quotes": int(eng_raw.get("quotes", 0)) if eng_raw.get("quotes") else None,
+ "likes": int(eng_raw["likes"]) if eng_raw.get("likes") is not None else None,
+ "reposts": int(eng_raw["reposts"]) if eng_raw.get("reposts") is not None else None,
+ "replies": int(eng_raw["replies"]) if eng_raw.get("replies") is not None else None,
+ "quotes": int(eng_raw["quotes"]) if eng_raw.get("quotes") is not None else None,
}
clean_item = {
"id": f"X{i+1}",
- "text": str(item.get("text", "")).strip()[:500], # Truncate long text
+ "text": _safe_text(item.get("text", "")).strip()[:500], # Truncate long text
"url": url,
- "author_handle": str(item.get("author_handle", "")).strip().lstrip("@"),
+ "author_handle": _safe_text(item.get("author_handle", "")).strip().lstrip("@"),
"date": item.get("date"),
"engagement": engagement,
- "why_relevant": str(item.get("why_relevant", "")).strip(),
+ "why_relevant": _safe_text(item.get("why_relevant", "")).strip(),
"relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))),
}
diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py
index 12fee3b..d77c3a3 100644
--- a/scripts/lib/youtube_yt.py
+++ b/scripts/lib/youtube_yt.py
@@ -1,4 +1,4 @@
-"""YouTube search and transcript extraction via yt-dlp for /last30days v2.1.
+"""YouTube search and transcript extraction via yt-dlp for the v3.0.0 pipeline.
Uses yt-dlp (https://github.com/yt-dlp/yt-dlp) for both YouTube search and
transcript extraction. No API keys needed — just have yt-dlp installed.
@@ -23,34 +23,42 @@ from typing import Any, Dict, List, Optional, Set, Tuple
# Depth configurations: how many videos to search / transcribe
DEPTH_CONFIG = {
- "quick": 10,
- "default": 20,
+ "quick": 6,
+ "default": 8,
"deep": 40,
}
TRANSCRIPT_LIMITS = {
- "quick": 3,
- "default": 5,
+ "quick": 0,
+ "default": 2,
"deep": 8,
}
# Max words to keep from each transcript
TRANSCRIPT_MAX_WORDS = 5000
+from . import http, log
from .relevance import token_overlap_relevance as _compute_relevance
-def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -> List[str]:
+def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -> list[str]:
"""Extract quotable highlights from a YouTube transcript.
- Similar to reddit_enrich.extract_comment_insights() but for
- continuous speech-to-text rather than threaded comments.
+ Filters filler (subscribe, welcome back, etc.), scores sentences by
+ specificity (numbers, proper nouns, topic relevance), and returns
+ the top highlights.
"""
if not transcript:
return []
sentences = re.split(r'(?<=[.!?])\s+', transcript)
+ # Fallback for punctuation-free transcripts (common with auto-captions):
+ # chunk into ~20-word segments so they pass the 8-50 word filter.
+ if len(sentences) <= 1 and len(transcript.split()) > 50:
+ words = transcript.split()
+ sentences = [' '.join(words[i:i+20]) for i in range(0, len(words), 20)]
+
filler = [
r"^(hey |hi |what's up|welcome back|in today's video|don't forget to)",
r"(subscribe|like and comment|hit the bell|check out the link|down below)",
@@ -87,9 +95,7 @@ def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -
def _log(msg: str):
- """Log to stderr."""
- sys.stderr.write(f"[YouTube] {msg}\n")
- sys.stderr.flush()
+ log.source_log("YouTube", msg, tty_only=False)
def is_ytdlp_installed() -> bool:
@@ -113,10 +119,75 @@ def _extract_core_subject(topic: str) -> str:
'recommendations', 'advice',
'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches',
+ # Temporal/meta words — planner generates these but they don't
+ # appear in YouTube titles, so strip them for better search.
+ 'last', 'days', 'recent', 'recently', 'month', 'week',
+ 'january', 'february', 'march', 'april', 'may', 'june',
+ 'july', 'august', 'september', 'october', 'november', 'december',
+ '2025', '2026', '2027',
+ 'music', 'public', 'appearances', 'developments', 'discussions', 'coverage',
})
return extract_core_subject(topic, noise=_YT_NOISE)
+def _infer_query_intent(topic: str) -> str:
+ """Tiny local intent classifier for YouTube query expansion."""
+ text = topic.lower().strip()
+ if re.search(r"\b(vs|versus|compare|difference between)\b", text):
+ return "comparison"
+ if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install|configure|troubleshoot|error|fix|debug)\b", text):
+ return "how_to"
+ if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
+ return "opinion"
+ if re.search(r"\b(pricing|feature|features|best .* for)\b", text):
+ return "product"
+ return "breaking_news"
+
+
+def expand_youtube_queries(topic: str, depth: str) -> List[str]:
+ """Generate multiple YouTube search queries from a topic.
+
+ Mirrors reddit.py's expand_reddit_queries() pattern:
+ 1. Extract core subject (strip noise words)
+ 2. Include original topic if different from core
+ 3. Add intent-specific OR-joined content-type variants
+ 4. Cap by depth: 1 for quick, 2 for default, 3 for deep
+
+ Returns 1-3 query strings depending on depth.
+ """
+ core = _extract_core_subject(topic)
+ queries = [core]
+
+ # Include cleaned original topic as variant if different from core
+ original_clean = topic.strip().rstrip('?!.')
+ if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
+ queries.append(original_clean)
+
+ qtype = _infer_query_intent(topic)
+
+ # Intent-specific YouTube content-type variants
+ if qtype == "opinion":
+ queries.append(f"{core} review OR reaction OR breakdown")
+ elif qtype == "product":
+ queries.append(f"{core} review OR comparison OR unboxing")
+ elif qtype == "comparison":
+ queries.append(f"{core} vs OR compared OR head to head")
+ elif qtype == "how_to":
+ queries.append(f"{core} tutorial OR guide OR explained")
+ else:
+ # breaking_news / general — YouTube content types
+ queries.append(f"{core} review OR reaction OR breakdown")
+
+ # Deep depth: add full-length content variant
+ if depth == "deep":
+ queries.append(f"{core} full OR complete OR official")
+
+ # Cap by depth budget
+ caps = {"quick": 1, "default": 2, "deep": 3}
+ cap = caps.get(depth, 2)
+ return queries[:cap]
+
+
def search_youtube(
topic: str,
from_date: str,
@@ -195,9 +266,9 @@ def search_youtube(
continue
video_id = video.get("id", "")
- view_count = video.get("view_count") or 0
- like_count = video.get("like_count") or 0
- comment_count = video.get("comment_count") or 0
+ view_count = video.get("view_count") if video.get("view_count") is not None else 0
+ like_count = video.get("like_count") if video.get("like_count") is not None else 0
+ comment_count = video.get("comment_count") if video.get("comment_count") is not None else 0
upload_date = video.get("upload_date", "") # YYYYMMDD
# Convert YYYYMMDD to YYYY-MM-DD
@@ -205,6 +276,7 @@ def search_youtube(
if upload_date and len(upload_date) == 8:
date_str = f"{upload_date[:4]}-{upload_date[4:6]}-{upload_date[6:8]}"
+ description = str(video.get("description", ""))[:500]
items.append({
"video_id": video_id,
"title": video.get("title", ""),
@@ -217,8 +289,9 @@ def search_youtube(
"comments": comment_count,
},
"duration": video.get("duration"),
- "relevance": _compute_relevance(core_topic, video.get("title", "")),
+ "relevance": _compute_relevance(core_topic, f"{video.get('title', '')} {description}"),
"why_relevant": f"YouTube: {video.get('title', core_topic)[:60]}",
+ "description": description,
})
# Soft date filter: prefer recent items but fall back to all if too few
@@ -433,11 +506,15 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
raw_vtt = None
if is_ytdlp_installed():
raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
+ if not raw_vtt:
+ _log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback")
+ raw_vtt = _fetch_transcript_direct(video_id)
else:
_log("yt-dlp not installed, using direct HTTP transcript fetch")
raw_vtt = _fetch_transcript_direct(video_id)
if not raw_vtt:
+ _log(f"No transcript available for {video_id} (no captions found)")
return None
transcript = _clean_vtt(raw_vtt)
@@ -479,11 +556,16 @@ def fetch_transcripts_parallel(
vid = futures[future]
try:
results[vid] = future.result()
- except Exception:
+ except (OSError, subprocess.SubprocessError) as exc:
+ _log(f"Transcript fetch error for {vid}: {exc}")
+ results[vid] = None
+ except Exception as exc:
+ _log(f"Unexpected transcript error for {vid}: {type(exc).__name__}: {exc}")
results[vid] = None
got = sum(1 for v in results.values() if v)
- _log(f"Got transcripts for {got}/{len(video_ids)} videos")
+ errors = sum(1 for v in results.values() if v is None)
+ _log(f"Got transcripts for {got}/{len(video_ids)} videos ({errors} failed)")
return results
@@ -495,6 +577,9 @@ def search_and_transcribe(
) -> Dict[str, Any]:
"""Full YouTube search: find videos, then fetch transcripts for top results.
+ Uses expand_youtube_queries() to generate multiple search queries,
+ runs yt-dlp for each, and merges/deduplicates results by video ID.
+
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
@@ -504,17 +589,37 @@ def search_and_transcribe(
Returns:
Dict with 'items' list. Each item has a 'transcript_snippet' field.
"""
- # Step 1: Search
- search_result = search_youtube(topic, from_date, to_date, depth)
- items = search_result.get("items", [])
+ # Step 1: Multi-query search — run yt-dlp for each expanded query
+ queries = expand_youtube_queries(topic, depth)
+ seen_ids: Set[str] = set()
+ items: List[Dict[str, Any]] = []
+ for q in queries:
+ search_result = search_youtube(q, from_date, to_date, depth)
+ for item in search_result.get("items", []):
+ vid = item.get("video_id", "")
+ if vid and vid not in seen_ids:
+ seen_ids.add(vid)
+ items.append(item)
+
+ # Sort merged results by views descending
+ items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True)
if not items:
return search_result
- # Step 2: Fetch transcripts for top N by views
+ # Step 2: Fetch transcripts for top videos by views.
+ # Try more candidates than the limit because some videos (music videos,
+ # short clips) lack captions. Attempt up to 3x the limit so we have a
+ # good chance of reaching the target number of successful transcripts.
transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
- top_ids = [item["video_id"] for item in items[:transcript_limit]]
- transcripts = fetch_transcripts_parallel(top_ids)
+ transcripts: Dict[str, Optional[str]] = {}
+ if transcript_limit > 0:
+ attempt_count = min(len(items), transcript_limit * 3)
+ candidate_ids = [item["video_id"] for item in items[:attempt_count]]
+ _log(f"Fetching transcripts for up to {attempt_count} videos (target: {transcript_limit}): {candidate_ids}")
+ transcripts = fetch_transcripts_parallel(candidate_ids)
+ else:
+ _log(f"Transcript limit is 0 for depth={depth}, skipping transcript fetch")
# Step 3: Attach transcripts and extract highlights
core_topic = _extract_core_subject(topic)
@@ -536,3 +641,335 @@ def parse_youtube_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
List of item dicts ready for normalization.
"""
return response.get("items", [])
+
+
+# ---------------------------------------------------------------------------
+# ScrapeCreators YouTube API support
+# ---------------------------------------------------------------------------
+
+SCRAPECREATORS_YT_BASE = "https://api.scrapecreators.com/v1/youtube"
+
+try:
+ import requests as _requests
+except ImportError:
+ _requests = None
+
+
+def _sc_headers(token: str) -> Dict[str, str]:
+ """Build ScrapeCreators request headers."""
+ return {
+ "x-api-key": token,
+ "Content-Type": "application/json",
+ }
+
+
+def _total_engagement(item: Dict[str, Any]) -> int:
+ """Combined engagement score for ranking which videos to enrich."""
+ eng = item.get("engagement", {})
+ views = eng.get("views", 0) or 0
+ likes = eng.get("likes", 0) or 0
+ comments = eng.get("comments", 0) or 0
+ return views + likes + comments
+
+
+def enrich_with_comments(
+ items: List[Dict[str, Any]],
+ token: str,
+ max_videos: int = 3,
+ max_comments: int = 5,
+) -> List[Dict[str, Any]]:
+ """Enrich top YouTube videos with comment data from ScrapeCreators.
+
+ For the top N videos by engagement, fetches comments via the SC API
+ and attaches them as a ``top_comments`` field on each item.
+
+ Args:
+ items: YouTube items from search_and_transcribe() or search_youtube_sc()
+ token: ScrapeCreators API key
+ max_videos: How many videos to enrich with comments
+ max_comments: Max comments to keep per video
+
+ Returns:
+ Items list (mutated in place) with top_comments added to enriched items.
+ """
+ if not items or not token or max_videos <= 0:
+ return items
+
+ ranked = sorted(items, key=_total_engagement, reverse=True)
+ top_items = ranked[:max_videos]
+ _log(f"Enriching comments for {len(top_items)} YouTube videos")
+
+ enriched_count = 0
+ for item in top_items:
+ video_id = item.get("video_id", "")
+ if not video_id:
+ continue
+ try:
+ comments = _fetch_video_comments(video_id, token, max_comments)
+ if comments:
+ item["top_comments"] = comments
+ enriched_count += 1
+ except Exception as exc:
+ _log(f"Comment enrichment failed for {video_id}: {exc}")
+
+ _log(f"Enriched {enriched_count}/{len(top_items)} videos with comments")
+ return items
+
+
+def _fetch_video_comments(
+ video_id: str,
+ token: str,
+ max_comments: int = 5,
+) -> List[Dict[str, Any]]:
+ """Fetch comments for a single YouTube video via ScrapeCreators.
+
+ Args:
+ video_id: YouTube video ID
+ token: ScrapeCreators API key
+ max_comments: Maximum comments to return
+
+ Returns:
+ List of comment dicts with author, text, likes, date.
+ """
+ if not _requests:
+ try:
+ from urllib.parse import urlencode
+ params = urlencode({"id": video_id})
+ url = f"{SCRAPECREATORS_YT_BASE}/video/comments?{params}"
+ headers = _sc_headers(token)
+ headers["User-Agent"] = http.USER_AGENT
+ data = http.get(url, headers=headers, timeout=30, retries=2)
+ except Exception as exc:
+ _log(f"Comment fetch error (urllib) for {video_id}: {exc}")
+ return []
+ else:
+ try:
+ resp = _requests.get(
+ f"{SCRAPECREATORS_YT_BASE}/video/comments",
+ params={"id": video_id},
+ headers=_sc_headers(token),
+ timeout=30,
+ )
+ resp.raise_for_status()
+ data = resp.json()
+ except Exception as exc:
+ _log(f"Comment fetch error for {video_id}: {exc}")
+ return []
+
+ raw_comments = data.get("comments", data.get("data", []))
+ comments = []
+ for c in raw_comments[:max_comments]:
+ text = c.get("text") or c.get("body") or c.get("content", "")
+ if not text:
+ continue
+ comments.append({
+ "author": c.get("author") or c.get("author_name", ""),
+ "text": text[:400],
+ "likes": c.get("likes") or c.get("vote_count", 0),
+ "date": c.get("date") or c.get("published_at", ""),
+ })
+
+ return comments
+
+
+def search_youtube_sc(
+ topic: str,
+ from_date: str,
+ to_date: str,
+ depth: str = "default",
+ token: str = None,
+) -> Dict[str, Any]:
+ """Search YouTube via ScrapeCreators API (fallback when yt-dlp is unavailable).
+
+ Uses SC keyword search to find videos and SC transcript endpoint to
+ fetch transcripts. Called by pipeline.py when yt-dlp fails.
+
+ Args:
+ topic: Search topic
+ from_date: Start date (YYYY-MM-DD)
+ to_date: End date (YYYY-MM-DD)
+ depth: 'quick', 'default', or 'deep'
+ token: ScrapeCreators API key
+
+ Returns:
+ Dict with 'items' list of video metadata dicts.
+ """
+ if not token:
+ return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
+
+ count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+ core_topic = _extract_core_subject(topic)
+ _log(f"Searching YouTube via ScrapeCreators for '{core_topic}' (depth={depth})")
+
+ # Step 1: Search
+ raw_items = _sc_youtube_search(core_topic, token)
+ if not raw_items:
+ _log("SC YouTube search returned 0 results")
+ return {"items": []}
+
+ # Parse into normalized items
+ items = []
+ for i, raw in enumerate(raw_items[:count]):
+ video_id = (
+ raw.get("id") or raw.get("video_id") or raw.get("videoId") or ""
+ )
+ title = raw.get("title", "")
+ channel = raw.get("channel") or raw.get("channel_name") or raw.get("uploader", "")
+ description = str(raw.get("description", ""))[:500]
+ view_count = raw.get("view_count") or raw.get("views", 0)
+ like_count = raw.get("like_count") or raw.get("likes", 0)
+ comment_count = raw.get("comment_count") or raw.get("comments", 0)
+
+ # Date: try multiple field names
+ date_str = raw.get("upload_date") or raw.get("date") or raw.get("published_at", "")
+ if date_str and len(date_str) == 8 and date_str.isdigit():
+ date_str = f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}"
+ elif date_str and "T" in date_str:
+ date_str = date_str[:10]
+
+ url = raw.get("url", "")
+ if not url and video_id:
+ url = f"https://www.youtube.com/watch?v={video_id}"
+
+ items.append({
+ "video_id": video_id,
+ "title": title,
+ "url": url,
+ "channel_name": channel,
+ "date": date_str if date_str else None,
+ "engagement": {
+ "views": view_count or 0,
+ "likes": like_count or 0,
+ "comments": comment_count or 0,
+ },
+ "duration": raw.get("duration"),
+ "relevance": _compute_relevance(core_topic, f"{title} {description}"),
+ "why_relevant": f"YouTube: {title[:60]}" if title else f"YouTube: {core_topic}",
+ "description": description,
+ })
+
+ # Soft date filter
+ recent = [i for i in items if i["date"] and i["date"] >= from_date]
+ if len(recent) >= 3:
+ items = recent
+ _log(f"Found {len(items)} videos within date range")
+ else:
+ _log(f"Found {len(items)} videos ({len(recent)} within date range, keeping all)")
+
+ # Sort by views
+ items.sort(key=lambda x: x["engagement"]["views"], reverse=True)
+
+ # Step 2: Fetch transcripts for top videos
+ transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
+ if transcript_limit > 0 and items:
+ attempt_count = min(len(items), transcript_limit * 3)
+ _log(f"Fetching SC transcripts for up to {attempt_count} videos (target: {transcript_limit})")
+ for item in items[:attempt_count]:
+ vid = item["video_id"]
+ if not vid:
+ continue
+ transcript = _sc_fetch_transcript(vid, token)
+ item["transcript_snippet"] = transcript or ""
+ item["transcript_highlights"] = extract_transcript_highlights(
+ transcript or "", core_topic,
+ )
+ else:
+ for item in items:
+ item["transcript_snippet"] = ""
+ item["transcript_highlights"] = []
+
+ _log(f"SC YouTube: {len(items)} videos returned")
+ return {"items": items}
+
+
+def _sc_youtube_search(keyword: str, token: str) -> List[Dict[str, Any]]:
+ """Call ScrapeCreators YouTube search endpoint.
+
+ Args:
+ keyword: Search keyword
+ token: ScrapeCreators API key
+
+ Returns:
+ List of raw video dicts from the API.
+ """
+ if not _requests:
+ try:
+ from urllib.parse import urlencode
+ params = urlencode({"keyword": keyword})
+ url = f"{SCRAPECREATORS_YT_BASE}/search?{params}"
+ headers = _sc_headers(token)
+ headers["User-Agent"] = http.USER_AGENT
+ data = http.get(url, headers=headers, timeout=30, retries=2)
+ return data.get("videos", data.get("data", data.get("items", [])))
+ except Exception as exc:
+ _log(f"SC YouTube search error (urllib): {exc}")
+ return []
+
+ try:
+ resp = _requests.get(
+ f"{SCRAPECREATORS_YT_BASE}/search",
+ params={"keyword": keyword},
+ headers=_sc_headers(token),
+ timeout=30,
+ )
+ resp.raise_for_status()
+ data = resp.json()
+ return data.get("videos", data.get("data", data.get("items", [])))
+ except Exception as exc:
+ _log(f"SC YouTube search error: {exc}")
+ return []
+
+
+def _sc_fetch_transcript(video_id: str, token: str) -> Optional[str]:
+ """Fetch transcript for a YouTube video via ScrapeCreators.
+
+ Args:
+ video_id: YouTube video ID
+ token: ScrapeCreators API key
+
+ Returns:
+ Plaintext transcript string, or None if unavailable.
+ """
+ if not _requests:
+ try:
+ from urllib.parse import urlencode
+ params = urlencode({"id": video_id})
+ url = f"{SCRAPECREATORS_YT_BASE}/video/transcript?{params}"
+ headers = _sc_headers(token)
+ headers["User-Agent"] = http.USER_AGENT
+ data = http.get(url, headers=headers, timeout=30, retries=2)
+ except Exception as exc:
+ _log(f"SC transcript error (urllib) for {video_id}: {exc}")
+ return None
+ else:
+ try:
+ resp = _requests.get(
+ f"{SCRAPECREATORS_YT_BASE}/video/transcript",
+ params={"id": video_id},
+ headers=_sc_headers(token),
+ timeout=30,
+ )
+ if resp.status_code != 200:
+ _log(f"SC transcript returned {resp.status_code} for {video_id}")
+ return None
+ data = resp.json()
+ except Exception as exc:
+ _log(f"SC transcript error for {video_id}: {exc}")
+ return None
+
+ transcript = data.get("transcript")
+ if not transcript:
+ return None
+
+ if isinstance(transcript, list):
+ transcript = " ".join(str(s) for s in transcript)
+
+ # Clean VTT formatting if present
+ transcript = _clean_vtt(transcript)
+
+ # Truncate to max words
+ words = transcript.split()
+ if len(words) > TRANSCRIPT_MAX_WORDS:
+ transcript = " ".join(words[:TRANSCRIPT_MAX_WORDS]) + "..."
+
+ return transcript if transcript else None
diff --git a/scripts/store.py b/scripts/store.py
index 02552ef..3994639 100644
--- a/scripts/store.py
+++ b/scripts/store.py
@@ -18,6 +18,11 @@ from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, Optional
+SCRIPT_DIR = Path(__file__).parent.resolve()
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from lib import schema
+
DB_DIR = Path.home() / ".local" / "share" / "last30days"
DB_PATH = DB_DIR / "research.db"
@@ -128,9 +133,7 @@ INSERT OR IGNORE INTO settings (key, value) VALUES ('default_schedule', '0 8 * *
"""
# Future migrations keyed by version number
-MIGRATIONS: Dict[int, str] = {
- # 2: "ALTER TABLE findings ADD COLUMN tags TEXT DEFAULT '[]';",
-}
+MIGRATIONS: Dict[int, str] = {}
def _connect(db_path: Optional[Path] = None) -> sqlite3.Connection:
@@ -577,6 +580,78 @@ def get_trending(days: int = 7) -> List[Dict[str, Any]]:
conn.close()
+def finding_from_candidate(candidate: schema.Candidate) -> Dict[str, Any]:
+ """Convert a ranked candidate into a persisted finding."""
+ primary_item = schema.candidate_primary_item(candidate)
+ corroborating_sources = [
+ source for source in schema.candidate_sources(candidate)
+ if source and source != candidate.source
+ ]
+ summary = candidate.explanation or candidate.snippet or ""
+ if corroborating_sources:
+ prefix = f"Also seen in: {', '.join(corroborating_sources)}."
+ summary = f"{prefix} {summary}".strip()
+ body = (
+ primary_item.body
+ if primary_item and primary_item.body
+ else candidate.snippet or candidate.title
+ )
+ author = primary_item.author if primary_item and primary_item.author else ""
+ return {
+ "source": candidate.source or "unknown",
+ "source_url": candidate.url,
+ "source_title": candidate.title,
+ "author": author,
+ "content": body,
+ "summary": summary,
+ "engagement_score": candidate.engagement or 0,
+ "relevance_score": candidate.final_score or candidate.rerank_score or candidate.local_relevance,
+ }
+
+
+def findings_from_report(
+ report: schema.Report,
+ *,
+ limit: Optional[int] = None,
+) -> List[Dict[str, Any]]:
+ """Convert report into persisted findings.
+
+ Uses ranked candidates (post-rerank) when available for quality scores and explanations.
+ Supplements with raw items from items_by_source for HN/PM that didn't rank highly
+ but are valuable for watchlist persistence.
+ """
+ findings = []
+ seen_urls = set()
+
+ # Phase 1: Process ranked candidates (high-quality data with explanations and corroboration)
+ for candidate in report.ranked_candidates:
+ finding = finding_from_candidate(candidate)
+ findings.append(finding)
+ seen_urls.add(candidate.url)
+
+ # Phase 2: Add HN/PM items not already captured in ranked candidates
+ for source_name in ["hackernews", "polymarket"]:
+ if source_name not in report.items_by_source:
+ continue
+ for item in report.items_by_source[source_name]:
+ if item.url in seen_urls:
+ continue # Already captured with rich data
+ findings.append({
+ "source": source_name,
+ "source_url": item.url,
+ "source_title": item.title,
+ "author": item.author or "",
+ "content": item.body or "",
+ "summary": item.snippet or (item.body[:500] if item.body else ""),
+ "engagement_score": item.engagement_score or 0.0,
+ "relevance_score": item.local_relevance or 0.5,
+ })
+ seen_urls.add(item.url)
+
+ # Apply global limit after collecting all findings (fix: was per-source, now global)
+ return findings[:limit] if limit is not None else findings
+
+
# --- CLI interface ---
diff --git a/scripts/sync.sh b/scripts/sync.sh
index e47eb74..a7f1dde 100755
--- a/scripts/sync.sh
+++ b/scripts/sync.sh
@@ -6,45 +6,64 @@ set -euo pipefail
SRC="$(cd "$(dirname "$0")/.." && pwd)"
echo "Source: $SRC"
-TARGETS=(
- "$HOME/.claude/skills/last30days"
+COMMON_TARGETS=(
+ # Claude Code plugin cache: marketplace installs overwrite on update,
+ # but local development needs the cache kept in sync with the repo.
+ # Do NOT add ~/.claude/skills/last30days - it creates a duplicate
+ # /last30days-3 in the slash command menu alongside the plugin version.
+ "$HOME/.claude/plugins/cache/last30days-skill-private/last30days-3/3.0.0-alpha"
+ "$HOME/.claude/plugins/cache/last30days-skill-private/last30days-3-nogem/3.0.0-nogem"
"$HOME/.agents/skills/last30days"
"$HOME/.codex/skills/last30days"
)
+OPENCLAW_TARGET="$HOME/.openclaw/skills/last30days"
+
+sync_target() {
+ local target="$1"
+ local skill_md="$2"
-for t in "${TARGETS[@]}"; do
echo ""
- echo "--- Syncing to $t ---"
- mkdir -p "$t/scripts/lib"
+ echo "--- Syncing to $target ---"
+ mkdir -p "$target/scripts/lib" "$target/variants/open/references"
- cp "$SRC/SKILL.md" "$t/"
+ cp "$skill_md" "$target/SKILL.md"
- # Main script + lib modules (rsync handles identical files gracefully)
- rsync -a "$SRC/scripts/last30days.py" "$t/scripts/"
- rsync -a "$SRC/scripts/lib/"*.py "$t/scripts/lib/"
+ rsync -a \
+ "$SRC/scripts/last30days.py" \
+ "$SRC/scripts/watchlist.py" \
+ "$SRC/scripts/briefing.py" \
+ "$SRC/scripts/store.py" \
+ "$target/scripts/"
+ rsync -a "$SRC/scripts/lib/"*.py "$target/scripts/lib/"
+ rsync -a "$SRC/variants/open/" "$target/variants/open/"
- # Vendor directory (bird-search CLI)
if [ -d "$SRC/scripts/lib/vendor" ]; then
- rsync -a "$SRC/scripts/lib/vendor" "$t/scripts/lib/"
+ rsync -a "$SRC/scripts/lib/vendor" "$target/scripts/lib/"
fi
- # Fixtures
if [ -d "$SRC/fixtures" ]; then
- mkdir -p "$t/fixtures"
- rsync -a "$SRC/fixtures/" "$t/fixtures/"
+ mkdir -p "$target/fixtures"
+ rsync -a "$SRC/fixtures/" "$target/fixtures/"
fi
- # Count and report
- mod_count=$(ls "$t/scripts/lib/"*.py 2>/dev/null | wc -l | tr -d ' ')
+ mod_count=$(ls "$target/scripts/lib/"*.py 2>/dev/null | wc -l | tr -d ' ')
echo " Copied $mod_count modules"
- # Verify imports
- if (cd "$t/scripts" && python3 -c "from lib import youtube_yt, bird_x, render, ui; print(' Import check: OK')" 2>&1); then
+ if (
+ cd "$target/scripts" &&
+ python3 -c "import briefing, store, watchlist; from lib import youtube_yt, bird_x, render, ui; print(' Import check: OK')"
+ ); then
true
else
echo " Import check FAILED"
fi
+}
+
+for t in "${COMMON_TARGETS[@]}"; do
+ sync_target "$t" "$SRC/SKILL.md"
done
+sync_target "$OPENCLAW_TARGET" "$SRC/variants/open/SKILL.md"
+
echo ""
echo "Sync complete."
diff --git a/scripts/test_device_auth.py b/scripts/test_device_auth.py
new file mode 100644
index 0000000..33bc02f
--- /dev/null
+++ b/scripts/test_device_auth.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""Test ScrapeCreators GitHub device auth flow from the CLI.
+
+Usage:
+ python3 scripts/test_device_auth.py
+
+Flow:
+ 1. Starts device code request
+ 2. Shows user code + opens GitHub auth URL in browser
+ 3. Polls for token until you complete auth
+ 4. Fetches your profile and prints your API key
+"""
+
+import json
+import sys
+import time
+import webbrowser
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError, URLError
+
+BASE = "https://api.scrapecreators.com/v1/github/device"
+
+
+def _post(url, data=None):
+ body = json.dumps(data).encode() if data else None
+ req = Request(url, data=body, method="POST")
+ req.add_header("Content-Type", "application/json")
+ with urlopen(req, timeout=15) as resp:
+ return json.loads(resp.read())
+
+
+def _get(url, token):
+ req = Request(url)
+ req.add_header("Authorization", f"Bearer {token}")
+ with urlopen(req, timeout=15) as resp:
+ return json.loads(resp.read())
+
+
+def main():
+ # Step 1: Start device flow
+ print("Starting ScrapeCreators GitHub device auth...\n")
+ try:
+ code_resp = _post(f"{BASE}/code")
+ except (HTTPError, URLError) as e:
+ print(f"Failed to start device flow: {e}")
+ sys.exit(1)
+
+ device_code = code_resp.get("device_code")
+ user_code = code_resp.get("user_code")
+ verification_uri = code_resp.get("verification_uri")
+ interval = code_resp.get("interval", 5)
+ expires_in = code_resp.get("expires_in", 900)
+
+ if not device_code or not user_code:
+ print(f"Unexpected response: {json.dumps(code_resp, indent=2)}")
+ sys.exit(1)
+
+ print(f"Your code: {user_code}")
+ print(f"Open: {verification_uri}")
+ print(f"Expires in: {expires_in}s\n")
+
+ # Open browser
+ if verification_uri:
+ webbrowser.open(verification_uri)
+ print("Opened browser. Enter the code above, then authorize.\n")
+
+ # Step 2: Poll for token
+ print("Waiting for authorization", end="", flush=True)
+ deadline = time.time() + expires_in
+ access_token = None
+
+ while time.time() < deadline:
+ time.sleep(interval)
+ print(".", end="", flush=True)
+ try:
+ token_resp = _post(f"{BASE}/token", {"device_code": device_code})
+ except HTTPError as e:
+ # Some APIs return 4xx while pending
+ if e.code in (400, 403, 428):
+ continue
+ print(f"\nPoll error: {e}")
+ sys.exit(1)
+ except URLError:
+ continue
+
+ if token_resp.get("access_token"):
+ access_token = token_resp["access_token"]
+ break
+
+ # Check for explicit error states
+ error = token_resp.get("error")
+ if error == "authorization_pending" or error == "slow_down":
+ if error == "slow_down":
+ interval = min(interval + 2, 30)
+ continue
+ if error in ("expired_token", "access_denied"):
+ print(f"\n\nAuth failed: {error}")
+ sys.exit(1)
+
+ if not access_token:
+ print("\n\nTimed out waiting for authorization.")
+ sys.exit(1)
+
+ print(f"\n\nAuthorized! Access token: {access_token[:12]}...\n")
+
+ # Step 3: Fetch profile
+ print("Fetching profile...")
+ try:
+ profile = _get(f"{BASE}/profile", access_token)
+ except (HTTPError, URLError) as e:
+ print(f"Failed to fetch profile: {e}")
+ print(f"(access_token was: {access_token})")
+ sys.exit(1)
+
+ print(f"\nProfile response:\n{json.dumps(profile, indent=2)}\n")
+
+ api_key = profile.get("api_key")
+ if api_key:
+ print("=" * 50)
+ print(f"Your ScrapeCreators API key: {api_key}")
+ print("=" * 50)
+ print(f"\nTo use it: echo 'SCRAPECREATORS_API_KEY={api_key}' >> ~/.config/last30days/.env")
+ else:
+ print("No api_key in profile response. Full response printed above.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/verify_v3.py b/scripts/verify_v3.py
new file mode 100644
index 0000000..7cbc193
--- /dev/null
+++ b/scripts/verify_v3.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Run the v3 verification bundle for last30days."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+PYTHON = sys.executable
+
+SMOKE_TOPIC = "openclaw skills"
+SMOKE_CASES = [
+ ("gemini", ["--quick", "--search=grounding,hackernews"]),
+ ("openai", ["--quick", "--search=reddit,hackernews"]),
+ ("xai", ["--quick", "--search=reddit,hackernews"]),
+ ("auto", ["--quick", "--search=reddit,grounding,hackernews"]),
+]
+
+LATENCY_TOPICS = [
+ "openclaw skills",
+ "codex vs claude code",
+ "anthropic odds",
+]
+LATENCY_PROFILES = [
+ ("quick", ["--quick", "--search=grounding,hackernews"]),
+ ("default", ["--search=grounding,hackernews"]),
+ ("deep", ["--deep", "--search=grounding,hackernews"]),
+]
+
+
+def run_command(cmd: list[str], *, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ cmd,
+ cwd=REPO_ROOT,
+ env=env,
+ text=True,
+ capture_output=True,
+ timeout=timeout,
+ check=True,
+ )
+
+
+def verify_unit() -> dict[str, str]:
+ run_command([PYTHON, "-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py"], timeout=600)
+ run_command(
+ [
+ PYTHON,
+ "-m",
+ "py_compile",
+ *subprocess.run(
+ ["rg", "--files", "scripts", "tests", "-g", "*.py", "-g", "!scripts/lib/vendor/**"],
+ cwd=REPO_ROOT,
+ text=True,
+ capture_output=True,
+ check=True,
+ ).stdout.split(),
+ ],
+ timeout=600,
+ )
+ return {"status": "ok"}
+
+
+def verify_diagnose() -> dict[str, object]:
+ result = run_command([PYTHON, "scripts/last30days.py", "--diagnose"], timeout=120)
+ return json.loads(result.stdout)
+
+
+def verify_smoke() -> list[dict[str, object]]:
+ rows: list[dict[str, object]] = []
+ for provider, extra in SMOKE_CASES:
+ env = os.environ.copy()
+ env["LAST30DAYS_REASONING_PROVIDER"] = provider
+ start = time.time()
+ result = run_command(
+ [PYTHON, "scripts/last30days.py", SMOKE_TOPIC, "--emit=json", *extra],
+ env=env,
+ timeout=240,
+ )
+ duration = round(time.time() - start, 2)
+ report = json.loads(result.stdout)
+ rows.append(
+ {
+ "provider": provider,
+ "duration_seconds": duration,
+ "reasoning_provider": (report.get("provider_runtime") or {}).get("reasoning_provider"),
+ "cluster_count": len(report.get("clusters") or []),
+ "candidate_count": len(report.get("ranked_candidates") or []),
+ "error_sources": sorted((report.get("errors_by_source") or {}).keys()),
+ }
+ )
+ return rows
+
+
+def verify_latency() -> dict[str, dict[str, object]]:
+ results: dict[str, dict[str, object]] = {}
+ for profile, extra in LATENCY_PROFILES:
+ timings = []
+ for topic in LATENCY_TOPICS:
+ start = time.time()
+ run_command(
+ [PYTHON, "scripts/last30days.py", topic, "--emit=json", *extra],
+ timeout=300,
+ )
+ timings.append(time.time() - start)
+ results[profile] = {
+ "times": [round(value, 2) for value in timings],
+ "median_seconds": round(statistics.median(timings), 2),
+ "max_seconds": round(max(timings), 2),
+ }
+ return results
+
+
+def verify_eval(
+ *,
+ baseline: str,
+ candidate: str,
+ output_dir: str,
+ quick: bool,
+ limit: int,
+ timeout: int,
+) -> dict[str, object]:
+ cmd = [
+ PYTHON,
+ "scripts/evaluate_search_quality.py",
+ f"--baseline={baseline}",
+ f"--candidate={candidate}",
+ f"--output-dir={output_dir}",
+ f"--limit={limit}",
+ f"--timeout={timeout}",
+ ]
+ if quick:
+ cmd.append("--quick")
+ run_command(cmd, timeout=max(timeout * 8, 600))
+ output = Path(output_dir)
+ metrics = json.loads((output / "metrics.json").read_text())
+ summary = (output / "summary.md").read_text()
+ return {"metrics": metrics, "summary": summary}
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Run the v3 verification bundle")
+ parser.add_argument("--skip-eval", action="store_true", help="Skip the judged evaluator")
+ parser.add_argument("--skip-latency", action="store_true", help="Skip live latency sampling")
+ parser.add_argument("--baseline", default="HEAD~1")
+ parser.add_argument("--candidate", default="WORKTREE")
+ parser.add_argument("--output-dir", default="/tmp/last30days-v3-verify")
+ parser.add_argument("--quick-eval", action="store_true", help="Use evaluator quick mode")
+ parser.add_argument("--eval-limit", type=int, default=20)
+ parser.add_argument("--eval-timeout", type=int, default=240)
+ return parser
+
+
+def main() -> int:
+ args = build_parser().parse_args()
+ summary: dict[str, object] = {}
+ summary["unit"] = verify_unit()
+ summary["diagnose"] = verify_diagnose()
+ summary["smoke"] = verify_smoke()
+ if not args.skip_latency:
+ summary["latency"] = verify_latency()
+ if not args.skip_eval:
+ summary["eval"] = verify_eval(
+ baseline=args.baseline,
+ candidate=args.candidate,
+ output_dir=args.output_dir,
+ quick=args.quick_eval,
+ limit=args.eval_limit,
+ timeout=args.eval_timeout,
+ )
+ print(json.dumps(summary, indent=2, sort_keys=True))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/watchlist.py b/scripts/watchlist.py
index b354e99..a768e1f 100644
--- a/scripts/watchlist.py
+++ b/scripts/watchlist.py
@@ -1,62 +1,114 @@
#!/usr/bin/env python3
-"""Topic watchlist management for last30days.
+"""Topic watchlist management for last30days."""
-CLI for adding, removing, and listing watched topics with auto-bootstrap.
-On first `add`, creates the SQLite database.
-
-Usage:
- python3 watchlist.py add "AI video tools" [--schedule "0 8 * * *"]
- python3 watchlist.py add "NVIDIA news" --weekly
- python3 watchlist.py remove "AI video tools"
- python3 watchlist.py list
- python3 watchlist.py run-all
- python3 watchlist.py run-one "AI video tools"
- python3 watchlist.py config delivery telegram
- python3 watchlist.py config budget 10.00
-"""
+from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
-from datetime import datetime
from pathlib import Path
+try:
+ import requests
+except ImportError:
+ requests = None
+
SCRIPT_DIR = Path(__file__).parent.resolve()
sys.path.insert(0, str(SCRIPT_DIR))
import store
+from lib import schema
+# --- Webhook Delivery Functions ---
+
+def _deliver_findings(topic_name: str, counts: dict) -> None:
+ """Send webhook notification if delivery is configured and there are new findings."""
+ channel = store.get_setting("delivery_channel", "")
+ if not channel or counts.get("new", 0) == 0:
+ return
+
+ mode = store.get_setting("delivery_mode", "announce")
+ message = _format_delivery_message(topic_name, counts, mode)
+
+ try:
+ if "hooks.slack.com" in channel:
+ _send_slack_webhook(channel, message)
+ elif channel.startswith("https://"):
+ _send_generic_webhook(channel, message)
+ except Exception as e:
+ # Don't fail the research run if delivery fails
+ print(f"Delivery failed: {e}", file=sys.stderr)
+
+
+def _format_delivery_message(topic: str, counts: dict, mode: str) -> str:
+ """Format notification message based on delivery mode."""
+ new = counts.get("new", 0)
+ updated = counts.get("updated", 0)
+
+ if mode == "announce":
+ return f"📰 *last30days update: {topic}*\n{new} new, {updated} updated"
+ elif mode == "silent":
+ return f"last30days: {new} new findings for '{topic}'"
+ else:
+ return f"last30days: Research complete for '{topic}'"
+
+
+def _send_slack_webhook(url: str, text: str) -> None:
+ """POST to Slack incoming webhook."""
+ if not requests:
+ raise RuntimeError("requests library not available for webhook delivery")
+
+ response = requests.post(
+ url,
+ json={"text": text},
+ headers={"Content-Type": "application/json"},
+ timeout=10,
+ )
+ response.raise_for_status()
+
+
+def _send_generic_webhook(url: str, text: str) -> None:
+ """POST JSON payload to generic webhook."""
+ if not requests:
+ raise RuntimeError("requests library not available for webhook delivery")
+
+ response = requests.post(
+ url,
+ json={
+ "message": text,
+ "source": "last30days",
+ "timestamp": time.time(),
+ },
+ headers={"Content-Type": "application/json"},
+ timeout=10,
+ )
+ response.raise_for_status()
+
+
+# --- Command Handlers ---
+
def cmd_add(args):
- """Add a topic to the watchlist."""
schedule = "0 8 * * 1" if args.weekly else (args.schedule or "0 8 * * *")
- queries = args.queries.split(",") if args.queries else None
-
+ queries = [query.strip() for query in (args.queries or "").split(",") if query.strip()] or None
topic = store.add_topic(args.topic, search_queries=queries, schedule=schedule)
-
sched_desc = "weekly (Mondays 8am)" if args.weekly else f"daily ({schedule})"
- result = {
+ print(json.dumps({
"action": "added",
"topic": topic["name"],
"schedule": sched_desc,
"message": f'Added "{topic["name"]}" to watchlist. Schedule: {sched_desc}.',
- }
-
- print(json.dumps(result, default=str))
+ }, default=str))
def cmd_remove(args):
- """Remove a topic from the watchlist."""
removed = store.remove_topic(args.topic)
-
if not removed:
print(json.dumps({"action": "not_found", "topic": args.topic, "message": f'Topic not found: "{args.topic}"'}))
return
-
remaining = store.list_topics()
-
print(json.dumps({
"action": "removed",
"topic": args.topic,
@@ -66,55 +118,43 @@ def cmd_remove(args):
def cmd_list(args):
- """List all watched topics with stats."""
+ del args
topics = store.list_topics()
budget_used = store.get_daily_cost()
- budget_limit = store.get_setting("daily_budget", "5.00")
-
- result = {
+ budget_limit = float(store.get_setting("daily_budget", "5.00"))
+ print(json.dumps({
"topics": topics,
"budget_used": budget_used,
- "budget_limit": float(budget_limit),
- }
- print(json.dumps(result, default=str))
+ "budget_limit": budget_limit,
+ }, default=str))
def cmd_run_one(args):
- """Run research for a single topic."""
topic = store.get_topic(args.topic)
if not topic:
print(json.dumps({"error": f'Topic not found: "{args.topic}"'}))
sys.exit(1)
-
- result = _run_topic(topic)
- print(json.dumps(result, default=str))
+ print(json.dumps(_run_topic(topic), default=str))
def cmd_run_all(args):
- """Run research for all enabled topics with budget guard."""
- topics = store.list_topics()
- enabled = [t for t in topics if t["enabled"]]
-
- if not enabled:
+ del args
+ topics = [topic for topic in store.list_topics() if topic["enabled"]]
+ if not topics:
print(json.dumps({"message": "No enabled topics to research."}))
return
budget_limit = float(store.get_setting("daily_budget", "5.00"))
results = []
-
- for topic in enabled:
- # Budget guard
- daily_cost = store.get_daily_cost()
- if daily_cost >= budget_limit:
+ for topic in topics:
+ if store.get_daily_cost() >= budget_limit:
results.append({
"topic": topic["name"],
"status": "skipped",
- "reason": f"Budget exceeded: ${daily_cost:.2f}/${budget_limit:.2f}",
+ "reason": f"Budget exceeded: ${store.get_daily_cost():.2f}/${budget_limit:.2f}",
})
continue
-
- result = _run_topic(topic)
- results.append(result)
+ results.append(_run_topic(topic))
print(json.dumps({
"action": "run_all",
@@ -125,32 +165,28 @@ def cmd_run_all(args):
def _run_topic(topic: dict) -> dict:
- """Run research for a single topic and store findings."""
start_time = time.time()
topic_id = topic["id"]
-
- # Record the run
- run_id = store.record_run(topic_id, source_mode="both", status="running")
+ run_id = store.record_run(topic_id, source_mode="v3", status="running")
try:
- # Prefer custom search_queries over topic name (#40)
search_queries = json.loads(topic["search_queries"]) if topic.get("search_queries") else None
search_term = search_queries[0] if search_queries else topic["name"]
- cmd = [
- sys.executable,
- str(SCRIPT_DIR / "last30days.py"),
- search_term,
- "--emit=json",
- ]
result = subprocess.run(
- cmd,
+ [
+ sys.executable,
+ str(SCRIPT_DIR / "last30days.py"),
+ search_term,
+ "--emit=json",
+ "--quick",
+ "--lookback-days",
+ "90",
+ ],
capture_output=True,
text=True,
timeout=300,
)
-
duration = time.time() - start_time
-
if result.returncode != 0:
store.update_run(
run_id,
@@ -165,66 +201,9 @@ def _run_topic(topic: dict) -> dict:
"duration": duration,
}
- # Parse research output
- data = json.loads(result.stdout)
-
- # Convert research items to findings format
- findings = []
- for item in data.get("reddit", []):
- findings.append({
- "source": "reddit",
- "url": item.get("url", ""),
- "title": item.get("title", ""),
- "author": item.get("author", ""),
- "content": item.get("title", ""),
- "summary": item.get("top_comments_summary", ""),
- "engagement_score": item.get("upvotes", 0),
- "relevance_score": item.get("relevance", 0),
- })
- for item in data.get("x", []):
- findings.append({
- "source": "x",
- "url": item.get("url", ""),
- "title": item.get("text", "")[:100],
- "author": item.get("author_handle", ""),
- "content": item.get("text", ""),
- "engagement_score": (item.get("engagement") or {}).get("likes", 0),
- "relevance_score": item.get("relevance", 0),
- })
- for item in data.get("youtube", []):
- findings.append({
- "source": "youtube",
- "url": item.get("url", ""),
- "title": item.get("title", ""),
- "author": item.get("channel_name", item.get("channel", "")),
- "content": item.get("transcript_snippet", "") or item.get("title", ""),
- "engagement_score": (item.get("engagement") or {}).get("views", 0),
- "relevance_score": item.get("relevance", 0),
- })
- for item in data.get("tiktok", []):
- findings.append({
- "source": "tiktok",
- "url": item.get("url", ""),
- "title": (item.get("caption_snippet", "") or "")[:120],
- "author": item.get("author", ""),
- "content": item.get("caption_snippet", ""),
- "engagement_score": (item.get("engagement") or {}).get("views", 0),
- "relevance_score": item.get("relevance", 0),
- })
- for item in data.get("instagram", []):
- findings.append({
- "source": "instagram",
- "url": item.get("url", ""),
- "title": (item.get("caption_snippet", "") or "")[:120],
- "author": item.get("author_name", ""),
- "content": item.get("caption_snippet", ""),
- "engagement_score": (item.get("engagement") or {}).get("views", 0),
- "relevance_score": item.get("relevance", 0),
- })
-
- # Store with dedup
+ report = schema.report_from_dict(json.loads(result.stdout))
+ findings = store.findings_from_report(report, limit=25)
counts = store.store_findings(run_id, topic_id, findings)
-
store.update_run(
run_id,
status="completed",
@@ -232,7 +211,10 @@ def _run_topic(topic: dict) -> dict:
findings_new=counts["new"],
findings_updated=counts["updated"],
)
-
+
+ # Deliver webhook notification if configured
+ _deliver_findings(topic["name"], counts)
+
return {
"topic": topic["name"],
"status": "completed",
@@ -240,90 +222,78 @@ def _run_topic(topic: dict) -> dict:
"updated": counts["updated"],
"duration": duration,
}
-
except subprocess.TimeoutExpired:
duration = time.time() - start_time
store.update_run(
- run_id, status="failed",
+ run_id,
+ status="failed",
error_message="Research timed out after 300s",
duration_seconds=duration,
)
return {"topic": topic["name"], "status": "failed", "error": "timeout"}
-
- except json.JSONDecodeError as e:
+ except json.JSONDecodeError as exc:
duration = time.time() - start_time
store.update_run(
- run_id, status="failed",
- error_message=f"Invalid JSON output: {e}",
+ run_id,
+ status="failed",
+ error_message=f"Invalid JSON output: {exc}",
duration_seconds=duration,
)
- return {"topic": topic["name"], "status": "failed", "error": f"parse error: {e}"}
-
- except Exception as e:
- duration = time.time() - start_time
- store.update_run(
- run_id, status="failed",
- error_message=str(e)[:500],
- duration_seconds=duration,
- )
- return {"topic": topic["name"], "status": "failed", "error": str(e)}
-
-
+ return {"topic": topic["name"], "status": "failed", "error": f"parse error: {exc}"}
def cmd_config(args):
- """Configure watchlist settings."""
- if args.setting == "delivery":
- store.set_setting("delivery_channel", args.value)
- print(json.dumps({"action": "config", "setting": "delivery_channel", "value": args.value}))
- elif args.setting == "budget":
- store.set_setting("daily_budget", args.value)
- print(json.dumps({"action": "config", "setting": "daily_budget", "value": args.value}))
- else:
- print(json.dumps({"error": f"Unknown setting: {args.setting}. Use 'delivery' or 'budget'."}))
+ if args.key == "budget":
+ store.set_setting("daily_budget", str(args.value))
+ print(json.dumps({"action": "config", "key": "daily_budget", "value": str(args.value)}))
+ return
+ if args.key == "delivery":
+ store.set_setting("delivery_channel", str(args.value))
+ print(json.dumps({"action": "config", "key": "delivery_channel", "value": str(args.value)}))
+ return
+ raise SystemExit(f"Unknown config key: {args.key}")
-def main():
- parser = argparse.ArgumentParser(description="Manage last30days topic watchlist")
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Manage the last30days watchlist")
sub = parser.add_subparsers(dest="command")
- # add
- a = sub.add_parser("add", help="Add a topic to the watchlist")
- a.add_argument("topic", help="Topic name")
- a.add_argument("--schedule", help="Cron expression (default: 0 8 * * *)")
- a.add_argument("--weekly", action="store_true", help="Run weekly instead of daily")
- a.add_argument("--queries", help="Comma-separated custom search queries")
- a.set_defaults(func=cmd_add)
+ add = sub.add_parser("add")
+ add.add_argument("topic")
+ add.add_argument("--schedule")
+ add.add_argument("--weekly", action="store_true")
+ add.add_argument("--queries")
+ add.set_defaults(func=cmd_add)
- # remove
- r = sub.add_parser("remove", help="Remove a topic from the watchlist")
- r.add_argument("topic", help="Topic name")
- r.set_defaults(func=cmd_remove)
+ remove = sub.add_parser("remove")
+ remove.add_argument("topic")
+ remove.set_defaults(func=cmd_remove)
- # list
- l = sub.add_parser("list", help="List all watched topics")
- l.set_defaults(func=cmd_list)
+ list_parser = sub.add_parser("list")
+ list_parser.set_defaults(func=cmd_list)
- # run-all
- ra = sub.add_parser("run-all", help="Run research for all enabled topics")
- ra.set_defaults(func=cmd_run_all)
+ run_one = sub.add_parser("run-one")
+ run_one.add_argument("topic")
+ run_one.set_defaults(func=cmd_run_one)
- # run-one
- ro = sub.add_parser("run-one", help="Run research for a single topic")
- ro.add_argument("topic", help="Topic name")
- ro.set_defaults(func=cmd_run_one)
+ run_all = sub.add_parser("run-all")
+ run_all.set_defaults(func=cmd_run_all)
- # config
- c = sub.add_parser("config", help="Configure watchlist settings")
- c.add_argument("setting", help="Setting name (delivery, budget)")
- c.add_argument("value", help="Setting value")
- c.set_defaults(func=cmd_config)
+ config = sub.add_parser("config")
+ config.add_argument("key", choices=["delivery", "budget"])
+ config.add_argument("value")
+ config.set_defaults(func=cmd_config)
+ return parser
+
+
+def main() -> int:
+ parser = build_parser()
args = parser.parse_args()
- if not args.command:
+ if not getattr(args, "command", None):
parser.print_help()
- sys.exit(1)
-
+ return 1
args.func(args)
+ return 0
if __name__ == "__main__":
- main()
+ raise SystemExit(main())
diff --git a/skills/last30days/SKILL.md b/skills/last30days-nux/SKILL.md
similarity index 100%
rename from skills/last30days/SKILL.md
rename to skills/last30days-nux/SKILL.md
diff --git a/skills/last30days-v3/SKILL.md b/skills/last30days-v3/SKILL.md
new file mode 100644
index 0000000..f0f4667
--- /dev/null
+++ b/skills/last30days-v3/SKILL.md
@@ -0,0 +1,219 @@
+---
+name: last30days-3
+version: "3.0.0-alpha"
+description: "[V3 DEV] Research any topic from the last 30 days. Side-by-side test version of the v3 pipeline."
+argument-hint: "last30days-v3 codex vs claude code"
+allowed-tools: Bash, Read, Write, WebSearch
+homepage: https://github.com/mvanhorn/last30days-skill
+repository: https://github.com/mvanhorn/last30days-skill
+author: mvanhorn
+license: MIT
+user-invocable: true
+---
+
+# last30days v3.0.0
+
+Use `last30days` when the user wants recent, cross-source evidence from the last 30 days.
+
+The runtime is a single v3 pipeline:
+
+1. plan the query
+2. retrieve per `(subquery, source)`
+3. normalize and dedupe
+4. extract best snippets
+5. fuse with weighted RRF
+6. rerank with one relevance score
+7. cluster evidence
+8. render ranked clusters
+
+## Setup: resolve the skill root
+
+```bash
+for dir in \
+ "." \
+ "${CLAUDE_PLUGIN_ROOT:-}" \
+ "${GEMINI_EXTENSION_DIR:-}" \
+ "$HOME/.openclaw/workspace/skills/last30days" \
+ "$HOME/.openclaw/skills/last30days" \
+ "$HOME/.claude/skills/last30days" \
+ "$HOME/.agents/skills/last30days" \
+ "$HOME/.codex/skills/last30days"; do
+ [ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
+done
+
+if [ -z "${SKILL_ROOT:-}" ]; then
+ echo "ERROR: Could not find scripts/last30days.py" >&2
+ exit 1
+fi
+```
+
+## Default command
+
+```bash
+python3 "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=compact
+```
+
+## Useful commands
+
+```bash
+python3 "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --emit=json
+python3 "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --quick
+python3 "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --deep
+python3 "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --search=reddit,x,grounding
+python3 "${SKILL_ROOT}/scripts/last30days.py" $ARGUMENTS --store
+python3 "${SKILL_ROOT}/scripts/last30days.py" --diagnose
+```
+
+## Runtime expectations
+
+- One reasoning provider is required: `GOOGLE_API_KEY` for Gemini, `OPENAI_API_KEY` for OpenAI, or `XAI_API_KEY` for xAI.
+- `BRAVE_API_KEY` enables Brave web search (recommended). `SERPER_API_KEY` is the web fallback.
+- `SCRAPECREATORS_API_KEY` enables Reddit, TikTok, and Instagram.
+- `XAI_API_KEY` enables xAI reasoning and X search.
+- `AUTH_TOKEN` plus `CT0` enables Bird-backed X search.
+- `yt-dlp` enables YouTube.
+- Planning and reranking fall back gracefully: Gemini -> OpenAI -> xAI -> deterministic/local.
+- Web retrieval stays within Brave/Serper dated results. Undated web hits are dropped.
+- For OpenClaw-specific watchlist, briefing, and history workflows, use `variants/open/SKILL.md`.
+
+## Output model
+
+- `compact` and `md`: cluster-first markdown
+- `json`: full v3 report
+- `context`: short synthesis-oriented context
+
+Important report fields:
+
+- `provider_runtime`
+- `query_plan`
+- `ranked_candidates`
+- `clusters`
+- `items_by_source`
+- `errors_by_source`
+
+## Usage guidance for agents
+
+- Prefer `--quick` for fast iteration.
+- Prefer default mode when the user wants a balanced answer.
+- Prefer `--deep` only when the user explicitly wants maximum recall or the topic is complex enough to justify extra latency.
+- Prefer `--emit=json` when downstream code or evaluation will consume the result.
+- Use `--search=` only when the user explicitly wants source restrictions.
+
+## X handle resolution
+
+If the topic could have its own X/Twitter account (people, brands, products, companies), do a quick WebSearch for their handle:
+```
+WebSearch("{TOPIC} X twitter handle site:x.com")
+```
+If you find a verified handle, pass `--x-handle={handle}` (without @). This searches their posts directly, finding content they posted that doesn't mention their own name. Skip this for generic concepts ("best headphones 2026", "how to use Docker").
+
+## Synthesis guidance
+
+### First: synthesize, don't summarize
+
+Extract key facts from the output first, then synthesize across sources. Lead with patterns that appear across multiple clusters. Present a unified narrative, not a source-by-source summary.
+
+### Ground in actual research, not pre-existing knowledge
+
+Use exact product/tool names, specific quotes, and what sources actually say. If research mentions "ClawdBot" and "@clawdbot", that is a different product than "Claude Code" -- read what the research actually says.
+
+**Anti-pattern to avoid:**
+- BAD: User asks "best Claude Code skills" and you respond with generic advice: "Skills are powerful. Keep them under 500 lines."
+- GOOD: You respond with specifics from the research: "Most mentioned: /commit (5 mentions), remotion skill (4x), git-worktree (3x). The Remotion announcement got 16K likes on X per @thedorbrothers."
+
+### Source weighting (highest to lowest signal)
+
+1. **Cross-cluster corroboration** -- same evidence across multiple sources is the strongest signal. Lead with it.
+2. **Reddit top comments** -- often the wittiest, most insightful take. Quote directly when upvotes are high.
+3. **YouTube transcript highlights** -- pre-extracted key moments. Quote and attribute to channel name.
+4. **X/Twitter @handles** -- real-time community signal. Quote with engagement context.
+5. **Polymarket odds** -- real money on outcomes cuts through opinion. Include specific odds AND movement.
+6. **TikTok/Instagram** -- viral/creator signal. Cite @creators with views/likes.
+7. **Hacker News** -- technical community perspective. Cite as "per HN."
+8. **Web (Brave/Serper)** -- cite only when social sources don't cover a fact.
+
+### Polymarket interpretation
+
+When Polymarket returns relevant markets:
+1. Prefer structural/long-term markets over near-term deadlines (championship odds > regular season, IPO > incremental update)
+2. Call out the specific outcome's odds and movement, not just that a market exists
+3. Weave odds into the narrative as supporting evidence, don't isolate them
+4. When multiple relevant markets exist, highlight 3-5 ordered by importance
+
+Domain importance ranking:
+- **Sports:** Championship/tournament > conference title > regular season > weekly matchup
+- **Geopolitics:** Regime change/structural > near-term strike deadlines > sanctions
+- **Tech/Business:** IPO, major product launch > incremental updates
+- **Elections:** Presidency > primary > individual state
+
+### Citation rules
+
+Cite the single strongest source per point in short format: "per @handle" or "per r/subreddit". Save engagement metrics for the stats section. Use the priority order from source weighting above. The tool's value is surfacing what PEOPLE are saying, not what journalists wrote.
+
+### Comparison queries
+
+For "X vs Y" queries, structure output as:
+
+```
+## Quick Verdict
+[1-2 sentences: which one the community prefers and why, with source counts]
+
+## [Entity A]
+**Community Sentiment:** [Positive/Mixed/Negative] (N mentions across sources)
+**Strengths:** [with source attribution]
+**Weaknesses:** [with source attribution]
+
+## [Entity B]
+[Same structure]
+
+## Head-to-Head
+| Dimension | Entity A | Entity B |
+|-----------|----------|----------|
+| [Key dim] | [position] | [position] |
+
+## Bottom Line
+Choose A if... Choose B if... (based on community data)
+```
+
+### Recommendation queries
+
+When users ask "best X" or "top X", extract SPECIFIC NAMES:
+
+```
+Most mentioned:
+[Name] -- Nx mentions
+ Sources: @handle1, r/subreddit, [YouTube channel]
+
+[Name] -- Nx mentions
+ Sources: @handle2, r/subreddit2
+
+Notable mentions: [others with 1-2 mentions]
+```
+
+### Edge cases
+
+- **Empty results from a source:** State what is missing. ("No Reddit discussion found for this topic.") Do not fill the gap with training data.
+- **Sources contradict each other:** Present both sides with attribution. ("Reddit r/fitness is bullish on X, while @DrExpert on X warns about Y.")
+- **All results are low-engagement or off-topic:** Acknowledge uncertainty. ("Limited recent discussion found -- these findings should be treated as preliminary.")
+
+### Follow-up conversations
+
+After research completes, treat yourself as an expert on this topic. Answer follow-ups from the research findings. Cite the specific threads, posts, and channels you found. Only run new research if the user asks about a DIFFERENT topic.
+
+## Security and permissions
+
+**What this skill does:**
+- Sends search queries to ScrapeCreators API for Reddit, TikTok, Instagram search
+- Sends search queries via xAI API or Bird client for X search
+- Sends search queries to Algolia HN Search API (free, no auth)
+- Sends search queries to Polymarket Gamma API (free, no auth)
+- Runs yt-dlp locally for YouTube search and transcript extraction (no API key)
+- Sends search queries to Brave Search API or Serper for web search (optional)
+- Uses Gemini, OpenAI, or xAI for LLM planning and reranking
+- Stores findings in local SQLite database (--store mode only)
+
+**What this skill does NOT do:**
+- Does not post, like, or modify content on any platform
+- Does not access your personal accounts on any platform
+- Does not share API keys between providers
+- Does not log or cache API keys in output files
diff --git a/tests/e2e_comparison.py b/tests/e2e_comparison.py
new file mode 100644
index 0000000..00dd513
--- /dev/null
+++ b/tests/e2e_comparison.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""E2E comparison: run sample queries on both v3 (current branch) and v2.9.5 (main).
+
+Usage:
+ python3 tests/e2e_comparison.py [--v2-script PATH]
+
+Outputs a markdown comparison table with per-query metrics.
+"""
+
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+V3_SCRIPT = str(REPO / "scripts" / "last30days.py")
+
+# v2.9.5 from plugin cache (main branch equivalent)
+V2_SCRIPT = str(
+ Path.home()
+ / ".claude/plugins/cache/last30days/last30days/2.9.5/scripts/last30days.py"
+)
+
+EVAL_TOPICS_FILE = REPO / "fixtures" / "eval_topics.json"
+
+
+def _load_queries() -> list[tuple[str, str]]:
+ if EVAL_TOPICS_FILE.exists():
+ rows = json.loads(EVAL_TOPICS_FILE.read_text())
+ return [(row["topic"], row["query_type"]) for row in rows]
+ return [
+ ("openclaw vs nanoclaw vs ironclaw", "comparison"),
+ ("how to deploy on Fly.io", "how_to"),
+ ("kanye west", "breaking_news"),
+ ("odds of recession", "prediction"),
+ ("explain transformer architecture", "concept"),
+ ]
+
+
+QUERIES = _load_queries()
+
+
+def run_query(script: str, topic: str, timeout: int = 180) -> dict:
+ """Run a query and return parsed JSON + timing."""
+ start = time.time()
+ try:
+ result = subprocess.run(
+ [sys.executable, script, topic, "--emit=json"],
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ )
+ elapsed = time.time() - start
+ if result.returncode != 0:
+ return {
+ "error": result.stderr[:200],
+ "elapsed": elapsed,
+ "sources": 0,
+ "candidates": 0,
+ "intent": "error",
+ "subqueries": 0,
+ }
+ data = json.loads(result.stdout)
+
+ # v3 shape
+ if "query_plan" in data:
+ items_by_source = data.get("items_by_source", {})
+ return {
+ "elapsed": elapsed,
+ "sources": sum(1 for v in items_by_source.values() if v),
+ "total_items": sum(len(v) for v in items_by_source.values()),
+ "candidates": len(data.get("ranked_candidates", [])),
+ "clusters": len(data.get("clusters", [])),
+ "intent": data["query_plan"].get("intent", "?"),
+ "subqueries": len(data["query_plan"].get("subqueries", [])),
+ "errors": list(data.get("errors_by_source", {}).keys()),
+ }
+
+ # v2 shape
+ sources_with_items = 0
+ total_items = 0
+ for key in ["reddit", "x", "youtube", "tiktok", "instagram", "hackernews",
+ "bluesky", "truthsocial", "polymarket", "web"]:
+ items = data.get(key, [])
+ if items:
+ sources_with_items += 1
+ total_items += len(items)
+ return {
+ "elapsed": elapsed,
+ "sources": sources_with_items,
+ "total_items": total_items,
+ "candidates": total_items,
+ "clusters": 0,
+ "intent": data.get("mode", "?"),
+ "subqueries": 0,
+ "errors": [k for k in ["reddit_error", "x_error", "youtube_error",
+ "tiktok_error", "instagram_error"]
+ if data.get(k)],
+ }
+ except subprocess.TimeoutExpired:
+ return {
+ "error": "timeout",
+ "elapsed": timeout,
+ "sources": 0,
+ "candidates": 0,
+ "intent": "timeout",
+ "subqueries": 0,
+ }
+ except Exception as exc:
+ return {
+ "error": str(exc)[:200],
+ "elapsed": time.time() - start,
+ "sources": 0,
+ "candidates": 0,
+ "intent": "error",
+ "subqueries": 0,
+ }
+
+
+def main():
+ v2_script = V2_SCRIPT
+ if len(sys.argv) > 2 and sys.argv[1] == "--v2-script":
+ v2_script = sys.argv[2]
+
+ if not Path(v2_script).exists():
+ print(f"v2 script not found at {v2_script}", file=sys.stderr)
+ print("Use --v2-script PATH to specify", file=sys.stderr)
+ sys.exit(1)
+
+ print("# E2E Comparison: v3.0.0 (branch) vs v2.9.5 (main)")
+ print()
+ print(f"- v3 script: {V3_SCRIPT}")
+ print(f"- v2 script: {v2_script}")
+ print(f"- Queries: {len(QUERIES)}")
+ print()
+
+ results = []
+ for i, (topic, expected_intent) in enumerate(QUERIES, 1):
+ print(f"[{i}/{len(QUERIES)}] {topic}", file=sys.stderr)
+ sys.stderr.flush()
+
+ print(f" v3...", end="", file=sys.stderr)
+ sys.stderr.flush()
+ v3 = run_query(V3_SCRIPT, topic)
+ print(f" {v3.get('elapsed', 0):.1f}s", file=sys.stderr)
+ sys.stderr.flush()
+
+ print(f" v2...", end="", file=sys.stderr)
+ sys.stderr.flush()
+ v2 = run_query(v2_script, topic)
+ print(f" {v2.get('elapsed', 0):.1f}s", file=sys.stderr)
+ sys.stderr.flush()
+
+ results.append({
+ "topic": topic,
+ "expected_intent": expected_intent,
+ "v3": v3,
+ "v2": v2,
+ })
+
+ # Print comparison table
+ print("| Query | Intent | v3 sources | v2 sources | v3 items | v2 items | v3 time | v2 time | v3 errors | v2 errors |")
+ print("|-------|--------|-----------|-----------|---------|---------|---------|---------|-----------|-----------|")
+ for r in results:
+ v3, v2 = r["v3"], r["v2"]
+ v3_err = ", ".join(v3.get("errors", [])) or "-"
+ v2_err = ", ".join(v2.get("errors", [])) or "-"
+ print(
+ f"| {r['topic'][:45]} | {v3.get('intent', '?')} | "
+ f"{v3.get('sources', 0)} | {v2.get('sources', 0)} | "
+ f"{v3.get('total_items', 0)} | {v2.get('total_items', 0)} | "
+ f"{v3.get('elapsed', 0):.1f}s | {v2.get('elapsed', 0):.1f}s | "
+ f"{v3_err} | {v2_err} |"
+ )
+
+ # Summary
+ print()
+ v3_total_sources = sum(r["v3"].get("sources", 0) for r in results)
+ v2_total_sources = sum(r["v2"].get("sources", 0) for r in results)
+ v3_total_items = sum(r["v3"].get("total_items", 0) for r in results)
+ v2_total_items = sum(r["v2"].get("total_items", 0) for r in results)
+ v3_total_time = sum(r["v3"].get("elapsed", 0) for r in results)
+ v2_total_time = sum(r["v2"].get("elapsed", 0) for r in results)
+ v3_errors = sum(len(r["v3"].get("errors", [])) for r in results)
+ v2_errors = sum(len(r["v2"].get("errors", [])) for r in results)
+
+ print("## Summary")
+ print()
+ print(f"| Metric | v3.0.0 | v2.9.5 | Delta |")
+ print(f"|--------|--------|--------|-------|")
+ print(f"| Total sources with items | {v3_total_sources} | {v2_total_sources} | {v3_total_sources - v2_total_sources:+d} |")
+ print(f"| Total items retrieved | {v3_total_items} | {v2_total_items} | {v3_total_items - v2_total_items:+d} |")
+ print(f"| Total wall time | {v3_total_time:.1f}s | {v2_total_time:.1f}s | {v3_total_time - v2_total_time:+.1f}s |")
+ print(f"| Source errors | {v3_errors} | {v2_errors} | {v3_errors - v2_errors:+d} |")
+ print(f"| Avg sources/query | {v3_total_sources/len(results):.1f} | {v2_total_sources/len(results):.1f} | |")
+ print(f"| Avg items/query | {v3_total_items/len(results):.1f} | {v2_total_items/len(results):.1f} | |")
+ print(f"| Avg time/query | {v3_total_time/len(results):.1f}s | {v2_total_time/len(results):.1f}s | |")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_adversarial_v3.py b/tests/test_adversarial_v3.py
new file mode 100644
index 0000000..1d5a9c1
--- /dev/null
+++ b/tests/test_adversarial_v3.py
@@ -0,0 +1,198 @@
+"""Adversarial query tests for the v3 planner.
+
+These target edge cases found during regression analysis: slash-separated
+comparisons, 'difference between X and Y' phrasing, trailing context
+leaking into entities, degenerate inputs, and false-positive resistance.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import planner
+
+
+class TestSlashSeparatedComparison(unittest.TestCase):
+ """'React/Vue/Svelte' should be detected as comparison intent
+ and produce entity subqueries."""
+
+ def test_slash_triggers_comparison_intent(self):
+ self.assertEqual(planner._infer_intent("React/Vue/Svelte"), "comparison")
+
+ def test_slash_extracts_entities(self):
+ entities = planner._comparison_entities("React/Vue/Svelte")
+ self.assertEqual(len(entities), 3)
+ self.assertIn("React", entities)
+ self.assertIn("Vue", entities)
+ self.assertIn("Svelte", entities)
+
+ def test_slash_forces_deterministic(self):
+ self.assertTrue(planner._should_force_deterministic_plan("React/Vue"))
+
+ def test_url_slash_does_not_trigger_comparison(self):
+ self.assertNotEqual(planner._infer_intent("https://example.com/path"), "comparison")
+
+ def test_slash_trailing_context_stripped(self):
+ entities = planner._comparison_entities("React/Vue/Svelte for frontend in 2026")
+ for entity in entities:
+ self.assertNotIn("frontend", entity.lower(),
+ f"Trailing context leaked: '{entity}'")
+
+
+class TestDifferenceBetweenPhrasing(unittest.TestCase):
+ """'difference between X and Y' should extract both entities."""
+
+ def test_intent_is_comparison(self):
+ self.assertEqual(
+ planner._infer_intent("difference between OpenClaw and NanoClaw"),
+ "comparison",
+ )
+
+ def test_entities_extracted(self):
+ entities = planner._comparison_entities("difference between OpenClaw and NanoClaw")
+ self.assertEqual(len(entities), 2)
+ self.assertIn("OpenClaw", entities)
+ self.assertIn("NanoClaw", entities)
+
+ def test_forces_deterministic(self):
+ self.assertTrue(
+ planner._should_force_deterministic_plan("difference between OpenClaw and NanoClaw")
+ )
+
+
+class TestAndFalsePositive(unittest.TestCase):
+ """'and' must not split entities outside 'difference between' context."""
+
+ def test_pros_and_cons_no_entities(self):
+ entities = planner._comparison_entities("pros and cons of AI")
+ self.assertEqual(entities, [])
+
+ def test_react_and_vue_no_entities(self):
+ # No "vs" or "difference between" -- just "and"
+ entities = planner._comparison_entities("React and Vue")
+ self.assertEqual(entities, [])
+
+
+class TestTrailingContextStripping(unittest.TestCase):
+ """Trailing preposition phrases must not leak into entity strings."""
+
+ def test_for_stripped(self):
+ entities = planner._comparison_entities("A vs B for production use")
+ self.assertNotIn("production", entities[-1].lower())
+
+ def test_in_stripped(self):
+ entities = planner._comparison_entities("A vs B in 2026")
+ self.assertNotIn("2026", entities[-1])
+
+ def test_with_stripped(self):
+ entities = planner._comparison_entities("A vs B with better security")
+ self.assertNotIn("security", entities[-1].lower())
+
+ def test_core_entity_preserved(self):
+ entities = planner._comparison_entities("Fly.io vs Railway.app for deployment")
+ self.assertTrue(any("Fly" in e for e in entities))
+ self.assertTrue(any("Railway" in e for e in entities))
+
+
+class TestDuplicateEntities(unittest.TestCase):
+
+ def test_deduped(self):
+ entities = planner._comparison_entities("OpenClaw vs OpenClaw")
+ self.assertEqual(len(entities), 1)
+
+
+class TestFiveWayComparison(unittest.TestCase):
+
+ def test_capped_at_max(self):
+ topic = "A vs B vs C vs D vs E vs F"
+ entities = planner._comparison_entities(topic)
+ self.assertLessEqual(len(entities), planner._max_subqueries("comparison"))
+
+ def test_does_not_crash(self):
+ plan = planner.plan_query(
+ topic="A vs B vs C vs D vs E",
+ available_sources=["reddit", "x", "grounding"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertLessEqual(len(plan.subqueries), 4)
+
+
+class TestDegenerateInputs(unittest.TestCase):
+
+ def test_single_word(self):
+ plan = planner.plan_query(
+ topic="Bitcoin",
+ available_sources=["reddit"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertGreater(len(plan.subqueries), 0)
+
+ def test_empty_vs_split(self):
+ plan = planner.plan_query(
+ topic="vs vs vs",
+ available_sources=["reddit"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ for sq in plan.subqueries:
+ self.assertTrue(sq.search_query.strip())
+
+ def test_very_long_comparison(self):
+ topic = " vs ".join(f"Tool{i}" for i in range(20))
+ plan = planner.plan_query(
+ topic=topic,
+ available_sources=["reddit", "x"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertLessEqual(len(plan.subqueries), 4)
+
+
+class TestMixedCaseAndPunctuation(unittest.TestCase):
+
+ def test_uppercase_vs_period(self):
+ self.assertEqual(
+ planner._infer_intent("OpenClaw VS. NanoClaw VS. IronClaw"),
+ "comparison",
+ )
+
+ def test_entities_preserved_with_mixed_case(self):
+ entities = planner._comparison_entities("OpenClaw VS. NanoClaw VS. IronClaw")
+ self.assertGreaterEqual(len(entities), 3)
+
+
+class TestSubstringEntity(unittest.TestCase):
+
+ def test_react_vs_react_native_not_collapsed(self):
+ entities = planner._comparison_entities("React vs React Native")
+ self.assertEqual(len(entities), 2)
+ self.assertTrue(any("Native" in e for e in entities))
+
+
+class TestNoiseWordEntities(unittest.TestCase):
+ """Entities that are also common English words (Swift, Rust, Go)
+ must survive entity extraction."""
+
+ def test_swift_preserved(self):
+ entities = planner._comparison_entities("Swift vs Rust vs Go")
+ self.assertGreaterEqual(len(entities), 3)
+
+ def test_go_not_stripped(self):
+ entities = planner._comparison_entities("Swift vs Rust vs Go")
+ self.assertTrue(any("Go" in e for e in entities))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_bird_x.py b/tests/test_bird_x.py
index 2e9e57a..b50c9b6 100644
--- a/tests/test_bird_x.py
+++ b/tests/test_bird_x.py
@@ -1,101 +1,64 @@
-"""Tests for bird_x module."""
-
import sys
import unittest
from pathlib import Path
-from unittest import mock
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
-from lib import bird_x
+from lib.bird_x import parse_bird_response
-class TestExtractCoreSubject(unittest.TestCase):
- def tearDown(self):
- bird_x._credentials.clear()
+class TestBirdXEngagementZero(unittest.TestCase):
+ def test_zero_likes_preserved(self):
+ tweets = [
+ {
+ "id": "1",
+ "text": "test",
+ "permanent_url": "https://x.com/u/status/1",
+ "likeCount": 0,
+ "retweetCount": 5,
+ }
+ ]
+ items = parse_bird_response(tweets, "test query")
+ self.assertEqual(0, items[0]["engagement"]["likes"])
+ self.assertEqual(5, items[0]["engagement"]["reposts"])
- def test_strips_trending_noise(self):
- result = bird_x._extract_core_subject("trendiest Claude Code skills")
- self.assertNotIn("trendiest", result)
- self.assertIn("claude", result.lower())
+ def test_none_likes_when_missing(self):
+ tweets = [
+ {
+ "id": "1",
+ "text": "test tweet with no engagement fields",
+ "permanent_url": "https://x.com/u/status/1",
+ # no likeCount, like_count, or favorite_count
+ }
+ ]
+ items = parse_bird_response(tweets, "test query")
+ self.assertIsNone(items[0]["engagement"]["likes"])
- def test_strips_tool_noise(self):
- result = bird_x._extract_core_subject("best AI tools for coding")
- self.assertNotIn("tools", result)
- self.assertNotIn("best", result)
+ def test_fallback_to_second_key(self):
+ tweets = [
+ {
+ "id": "1",
+ "text": "test",
+ "permanent_url": "https://x.com/u/status/1",
+ "like_count": 7,
+ }
+ ]
+ items = parse_bird_response(tweets, "test query")
+ self.assertEqual(7, items[0]["engagement"]["likes"])
- def test_strips_skill_noise(self):
- result = bird_x._extract_core_subject("top claude code skills")
- self.assertNotIn("skills", result)
- self.assertNotIn("top", result)
-
-
-class TestBirdSearchRetries(unittest.TestCase):
- def tearDown(self):
- bird_x._credentials.clear()
-
- def test_last_chance_retry_uses_strongest_token(self):
- """When shorter retry also returns 0, uses longest non-noise token."""
- empty = {"items": []}
- with mock.patch.object(bird_x, "_extract_core_subject", return_value="best codex skill plugin"), \
- mock.patch.object(bird_x, "parse_bird_response", return_value=[]), \
- mock.patch.object(bird_x, "_run_bird_search", return_value=empty) as run_mock:
- bird_x.search_x("best codex skill plugin", "2026-01-01", "2026-01-31", depth="quick")
-
- # Should try: original, shorter (2-word), last-chance (strongest token)
- self.assertEqual(run_mock.call_count, 3)
- queries = [call.args[0] for call in run_mock.call_args_list]
- # Last call should use "codex" (longest non-noise word)
- self.assertIn("codex", queries[2])
-
- def test_no_retry_when_first_query_has_results(self):
- """No retry when first query succeeds."""
- result = {"items": [{"id": "1"}]}
- with mock.patch.object(bird_x, "_extract_core_subject", return_value="nano banana"), \
- mock.patch.object(bird_x, "parse_bird_response", return_value=[{"id": "1"}]), \
- mock.patch.object(bird_x, "_run_bird_search", return_value=result) as run_mock:
- bird_x.search_x("nano banana prompting", "2026-01-01", "2026-01-31")
-
- self.assertEqual(run_mock.call_count, 1)
-
-
-class TestBirdAuthEnvironment(unittest.TestCase):
- def tearDown(self):
- bird_x._credentials.clear()
-
- def test_subprocess_env_disables_browser_cookie_fallback_when_injected(self):
- bird_x.set_credentials("auth-token", "ct0-token")
-
- env = bird_x._subprocess_env()
-
- self.assertEqual(env["AUTH_TOKEN"], "auth-token")
- self.assertEqual(env["CT0"], "ct0-token")
- self.assertEqual(env["BIRD_DISABLE_BROWSER_COOKIES"], "1")
-
- def test_is_bird_authenticated_short_circuits_when_credentials_injected(self):
- bird_x.set_credentials("auth-token", "ct0-token")
-
- with mock.patch.object(bird_x, "is_bird_installed", return_value=True), \
- mock.patch.object(bird_x.subprocess, "run") as run_mock:
- result = bird_x.is_bird_authenticated()
-
- self.assertEqual(result, "env AUTH_TOKEN")
- run_mock.assert_not_called()
-
- def test_search_handles_passes_injected_credentials_to_subprocess(self):
- bird_x.set_credentials("auth-token", "ct0-token")
-
- proc = mock.Mock()
- proc.communicate.return_value = ("[]", "")
- proc.returncode = 0
-
- with mock.patch.object(bird_x.subprocess, "Popen", return_value=proc) as popen_mock:
- bird_x.search_handles(["openai"], "codex vs claude code", "2026-01-01", count_per=1)
-
- env = popen_mock.call_args.kwargs["env"]
- self.assertEqual(env["AUTH_TOKEN"], "auth-token")
- self.assertEqual(env["CT0"], "ct0-token")
- self.assertEqual(env["BIRD_DISABLE_BROWSER_COOKIES"], "1")
+ def test_zero_does_not_fall_through(self):
+ """likeCount=0 should not fall through to like_count=10."""
+ tweets = [
+ {
+ "id": "1",
+ "text": "test",
+ "permanent_url": "https://x.com/u/status/1",
+ "likeCount": 0,
+ "like_count": 10,
+ }
+ ]
+ items = parse_bird_response(tweets, "test query")
+ self.assertEqual(0, items[0]["engagement"]["likes"])
if __name__ == "__main__":
diff --git a/tests/test_brave_search.py b/tests/test_brave_search.py
deleted file mode 100644
index e8d9255..0000000
--- a/tests/test_brave_search.py
+++ /dev/null
@@ -1,197 +0,0 @@
-"""Tests for Brave Search module, including LLM Context endpoint."""
-
-import sys
-import os
-import unittest
-
-# Ensure scripts/ is on path
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts'))
-
-from lib.brave_search import (
- _normalize_results,
- _normalize_llm_context,
- _days_between,
- _brave_freshness,
- _parse_brave_date,
- EXCLUDED_DOMAINS,
-)
-
-
-class TestDaysBetween(unittest.TestCase):
- def test_same_day(self):
- self.assertEqual(_days_between("2026-03-01", "2026-03-01"), 1)
-
- def test_one_week(self):
- self.assertEqual(_days_between("2026-03-01", "2026-03-08"), 7)
-
- def test_invalid_dates(self):
- self.assertEqual(_days_between("bad", "dates"), 30)
-
-
-class TestBraveFreshness(unittest.TestCase):
- def test_one_day(self):
- self.assertEqual(_brave_freshness(1), "pd")
-
- def test_one_week(self):
- self.assertEqual(_brave_freshness(7), "pw")
-
- def test_one_month(self):
- self.assertEqual(_brave_freshness(31), "pm")
-
- def test_longer_returns_range(self):
- result = _brave_freshness(60)
- self.assertIn("to", result)
-
- def test_none(self):
- self.assertIsNone(_brave_freshness(None))
-
-
-class TestParseBraveDate(unittest.TestCase):
- def test_hours_ago(self):
- result = _parse_brave_date("3 hours ago", None)
- self.assertIsNotNone(result)
- self.assertRegex(result, r"\d{4}-\d{2}-\d{2}")
-
- def test_days_ago(self):
- result = _parse_brave_date("5 days ago", None)
- self.assertIsNotNone(result)
-
- def test_weeks_ago(self):
- result = _parse_brave_date("2 weeks ago", None)
- self.assertIsNotNone(result)
-
- def test_iso_date(self):
- self.assertEqual(_parse_brave_date("2026-03-10T12:00:00", None), "2026-03-10")
-
- def test_none(self):
- self.assertIsNone(_parse_brave_date(None, None))
-
-
-class TestNormalizeResults(unittest.TestCase):
- def test_merges_news_and_web(self):
- response = {
- "news": {"results": [
- {"url": "https://news.example.com/a", "title": "News A", "description": "News desc"},
- ]},
- "web": {"results": [
- {"url": "https://blog.example.com/b", "title": "Blog B", "description": "Blog desc"},
- ]},
- }
- items = _normalize_results(response, "2026-03-01", "2026-03-10")
- self.assertEqual(len(items), 2)
- self.assertEqual(items[0]["title"], "News A")
- self.assertEqual(items[1]["title"], "Blog B")
-
- def test_excludes_reddit_and_x(self):
- response = {
- "web": {"results": [
- {"url": "https://www.reddit.com/r/test/123", "title": "Reddit", "description": "text"},
- {"url": "https://x.com/user/status/1", "title": "X post", "description": "text"},
- {"url": "https://example.com/ok", "title": "OK", "description": "text"},
- ]},
- }
- items = _normalize_results(response, "2026-03-01", "2026-03-10")
- self.assertEqual(len(items), 1)
- self.assertEqual(items[0]["title"], "OK")
-
- def test_default_relevance(self):
- response = {"web": {"results": [
- {"url": "https://a.com", "title": "A", "description": "desc"},
- ]}}
- items = _normalize_results(response, "2026-03-01", "2026-03-10")
- self.assertEqual(items[0]["relevance"], 0.6)
-
-
-class TestNormalizeLlmContext(unittest.TestCase):
- def _make_response(self, generic=None, sources=None):
- return {
- "grounding": {"generic": generic or []},
- "sources": sources or {},
- }
-
- def test_basic_result(self):
- resp = self._make_response(
- generic=[{
- "url": "https://docs.example.com/page",
- "title": "Example Page",
- "snippets": ["First chunk of text.", "Second chunk of text."],
- }],
- sources={
- "https://docs.example.com/page": {
- "title": "Example Page",
- "hostname": "docs.example.com",
- "age": ["2026-03-05", "5 days ago"],
- }
- },
- )
- items = _normalize_llm_context(resp)
- self.assertEqual(len(items), 1)
- item = items[0]
- self.assertEqual(item["title"], "Example Page")
- self.assertEqual(item["url"], "https://docs.example.com/page")
- self.assertIn("First chunk", item["snippet"])
- self.assertIn("Second chunk", item["snippet"])
- self.assertEqual(item["date"], "2026-03-05")
- self.assertEqual(item["date_confidence"], "med")
- self.assertEqual(item["relevance"], 0.7)
- self.assertEqual(item["source_domain"], "docs.example.com")
-
- def test_excludes_reddit(self):
- resp = self._make_response(
- generic=[
- {"url": "https://www.reddit.com/r/test", "title": "Reddit", "snippets": ["text"]},
- {"url": "https://example.com", "title": "OK", "snippets": ["text"]},
- ],
- )
- items = _normalize_llm_context(resp)
- self.assertEqual(len(items), 1)
- self.assertEqual(items[0]["title"], "OK")
-
- def test_empty_grounding(self):
- resp = self._make_response()
- items = _normalize_llm_context(resp)
- self.assertEqual(items, [])
-
- def test_snippet_truncation(self):
- long_snippet = "x" * 2000
- resp = self._make_response(
- generic=[{"url": "https://a.com", "title": "A", "snippets": [long_snippet]}],
- )
- items = _normalize_llm_context(resp)
- self.assertLessEqual(len(items[0]["snippet"]), 1500)
-
- def test_no_date_gives_low_confidence(self):
- resp = self._make_response(
- generic=[{"url": "https://a.com", "title": "A", "snippets": ["text"]}],
- sources={"https://a.com": {"hostname": "a.com", "age": None}},
- )
- items = _normalize_llm_context(resp)
- self.assertIsNone(items[0]["date"])
- self.assertEqual(items[0]["date_confidence"], "low")
-
- def test_multiple_age_entries_picks_first_valid(self):
- resp = self._make_response(
- generic=[{"url": "https://a.com", "title": "A", "snippets": ["text"]}],
- sources={"https://a.com": {
- "hostname": "a.com",
- "age": ["Monday, March 10, 2026", "2026-03-10", "1 day ago"],
- }},
- )
- items = _normalize_llm_context(resp)
- self.assertEqual(items[0]["date"], "2026-03-10")
-
- def test_ids_are_sequential(self):
- resp = self._make_response(
- generic=[
- {"url": "https://a.com", "title": "A", "snippets": ["a"]},
- {"url": "https://b.com", "title": "B", "snippets": ["b"]},
- {"url": "https://c.com", "title": "C", "snippets": ["c"]},
- ],
- )
- items = _normalize_llm_context(resp)
- ids = [item["id"] for item in items]
- self.assertEqual(ids, ["W1", "W2", "W3"])
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_briefing_v3.py b/tests/test_briefing_v3.py
new file mode 100644
index 0000000..cd1cec4
--- /dev/null
+++ b/tests/test_briefing_v3.py
@@ -0,0 +1,35 @@
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+import briefing
+import store
+
+
+class BriefingV3Tests(unittest.TestCase):
+ def test_generate_daily_uses_utc_for_last_run(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ db_path = Path(tmpdir) / "research.db"
+ briefs_dir = Path(tmpdir) / "briefs"
+ old_db_override = store._db_override
+ old_briefs_dir = briefing.BRIEFS_DIR
+ try:
+ store._db_override = db_path
+ briefing.BRIEFS_DIR = briefs_dir
+ topic = store.add_topic("test topic")
+ store.record_run(topic["id"], source_mode="v3", status="completed")
+ result = briefing.generate_daily()
+ self.assertEqual(result["status"], "ok")
+ self.assertEqual(result["topics"][0]["name"], "test topic")
+ self.assertIsNotNone(result["topics"][0]["hours_ago"])
+ self.assertGreaterEqual(result["topics"][0]["hours_ago"], 0.0)
+ finally:
+ store._db_override = old_db_override
+ briefing.BRIEFS_DIR = old_briefs_dir
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_cache.py b/tests/test_cache.py
deleted file mode 100644
index fe1d9a0..0000000
--- a/tests/test_cache.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""Tests for cache module."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import cache
-
-
-class TestGetCacheKey(unittest.TestCase):
- def test_returns_string(self):
- result = cache.get_cache_key("test topic", "2026-01-01", "2026-01-31", "both")
- self.assertIsInstance(result, str)
-
- def test_consistent_for_same_inputs(self):
- key1 = cache.get_cache_key("test topic", "2026-01-01", "2026-01-31", "both")
- key2 = cache.get_cache_key("test topic", "2026-01-01", "2026-01-31", "both")
- self.assertEqual(key1, key2)
-
- def test_different_for_different_inputs(self):
- key1 = cache.get_cache_key("topic a", "2026-01-01", "2026-01-31", "both")
- key2 = cache.get_cache_key("topic b", "2026-01-01", "2026-01-31", "both")
- self.assertNotEqual(key1, key2)
-
- def test_key_length(self):
- key = cache.get_cache_key("test", "2026-01-01", "2026-01-31", "both")
- self.assertEqual(len(key), 16)
-
-
-class TestCachePath(unittest.TestCase):
- def test_returns_path(self):
- result = cache.get_cache_path("abc123")
- self.assertIsInstance(result, Path)
-
- def test_has_json_extension(self):
- result = cache.get_cache_path("abc123")
- self.assertEqual(result.suffix, ".json")
-
-
-class TestCacheValidity(unittest.TestCase):
- def test_nonexistent_file_is_invalid(self):
- fake_path = Path("/nonexistent/path/file.json")
- result = cache.is_cache_valid(fake_path)
- self.assertFalse(result)
-
-
-class TestModelCache(unittest.TestCase):
- def test_get_cached_model_returns_none_for_missing(self):
- # Clear any existing cache first
- result = cache.get_cached_model("nonexistent_provider")
- # May be None or a cached value, but should not error
- self.assertTrue(result is None or isinstance(result, str))
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_cli_v3.py b/tests/test_cli_v3.py
new file mode 100644
index 0000000..e09b68d
--- /dev/null
+++ b/tests/test_cli_v3.py
@@ -0,0 +1,190 @@
+# ruff: noqa: E402
+import json
+import io
+import tempfile
+import subprocess
+import sys
+import types
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+from unittest import mock
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import last30days as cli
+from lib import schema
+
+
+class CliV3Tests(unittest.TestCase):
+ def make_report(self) -> schema.Report:
+ return schema.Report(
+ topic="OpenClaw vs NanoClaw",
+ range_from="2026-02-14",
+ range_to="2026-03-16",
+ generated_at="2026-03-16T00:00:00+00:00",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="gemini",
+ planner_model="gemini-3.1-flash-lite-preview",
+ rerank_model="gemini-3.1-flash-lite-preview",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="comparison",
+ freshness_mode="balanced_recent",
+ cluster_mode="debate",
+ raw_topic="OpenClaw vs NanoClaw",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="openclaw vs nanoclaw",
+ ranking_query="How does OpenClaw compare to NanoClaw?",
+ sources=["grounding"],
+ )
+ ],
+ source_weights={"grounding": 1.0},
+ ),
+ clusters=[],
+ ranked_candidates=[],
+ items_by_source={"grounding": []},
+ errors_by_source={},
+ )
+
+ def test_mock_json_cli(self):
+ result = subprocess.run(
+ [sys.executable, "scripts/last30days.py", "test topic", "--mock", "--emit=json"],
+ cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ self.assertEqual(0, result.returncode, result.stderr)
+ payload = json.loads(result.stdout)
+ self.assertIn("query_plan", payload)
+ self.assertIn("ranked_candidates", payload)
+ self.assertIn("clusters", payload)
+
+ def test_parse_search_flag_normalizes_aliases_and_dedupes(self):
+ self.assertEqual(
+ ["grounding", "reddit", "hackernews"],
+ cli.parse_search_flag("web, reddit, hn, web"),
+ )
+
+ def test_parse_search_flag_rejects_invalid_or_empty_inputs(self):
+ with self.assertRaises(SystemExit):
+ cli.parse_search_flag("unknown")
+ with self.assertRaises(SystemExit):
+ cli.parse_search_flag(" , ")
+
+ def test_missing_sources_for_promo_prefers_reddit_x_then_web(self):
+ self.assertEqual(
+ "both",
+ cli._missing_sources_for_promo({"available_sources": ["youtube"]}),
+ )
+ self.assertEqual(
+ "web",
+ cli._missing_sources_for_promo({"available_sources": ["reddit", "x"]}),
+ )
+ self.assertIsNone(
+ cli._missing_sources_for_promo({"available_sources": ["reddit", "x", "grounding"]}),
+ )
+
+ def test_slugify_and_emit_output_cover_supported_modes(self):
+ report = self.make_report()
+ self.assertEqual("openclaw-vs-nanoclaw", cli.slugify(report.topic))
+
+ compact = cli.emit_output(report, "compact")
+ json_output = cli.emit_output(report, "json")
+ context = cli.emit_output(report, "context")
+
+ self.assertIn("# last30days-3 v3.0.0-alpha", compact)
+ self.assertIn('"topic": "OpenClaw vs NanoClaw"', json_output)
+ self.assertIsInstance(context, str)
+
+ with self.assertRaises(SystemExit):
+ cli.emit_output(report, "bad-mode")
+
+ def test_save_output_writes_expected_extension(self):
+ report = self.make_report()
+ with tempfile.TemporaryDirectory() as tmp:
+ path = cli.save_output(report, "json", tmp)
+ self.assertEqual(".json", path.suffix)
+ payload = json.loads(path.read_text())
+ self.assertEqual("OpenClaw vs NanoClaw", payload["topic"])
+
+ def test_persist_report_updates_run_status_on_success_and_failure(self):
+ report = self.make_report()
+
+ success_store = types.SimpleNamespace(
+ init_db=mock.Mock(),
+ add_topic=mock.Mock(return_value={"id": 7}),
+ record_run=mock.Mock(return_value=11),
+ findings_from_report=mock.Mock(return_value=[{"title": "x"}]),
+ store_findings=mock.Mock(return_value={"new": 2, "updated": 1}),
+ update_run=mock.Mock(),
+ )
+ with mock.patch.dict(sys.modules, {"store": success_store}):
+ counts = cli.persist_report(report)
+ self.assertEqual({"new": 2, "updated": 1}, counts)
+ success_store.update_run.assert_called_once_with(
+ 11,
+ status="completed",
+ findings_new=2,
+ findings_updated=1,
+ )
+
+ failure_store = types.SimpleNamespace(
+ init_db=mock.Mock(),
+ add_topic=mock.Mock(return_value={"id": 7}),
+ record_run=mock.Mock(return_value=12),
+ findings_from_report=mock.Mock(side_effect=RuntimeError("boom")),
+ store_findings=mock.Mock(),
+ update_run=mock.Mock(),
+ )
+ with mock.patch.dict(sys.modules, {"store": failure_store}):
+ with self.assertRaises(RuntimeError):
+ cli.persist_report(report)
+ failure_store.update_run.assert_called_once()
+ _, kwargs = failure_store.update_run.call_args
+ self.assertEqual("failed", kwargs["status"])
+ self.assertIn("boom", kwargs["error_message"])
+
+ def test_main_wires_banner_and_progress_display(self):
+ report = self.make_report()
+ diag = {
+ "available_sources": ["grounding", "youtube"],
+ "providers": {"google": True, "openai": False, "xai": False},
+ "x_backend": None,
+ "bird_installed": True,
+ "bird_authenticated": False,
+ "bird_username": None,
+ "native_web_backend": "brave",
+ }
+ fake_progress = mock.Mock()
+ with mock.patch.object(cli.env, "get_config", return_value={}), \
+ mock.patch.object(cli.pipeline, "diagnose", return_value=diag), \
+ mock.patch.object(cli.pipeline, "run", return_value=report), \
+ mock.patch.object(cli.ui, "show_diagnostic_banner") as banner, \
+ mock.patch.object(cli.ui, "ProgressDisplay", return_value=fake_progress) as progress_cls, \
+ mock.patch.object(cli, "emit_output", return_value="# rendered"), \
+ mock.patch.object(sys, "argv", ["last30days.py", "test", "topic"]):
+ stdout = io.StringIO()
+ stderr = io.StringIO()
+ with redirect_stdout(stdout), redirect_stderr(stderr):
+ rc = cli.main()
+ self.assertEqual(0, rc)
+ banner.assert_not_called() # Banner moved to post-research
+ progress_cls.assert_called_once_with("test topic", show_banner=True)
+ fake_progress.start_processing.assert_called_once()
+ fake_progress.end_processing.assert_called_once()
+ fake_progress.show_complete.assert_called_once_with(
+ source_counts={"grounding": 0},
+ display_sources=["grounding"],
+ )
+ fake_progress.show_promo.assert_called_once_with("both", diag=diag)
+ self.assertIn("# rendered", stdout.getvalue())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_cluster_v3.py b/tests/test_cluster_v3.py
new file mode 100644
index 0000000..86b7583
--- /dev/null
+++ b/tests/test_cluster_v3.py
@@ -0,0 +1,201 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import cluster, schema
+
+
+def make_candidate(candidate_id: str, source: str, title: str, snippet: str, score: float) -> schema.Candidate:
+ return schema.Candidate(
+ candidate_id=candidate_id,
+ item_id=candidate_id,
+ source=source,
+ title=title,
+ url=f"https://example.com/{candidate_id}",
+ snippet=snippet,
+ subquery_labels=["primary"],
+ native_ranks={"primary:reddit": 1},
+ local_relevance=0.8,
+ freshness=80,
+ engagement=10,
+ source_quality=0.7,
+ rrf_score=0.02,
+ rerank_score=score,
+ final_score=score,
+ )
+
+
+class ClusterV3Tests(unittest.TestCase):
+ def test_singleton_clusters_for_non_clustered_plan(self):
+ plan = schema.QueryPlan(
+ intent="how_to",
+ freshness_mode="balanced_recent",
+ cluster_mode="none",
+ raw_topic="docker setup",
+ subqueries=[schema.SubQuery(label="primary", search_query="docker setup", ranking_query="How do I set up Docker?", sources=["reddit"])],
+ source_weights={"reddit": 1.0},
+ )
+ candidates = [
+ make_candidate("c1", "reddit", "Docker setup guide", "Step by step setup", 80),
+ make_candidate("c2", "youtube", "Docker install video", "Video walkthrough", 75),
+ ]
+ clusters = cluster.cluster_candidates(candidates, plan)
+ self.assertEqual(2, len(clusters))
+ self.assertEqual(["c1"], clusters[0].representative_ids)
+ self.assertEqual(["c2"], clusters[1].representative_ids)
+
+ def test_breaking_news_clusters_related_items(self):
+ plan = schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="model launch",
+ subqueries=[schema.SubQuery(label="primary", search_query="model launch", ranking_query="What happened in the model launch?", sources=["reddit", "x"])],
+ source_weights={"reddit": 0.5, "x": 0.5},
+ )
+ candidates = [
+ make_candidate("c1", "reddit", "Open model launch reactions", "People are reacting to the open model launch today.", 88),
+ make_candidate("c2", "x", "Open model launch update", "People are reacting to the open model launch today on X.", 84),
+ make_candidate("c3", "youtube", "Different topic", "A separate discussion about hardware benchmarks.", 70),
+ ]
+ clusters = cluster.cluster_candidates(candidates, plan)
+ self.assertEqual(2, len(clusters))
+ self.assertEqual(2, len(clusters[0].candidate_ids))
+ self.assertIn("c1", clusters[0].candidate_ids)
+ self.assertIn("c2", clusters[0].candidate_ids)
+
+
+class TestCrossSourceMerging(unittest.TestCase):
+ """Test the entity-based second pass that merges same-story clusters across sources."""
+
+ def _plan(self, intent="breaking_news"):
+ return schema.QueryPlan(
+ intent=intent,
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test",
+ subqueries=[schema.SubQuery(label="primary", search_query="test", ranking_query="test", sources=["reddit", "x", "tiktok"])],
+ source_weights={"reddit": 0.5, "x": 0.5, "tiktok": 0.5},
+ )
+
+ def test_same_story_different_phrasing_merges(self):
+ """Wireless Festival example: same event, different wording, different sources."""
+ candidates = [
+ make_candidate("c1", "reddit", "Kanye West to headline all three nights of Wireless Festival 2026", "Big announcement for Wireless.", 80),
+ make_candidate("c2", "x", "BREAKING: Kanye West is making his massive UK comeback at Wireless Festival this July", "Ye returns to UK.", 75),
+ make_candidate("c3", "youtube", "Kanye West BULLY Album Review - Knox Hill Reacts", "Full album reaction and breakdown.", 70),
+ ]
+ clusters = cluster.cluster_candidates(candidates, self._plan())
+ # c1 and c2 should merge (Kanye + Wireless + Festival overlap), c3 should stay separate
+ self.assertEqual(2, len(clusters))
+ wireless_cluster = next(cl for cl in clusters if len(cl.candidate_ids) == 2)
+ self.assertIn("c1", wireless_cluster.candidate_ids)
+ self.assertIn("c2", wireless_cluster.candidate_ids)
+ self.assertEqual(sorted(["reddit", "x"]), wireless_cluster.sources)
+ # Multi-source cluster should not have "single-source" uncertainty
+ self.assertNotEqual("single-source", wireless_cluster.uncertainty)
+
+ def test_different_stories_dont_merge(self):
+ """Different topics should stay separate even with some entity overlap (e.g., 'Kanye')."""
+ candidates = [
+ make_candidate("c1", "reddit", "Kanye West BULLY Album First Impressions Thread", "What do you think of BULLY?", 80),
+ make_candidate("c2", "x", "Kanye West apology for antisemitism in Wall Street Journal ad", "Full page WSJ ad.", 75),
+ make_candidate("c3", "tiktok", "Kanye West Wireless Festival ticket prices breakdown", "How much for Wireless tickets?", 70),
+ ]
+ clusters = cluster.cluster_candidates(candidates, self._plan())
+ # These are 3 different stories, should remain as 3 clusters
+ self.assertEqual(3, len(clusters))
+
+ def test_same_source_clusters_dont_merge(self):
+ """Two single-source clusters from the same source should not merge via entity pass."""
+ candidates = [
+ make_candidate("c1", "reddit", "Kanye West Wireless Festival headline announcement", "Three nights!", 80),
+ make_candidate("c2", "reddit", "Kanye West returning to Wireless Festival confirmed", "UK comeback.", 70),
+ ]
+ clusters = cluster.cluster_candidates(candidates, self._plan())
+ # The initial greedy pass may or may not merge these (depends on token similarity).
+ # But if they end up as separate clusters, the entity pass should NOT merge them
+ # since they're both from reddit.
+ for cl in clusters:
+ self.assertTrue(len(cl.sources) >= 1) # basic sanity
+
+
+class TestPolymarketIsolation(unittest.TestCase):
+ """Polymarket clusters must not merge with non-Polymarket clusters via entity overlap."""
+
+ def _plan(self):
+ return schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test",
+ subqueries=[schema.SubQuery(label="primary", search_query="test", ranking_query="test", sources=["reddit", "x", "polymarket"])],
+ source_weights={"reddit": 0.5, "x": 0.5, "polymarket": 0.5},
+ )
+
+ def test_polymarket_does_not_merge_into_news_cluster(self):
+ """A Polymarket prediction about Sam Altman should not merge into a news cluster about Sam Altman."""
+ candidates = [
+ make_candidate("c1", "reddit", "Sam Altman personal rivalry with Elon Musk escalates", "The feud between Sam Altman and Elon Musk continues.", 80),
+ make_candidate("c2", "polymarket", "Sam Altman equity stake in OpenAI valued at $500M", "Will Sam Altman receive equity in OpenAI restructuring?", 75),
+ ]
+ clusters = cluster.cluster_candidates(candidates, self._plan())
+ self.assertEqual(2, len(clusters), "Polymarket and news clusters should remain separate")
+ # Each cluster should have exactly one candidate
+ for cl in clusters:
+ self.assertEqual(1, len(cl.candidate_ids))
+
+ def test_two_polymarket_clusters_not_blocked_by_poly_guard(self):
+ """Two Polymarket items about the same topic are not blocked by the Polymarket guard.
+
+ Note: same-source clusters are still blocked by the existing same-source
+ guard, so we verify the poly guard specifically by checking that two
+ polymarket items with high text similarity merge via the greedy pass.
+ """
+ candidates = [
+ make_candidate("c1", "polymarket", "Sam Altman equity stake in OpenAI restructuring", "Will Sam Altman get equity in the OpenAI restructuring deal?", 80),
+ make_candidate("c2", "polymarket", "Sam Altman equity stake in OpenAI restructuring odds", "Will Sam Altman get equity in the OpenAI restructuring deal? Current odds.", 75),
+ ]
+ clusters = cluster.cluster_candidates(candidates, self._plan())
+ # High text similarity means greedy pass merges them
+ self.assertEqual(1, len(clusters))
+ self.assertEqual(2, len(clusters[0].candidate_ids))
+
+ def test_neither_polymarket_still_merges(self):
+ """Non-Polymarket clusters with entity overlap should still merge (existing behavior)."""
+ candidates = [
+ make_candidate("c1", "reddit", "Sam Altman OpenAI restructuring announcement details", "Sam Altman announces major OpenAI restructuring.", 80),
+ make_candidate("c2", "x", "Sam Altman reveals OpenAI restructuring plan for 2026", "Major OpenAI restructuring coming says Sam Altman.", 75),
+ ]
+ clusters = cluster.cluster_candidates(candidates, self._plan())
+ self.assertEqual(1, len(clusters))
+ self.assertEqual(2, len(clusters[0].candidate_ids))
+
+
+class TestClusterUncertainty(unittest.TestCase):
+ def test_single_source_returns_single_source(self):
+ candidates = [make_candidate("c1", "reddit", "Title", "Body", 80)]
+ result = cluster._cluster_uncertainty(candidates)
+ self.assertEqual("single-source", result)
+
+ def test_multi_source_high_score_returns_none(self):
+ candidates = [
+ make_candidate("c1", "reddit", "Title", "Body", 80),
+ make_candidate("c2", "x", "Title2", "Body2", 70),
+ ]
+ result = cluster._cluster_uncertainty(candidates)
+ self.assertIsNone(result)
+
+ def test_multi_source_low_score_returns_thin_evidence(self):
+ candidates = [
+ make_candidate("c1", "reddit", "Title", "Body", 30),
+ make_candidate("c2", "x", "Title2", "Body2", 40),
+ ]
+ result = cluster._cluster_uncertainty(candidates)
+ self.assertEqual("thin-evidence", result)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_codex_auth.py b/tests/test_codex_auth.py
deleted file mode 100644
index 0cbe72d..0000000
--- a/tests/test_codex_auth.py
+++ /dev/null
@@ -1,203 +0,0 @@
-"""Tests for Codex auth integration (env.py + openai_reddit.py)."""
-
-import base64
-import json
-import os
-import sys
-import time
-import unittest
-from pathlib import Path
-from unittest.mock import patch
-
-# Add scripts directory to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import env, openai_reddit
-
-
-def _make_jwt(payload: dict) -> str:
- """Build a fake JWT with the given payload (no signature verification)."""
- header = base64.urlsafe_b64encode(json.dumps({"alg": "none"}).encode()).rstrip(b"=")
- body = base64.urlsafe_b64encode(json.dumps(payload).encode()).rstrip(b"=")
- return f"{header.decode()}.{body.decode()}.fakesig"
-
-
-class TestDecodeJwtPayload(unittest.TestCase):
-
- def test_valid_jwt(self):
- token = _make_jwt({"sub": "user123", "exp": 9999999999})
- result = env._decode_jwt_payload(token)
- self.assertEqual(result["sub"], "user123")
-
- def test_invalid_jwt(self):
- self.assertIsNone(env._decode_jwt_payload("not-a-jwt"))
-
- def test_empty_string(self):
- self.assertIsNone(env._decode_jwt_payload(""))
-
-
-class TestTokenExpired(unittest.TestCase):
-
- def test_not_expired(self):
- token = _make_jwt({"exp": int(time.time()) + 3600})
- self.assertFalse(env._token_expired(token))
-
- def test_expired(self):
- token = _make_jwt({"exp": int(time.time()) - 100})
- self.assertTrue(env._token_expired(token))
-
- def test_no_exp_claim(self):
- token = _make_jwt({"sub": "user"})
- self.assertFalse(env._token_expired(token))
-
-
-class TestExtractChatgptAccountId(unittest.TestCase):
-
- def test_extracts_account_id(self):
- token = _make_jwt({
- "https://api.openai.com/auth": {
- "chatgpt_account_id": "acct_abc123"
- }
- })
- self.assertEqual(env.extract_chatgpt_account_id(token), "acct_abc123")
-
- def test_missing_auth_claim(self):
- token = _make_jwt({"sub": "user"})
- self.assertIsNone(env.extract_chatgpt_account_id(token))
-
- def test_missing_account_id_in_claim(self):
- token = _make_jwt({
- "https://api.openai.com/auth": {"other_field": "value"}
- })
- self.assertIsNone(env.extract_chatgpt_account_id(token))
-
-
-class TestGetOpenaiAuth(unittest.TestCase):
-
- @patch.dict(os.environ, {}, clear=True)
- def test_api_key_takes_priority(self):
- """OPENAI_API_KEY in env file is used when env var is not set."""
- file_env = {"OPENAI_API_KEY": "sk-test123"}
- auth = env.get_openai_auth(file_env)
- self.assertEqual(auth.source, "api_key")
- self.assertEqual(auth.status, "ok")
- self.assertEqual(auth.token, "sk-test123")
- self.assertIsNone(auth.account_id)
-
- @patch.dict(os.environ, {"OPENAI_API_KEY": "sk-from-env"}, clear=False)
- def test_env_var_takes_priority(self):
- """OPENAI_API_KEY env var should be preferred over file."""
- file_env = {}
- auth = env.get_openai_auth(file_env)
- self.assertEqual(auth.source, "api_key")
- self.assertEqual(auth.token, "sk-from-env")
-
- def test_no_keys_returns_none_source(self):
- """No API key and no Codex auth → source=none."""
- fake_path = Path("/tmp/nonexistent_codex_auth_test.json")
- with patch.object(env, 'CODEX_AUTH_FILE', fake_path):
- # Also patch get_codex_access_token to avoid reading real auth file
- with patch.object(env, 'get_codex_access_token', return_value=(None, "missing")):
- environ_copy = {k: v for k, v in os.environ.items() if k != "OPENAI_API_KEY"}
- with patch.dict(os.environ, environ_copy, clear=True):
- auth = env.get_openai_auth({})
- self.assertEqual(auth.source, "none")
- self.assertIsNone(auth.token)
-
-
-class TestLoadCodexAuth(unittest.TestCase):
-
- def test_nonexistent_file(self):
- result = env.load_codex_auth(Path("/tmp/nonexistent_codex_auth.json"))
- self.assertEqual(result, {})
-
- def test_valid_json(self):
- import tempfile
- data = {"tokens": {"access_token": "tok123"}}
- with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
- json.dump(data, f)
- f.flush()
- result = env.load_codex_auth(Path(f.name))
- os.unlink(f.name)
- self.assertEqual(result["tokens"]["access_token"], "tok123")
-
-
-class TestGetAvailableSourcesWithAuth(unittest.TestCase):
-
- @patch("lib.bird_x.is_bird_installed", return_value=False)
- def test_codex_auth_ok_counts_as_openai(self, _mock_bird):
- config = {
- "OPENAI_API_KEY": "codex-token",
- "OPENAI_AUTH_STATUS": "ok",
- "XAI_API_KEY": None,
- }
- result = env.get_available_sources(config)
- self.assertIn("reddit", result)
-
- @patch("lib.bird_x.is_bird_installed", return_value=False)
- def test_codex_auth_expired_not_counted(self, _mock_bird):
- config = {
- "OPENAI_API_KEY": None,
- "OPENAI_AUTH_STATUS": "expired",
- "XAI_API_KEY": None,
- }
- result = env.get_available_sources(config)
- # Reddit is available via public JSON fallback even without OpenAI auth
- self.assertEqual(result, "reddit")
-
-
-class TestParseCodexStream(unittest.TestCase):
-
- def test_response_completed_event(self):
- """Should extract response from response.completed SSE event."""
- sse = (
- 'data: {"type":"response.created","response":{"id":"r1"}}\n\n'
- 'data: {"type":"response.completed","response":{"id":"r1","output":[{"type":"message","content":[{"type":"output_text","text":"hello"}]}]}}\n\n'
- )
- result = openai_reddit._parse_codex_stream(sse)
- self.assertIn("output", result)
-
- def test_delta_fallback(self):
- """Should reconstruct text from delta events."""
- sse = (
- 'data: {"delta":"hel"}\n\n'
- 'data: {"delta":"lo"}\n\n'
- )
- result = openai_reddit._parse_codex_stream(sse)
- self.assertIn("output", result)
- text = result["output"][0]["content"][0]["text"]
- self.assertEqual(text, "hello")
-
- def test_empty_stream(self):
- result = openai_reddit._parse_codex_stream("")
- self.assertEqual(result, {})
-
-
-class TestBuildPayload(unittest.TestCase):
-
- def test_api_key_payload(self):
- payload = openai_reddit._build_payload(
- "gpt-4o", "instructions", "input text", "api_key"
- )
- self.assertEqual(payload["model"], "gpt-4o")
- self.assertEqual(payload["input"], "input text")
- self.assertNotIn("stream", payload)
-
- def test_codex_payload_has_stream(self):
- payload = openai_reddit._build_payload(
- "gpt-4o", "instructions", "input text", env.AUTH_SOURCE_CODEX
- )
- self.assertTrue(payload["stream"])
- # Input should be structured message format for Codex
- self.assertIsInstance(payload["input"], list)
- self.assertEqual(payload["input"][0]["role"], "user")
-
- def test_codex_payload_has_store_false(self):
- payload = openai_reddit._build_payload(
- "gpt-4o", "inst", "text", env.AUTH_SOURCE_CODEX
- )
- self.assertFalse(payload["store"])
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_cross_source.py b/tests/test_cross_source.py
deleted file mode 100644
index 70ecec7..0000000
--- a/tests/test_cross_source.py
+++ /dev/null
@@ -1,186 +0,0 @@
-"""Tests for cross-source linking."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import dedupe, schema
-
-
-class TestCrossSourceLink(unittest.TestCase):
- def _make_reddit(self, id, title, score=50):
- item = schema.RedditItem(id=id, title=title, url="", subreddit="test")
- item.score = score
- return item
-
- def _make_hn(self, id, title, score=50):
- item = schema.HackerNewsItem(id=id, title=title, url="", hn_url="", author="user")
- item.score = score
- return item
-
- def _make_x(self, id, text, score=50):
- item = schema.XItem(id=id, text=text, url="", author_handle="user")
- item.score = score
- return item
-
- def _make_yt(self, id, title, score=50):
- item = schema.YouTubeItem(id=id, title=title, url="", channel_name="ch")
- item.score = score
- return item
-
- def _make_web(self, id, title, score=50):
- item = schema.WebSearchItem(id=id, title=title, url="", source_domain="example.com", snippet="")
- item.score = score
- return item
-
- def test_no_crossrefs_for_unrelated(self):
- reddit = [self._make_reddit("R1", "Best Claude Code Tips")]
- hn = [self._make_hn("HN1", "Python Django Release Notes")]
- dedupe.cross_source_link(reddit, hn)
- self.assertEqual(reddit[0].cross_refs, [])
- self.assertEqual(hn[0].cross_refs, [])
-
- def test_bidirectional_link(self):
- reddit = [self._make_reddit("R1", "OpenAI launches GPT-5 with new features")]
- hn = [self._make_hn("HN1", "OpenAI launches GPT-5 with new features")]
- dedupe.cross_source_link(reddit, hn)
- self.assertIn("HN1", reddit[0].cross_refs)
- self.assertIn("R1", hn[0].cross_refs)
-
- def test_multi_source_link(self):
- reddit = [self._make_reddit("R1", "Claude Code gets new skill system")]
- hn = [self._make_hn("HN1", "Claude Code gets new skill system")]
- yt = [self._make_yt("YT1", "Claude Code gets new skill system")]
- dedupe.cross_source_link(reddit, hn, yt)
- # All three should reference each other
- self.assertEqual(len(reddit[0].cross_refs), 2)
- self.assertEqual(len(hn[0].cross_refs), 2)
- self.assertEqual(len(yt[0].cross_refs), 2)
-
- def test_same_source_not_linked(self):
- reddit = [
- self._make_reddit("R1", "OpenAI GPT-5 launch details"),
- self._make_reddit("R2", "OpenAI GPT-5 launch details"),
- ]
- dedupe.cross_source_link(reddit)
- self.assertEqual(reddit[0].cross_refs, [])
- self.assertEqual(reddit[1].cross_refs, [])
-
- def test_x_text_truncation_helps(self):
- # Truncation increases similarity vs full tweet.
- # A near-identical short tweet should match a Reddit title.
- reddit = [self._make_reddit("R1", "Anthropic releases Claude 4 model")]
- x_short = [self._make_x("X1", "Anthropic releases Claude 4 model today!")]
- dedupe.cross_source_link(reddit, x_short)
- self.assertIn("X1", reddit[0].cross_refs)
- self.assertIn("R1", x_short[0].cross_refs)
-
- def test_long_x_text_may_not_match(self):
- # When a tweet diverges significantly after the shared prefix,
- # Jaccard drops below 0.5 even with truncation. This is expected.
- reddit = [self._make_reddit("R1", "Anthropic releases Claude 4 model")]
- x_long = [self._make_x("X1",
- "Anthropic releases Claude 4 model and it's incredible. "
- "The reasoning capabilities are next level. Just tested it "
- "on my entire codebase and it understood everything."
- )]
- dedupe.cross_source_link(reddit, x_long)
- # May or may not match depending on trigram overlap - just verify no crash
- self.assertIsInstance(reddit[0].cross_refs, list)
-
- def test_empty_lists(self):
- # Should not crash
- dedupe.cross_source_link([], [], [])
-
- def test_single_item(self):
- reddit = [self._make_reddit("R1", "Test item")]
- dedupe.cross_source_link(reddit)
- self.assertEqual(reddit[0].cross_refs, [])
-
- def test_no_duplicate_refs(self):
- reddit = [self._make_reddit("R1", "Same exact title repeated")]
- hn = [self._make_hn("HN1", "Same exact title repeated")]
- # Call twice - should not duplicate refs
- dedupe.cross_source_link(reddit, hn)
- dedupe.cross_source_link(reddit, hn)
- self.assertEqual(reddit[0].cross_refs.count("HN1"), 1)
- self.assertEqual(hn[0].cross_refs.count("R1"), 1)
-
- def test_web_items_linked(self):
- web = [self._make_web("W1", "Claude Code skill system overview")]
- hn = [self._make_hn("HN1", "Claude Code skill system overview")]
- dedupe.cross_source_link(web, hn)
- self.assertIn("HN1", web[0].cross_refs)
- self.assertIn("W1", hn[0].cross_refs)
-
- def _make_pm(self, id, title, score=50):
- item = schema.PolymarketItem(id=id, title=title, question="Q?", url="")
- item.score = score
- return item
-
- def test_polymarket_to_reddit_link(self):
- reddit = [self._make_reddit("R1", "Will Arizona win the Big 12 Championship?")]
- pm = [self._make_pm("PM1", "Will Arizona win the Big 12 Championship?")]
- dedupe.cross_source_link(reddit, pm)
- self.assertIn("PM1", reddit[0].cross_refs)
- self.assertIn("R1", pm[0].cross_refs)
-
- def test_polymarket_multi_source(self):
- reddit = [self._make_reddit("R1", "Iran nuclear deal prediction markets")]
- hn = [self._make_hn("HN1", "Iran nuclear deal prediction markets")]
- pm = [self._make_pm("PM1", "Iran nuclear deal prediction markets")]
- dedupe.cross_source_link(reddit, hn, pm)
- self.assertEqual(len(reddit[0].cross_refs), 2)
- self.assertEqual(len(pm[0].cross_refs), 2)
-
-
-class TestCrossRefsSchemaRoundTrip(unittest.TestCase):
- def test_reddit_roundtrip(self):
- item = schema.RedditItem(id="R1", title="Test", url="", subreddit="test",
- cross_refs=["HN1", "X2"])
- d = item.to_dict()
- self.assertEqual(d['cross_refs'], ["HN1", "X2"])
-
- def test_reddit_empty_crossrefs_omitted(self):
- item = schema.RedditItem(id="R1", title="Test", url="", subreddit="test")
- d = item.to_dict()
- self.assertNotIn('cross_refs', d)
-
- def test_report_roundtrip(self):
- report = schema.Report(
- topic="test", range_from="2026-01-01", range_to="2026-02-01",
- generated_at="2026-02-01T00:00:00Z", mode="both",
- reddit=[schema.RedditItem(id="R1", title="T", url="", subreddit="s",
- cross_refs=["HN1"])],
- hackernews=[schema.HackerNewsItem(id="HN1", title="T", url="", hn_url="",
- author="u", cross_refs=["R1"])],
- )
- d = report.to_dict()
- restored = schema.Report.from_dict(d)
- self.assertEqual(restored.reddit[0].cross_refs, ["HN1"])
- self.assertEqual(restored.hackernews[0].cross_refs, ["R1"])
-
-
-class TestGetCrossSourceText(unittest.TestCase):
- def test_x_truncated(self):
- item = schema.XItem(id="X1", text="A" * 200, url="", author_handle="u")
- result = dedupe._get_cross_source_text(item)
- self.assertEqual(len(result), 100)
-
- def test_reddit_uses_title(self):
- item = schema.RedditItem(id="R1", title="My Title", url="", subreddit="s")
- result = dedupe._get_cross_source_text(item)
- self.assertEqual(result, "My Title")
-
- def test_web_uses_title(self):
- item = schema.WebSearchItem(id="W1", title="Web Title", url="",
- source_domain="example.com", snippet="snip")
- result = dedupe._get_cross_source_text(item)
- self.assertEqual(result, "Web Title")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_dates_v3.py b/tests/test_dates_v3.py
new file mode 100644
index 0000000..6d445ef
--- /dev/null
+++ b/tests/test_dates_v3.py
@@ -0,0 +1,63 @@
+import sys
+import unittest
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import dates
+
+
+class DatesV3Tests(unittest.TestCase):
+ def test_get_date_range_returns_iso_window(self):
+ start, end = dates.get_date_range(7)
+ self.assertRegex(start, r"^\d{4}-\d{2}-\d{2}$")
+ self.assertRegex(end, r"^\d{4}-\d{2}-\d{2}$")
+ self.assertEqual(7, (datetime.fromisoformat(end) - datetime.fromisoformat(start)).days)
+
+ def test_parse_date_supports_timestamps_and_iso_variants(self):
+ unix_parsed = dates.parse_date("1710460800")
+ self.assertEqual("2024-03-15T00:00:00+00:00", unix_parsed.isoformat())
+
+ plain = dates.parse_date("2026-03-16")
+ self.assertEqual("2026-03-16T00:00:00+00:00", plain.isoformat())
+
+ zulu = dates.parse_date("2026-03-16T12:34:56Z")
+ self.assertEqual("2026-03-16T12:34:56+00:00", zulu.isoformat())
+
+ fractional = dates.parse_date("2026-03-16T12:34:56.123456+02:00")
+ self.assertEqual("2026-03-16T10:34:56.123456+00:00", fractional.isoformat())
+
+ def test_parse_date_rejects_empty_and_invalid_values(self):
+ self.assertIsNone(dates.parse_date(None))
+ self.assertIsNone(dates.parse_date(""))
+ self.assertIsNone(dates.parse_date("not-a-date"))
+
+ def test_timestamp_to_date_handles_valid_and_invalid_values(self):
+ self.assertEqual("2024-03-15", dates.timestamp_to_date(1710460800))
+ self.assertIsNone(dates.timestamp_to_date(None))
+ self.assertIsNone(dates.timestamp_to_date("bad"))
+
+ def test_get_date_confidence_distinguishes_in_range_and_invalid(self):
+ self.assertEqual("high", dates.get_date_confidence("2026-03-10", "2026-03-01", "2026-03-16"))
+ self.assertEqual("low", dates.get_date_confidence("2026-02-10", "2026-03-01", "2026-03-16"))
+ self.assertEqual("low", dates.get_date_confidence("2026-03-20", "2026-03-01", "2026-03-16"))
+ self.assertEqual("low", dates.get_date_confidence("bad", "2026-03-01", "2026-03-16"))
+ self.assertEqual("low", dates.get_date_confidence(None, "2026-03-01", "2026-03-16"))
+
+ def test_days_ago_and_recency_score_cover_edge_cases(self):
+ today = datetime.now(timezone.utc).date()
+ old = (today - timedelta(days=45)).isoformat()
+ future = (today + timedelta(days=1)).isoformat()
+
+ self.assertEqual(0, dates.days_ago(today.isoformat()))
+ self.assertIsNone(dates.days_ago("bad"))
+
+ self.assertEqual(100, dates.recency_score(today.isoformat()))
+ self.assertEqual(0, dates.recency_score(old))
+ self.assertEqual(100, dates.recency_score(future))
+ self.assertEqual(0, dates.recency_score(None))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_dedupe.py b/tests/test_dedupe.py
deleted file mode 100644
index a790db5..0000000
--- a/tests/test_dedupe.py
+++ /dev/null
@@ -1,111 +0,0 @@
-"""Tests for dedupe module."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import dedupe, schema
-
-
-class TestNormalizeText(unittest.TestCase):
- def test_lowercase(self):
- result = dedupe.normalize_text("HELLO World")
- self.assertEqual(result, "hello world")
-
- def test_removes_punctuation(self):
- result = dedupe.normalize_text("Hello, World!")
- # Punctuation replaced with space, then whitespace collapsed
- self.assertEqual(result, "hello world")
-
- def test_collapses_whitespace(self):
- result = dedupe.normalize_text("hello world")
- self.assertEqual(result, "hello world")
-
-
-class TestGetNgrams(unittest.TestCase):
- def test_short_text(self):
- result = dedupe.get_ngrams("ab", n=3)
- self.assertEqual(result, {"ab"})
-
- def test_normal_text(self):
- result = dedupe.get_ngrams("hello", n=3)
- self.assertIn("hel", result)
- self.assertIn("ell", result)
- self.assertIn("llo", result)
-
-
-class TestJaccardSimilarity(unittest.TestCase):
- def test_identical_sets(self):
- set1 = {"a", "b", "c"}
- result = dedupe.jaccard_similarity(set1, set1)
- self.assertEqual(result, 1.0)
-
- def test_disjoint_sets(self):
- set1 = {"a", "b", "c"}
- set2 = {"d", "e", "f"}
- result = dedupe.jaccard_similarity(set1, set2)
- self.assertEqual(result, 0.0)
-
- def test_partial_overlap(self):
- set1 = {"a", "b", "c"}
- set2 = {"b", "c", "d"}
- result = dedupe.jaccard_similarity(set1, set2)
- self.assertEqual(result, 0.5) # 2 overlap / 4 union
-
- def test_empty_sets(self):
- result = dedupe.jaccard_similarity(set(), set())
- self.assertEqual(result, 0.0)
-
-
-class TestFindDuplicates(unittest.TestCase):
- def test_no_duplicates(self):
- items = [
- schema.RedditItem(id="R1", title="Completely different topic A", url="", subreddit=""),
- schema.RedditItem(id="R2", title="Another unrelated subject B", url="", subreddit=""),
- ]
- result = dedupe.find_duplicates(items)
- self.assertEqual(result, [])
-
- def test_finds_duplicates(self):
- items = [
- schema.RedditItem(id="R1", title="Best practices for Claude Code skills", url="", subreddit=""),
- schema.RedditItem(id="R2", title="Best practices for Claude Code skills guide", url="", subreddit=""),
- ]
- result = dedupe.find_duplicates(items, threshold=0.7)
- self.assertEqual(len(result), 1)
- self.assertEqual(result[0], (0, 1))
-
-
-class TestDedupeItems(unittest.TestCase):
- def test_keeps_higher_scored(self):
- items = [
- schema.RedditItem(id="R1", title="Best practices for skills", url="", subreddit="", score=90),
- schema.RedditItem(id="R2", title="Best practices for skills guide", url="", subreddit="", score=50),
- ]
- result = dedupe.dedupe_items(items, threshold=0.6)
- self.assertEqual(len(result), 1)
- self.assertEqual(result[0].id, "R1")
-
- def test_keeps_all_unique(self):
- items = [
- schema.RedditItem(id="R1", title="Topic about apples", url="", subreddit="", score=90),
- schema.RedditItem(id="R2", title="Discussion of oranges", url="", subreddit="", score=50),
- ]
- result = dedupe.dedupe_items(items)
- self.assertEqual(len(result), 2)
-
- def test_empty_list(self):
- result = dedupe.dedupe_items([])
- self.assertEqual(result, [])
-
- def test_single_item(self):
- items = [schema.RedditItem(id="R1", title="Test", url="", subreddit="")]
- result = dedupe.dedupe_items(items)
- self.assertEqual(len(result), 1)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_dedupe_v3.py b/tests/test_dedupe_v3.py
new file mode 100644
index 0000000..3cb235f
--- /dev/null
+++ b/tests/test_dedupe_v3.py
@@ -0,0 +1,217 @@
+"""Unit tests for dedupe.py: text normalization, similarity metrics, and deduplication."""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import dedupe
+from lib.schema import SourceItem
+
+
+def _item(title: str, body: str = "", source: str = "reddit", item_id: str = "t1") -> SourceItem:
+ return SourceItem(
+ item_id=item_id, source=source, title=title, body=body,
+ url="https://example.com", engagement={}, metadata={},
+ )
+
+
+# ---------------------------------------------------------------------------
+# normalize_text
+# ---------------------------------------------------------------------------
+
+class TestNormalizeText(unittest.TestCase):
+
+ def test_lowercases(self):
+ self.assertEqual(dedupe.normalize_text("Hello World"), "hello world")
+
+ def test_strips_punctuation(self):
+ self.assertEqual(dedupe.normalize_text("it's a test!"), "it s a test")
+
+ def test_collapses_whitespace(self):
+ self.assertEqual(dedupe.normalize_text("a b\t\nc"), "a b c")
+
+ def test_empty_string(self):
+ self.assertEqual(dedupe.normalize_text(""), "")
+
+
+# ---------------------------------------------------------------------------
+# get_ngrams
+# ---------------------------------------------------------------------------
+
+class TestGetNgrams(unittest.TestCase):
+
+ def test_simple_trigrams(self):
+ ngrams = dedupe.get_ngrams("abcde")
+ self.assertEqual(ngrams, {"abc", "bcd", "cde"})
+
+ def test_short_text_returns_whole(self):
+ ngrams = dedupe.get_ngrams("ab")
+ self.assertEqual(ngrams, {"ab"})
+
+ def test_empty_returns_empty_set(self):
+ self.assertEqual(dedupe.get_ngrams(""), set())
+
+ def test_normalizes_before_ngrams(self):
+ # "A!B" -> "a b" after normalization -> {"a b"}
+ ngrams = dedupe.get_ngrams("A!B")
+ self.assertEqual(ngrams, {"a b"})
+
+
+# ---------------------------------------------------------------------------
+# jaccard_similarity
+# ---------------------------------------------------------------------------
+
+class TestJaccardSimilarity(unittest.TestCase):
+
+ def test_identical_sets(self):
+ self.assertAlmostEqual(dedupe.jaccard_similarity({"a", "b"}, {"a", "b"}), 1.0)
+
+ def test_disjoint_sets(self):
+ self.assertAlmostEqual(dedupe.jaccard_similarity({"a"}, {"b"}), 0.0)
+
+ def test_partial_overlap(self):
+ result = dedupe.jaccard_similarity({"a", "b", "c"}, {"b", "c", "d"})
+ self.assertAlmostEqual(result, 2.0 / 4.0)
+
+ def test_empty_left(self):
+ self.assertAlmostEqual(dedupe.jaccard_similarity(set(), {"a"}), 0.0)
+
+ def test_both_empty(self):
+ self.assertAlmostEqual(dedupe.jaccard_similarity(set(), set()), 0.0)
+
+
+# ---------------------------------------------------------------------------
+# token_jaccard
+# ---------------------------------------------------------------------------
+
+class TestTokenJaccard(unittest.TestCase):
+
+ def test_identical_texts(self):
+ self.assertAlmostEqual(dedupe.token_jaccard("hello world", "hello world"), 1.0)
+
+ def test_completely_different(self):
+ self.assertAlmostEqual(dedupe.token_jaccard("alpha beta", "gamma delta"), 0.0)
+
+ def test_filters_stopwords(self):
+ # "the" and "a" are stopwords, so "big cat" vs "big dog" should compare on {big, cat} vs {big, dog}
+ result = dedupe.token_jaccard("the big cat", "a big dog")
+ self.assertAlmostEqual(result, 1.0 / 3.0) # {big} / {big, cat, dog}
+
+ def test_filters_single_char_tokens(self):
+ # Single char tokens like "I" are filtered (len > 1)
+ result = dedupe.token_jaccard("I am great", "I am terrible")
+ # "am" is len 2, "great"/"terrible" are content
+ self.assertGreater(result, 0.0)
+
+
+# ---------------------------------------------------------------------------
+# hybrid_similarity
+# ---------------------------------------------------------------------------
+
+class TestHybridSimilarity(unittest.TestCase):
+
+ def test_identical_texts(self):
+ self.assertAlmostEqual(dedupe.hybrid_similarity("same text", "same text"), 1.0)
+
+ def test_completely_different(self):
+ result = dedupe.hybrid_similarity("aaaaaa", "zzzzzz")
+ self.assertLess(result, 0.1)
+
+ def test_takes_max_of_both_methods(self):
+ text_a = "OpenClaw security issues discussion"
+ text_b = "OpenClaw security issues thread"
+ ngram_sim = dedupe.jaccard_similarity(
+ dedupe.get_ngrams(text_a), dedupe.get_ngrams(text_b)
+ )
+ token_sim = dedupe.token_jaccard(text_a, text_b)
+ self.assertAlmostEqual(
+ dedupe.hybrid_similarity(text_a, text_b),
+ max(ngram_sim, token_sim),
+ )
+
+
+# ---------------------------------------------------------------------------
+# item_text
+# ---------------------------------------------------------------------------
+
+class TestItemText(unittest.TestCase):
+
+ def test_combines_fields(self):
+ item = _item("My Title", "My Body")
+ text = dedupe.item_text(item)
+ self.assertIn("My Title", text)
+ self.assertIn("My Body", text)
+
+ def test_skips_none_fields(self):
+ item = _item("Title", "")
+ item.author = None
+ item.container = None
+ text = dedupe.item_text(item)
+ self.assertEqual(text, "Title")
+
+ def test_includes_author_and_container(self):
+ item = _item("Title", "Body")
+ item.author = "john"
+ item.container = "r/python"
+ text = dedupe.item_text(item)
+ self.assertIn("john", text)
+ self.assertIn("r/python", text)
+
+
+# ---------------------------------------------------------------------------
+# dedupe_items
+# ---------------------------------------------------------------------------
+
+class TestDedupeItems(unittest.TestCase):
+
+ def test_keeps_unique_items(self):
+ items = [
+ _item("OpenClaw is amazing", item_id="a"),
+ _item("NanoClaw security review", item_id="b"),
+ _item("IronClaw Rust architecture", item_id="c"),
+ ]
+ result = dedupe.dedupe_items(items)
+ self.assertEqual(len(result), 3)
+
+ def test_removes_near_duplicates(self):
+ items = [
+ _item("OpenClaw vs NanoClaw comparison review", item_id="a"),
+ _item("OpenClaw vs NanoClaw comparison review thread", item_id="b"),
+ ]
+ result = dedupe.dedupe_items(items)
+ self.assertEqual(len(result), 1)
+ self.assertEqual(result[0].item_id, "a") # keeps first
+
+ def test_keeps_first_of_duplicates(self):
+ items = [
+ _item("Same title here", item_id="first"),
+ _item("Same title here", item_id="second"),
+ ]
+ result = dedupe.dedupe_items(items)
+ self.assertEqual(result[0].item_id, "first")
+
+ def test_empty_body_items_kept(self):
+ item = SourceItem(
+ item_id="empty", source="reddit", title="", body="",
+ url="", engagement={}, metadata={},
+ )
+ result = dedupe.dedupe_items([item])
+ self.assertEqual(len(result), 1)
+
+ def test_threshold_respected(self):
+ items = [
+ _item("OpenClaw security analysis", item_id="a"),
+ _item("OpenClaw security review", item_id="b"),
+ ]
+ # With threshold=1.0, only exact matches are removed
+ result = dedupe.dedupe_items(items, threshold=1.0)
+ self.assertEqual(len(result), 2)
+ # With threshold=0.3, these similar items collapse
+ result_loose = dedupe.dedupe_items(items, threshold=0.3)
+ self.assertEqual(len(result_loose), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_entity_extract_v3.py b/tests/test_entity_extract_v3.py
new file mode 100644
index 0000000..b779d2e
--- /dev/null
+++ b/tests/test_entity_extract_v3.py
@@ -0,0 +1,58 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import entity_extract
+
+
+class TestExtractSubreddits(unittest.TestCase):
+ def test_extracts_primary_subreddit(self):
+ items = [{"subreddit": "r/MachineLearning"}]
+ result = entity_extract._extract_subreddits(items)
+ self.assertIn("MachineLearning", result)
+
+ def test_extracts_subreddit_without_prefix(self):
+ items = [{"subreddit": "localLLaMA"}]
+ result = entity_extract._extract_subreddits(items)
+ self.assertIn("localLLaMA", result)
+
+ def test_extracts_cross_references_from_comment_insights(self):
+ items = [
+ {
+ "subreddit": "technology",
+ "comment_insights": ["check out r/MachineLearning and r/LocalLLaMA for more"],
+ }
+ ]
+ result = entity_extract._extract_subreddits(items)
+ self.assertIn("MachineLearning", result)
+ self.assertIn("LocalLLaMA", result)
+
+ def test_extracts_cross_references_from_top_comments(self):
+ items = [
+ {
+ "subreddit": "AI",
+ "top_comments": [
+ {"excerpt": "see r/StableDiffusion for image gen stuff"},
+ ],
+ }
+ ]
+ result = entity_extract._extract_subreddits(items)
+ self.assertIn("StableDiffusion", result)
+
+ def test_ranks_by_frequency(self):
+ items = [
+ {"subreddit": "A"},
+ {"subreddit": "A"},
+ {"subreddit": "B"},
+ ]
+ result = entity_extract._extract_subreddits(items)
+ self.assertEqual(result[0], "A")
+
+ def test_empty_items(self):
+ self.assertEqual([], entity_extract._extract_subreddits([]))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_env_cookies.py b/tests/test_env_cookies.py
index aabb00c..4be7d69 100644
--- a/tests/test_env_cookies.py
+++ b/tests/test_env_cookies.py
@@ -1,15 +1,16 @@
"""Tests for browser cookie extraction integration in env.py."""
-from unittest.mock import patch, MagicMock
+import os
+import sys
+from pathlib import Path
+from unittest.mock import patch
import pytest
-from scripts.lib.env import extract_browser_credentials, COOKIE_DOMAINS
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+from lib.env import extract_browser_credentials, COOKIE_DOMAINS
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
def _base_config(**overrides):
"""Return a minimal config dict with common defaults."""
@@ -24,155 +25,105 @@ def _base_config(**overrides):
return cfg
-# ---------------------------------------------------------------------------
-# Tests
-# ---------------------------------------------------------------------------
-
class TestExtractBrowserCredentials:
"""Unit tests for extract_browser_credentials()."""
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- def test_auto_with_setup_complete_populates_credentials(self, mock_extract):
- """FROM_BROWSER=auto, SETUP_COMPLETE=true, mock returns valid cookies
- -> config now contains AUTH_TOKEN and CT0."""
- mock_extract.return_value = ({"auth_token": "tok123", "ct0": "ct0val"}, "chrome")
-
- config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
+ @patch("lib.cookie_extract.extract_cookies")
+ def test_auto_populates_credentials(self, mock_extract):
+ mock_extract.return_value = {"auth_token": "tok123", "ct0": "ct0val"}
+ config = _base_config(FROM_BROWSER="auto")
result = extract_browser_credentials(config)
-
assert result["AUTH_TOKEN"] == "tok123"
assert result["CT0"] == "ct0val"
- # Should have been called for x domain
- mock_extract.assert_any_call("auto", ".x.com", ["auth_token", "ct0"])
+ # auto mode tries firefox first, then safari, then chrome
+ mock_extract.assert_any_call("firefox", ".x.com", ["auth_token", "ct0"])
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
+ @patch("lib.cookie_extract.extract_cookies")
def test_explicit_auth_token_skips_x_extraction(self, mock_extract):
- """Config already has AUTH_TOKEN and CT0 from env var
- -> cookie extraction skipped for X (explicit takes priority)."""
- # Return None for any non-X domains that still get checked (e.g. Truth Social)
mock_extract.return_value = None
config = _base_config(
- AUTH_TOKEN="explicit_token",
- CT0="explicit_ct0",
+ AUTH_TOKEN="explicit_token", CT0="explicit_ct0",
FROM_BROWSER="auto",
- SETUP_COMPLETE="true",
)
result = extract_browser_credentials(config)
-
- # X cookies should not appear in result (already set)
assert "AUTH_TOKEN" not in result
assert "CT0" not in result
- # extract_cookies should NOT have been called for .x.com
for call in mock_extract.call_args_list:
- assert call[0][1] != ".x.com", "Should not extract cookies for X when credentials already set"
+ assert call[0][1] != ".x.com"
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
+ @patch("lib.cookie_extract.extract_cookies")
def test_from_browser_off_skips_all(self, mock_extract):
- """FROM_BROWSER=off -> no cookie extraction attempted."""
- config = _base_config(FROM_BROWSER="off", SETUP_COMPLETE="true")
+ config = _base_config(FROM_BROWSER="off")
result = extract_browser_credentials(config)
-
assert result == {}
mock_extract.assert_not_called()
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- def test_no_setup_complete_no_from_browser_defaults_off(self, mock_extract):
- """FROM_BROWSER not set and SETUP_COMPLETE not set
- -> no extraction (wizard hasn't run)."""
- config = _base_config() # both None
+ @patch("lib.cookie_extract.extract_cookies")
+ def test_no_from_browser_defaults_to_silent(self, mock_extract):
+ """Default (no FROM_BROWSER): tries Firefox and Safari only, skips Chrome."""
+ mock_extract.return_value = None
+ config = _base_config()
result = extract_browser_credentials(config)
-
assert result == {}
- mock_extract.assert_not_called()
+ # Should try firefox and safari but NOT chrome
+ browser_args = [call[0][0] for call in mock_extract.call_args_list]
+ assert "firefox" in browser_args
+ assert "safari" in browser_args
+ assert "chrome" not in browser_args
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- def test_setup_complete_no_from_browser_defaults_auto(self, mock_extract):
- """SETUP_COMPLETE is set but FROM_BROWSER is not
- -> defaults to 'auto'."""
- mock_extract.return_value = ({"auth_token": "found", "ct0": "found_ct0"}, "firefox")
-
- config = _base_config(SETUP_COMPLETE="true") # FROM_BROWSER=None
- result = extract_browser_credentials(config)
-
- assert result["AUTH_TOKEN"] == "found"
- # extract_cookies should have been called with 'auto'
- mock_extract.assert_any_call("auto", ".x.com", ["auth_token", "ct0"])
-
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
+ @patch("lib.cookie_extract.extract_cookies")
def test_from_browser_firefox_only(self, mock_extract):
- """FROM_BROWSER=firefox -> only Firefox extraction attempted."""
- mock_extract.return_value = ({"auth_token": "ff_tok", "ct0": "ff_ct0"}, "firefox")
-
- config = _base_config(FROM_BROWSER="firefox", SETUP_COMPLETE="true")
+ mock_extract.return_value = {"auth_token": "ff_tok", "ct0": "ff_ct0"}
+ config = _base_config(FROM_BROWSER="firefox")
result = extract_browser_credentials(config)
-
assert result["AUTH_TOKEN"] == "ff_tok"
- # All calls should use 'firefox' as the browser arg
for call in mock_extract.call_args_list:
assert call[0][0] == "firefox"
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
+ @patch("lib.cookie_extract.extract_cookies")
def test_extraction_returns_none_config_unchanged(self, mock_extract):
- """Cookie extraction returns None for X -> config unchanged for AUTH_TOKEN."""
mock_extract.return_value = None
-
- config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
+ config = _base_config(FROM_BROWSER="auto")
result = extract_browser_credentials(config)
-
assert "AUTH_TOKEN" not in result
assert "CT0" not in result
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- def test_extraction_raises_exception_config_unchanged(self, mock_extract):
- """Cookie extraction raises exception -> caught, config unchanged."""
+ @patch("lib.cookie_extract.extract_cookies")
+ def test_extraction_raises_exception_caught(self, mock_extract):
mock_extract.side_effect = RuntimeError("database locked")
-
- config = _base_config(FROM_BROWSER="auto", SETUP_COMPLETE="true")
+ config = _base_config(FROM_BROWSER="auto")
result = extract_browser_credentials(config)
-
- # Should not raise, and no credentials populated
assert "AUTH_TOKEN" not in result
assert "CT0" not in result
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
+ @patch("lib.cookie_extract.extract_cookies")
def test_partial_credentials_only_fills_missing(self, mock_extract):
- """If AUTH_TOKEN is set but CT0 is not, only CT0 gets filled."""
- mock_extract.return_value = ({"auth_token": "cookie_tok", "ct0": "cookie_ct0"}, "chrome")
-
+ mock_extract.return_value = {"auth_token": "cookie_tok", "ct0": "cookie_ct0"}
config = _base_config(
- AUTH_TOKEN="explicit",
- CT0=None,
+ AUTH_TOKEN="explicit", CT0=None,
FROM_BROWSER="auto",
- SETUP_COMPLETE="true",
)
result = extract_browser_credentials(config)
-
- # AUTH_TOKEN already set, should not be overridden
assert "AUTH_TOKEN" not in result
- # CT0 was missing, should be filled
assert result["CT0"] == "cookie_ct0"
class TestGetConfigCookieIntegration:
"""Integration test: get_config() calls extract_browser_credentials."""
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- @patch("scripts.lib.env._find_project_env", return_value=None)
- @patch("scripts.lib.env.load_env_file", return_value={})
- @patch("scripts.lib.env.get_openai_auth")
+ @patch("lib.cookie_extract.extract_cookies")
+ @patch("lib.env._find_project_env", return_value=None)
+ @patch("lib.env.load_env_file", return_value={})
+ @patch("lib.env.get_openai_auth")
def test_get_config_injects_cookies(
self, mock_openai, mock_load, mock_proj, mock_extract
):
- """get_config merges browser cookies into the returned config."""
- from scripts.lib.env import get_config, OpenAIAuth
-
+ from lib.env import get_config, OpenAIAuth
mock_openai.return_value = OpenAIAuth(
token=None, source="none", status="missing",
account_id=None, codex_auth_file="/fake",
)
- mock_extract.return_value = ({"auth_token": "browser_tok", "ct0": "browser_ct0"}, "firefox")
-
- import os
+ mock_extract.return_value = {"auth_token": "browser_tok", "ct0": "browser_ct0"}
env_patch = {
"SETUP_COMPLETE": "true",
"FROM_BROWSER": "auto",
@@ -180,6 +131,5 @@ class TestGetConfigCookieIntegration:
}
with patch.dict(os.environ, env_patch, clear=False):
config = get_config()
-
assert config["AUTH_TOKEN"] == "browser_tok"
assert config["CT0"] == "browser_ct0"
diff --git a/tests/test_env_project.py b/tests/test_env_project.py
deleted file mode 100644
index 5d3e698..0000000
--- a/tests/test_env_project.py
+++ /dev/null
@@ -1,236 +0,0 @@
-"""Tests for per-project .env config discovery and precedence."""
-
-import os
-import sys
-import unittest
-from pathlib import Path
-from unittest.mock import patch
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import env
-
-
-class TestFindProjectEnv(unittest.TestCase):
- """Tests for _find_project_env() directory walking."""
-
- def test_finds_env_in_cwd(self, ):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- claude_dir = Path(tmpdir) / ".claude"
- claude_dir.mkdir()
- env_file = claude_dir / "last30days.env"
- env_file.write_text("OPENAI_API_KEY=sk-test\n")
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)):
- result = env._find_project_env()
- self.assertIsNotNone(result)
- self.assertEqual(result.name, "last30days.env")
-
- def test_returns_none_when_no_config(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)):
- result = env._find_project_env()
- self.assertIsNone(result)
-
-
-class TestConfigPrecedence(unittest.TestCase):
- """Tests for config source priority."""
-
- def test_project_overrides_global(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- # Create global config
- global_dir = Path(tmpdir) / "global"
- global_dir.mkdir()
- global_env = global_dir / ".env"
- global_env.write_text("BRAVE_API_KEY=global-key\nPARALLEL_API_KEY=global-only\n")
-
- # Create project config
- project_dir = Path(tmpdir) / "project" / ".claude"
- project_dir.mkdir(parents=True)
- project_env = project_dir / "last30days.env"
- project_env.write_text("BRAVE_API_KEY=project-key\n")
-
- with patch.object(Path, 'cwd', return_value=Path(tmpdir) / "project"), \
- patch.object(env, 'CONFIG_FILE', global_env), \
- patch.dict(os.environ, {}, clear=False):
- # Remove any env vars that would override
- for k in ('BRAVE_API_KEY', 'PARALLEL_API_KEY', 'OPENAI_API_KEY'):
- os.environ.pop(k, None)
- config = env.get_config()
- # Project value wins over global
- self.assertEqual(config['BRAVE_API_KEY'], 'project-key')
- # Global value still available for keys not in project
- self.assertEqual(config['PARALLEL_API_KEY'], 'global-only')
-
- def test_env_var_overrides_project(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- project_dir = Path(tmpdir) / ".claude"
- project_dir.mkdir()
- project_env = project_dir / "last30days.env"
- project_env.write_text("BRAVE_API_KEY=project-key\n")
-
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', None), \
- patch.dict(os.environ, {'BRAVE_API_KEY': 'env-key'}):
- config = env.get_config()
- self.assertEqual(config['BRAVE_API_KEY'], 'env-key')
-
- def test_gemini_keys_load_from_project_env(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- project_dir = Path(tmpdir) / ".claude"
- project_dir.mkdir()
- project_env = project_dir / "last30days.env"
- project_env.write_text("GEMINI_API_KEY=gem-key\nGEMINI_MODEL=gemini-3-pro-preview\n")
-
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', None), \
- patch.dict(os.environ, {}, clear=False):
- os.environ.pop('GEMINI_API_KEY', None)
- os.environ.pop('GEMINI_MODEL', None)
- config = env.get_config()
- self.assertEqual(config['GEMINI_API_KEY'], 'gem-key')
- self.assertEqual(config['GEMINI_MODEL'], 'gemini-3-pro-preview')
-
- def test_google_api_key_loads_from_project_env(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- project_dir = Path(tmpdir) / ".claude"
- project_dir.mkdir()
- project_env = project_dir / "last30days.env"
- project_env.write_text("GOOGLE_API_KEY=google-key\n")
-
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', None), \
- patch.dict(os.environ, {}, clear=False):
- os.environ.pop('GOOGLE_API_KEY', None)
- config = env.get_config()
- self.assertEqual(config['GOOGLE_API_KEY'], 'google-key')
-
-
-class TestConfigSource(unittest.TestCase):
- """Tests for _CONFIG_SOURCE tracking."""
-
- def test_tracks_project_source(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- project_dir = Path(tmpdir) / ".claude"
- project_dir.mkdir()
- project_env = project_dir / "last30days.env"
- project_env.write_text("BRAVE_API_KEY=test\n")
-
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', None):
- config = env.get_config()
- self.assertIn('project:', config['_CONFIG_SOURCE'])
-
- def test_tracks_global_source(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- global_env = Path(tmpdir) / ".env"
- global_env.write_text("BRAVE_API_KEY=test\n")
-
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', global_env):
- config = env.get_config()
- self.assertIn('global:', config['_CONFIG_SOURCE'])
-
- def test_tracks_env_only(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', None):
- config = env.get_config()
- self.assertEqual(config['_CONFIG_SOURCE'], 'env_only')
-
-
-class TestConfigExists(unittest.TestCase):
- """Tests for config_exists() with project support."""
-
- def test_true_with_project_env(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- project_dir = Path(tmpdir) / ".claude"
- project_dir.mkdir()
- (project_dir / "last30days.env").write_text("KEY=val\n")
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)):
- self.assertTrue(env.config_exists())
-
- def test_true_with_global_env(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- global_env = Path(tmpdir) / ".env"
- global_env.write_text("KEY=val\n")
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', global_env):
- self.assertTrue(env.config_exists())
-
- def test_false_with_nothing(self):
- import tempfile
- with tempfile.TemporaryDirectory() as tmpdir:
- with patch.object(Path, 'cwd', return_value=Path(tmpdir)), \
- patch.object(env, 'CONFIG_FILE', None):
- self.assertFalse(env.config_exists())
-
-
-class TestFilePermissions(unittest.TestCase):
- """Tests for _check_file_permissions() warnings."""
-
- def test_warns_on_world_readable(self):
- import tempfile
- import io
- with tempfile.TemporaryDirectory() as tmpdir:
- f = Path(tmpdir) / ".env"
- f.write_text("KEY=val\n")
- f.chmod(0o644)
- stderr = io.StringIO()
- with patch('sys.stderr', stderr):
- env._check_file_permissions(f)
- self.assertIn("WARNING", stderr.getvalue())
-
- def test_no_warning_on_600(self):
- import tempfile
- import io
- with tempfile.TemporaryDirectory() as tmpdir:
- f = Path(tmpdir) / ".env"
- f.write_text("KEY=val\n")
- f.chmod(0o600)
- stderr = io.StringIO()
- with patch('sys.stderr', stderr):
- env._check_file_permissions(f)
- self.assertEqual(stderr.getvalue(), "")
-
-
-class TestXSourceSelection(unittest.TestCase):
- """Tests for supported X backend selection."""
-
- def test_get_x_source_ignores_scrapecreators_key(self):
- config = {'SCRAPECREATORS_API_KEY': 'sc-key'}
-
- with patch('lib.bird_x.is_bird_installed', return_value=False):
- self.assertIsNone(env.get_x_source(config))
-
- def test_get_x_source_status_ignores_scrapecreators_key(self):
- config = {'SCRAPECREATORS_API_KEY': 'sc-key'}
- bird_status = {
- 'installed': True,
- 'authenticated': False,
- 'username': None,
- 'can_install': False,
- }
-
- with patch('lib.bird_x.get_bird_status', return_value=bird_status), \
- patch('lib.bird_x.is_bird_installed', return_value=True), \
- patch('lib.bird_x.is_bird_authenticated', return_value=None):
- status = env.get_x_source_status(config)
-
- self.assertIsNone(status['source'])
- self.assertFalse(status['xai_available'])
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_env_v3.py b/tests/test_env_v3.py
new file mode 100644
index 0000000..ab8a70c
--- /dev/null
+++ b/tests/test_env_v3.py
@@ -0,0 +1,43 @@
+import os
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import bird_x, env
+
+
+class EnvV3Tests(unittest.TestCase):
+ def setUp(self):
+ self._saved_credentials = dict(bird_x._credentials)
+
+ def tearDown(self):
+ bird_x._credentials.clear()
+ bird_x._credentials.update(self._saved_credentials)
+
+ def test_x_source_prefers_xai_without_bird_probe(self):
+ with mock.patch("lib.bird_x.is_bird_authenticated", side_effect=AssertionError("should not probe bird auth")):
+ source = env.get_x_source({"XAI_API_KEY": "test"})
+ self.assertEqual("xai", source)
+
+ def test_x_source_uses_bird_with_explicit_cookies(self):
+ with mock.patch("lib.bird_x.is_bird_installed", return_value=True):
+ source = env.get_x_source({"AUTH_TOKEN": "a", "CT0": "b"})
+ self.assertEqual("bird", source)
+ self.assertEqual("a", bird_x._credentials["AUTH_TOKEN"])
+ self.assertEqual("b", bird_x._credentials["CT0"])
+
+ def test_bird_auth_never_checks_browser_cookies(self):
+ with mock.patch("lib.bird_x.is_bird_installed", return_value=True), mock.patch(
+ "lib.bird_x.subprocess.run",
+ side_effect=AssertionError("browser-cookie whoami should not run"),
+ ):
+ bird_x._credentials.clear()
+ with mock.patch.dict(os.environ, {}, clear=False):
+ self.assertIsNone(bird_x.is_bird_authenticated())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_evaluate_search_quality.py b/tests/test_evaluate_search_quality.py
deleted file mode 100644
index 1b94ffa..0000000
--- a/tests/test_evaluate_search_quality.py
+++ /dev/null
@@ -1,134 +0,0 @@
-"""Tests for the local search-quality evaluation harness."""
-
-import sys
-import tempfile
-import unittest
-from pathlib import Path
-from unittest.mock import patch
-
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-import evaluate_search_quality as evalsq
-
-
-class TestMetrics(unittest.TestCase):
- def test_jaccard(self):
- self.assertAlmostEqual(evalsq.jaccard({"a", "b"}, {"b", "c"}), 1 / 3)
-
- def test_retention(self):
- self.assertAlmostEqual(evalsq.retention({"a", "b"}, {"b", "c"}), 0.5)
-
- def test_precision_at_k(self):
- ranking = [
- {"key": "a", "source": "reddit"},
- {"key": "b", "source": "x"},
- {"key": "c", "source": "youtube"},
- ]
- judgments = {"a": 3, "b": 1, "c": 2}
- self.assertAlmostEqual(evalsq.precision_at_k(ranking, judgments, 2), 0.5)
-
- def test_ndcg_at_k(self):
- ranking = [
- {"key": "a", "source": "reddit"},
- {"key": "b", "source": "x"},
- {"key": "c", "source": "youtube"},
- ]
- judgments = {"a": 3, "b": 0, "c": 2}
- self.assertGreater(evalsq.ndcg_at_k(ranking, judgments, 3), 0.8)
-
- def test_ndcg_at_k_uses_best_items_from_judged_pool(self):
- ranking = [
- {"key": "a", "source": "reddit"},
- {"key": "b", "source": "x"},
- {"key": "c", "source": "youtube"},
- ]
- judged_pool = ranking + [
- {"key": "d", "source": "reddit"},
- {"key": "e", "source": "x"},
- ]
- judgments = {"a": 3, "b": 0, "c": 0, "d": 3, "e": 2}
- self.assertLess(
- evalsq.ndcg_at_k(ranking, judgments, 3, judged_pool),
- 1.0,
- )
-
- def test_source_coverage_recall_uses_union_pool(self):
- judged_pool = [
- {"key": "a", "source": "reddit"},
- {"key": "b", "source": "x"},
- {"key": "c", "source": "youtube"},
- ]
- ranking = [
- {"key": "a", "source": "reddit"},
- {"key": "b", "source": "x"},
- ]
- judgments = {"a": 3, "b": 0, "c": 2}
- self.assertAlmostEqual(evalsq.source_coverage_recall(ranking, judged_pool, judgments), 0.5)
-
-
-class TestRankedItems(unittest.TestCase):
- def test_build_ranked_items_sorts_by_score(self):
- report = {
- "reddit": [{"id": "R1", "title": "Low", "url": "r1", "score": 20}],
- "x": [{"id": "X1", "text": "High", "url": "x1", "score": 90}],
- "youtube": [],
- "tiktok": [],
- "instagram": [],
- "hackernews": [],
- "bluesky": [],
- "truthsocial": [],
- "polymarket": [],
- "websearch": [],
- }
- ranked = evalsq.build_ranked_items(report, per_source_limit=5)
- self.assertEqual(ranked[0]["key"], "x1")
-
-
-class TestPathWithoutNode(unittest.TestCase):
- def test_removes_node_entries(self):
- path = "/usr/bin:/tmp/node-bin:/opt/homebrew/bin"
-
- def fake_exists(path_obj):
- return str(path_obj).endswith("/tmp/node-bin/node")
-
- with patch.object(evalsq.Path, "exists", fake_exists):
- filtered = evalsq.path_without_node(path)
- self.assertEqual(filtered, "/usr/bin:/opt/homebrew/bin")
-
-
-class TestEvalToolPath(unittest.TestCase):
- def test_wraps_ytdlp_with_ignore_config(self):
- with tempfile.TemporaryDirectory() as tmpdir:
- eval_home = Path(tmpdir)
- with patch.object(evalsq.shutil, "which", return_value="/opt/homebrew/bin/yt-dlp"):
- path_value = evalsq.create_eval_tool_path(eval_home, "/usr/bin")
- wrapper = eval_home / "bin" / "yt-dlp"
- self.assertTrue(wrapper.exists())
- text = wrapper.read_text()
- self.assertIn("--ignore-config", text)
- self.assertIn("--no-cookies-from-browser", text)
- self.assertEqual(path_value, f"{eval_home / 'bin'}:/usr/bin")
-
-
-class TestJudgeKeyResolution(unittest.TestCase):
- def test_prefers_google_api_key(self):
- config = {
- "GOOGLE_API_KEY": "google-key",
- "GEMINI_API_KEY": "gem-key",
- "GOOGLE_GENAI_API_KEY": "genai-key",
- }
- self.assertEqual(evalsq.resolve_google_judge_api_key(config), "google-key")
-
- def test_falls_back_to_gemini_aliases(self):
- self.assertEqual(
- evalsq.resolve_google_judge_api_key({"GEMINI_API_KEY": "gem-key"}),
- "gem-key",
- )
- self.assertEqual(
- evalsq.resolve_google_judge_api_key({"GOOGLE_GENAI_API_KEY": "genai-key"}),
- "genai-key",
- )
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_evaluator_v3.py b/tests/test_evaluator_v3.py
new file mode 100644
index 0000000..604ef32
--- /dev/null
+++ b/tests/test_evaluator_v3.py
@@ -0,0 +1,203 @@
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+import evaluate_search_quality as evaluator
+
+
+class EvaluatorV3Tests(unittest.TestCase):
+ def test_build_ranked_items_uses_multi_source_provenance_and_best_date(self):
+ report = {
+ "ranked_candidates": [
+ {
+ "candidate_id": "c1",
+ "item_id": "i1",
+ "source": "grounding",
+ "sources": ["grounding", "reddit"],
+ "title": "Title",
+ "url": "https://example.com",
+ "snippet": "Snippet",
+ "subquery_labels": ["primary"],
+ "native_ranks": {"primary:grounding": 1},
+ "local_relevance": 0.8,
+ "freshness": 90,
+ "engagement": None,
+ "source_quality": 1.0,
+ "rrf_score": 0.02,
+ "final_score": 88.0,
+ "source_items": [
+ {"item_id": "i1", "source": "grounding", "title": "Title", "body": "Body", "url": "https://example.com", "published_at": "2026-03-10"},
+ {"item_id": "i2", "source": "reddit", "title": "Title", "body": "Body", "url": "https://example.com", "published_at": "2026-03-12"},
+ ],
+ }
+ ]
+ }
+ items = evaluator.build_ranked_items(report, 10)
+ self.assertEqual(["grounding", "reddit"], items[0]["sources"])
+ self.assertEqual("grounding, reddit", items[0]["source"])
+ self.assertEqual("2026-03-12", items[0]["date"])
+
+ grouped = evaluator.source_sets(report, 10)
+ self.assertEqual({"c1"}, grouped["grounding"])
+ self.assertEqual({"c1"}, grouped["reddit"])
+
+ def test_write_failure_summary_persists_failures(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ output_dir = Path(tmp)
+ evaluator.write_failure_summary(
+ output_dir,
+ "HEAD~1",
+ "HEAD",
+ summaries=[{
+ "topic": "test topic",
+ "baseline": {"precision_at_5": 0.5, "ndcg_at_5": 0.6, "source_coverage_recall": 1.0},
+ "candidate": {"precision_at_5": 0.7, "ndcg_at_5": 0.8, "source_coverage_recall": 1.0},
+ "stability": {"overall_jaccard": 0.4, "overall_retention_vs_baseline": 0.9},
+ }],
+ failures=[{"topic": "broken topic", "error": "timeout"}],
+ )
+ metrics = json.loads((output_dir / "metrics.json").read_text())
+ summary = (output_dir / "summary.md").read_text()
+ self.assertEqual(1, len(metrics["failures"]))
+ self.assertIn("broken topic", summary)
+ self.assertIn("## Failures", summary)
+
+ def test_resolve_repo_dir_keeps_live_worktree(self):
+ repo_dir, is_temp = evaluator.resolve_repo_dir("WORKTREE")
+ self.assertEqual(evaluator.REPO_ROOT, repo_dir)
+ self.assertFalse(is_temp)
+
+ def test_resolve_repo_dir_materializes_git_ref_in_temp_worktree(self):
+ fake_dir = Path("/tmp/last30days-eval-fake")
+ with mock.patch.object(evaluator, "create_worktree", return_value=fake_dir) as create_worktree:
+ repo_dir, is_temp = evaluator.resolve_repo_dir("HEAD~2")
+ create_worktree.assert_called_once_with("HEAD~2")
+ self.assertEqual(fake_dir, repo_dir)
+ self.assertTrue(is_temp)
+
+ def test_metric_helpers_cover_empty_and_ranked_cases(self):
+ ranking = [
+ {"key": "a", "sources": ["grounding"]},
+ {"key": "b", "sources": ["reddit"]},
+ ]
+ judged = [{"key": "a", "sources": ["grounding"]}, {"key": "b", "sources": ["reddit"]}]
+ judgments = {"a": 3, "b": 1}
+
+ self.assertEqual(1.0, evaluator.jaccard(set(), set()))
+ self.assertEqual(1.0, evaluator.retention(set(), {"a"}))
+ self.assertEqual(0.5, evaluator.precision_at_k(ranking, judgments, 2))
+ self.assertGreater(evaluator.ndcg_at_k(ranking, judgments, 2, judged), 0.0)
+ self.assertEqual(1.0, evaluator.source_coverage_recall(ranking, judged, judgments))
+ self.assertEqual(0.0, evaluator.precision_at_k([], judgments, 5))
+ self.assertEqual(0.0, evaluator.ndcg_at_k([], judgments, 5, judged))
+
+ def test_resolve_google_judge_api_key_prefers_google_key(self):
+ with mock.patch.dict("os.environ", {"GOOGLE_API_KEY": "google", "GEMINI_API_KEY": "gemini"}, clear=False):
+ self.assertEqual("google", evaluator.resolve_google_judge_api_key({}))
+ self.assertEqual("fallback", evaluator.resolve_google_judge_api_key({"GOOGLE_GENAI_API_KEY": "fallback"}))
+
+ def test_extract_gemini_text_raises_when_missing(self):
+ self.assertEqual(
+ "hello",
+ evaluator.extract_gemini_text({"candidates": [{"content": {"parts": [{"text": "hello"}]}}]}),
+ )
+ with self.assertRaises(ValueError):
+ evaluator.extract_gemini_text({"candidates": [{"content": {"parts": [{}]}}]})
+
+ def test_get_judgments_uses_cache_and_skips_when_not_configured(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ output_dir = Path(tmp)
+ cache_dir = output_dir / "judgments"
+ cache_dir.mkdir()
+ (cache_dir / "topic.json").write_text(json.dumps({"judgments": [{"id": "a", "grade": 3}]}))
+ cached = evaluator.get_judgments(
+ output_dir=output_dir,
+ slug="topic",
+ topic="test topic",
+ query_type="general",
+ items=[{"key": "a"}],
+ judge_model="gemini-3.1-flash-lite-preview",
+ gemini_api_key="key",
+ )
+ self.assertEqual({"a": 3}, cached)
+
+ skipped = evaluator.get_judgments(
+ output_dir=output_dir,
+ slug="fresh",
+ topic="test topic",
+ query_type="general",
+ items=[],
+ judge_model="gemini-3.1-flash-lite-preview",
+ gemini_api_key=None,
+ )
+ self.assertEqual({}, skipped)
+
+ def test_create_eval_env_and_run_last30days(self):
+ with mock.patch.object(evaluator.envlib, "get_config", return_value={"OPENAI_API_KEY": "config-openai"}):
+ with mock.patch.dict("os.environ", {"PATH": "/bin", "GOOGLE_API_KEY": "env-google"}, clear=False):
+ created = evaluator.create_eval_env()
+ self.assertEqual("/bin", created["PATH"])
+ self.assertEqual("env-google", created["GOOGLE_API_KEY"])
+ self.assertEqual("config-openai", created["OPENAI_API_KEY"])
+ self.assertEqual("", created["LAST30DAYS_CONFIG_DIR"])
+
+ with mock.patch.object(evaluator.subprocess, "run", return_value=mock.Mock(returncode=0, stdout='{"topic":"x"}', stderr="")):
+ payload = evaluator.run_last30days(
+ Path("/tmp/repo"),
+ "topic",
+ search="reddit",
+ timeout_seconds=30,
+ quick=True,
+ mock=True,
+ env={"PATH": "/bin"},
+ )
+ self.assertEqual("x", payload["topic"])
+
+ with mock.patch.object(evaluator.subprocess, "run", return_value=mock.Mock(returncode=2, stdout="", stderr="bad run")):
+ with self.assertRaises(RuntimeError):
+ evaluator.run_last30days(
+ Path("/tmp/repo"),
+ "topic",
+ search="reddit",
+ timeout_seconds=30,
+ quick=False,
+ mock=False,
+ env={"PATH": "/bin"},
+ )
+
+ def test_parse_topics_file_and_summary_writer(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ tmp_path = Path(tmp)
+ topics_path = tmp_path / "topics.json"
+ topics_path.write_text(json.dumps([{"topic": "topic a", "query_type": "comparison"}, {"topic": "topic b"}]))
+ self.assertEqual(
+ [("topic a", "comparison"), ("topic b", "general")],
+ evaluator.parse_topics_file(topics_path),
+ )
+
+ evaluator.write_summary(
+ tmp_path,
+ "HEAD~1",
+ "WORKTREE",
+ [
+ {
+ "topic": "topic a",
+ "baseline": {"precision_at_5": 0.1, "ndcg_at_5": 0.2, "source_coverage_recall": 0.5},
+ "candidate": {"precision_at_5": 0.3, "ndcg_at_5": 0.4, "source_coverage_recall": 0.8},
+ "stability": {"overall_jaccard": 0.6, "overall_retention_vs_baseline": 0.7},
+ }
+ ],
+ )
+ summary = (tmp_path / "summary.md").read_text()
+ metrics = json.loads((tmp_path / "metrics.json").read_text())
+ self.assertIn("| topic a | 0.10 | 0.30 |", summary)
+ self.assertEqual("HEAD~1", metrics["baseline"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_exa_search.py b/tests/test_exa_search.py
index 3692428..8473347 100644
--- a/tests/test_exa_search.py
+++ b/tests/test_exa_search.py
@@ -12,7 +12,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts'))
os.environ['LAST30DAYS_CONFIG_DIR'] = ''
from lib.exa_search import search_web, _normalize_results, _parse_exa_date, EXCLUDED_DOMAINS
-from lib import env, http
+from lib import http
class TestNormalizeResults(unittest.TestCase):
@@ -200,49 +200,27 @@ class TestSearchWebIntegration(unittest.TestCase):
self.assertEqual(results, [])
-class TestEnvExaPriority(unittest.TestCase):
- """Test that Exa is prioritized correctly in env.py."""
+class TestExaNormalizationV3Keys(unittest.TestCase):
+ """Test that Exa results include v3 normalization keys."""
- def test_no_exa_key_not_selected(self):
- config = {}
- self.assertIsNone(env.get_web_search_source(config))
-
- def test_exa_key_selected(self):
- config = {"EXA_API_KEY": "exa-test-key"}
- self.assertEqual(env.get_web_search_source(config), "exa")
-
- def test_exa_takes_priority_over_brave(self):
- config = {"EXA_API_KEY": "exa-key", "BRAVE_API_KEY": "brave-key"}
- self.assertEqual(env.get_web_search_source(config), "exa")
-
- def test_exa_takes_priority_over_parallel(self):
- config = {"EXA_API_KEY": "exa-key", "PARALLEL_API_KEY": "parallel-key"}
- self.assertEqual(env.get_web_search_source(config), "exa")
-
- def test_exa_takes_priority_over_openrouter(self):
- config = {"EXA_API_KEY": "exa-key", "OPENROUTER_API_KEY": "or-key"}
- self.assertEqual(env.get_web_search_source(config), "exa")
-
- def test_exa_takes_priority_over_all(self):
- config = {
- "EXA_API_KEY": "exa-key",
- "PARALLEL_API_KEY": "parallel-key",
- "BRAVE_API_KEY": "brave-key",
- "OPENROUTER_API_KEY": "or-key",
+ def test_results_include_engagement_and_metadata(self):
+ response = {
+ "results": [
+ {
+ "title": "Test Article",
+ "url": "https://example.com/test",
+ "text": "Content here",
+ "publishedDate": "2026-03-15T00:00:00.000Z",
+ "score": 0.8,
+ },
+ ]
}
- self.assertEqual(env.get_web_search_source(config), "exa")
-
- def test_fallback_to_parallel_without_exa(self):
- config = {"PARALLEL_API_KEY": "parallel-key", "BRAVE_API_KEY": "brave-key"}
- self.assertEqual(env.get_web_search_source(config), "parallel")
-
- def test_has_web_search_keys_with_exa(self):
- config = {"EXA_API_KEY": "exa-key"}
- self.assertTrue(env.has_web_search_keys(config))
-
- def test_has_web_search_keys_without_any(self):
- config = {}
- self.assertFalse(env.has_web_search_keys(config))
+ items = _normalize_results(response)
+ self.assertEqual(len(items), 1)
+ self.assertIn("engagement", items[0])
+ self.assertIn("metadata", items[0])
+ self.assertEqual(items[0]["engagement"], {})
+ self.assertEqual(items[0]["metadata"], {})
if __name__ == "__main__":
diff --git a/tests/test_fusion_v3.py b/tests/test_fusion_v3.py
new file mode 100644
index 0000000..618caf9
--- /dev/null
+++ b/tests/test_fusion_v3.py
@@ -0,0 +1,444 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import fusion, schema
+
+
+def make_item(item_id: str, source: str, url: str, title: str, rank_score: float) -> schema.SourceItem:
+ return schema.SourceItem(
+ item_id=item_id,
+ source=source,
+ title=title,
+ body=title,
+ url=url,
+ relevance_hint=rank_score,
+ snippet=title,
+ metadata={
+ "local_relevance": rank_score,
+ "freshness": 80,
+ "engagement_score": 5,
+ "source_quality": 0.7,
+ },
+ )
+
+
+class FusionV3Tests(unittest.TestCase):
+ def test_weighted_rrf_merges_duplicate_urls(self):
+ plan = schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test",
+ subqueries=[
+ schema.SubQuery(label="primary", search_query="test", ranking_query="What happened in test?", sources=["reddit", "x"], weight=0.7),
+ schema.SubQuery(label="reaction", search_query="test reaction", ranking_query="What are the reactions to test?", sources=["x"], weight=0.3),
+ ],
+ source_weights={"reddit": 0.4, "x": 0.6},
+ )
+ shared = "https://example.com/shared"
+ streams = {
+ ("primary", "reddit"): [make_item("r1", "reddit", shared, "Shared item", 0.8)],
+ ("primary", "x"): [make_item("x1", "x", shared, "Shared item", 0.9)],
+ ("reaction", "x"): [make_item("x2", "x", "https://example.com/unique", "Unique item", 0.7)],
+ }
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=10)
+ self.assertEqual(2, len(candidates))
+ merged = next(candidate for candidate in candidates if candidate.url == shared)
+ self.assertEqual({"primary"}, set(merged.subquery_labels))
+ self.assertEqual(2, len(merged.native_ranks))
+ self.assertEqual({"reddit", "x"}, set(merged.sources))
+ self.assertEqual(2, len(merged.source_items))
+
+
+ def test_diversify_pool_guarantees_min_per_qualifying_source(self):
+ """Every qualifying source (local_relevance >= 0.25) gets at least 2
+ items in the fused pool.
+
+ Dominant sources (x, tiktok) get high weights, so pure-RRF truncation
+ would squeeze out low-weight sources entirely. The diversity guarantee
+ must reserve at least 2 slots per qualifying active source. All sources
+ here have rank_score=0.8 (well above the 0.25 threshold), so every
+ source qualifies for reserved slots.
+ """
+ sources = ["reddit", "hackernews", "x", "tiktok", "bluesky", "youtube"]
+ # Heavily skewed weights: x and tiktok dominate.
+ weights = {
+ "x": 3.0,
+ "tiktok": 2.5,
+ "reddit": 0.5,
+ "hackernews": 0.4,
+ "bluesky": 0.3,
+ "youtube": 0.3,
+ }
+ plan = schema.QueryPlan(
+ intent="concept",
+ freshness_mode="relaxed",
+ cluster_mode="concept",
+ raw_topic="RAG",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="RAG",
+ ranking_query="What is RAG?",
+ sources=sources,
+ weight=1.0,
+ ),
+ ],
+ source_weights=weights,
+ )
+ streams: dict[tuple[str, str], list[schema.SourceItem]] = {}
+ for src in sources:
+ items = []
+ for rank in range(4):
+ items.append(
+ make_item(
+ item_id=f"{src}_{rank}",
+ source=src,
+ url=f"https://{src}.example.com/{rank}",
+ title=f"{src} item {rank}",
+ rank_score=0.8,
+ )
+ )
+ streams[("primary", src)] = items
+
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=12)
+ self.assertEqual(12, len(candidates))
+
+ source_counts: dict[str, int] = {}
+ for c in candidates:
+ source_counts[c.source] = source_counts.get(c.source, 0) + 1
+
+ for src in sources:
+ self.assertGreaterEqual(
+ source_counts.get(src, 0),
+ 2,
+ f"Source '{src}' has {source_counts.get(src, 0)} items, expected >= 2",
+ )
+
+
+ def test_diversify_pool_denies_slots_for_low_relevance_source(self):
+ """Sources with best local_relevance < 0.25 do not get reserved slots.
+
+ Create two sources: 'x' with local_relevance=0.5 (qualifies) and
+ 'reddit' with local_relevance=0.1 (below threshold). With a tight
+ pool_limit, the high-relevance source gets reserved slots while
+ the low-relevance source must compete on RRF merit alone.
+ """
+ plan = schema.QueryPlan(
+ intent="concept",
+ freshness_mode="relaxed",
+ cluster_mode="concept",
+ raw_topic="test",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="test",
+ ranking_query="What is test?",
+ sources=["x", "reddit"],
+ weight=1.0,
+ ),
+ ],
+ source_weights={"x": 1.0, "reddit": 1.0},
+ )
+
+ # x items: high relevance (0.5) -- qualifies for diversity reservation
+ x_items = [
+ make_item(f"x_{i}", "x", f"https://x.example.com/{i}", f"x item {i}", 0.5)
+ for i in range(4)
+ ]
+
+ # reddit items: low relevance (0.1) -- below threshold, no reserved slots
+ reddit_items = [
+ make_item(f"r_{i}", "reddit", f"https://reddit.example.com/{i}", f"reddit item {i}", 0.1)
+ for i in range(4)
+ ]
+
+ streams = {
+ ("primary", "x"): x_items,
+ ("primary", "reddit"): reddit_items,
+ }
+
+ # pool_limit=3: x gets 2 reserved + 1 more by RRF. Reddit has no
+ # reserved slots, so it must out-score x items in the remainder.
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=3)
+ self.assertEqual(3, len(candidates))
+
+ # x must have at least 2 (reserved slots)
+ x_count = sum(1 for c in candidates if c.source == "x")
+ self.assertGreaterEqual(x_count, 2, "x should have at least 2 reserved slots")
+
+ def test_diversify_pool_no_reservation_when_all_below_threshold(self):
+ """When all sources are below the relevance threshold, no reserved slots
+ are granted. The pool is filled purely by RRF score order."""
+ plan = schema.QueryPlan(
+ intent="concept",
+ freshness_mode="relaxed",
+ cluster_mode="concept",
+ raw_topic="test",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="test",
+ ranking_query="What is test?",
+ sources=["x", "reddit", "hackernews"],
+ weight=1.0,
+ ),
+ ],
+ # Give x a much higher weight so its items get higher RRF scores
+ source_weights={"x": 3.0, "reddit": 0.3, "hackernews": 0.3},
+ )
+
+ streams: dict[tuple[str, str], list[schema.SourceItem]] = {}
+ # All sources below threshold (local_relevance = 0.1)
+ for src in ["x", "reddit", "hackernews"]:
+ items = [
+ make_item(f"{src}_{i}", src, f"https://{src}.example.com/{i}", f"{src} item {i}", 0.1)
+ for i in range(4)
+ ]
+ streams[("primary", src)] = items
+
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=4)
+ self.assertEqual(4, len(candidates))
+
+ # With no diversity reservation and x having 3x the weight,
+ # x should dominate the top slots purely on RRF score
+ source_counts: dict[str, int] = {}
+ for c in candidates:
+ source_counts[c.source] = source_counts.get(c.source, 0) + 1
+
+ # x has 3x weight so its RRF scores are ~3x higher than reddit/hn.
+ # All 4 x items should beat all reddit/hackernews items.
+ self.assertEqual(
+ source_counts.get("x", 0),
+ 4,
+ f"Expected x to take all 4 slots on pure RRF merit, got {source_counts}",
+ )
+
+ def test_diversify_pool_threshold_boundary(self):
+ """Source with best local_relevance exactly at the threshold (0.25)
+ qualifies for reserved slots."""
+ plan = schema.QueryPlan(
+ intent="concept",
+ freshness_mode="relaxed",
+ cluster_mode="concept",
+ raw_topic="boundary",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="boundary",
+ ranking_query="What is boundary?",
+ sources=["x", "reddit"],
+ weight=1.0,
+ ),
+ ],
+ # Give x much higher weight so it would dominate without reservation
+ source_weights={"x": 5.0, "reddit": 0.1},
+ )
+
+ x_items = [
+ make_item(f"x_{i}", "x", f"https://x.example.com/{i}", f"x item {i}", 0.8)
+ for i in range(6)
+ ]
+
+ # reddit at exactly the threshold
+ reddit_items = [
+ make_item(f"r_{i}", "reddit", f"https://reddit.example.com/{i}", f"reddit item {i}", 0.25)
+ for i in range(3)
+ ]
+
+ streams = {
+ ("primary", "x"): x_items,
+ ("primary", "reddit"): reddit_items,
+ }
+
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=6)
+ self.assertEqual(6, len(candidates))
+
+ reddit_count = sum(1 for c in candidates if c.source == "reddit")
+ self.assertGreaterEqual(
+ reddit_count,
+ 2,
+ f"reddit (local_relevance=0.25, at threshold) should get 2 reserved slots, got {reddit_count}",
+ )
+
+
+def make_item_with_author(
+ item_id: str, source: str, url: str, title: str, rank_score: float, author: str | None = None,
+) -> schema.SourceItem:
+ return schema.SourceItem(
+ item_id=item_id,
+ source=source,
+ title=title,
+ body=title,
+ url=url,
+ author=author,
+ relevance_hint=rank_score,
+ snippet=title,
+ metadata={
+ "local_relevance": rank_score,
+ "freshness": 80,
+ "engagement_score": 5,
+ "source_quality": 0.7,
+ },
+ )
+
+
+class TestPerAuthorCap(unittest.TestCase):
+ """Per-author cap: no single author should have more than 3 items in fused pool."""
+
+ def _make_plan(self, sources: list[str]) -> schema.QueryPlan:
+ return schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="test",
+ ranking_query="test",
+ sources=sources,
+ weight=1.0,
+ ),
+ ],
+ source_weights={s: 1.0 for s in sources},
+ )
+
+ def test_author_with_8_items_capped_to_3(self):
+ """@grok scenario: 8 items from the same author, only best 3 survive."""
+ plan = self._make_plan(["x"])
+ items = [
+ make_item_with_author(
+ f"x_{i}", "x", f"https://x.com/{i}", f"grok summary {i}", 0.7, author="@grok",
+ )
+ for i in range(8)
+ ]
+ streams = {("primary", "x"): items}
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
+ grok_count = sum(
+ 1 for c in candidates
+ if any(si.author == "@grok" for si in c.source_items)
+ )
+ self.assertLessEqual(grok_count, 3, f"@grok should be capped at 3, got {grok_count}")
+
+ def test_author_with_3_items_all_kept(self):
+ """Author with exactly 3 items should keep all of them."""
+ plan = self._make_plan(["x"])
+ items = [
+ make_item_with_author(
+ f"x_{i}", "x", f"https://x.com/{i}", f"author3 post {i}", 0.7, author="@author3",
+ )
+ for i in range(3)
+ ]
+ streams = {("primary", "x"): items}
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
+ count = sum(
+ 1 for c in candidates
+ if any(si.author == "@author3" for si in c.source_items)
+ )
+ self.assertEqual(count, 3)
+
+ def test_items_without_author_not_capped(self):
+ """Items with no author field should never be dropped by the cap."""
+ plan = self._make_plan(["reddit"])
+ items = [
+ make_item_with_author(
+ f"r_{i}", "reddit", f"https://reddit.com/{i}", f"post {i}", 0.7, author=None,
+ )
+ for i in range(6)
+ ]
+ streams = {("primary", "reddit"): items}
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
+ self.assertEqual(len(candidates), 6)
+
+ def test_multiple_authors_capped_independently(self):
+ """Two prolific authors each get capped to 3 independently."""
+ plan = self._make_plan(["x"])
+ items = []
+ for i in range(5):
+ items.append(make_item_with_author(
+ f"grok_{i}", "x", f"https://x.com/grok/{i}", f"grok {i}", 0.7, author="@grok",
+ ))
+ for i in range(5):
+ items.append(make_item_with_author(
+ f"spam_{i}", "x", f"https://x.com/spam/{i}", f"spam {i}", 0.6, author="@spammer",
+ ))
+ streams = {("primary", "x"): items}
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
+ grok_count = sum(1 for c in candidates if any(si.author == "@grok" for si in c.source_items))
+ spam_count = sum(1 for c in candidates if any(si.author == "@spammer" for si in c.source_items))
+ self.assertLessEqual(grok_count, 3)
+ self.assertLessEqual(spam_count, 3)
+
+ def test_cap_keeps_best_items_by_rrf_order(self):
+ """The cap should keep the first (highest-ranked) items per author."""
+ plan = self._make_plan(["x"])
+ # Items with decreasing relevance scores so ranking is deterministic
+ items = [
+ make_item_with_author(
+ f"x_{i}", "x", f"https://x.com/{i}", f"post {i}", 0.9 - (i * 0.05), author="@prolific",
+ )
+ for i in range(5)
+ ]
+ streams = {("primary", "x"): items}
+ candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
+ kept_ids = {c.item_id for c in candidates if any(si.author == "@prolific" for si in c.source_items)}
+ # The top 3 items (x_0, x_1, x_2) should be kept
+ self.assertLessEqual(len(kept_ids), 3)
+
+
+class TestUrlNormalization(unittest.TestCase):
+ def test_strips_www(self):
+ from lib.fusion import _normalize_url
+ self.assertEqual(
+ _normalize_url("https://www.reddit.com/r/test"),
+ _normalize_url("https://reddit.com/r/test"),
+ )
+
+ def test_strips_old_prefix(self):
+ from lib.fusion import _normalize_url
+ self.assertEqual(
+ _normalize_url("https://old.reddit.com/r/test"),
+ _normalize_url("https://reddit.com/r/test"),
+ )
+
+ def test_strips_mobile_prefix(self):
+ from lib.fusion import _normalize_url
+ self.assertEqual(
+ _normalize_url("https://m.youtube.com/watch?v=abc"),
+ _normalize_url("https://youtube.com/watch?v=abc"),
+ )
+
+ def test_strips_utm_params(self):
+ from lib.fusion import _normalize_url
+ self.assertEqual(
+ _normalize_url("https://example.com/page?utm_source=twitter&id=5"),
+ _normalize_url("https://example.com/page?id=5"),
+ )
+
+ def test_strips_trailing_slash(self):
+ from lib.fusion import _normalize_url
+ self.assertEqual(
+ _normalize_url("https://example.com/page/"),
+ _normalize_url("https://example.com/page"),
+ )
+
+ def test_preserves_non_tracking_params(self):
+ from lib.fusion import _normalize_url
+ result = _normalize_url("https://example.com/page?id=5&sort=new")
+ self.assertIn("id=5", result)
+ self.assertIn("sort=new", result)
+
+ def test_case_insensitive(self):
+ from lib.fusion import _normalize_url
+ self.assertEqual(
+ _normalize_url("https://Reddit.com/r/Test"),
+ _normalize_url("https://reddit.com/r/test"),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_generate_synthesis_inputs_v3.py b/tests/test_generate_synthesis_inputs_v3.py
new file mode 100644
index 0000000..7fd1a44
--- /dev/null
+++ b/tests/test_generate_synthesis_inputs_v3.py
@@ -0,0 +1,123 @@
+import importlib.util
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import schema
+
+
+SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "generate-synthesis-inputs.py"
+
+
+def load_module():
+ spec = importlib.util.spec_from_file_location("generate_synthesis_inputs", SCRIPT_PATH)
+ module = importlib.util.module_from_spec(spec)
+ assert spec and spec.loader
+ spec.loader.exec_module(module)
+ return module
+
+
+class GenerateSynthesisInputsV3Tests(unittest.TestCase):
+ def test_main_uses_v3_report_deserializer(self):
+ module = load_module()
+ report = schema.Report(
+ topic="test topic",
+ range_from="2026-02-14",
+ range_to="2026-03-16",
+ generated_at="2026-03-16T00:00:00+00:00",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="gemini",
+ planner_model="gemini-3.1-flash-lite-preview",
+ rerank_model="gemini-3.1-flash-lite-preview",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test topic",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="test topic",
+ ranking_query="What happened with test topic?",
+ sources=["grounding"],
+ )
+ ],
+ source_weights={"grounding": 1.0},
+ ),
+ clusters=[
+ schema.Cluster(
+ cluster_id="cluster-1",
+ title="Title",
+ candidate_ids=["c1"],
+ representative_ids=["c1"],
+ sources=["grounding"],
+ score=90.0,
+ )
+ ],
+ ranked_candidates=[
+ schema.Candidate(
+ candidate_id="c1",
+ item_id="i1",
+ source="grounding",
+ sources=["grounding"],
+ title="Title",
+ url="https://example.com",
+ snippet="Snippet",
+ subquery_labels=["primary"],
+ native_ranks={"primary:grounding": 1},
+ local_relevance=0.8,
+ freshness=90,
+ engagement=None,
+ source_quality=1.0,
+ rrf_score=0.02,
+ rerank_score=91.0,
+ final_score=90.0,
+ source_items=[
+ schema.SourceItem(
+ item_id="i1",
+ source="grounding",
+ title="Title",
+ body="Body",
+ url="https://example.com",
+ published_at="2026-03-16",
+ )
+ ],
+ )
+ ],
+ items_by_source={
+ "grounding": [
+ schema.SourceItem(
+ item_id="i1",
+ source="grounding",
+ title="Title",
+ body="Body",
+ url="https://example.com",
+ )
+ ]
+ },
+ errors_by_source={},
+ )
+
+ with tempfile.TemporaryDirectory() as tmp:
+ json_dir = Path(tmp) / "json"
+ compact_dir = Path(tmp) / "compact"
+ json_dir.mkdir()
+ (json_dir / "sample.json").write_text(json.dumps(schema.to_dict(report)))
+
+ module.JSON_DIR = json_dir
+ module.COMPACT_DIR = compact_dir
+
+ result = module.main()
+
+ self.assertEqual(0, result)
+ output = (compact_dir / "sample.md").read_text()
+ self.assertIn("# last30days-3 v3.0.0-alpha: test topic", output)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_github.py b/tests/test_github.py
new file mode 100644
index 0000000..7ab9099
--- /dev/null
+++ b/tests/test_github.py
@@ -0,0 +1,147 @@
+"""Tests for GitHub source module."""
+
+import json
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+from lib import github
+
+
+class TestResolveToken(unittest.TestCase):
+ def test_explicit_token(self):
+ self.assertEqual(github._resolve_token("my-token"), "my-token")
+
+ @patch.dict("os.environ", {"GITHUB_TOKEN": "env-token"})
+ def test_env_token(self):
+ self.assertEqual(github._resolve_token(), "env-token")
+
+ @patch.dict("os.environ", {}, clear=True)
+ @patch("subprocess.run")
+ def test_gh_cli_fallback(self, mock_run):
+ mock_run.return_value = MagicMock(returncode=0, stdout="gh-token\n")
+ # Clear GITHUB_TOKEN from env for this test
+ result = github._resolve_token()
+ self.assertEqual(result, "gh-token")
+
+ @patch.dict("os.environ", {}, clear=True)
+ @patch("subprocess.run", side_effect=FileNotFoundError)
+ def test_no_token_available(self, mock_run):
+ result = github._resolve_token()
+ self.assertIsNone(result)
+
+
+class TestParseRepoFromUrl(unittest.TestCase):
+ def test_issue_url(self):
+ url = "https://github.com/facebook/react/issues/123"
+ self.assertEqual(github._parse_repo_from_url(url), "facebook/react")
+
+ def test_pr_url(self):
+ url = "https://github.com/vercel/next.js/pull/456"
+ self.assertEqual(github._parse_repo_from_url(url), "vercel/next.js")
+
+ def test_empty(self):
+ self.assertEqual(github._parse_repo_from_url(""), "")
+
+
+class TestParseDate(unittest.TestCase):
+ def test_iso_date(self):
+ self.assertEqual(github._parse_date("2026-03-15T12:00:00Z"), "2026-03-15")
+
+ def test_none(self):
+ self.assertIsNone(github._parse_date(None))
+
+ def test_empty(self):
+ self.assertIsNone(github._parse_date(""))
+
+
+class TestSearchGithub(unittest.TestCase):
+ @patch.dict("os.environ", {}, clear=True)
+ @patch("subprocess.run", side_effect=FileNotFoundError)
+ def test_no_token_returns_empty(self, mock_run):
+ result = github.search_github("react", "2026-03-01", "2026-03-31", token=None)
+ self.assertEqual(result, [])
+
+ @patch.object(github, "_fetch_json")
+ @patch.object(github, "_resolve_token", return_value="test-token")
+ def test_search_returns_items(self, mock_token, mock_fetch):
+ mock_fetch.return_value = {
+ "total_count": 1,
+ "items": [
+ {
+ "html_url": "https://github.com/facebook/react/issues/42",
+ "title": "React Server Components bug",
+ "body": "There is a bug when using RSC with streaming...",
+ "created_at": "2026-03-15T10:00:00Z",
+ "state": "open",
+ "comments": 12,
+ "reactions": {"total_count": 8},
+ "labels": [{"name": "bug"}, {"name": "rsc"}],
+ "user": {"login": "testuser"},
+ },
+ ],
+ }
+ result = github.search_github("react", "2026-03-01", "2026-03-31")
+ self.assertEqual(len(result), 1)
+ item = result[0]
+ self.assertEqual(item["source"], "github")
+ self.assertEqual(item["container"], "facebook/react")
+ self.assertEqual(item["title"], "React Server Components bug")
+ self.assertEqual(item["date"], "2026-03-15")
+ self.assertEqual(item["author"], "testuser")
+ self.assertIn("bug", item["metadata"]["labels"])
+ self.assertEqual(item["metadata"]["state"], "open")
+ self.assertEqual(item["metadata"]["comment_count"], 12)
+ self.assertEqual(item["metadata"]["reactions"], 8)
+ self.assertEqual(item["engagement"]["reactions"], 8)
+ self.assertEqual(item["engagement"]["comments"], 12)
+ self.assertFalse(item["metadata"]["is_pr"])
+
+ @patch.object(github, "_fetch_json", return_value=None)
+ @patch.object(github, "_resolve_token", return_value="test-token")
+ def test_rate_limit_returns_empty(self, mock_token, mock_fetch):
+ """403 rate limit returns empty list gracefully."""
+ result = github.search_github("react", "2026-03-01", "2026-03-31")
+ self.assertEqual(result, [])
+
+ @patch.object(github, "_fetch_json")
+ @patch.object(github, "_resolve_token", return_value="test-token")
+ def test_pr_detected(self, mock_token, mock_fetch):
+ mock_fetch.return_value = {
+ "total_count": 1,
+ "items": [
+ {
+ "html_url": "https://github.com/vercel/next.js/pull/99",
+ "title": "Add streaming support",
+ "body": "This PR adds...",
+ "created_at": "2026-03-20T10:00:00Z",
+ "state": "open",
+ "comments": 5,
+ "reactions": {"total_count": 3},
+ "labels": [],
+ "user": {"login": "dev"},
+ "pull_request": {"url": "..."},
+ },
+ ],
+ }
+ result = github.search_github("next.js", "2026-03-01", "2026-03-31")
+ self.assertEqual(len(result), 1)
+ self.assertTrue(result[0]["metadata"]["is_pr"])
+
+
+class TestComputeRelevance(unittest.TestCase):
+ def test_basic_relevance(self):
+ score = github._compute_relevance("react hooks", "React Hooks Tutorial", 0, 10, 5)
+ self.assertGreater(score, 0.5)
+ self.assertLessEqual(score, 1.0)
+
+ def test_lower_rank_lower_score(self):
+ high = github._compute_relevance("react", "React", 0, 0, 0)
+ low = github._compute_relevance("react", "React", 20, 0, 0)
+ self.assertGreater(high, low)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_grounding_v3.py b/tests/test_grounding_v3.py
new file mode 100644
index 0000000..e428c39
--- /dev/null
+++ b/tests/test_grounding_v3.py
@@ -0,0 +1,194 @@
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import grounding
+
+
+class BraveSearchTests(unittest.TestCase):
+ def test_brave_search_applies_freshness_and_filters_to_in_range_dated_items(self):
+ mock_response = {
+ "web": {
+ "results": [
+ {
+ "title": "Test Article",
+ "url": "https://example.com/article",
+ "description": "A test snippet",
+ "page_age": "2026-03-10T00:00:00",
+ },
+ {
+ "title": "Old Article",
+ "url": "https://example.com/old",
+ "description": "Should be filtered",
+ "page_age": "2025-12-10T00:00:00",
+ },
+ {
+ "title": "Undated Article",
+ "url": "https://example.com/undated",
+ "description": "Should also be filtered",
+ }
+ ]
+ }
+ }
+ with patch("lib.grounding.http.request", return_value=mock_response) as mock_req:
+ items, artifact = grounding.brave_search("test", ("2026-02-25", "2026-03-27"), "fake-key")
+ self.assertEqual(1, len(items))
+ self.assertEqual("Test Article", items[0]["title"])
+ self.assertEqual("https://example.com/article", items[0]["url"])
+ self.assertEqual("2026-03-10", items[0]["date"])
+ self.assertEqual("brave", artifact["label"])
+ call_url = mock_req.call_args.args[1]
+ self.assertIn("freshness=2026-02-25to2026-03-27", call_url)
+
+
+class SerperSearchTests(unittest.TestCase):
+ def test_serper_search_filters_to_in_range_dated_items(self):
+ mock_response = {
+ "organic": [
+ {
+ "title": "Serper Result",
+ "link": "https://example.com/serper",
+ "snippet": "A serper snippet",
+ "date": "Mar 15, 2026",
+ },
+ {
+ "title": "Old Result",
+ "link": "https://example.com/old",
+ "snippet": "Should be filtered",
+ "date": "Jan 15, 2026",
+ },
+ {
+ "title": "Undated Result",
+ "link": "https://example.com/undated",
+ "snippet": "Should also be filtered",
+ }
+ ]
+ }
+ with patch("lib.grounding.http.request", return_value=mock_response):
+ items, artifact = grounding.serper_search("test", ("2026-02-25", "2026-03-27"), "fake-key")
+ self.assertEqual(1, len(items))
+ self.assertEqual("Serper Result", items[0]["title"])
+ self.assertEqual("2026-03-15", items[0]["date"])
+ self.assertEqual("serper", artifact["label"])
+
+
+class ExaSearchTests(unittest.TestCase):
+ def test_exa_search_filters_to_in_range_dated_items(self):
+ mock_response = {
+ "results": [
+ {
+ "title": "Exa Result",
+ "url": "https://example.com/exa",
+ "text": "An exa snippet about AI trends",
+ "publishedDate": "2026-03-15T00:00:00.000Z",
+ "score": 0.85,
+ },
+ {
+ "title": "Old Exa Result",
+ "url": "https://example.com/old-exa",
+ "text": "Should be filtered out",
+ "publishedDate": "2025-12-01T00:00:00.000Z",
+ "score": 0.7,
+ },
+ {
+ "title": "Undated Exa Result",
+ "url": "https://example.com/undated-exa",
+ "text": "No date means filtered",
+ },
+ ]
+ }
+ with patch("lib.grounding.http.request", return_value=mock_response) as mock_req:
+ items, artifact = grounding.exa_search("test", ("2026-02-25", "2026-03-27"), "fake-exa-key")
+ self.assertEqual(1, len(items))
+ self.assertEqual("Exa Result", items[0]["title"])
+ self.assertEqual("https://example.com/exa", items[0]["url"])
+ self.assertEqual("2026-03-15", items[0]["date"])
+ self.assertTrue(items[0]["id"].startswith("WE"))
+ self.assertEqual("exa", artifact["label"])
+ self.assertEqual(1, artifact["resultCount"])
+ # Verify API call
+ call_args = mock_req.call_args
+ self.assertEqual("POST", call_args.args[0])
+ self.assertEqual("https://api.exa.ai/search", call_args.args[1])
+ self.assertEqual("fake-exa-key", call_args.kwargs["headers"]["x-api-key"])
+
+ def test_exa_search_returns_empty_for_no_results(self):
+ with patch("lib.grounding.http.request", return_value={"results": []}):
+ items, artifact = grounding.exa_search("test", ("2026-02-25", "2026-03-27"), "key")
+ self.assertEqual([], items)
+ self.assertEqual(0, artifact["resultCount"])
+
+
+class WebSearchDispatchTests(unittest.TestCase):
+ def test_auto_selects_brave_when_key_present(self):
+ config = {"BRAVE_API_KEY": "test-key"}
+ with patch("lib.grounding.brave_search", return_value=([], {})) as mock:
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
+ mock.assert_called_once()
+
+ def test_auto_selects_exa_when_only_exa_key(self):
+ config = {"EXA_API_KEY": "test-key"}
+ with patch("lib.grounding.exa_search", return_value=([], {})) as mock:
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
+ mock.assert_called_once()
+
+ def test_auto_selects_serper_when_only_serper_key(self):
+ config = {"SERPER_API_KEY": "test-key"}
+ with patch("lib.grounding.serper_search", return_value=([], {})) as mock:
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
+ mock.assert_called_once()
+
+ def test_auto_returns_empty_when_no_keys(self):
+ items, artifact = grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="auto")
+ self.assertEqual([], items)
+ self.assertEqual({}, artifact)
+
+ def test_none_returns_empty(self):
+ config = {"BRAVE_API_KEY": "test-key"}
+ items, artifact = grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="none")
+ self.assertEqual([], items)
+
+ def test_auto_prefers_brave_over_exa(self):
+ config = {"BRAVE_API_KEY": "brave-key", "EXA_API_KEY": "exa-key"}
+ with patch("lib.grounding.brave_search", return_value=([], {})) as mock_brave, \
+ patch("lib.grounding.exa_search", return_value=([], {})) as mock_exa:
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
+ mock_brave.assert_called_once()
+ mock_exa.assert_not_called()
+
+ def test_auto_prefers_exa_over_serper(self):
+ config = {"EXA_API_KEY": "exa-key", "SERPER_API_KEY": "serper-key"}
+ with patch("lib.grounding.exa_search", return_value=([], {})) as mock_exa, \
+ patch("lib.grounding.serper_search", return_value=([], {})) as mock_serper:
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
+ mock_exa.assert_called_once()
+ mock_serper.assert_not_called()
+
+ def test_auto_prefers_brave_when_all_keys_present(self):
+ config = {"BRAVE_API_KEY": "brave-key", "EXA_API_KEY": "exa-key", "SERPER_API_KEY": "serper-key"}
+ with patch("lib.grounding.brave_search", return_value=([], {})) as mock_brave, \
+ patch("lib.grounding.exa_search", return_value=([], {})) as mock_exa, \
+ patch("lib.grounding.serper_search", return_value=([], {})) as mock_serper:
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto")
+ mock_brave.assert_called_once()
+ mock_exa.assert_not_called()
+ mock_serper.assert_not_called()
+
+ def test_explicit_exa_without_key_raises(self):
+ with self.assertRaises(RuntimeError):
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="exa")
+
+ def test_explicit_brave_without_key_raises(self):
+ with self.assertRaises(RuntimeError):
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="brave")
+
+ def test_unsupported_backend_raises(self):
+ with self.assertRaises(ValueError):
+ grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="google")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_hackernews.py b/tests/test_hackernews.py
index ff776dd..0b0132f 100644
--- a/tests/test_hackernews.py
+++ b/tests/test_hackernews.py
@@ -1,219 +1,408 @@
-"""Tests for Hacker News source module."""
+"""Tests for hackernews.py - HN search via Algolia API."""
+import json
import sys
-import unittest
+from datetime import datetime, timezone
from pathlib import Path
+from unittest.mock import Mock, patch
+
+import pytest
-# Add lib to path
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-from lib import hackernews, normalize, schema, score
+from lib import hackernews
-class TestDateToUnix(unittest.TestCase):
- def test_known_date(self):
- # 2026-01-01 00:00:00 UTC
- result = hackernews._date_to_unix("2026-01-01")
- self.assertIsInstance(result, int)
- self.assertGreater(result, 0)
+# === Helper Functions ===
- def test_roundtrip(self):
- ts = hackernews._date_to_unix("2026-02-15")
- back = hackernews._unix_to_date(ts)
- self.assertEqual(back, "2026-02-15")
-
-
-class TestStripHtml(unittest.TestCase):
- def test_basic_html(self):
- result = hackernews._strip_html("Hello world
")
- self.assertIn("Hello", result)
- self.assertIn("world", result)
- self.assertNotIn("<", result)
-
- def test_html_entities(self):
- result = hackernews._strip_html("& test")
- self.assertIn("&", result)
- self.assertIn("test", result)
-
- def test_empty(self):
- result = hackernews._strip_html("")
- self.assertEqual(result, "")
-
-
-class TestParseHackernewsResponse(unittest.TestCase):
- SAMPLE_RESPONSE = {
- "hits": [
- {
- "objectID": "12345",
- "title": "Show HN: A new AI coding assistant",
- "url": "https://example.com/article",
- "author": "pg",
- "points": 350,
- "num_comments": 127,
- "created_at_i": 1739836800, # 2025-02-18
- },
- {
- "objectID": "12346",
- "title": "Ask HN: Best practices for LLM apps?",
- "url": "",
- "author": "dang",
- "points": 80,
- "num_comments": 45,
- "created_at_i": 1739750400, # 2025-02-17
- },
- ],
+def create_mock_hit(
+ object_id="12345",
+ title="Test HN Story",
+ points=100,
+ num_comments=50,
+ created_at_i=None,
+ author="testuser",
+ url="https://example.com",
+):
+ """Create a mock Algolia hit object."""
+ if created_at_i is None:
+ # Default to 30 days ago
+ dt = datetime.now(timezone.utc)
+ created_at_i = int(dt.timestamp()) - (30 * 86400)
+
+ return {
+ "objectID": object_id,
+ "title": title,
+ "points": points,
+ "num_comments": num_comments,
+ "created_at_i": created_at_i,
+ "author": author,
+ "url": url,
}
- def test_parses_hits(self):
- items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE)
- self.assertEqual(len(items), 2)
- def test_item_fields(self):
- items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE)
- item = items[0]
- self.assertEqual(item["object_id"], "12345")
- self.assertEqual(item["title"], "Show HN: A new AI coding assistant")
- self.assertEqual(item["url"], "https://example.com/article")
- self.assertEqual(item["hn_url"], "https://news.ycombinator.com/item?id=12345")
- self.assertEqual(item["author"], "pg")
- self.assertEqual(item["engagement"]["points"], 350)
- self.assertEqual(item["engagement"]["num_comments"], 127)
+# === Tests for _date_to_unix() ===
- def test_date_conversion(self):
- items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE)
- self.assertIsNotNone(items[0]["date"])
- # Should be a valid YYYY-MM-DD string
- self.assertRegex(items[0]["date"], r"^\d{4}-\d{2}-\d{2}$")
-
- def test_hn_url_for_askhn(self):
- """Ask HN posts have no article URL, but should have hn_url."""
- items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE)
- ask_hn = items[1]
- self.assertEqual(ask_hn["url"], "")
- self.assertIn("news.ycombinator.com", ask_hn["hn_url"])
-
- def test_relevance_range(self):
- items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE)
- for item in items:
- self.assertGreaterEqual(item["relevance"], 0.0)
- self.assertLessEqual(item["relevance"], 1.0)
-
- def test_empty_response(self):
- items = hackernews.parse_hackernews_response({"hits": []})
- self.assertEqual(items, [])
-
- def test_missing_hits(self):
- items = hackernews.parse_hackernews_response({})
- self.assertEqual(items, [])
+def test_date_to_unix_basic():
+ """Test converting YYYY-MM-DD to Unix timestamp."""
+ result = hackernews._date_to_unix("2026-01-01")
+
+ # Should be midnight UTC on Jan 1, 2026
+ expected = datetime(2026, 1, 1, tzinfo=timezone.utc).timestamp()
+ assert result == int(expected)
-class TestNormalizeHackernewsItems(unittest.TestCase):
- def test_normalize(self):
- raw_items = [
- {
- "object_id": "99999",
- "title": "Test Story",
- "url": "https://example.com",
- "hn_url": "https://news.ycombinator.com/item?id=99999",
- "author": "testuser",
- "date": "2026-02-15",
- "engagement": {"points": 100, "num_comments": 50},
- "relevance": 0.8,
- "why_relevant": "Test",
- }
+def test_date_to_unix_leap_day():
+ """Test date conversion with leap day."""
+ result = hackernews._date_to_unix("2024-02-29")
+
+ expected = datetime(2024, 2, 29, tzinfo=timezone.utc).timestamp()
+ assert result == int(expected)
+
+
+# === Tests for _unix_to_date() ===
+
+def test_unix_to_date_basic():
+ """Test converting Unix timestamp to YYYY-MM-DD."""
+ ts = int(datetime(2026, 1, 15, tzinfo=timezone.utc).timestamp())
+ result = hackernews._unix_to_date(ts)
+
+ assert result == "2026-01-15"
+
+
+def test_unix_to_date_with_time():
+ """Test that time component is stripped."""
+ ts = int(datetime(2026, 1, 15, 14, 30, 45, tzinfo=timezone.utc).timestamp())
+ result = hackernews._unix_to_date(ts)
+
+ assert result == "2026-01-15"
+
+
+# === Tests for _strip_html() ===
+
+def test_strip_html_basic():
+ """Test HTML stripping and entity decoding."""
+ html_text = "Hello & goodbye
"
+ result = hackernews._strip_html(html_text)
+
+ assert result == "Hello & goodbye"
+
+
+def test_strip_html_paragraph_tags():
+ """Test that tags are converted to newlines."""
+ html_text = "First
Second
Third"
+ result = hackernews._strip_html(html_text)
+
+ assert "First\n" in result
+ assert "Second\n" in result
+
+
+def test_strip_html_nested_tags():
+ """Test stripping nested HTML tags."""
+ html_text = "
"
+ result = hackernews._strip_html(html_text)
+
+ assert result == "Link text bold"
+
+
+def test_strip_html_entities():
+ """Test HTML entity decoding and tag stripping."""
+ html_text = "Text & "test""
+ result = hackernews._strip_html(html_text)
+
+ # Entities are decoded
+ assert "&" in result or "test" in result
+
+
+# === Tests for _title_matches_query() ===
+
+def test_title_matches_query_basic():
+ """Test basic query matching."""
+ title = "New AI framework for developers"
+ query = "AI framework"
+
+ assert hackernews._title_matches_query(title, query) is True
+
+
+def test_title_matches_query_case_insensitive():
+ """Test that matching is case-insensitive."""
+ title = "NEW AI FRAMEWORK"
+ query = "ai framework"
+
+ assert hackernews._title_matches_query(title, query) is True
+
+
+def test_title_matches_query_with_prefix():
+ """Test matching with HN prefix stripped."""
+ title = "Show HN: My new AI framework"
+ query = "AI framework"
+
+ # Should match "AI framework" in the content, not the "Show HN:" prefix
+ assert hackernews._title_matches_query(title, query) is True
+
+
+def test_title_matches_query_prefix_only():
+ """Test that matching prefix-only returns False."""
+ title = "Show HN: Something else entirely"
+ query = "Show HN"
+
+ # "Show HN" is a prefix, not real content
+ # After stripping, "Show HN" won't be in the stripped title
+ assert hackernews._title_matches_query(title, query) is False
+
+
+def test_title_matches_query_empty_query():
+ """Test that empty query always matches."""
+ title = "Any title"
+ query = ""
+
+ assert hackernews._title_matches_query(title, query) is True
+
+
+def test_title_matches_query_partial_match():
+ """Test that all query words must match."""
+ title = "New AI framework"
+ query = "AI blockchain"
+
+ # "blockchain" is not in title, so should fail
+ assert hackernews._title_matches_query(title, query) is False
+
+
+# === Tests for search_hackernews() ===
+
+@patch('lib.hackernews.http.request')
+def test_search_hackernews_basic(mock_request):
+ """Test basic HN search."""
+ mock_request.return_value = {
+ "hits": [create_mock_hit()],
+ "nbHits": 1,
+ }
+
+ result = hackernews.search_hackernews(
+ "AI framework",
+ "2026-01-01",
+ "2026-01-31",
+ depth="quick"
+ )
+
+ assert "hits" in result
+ assert len(result["hits"]) == 1
+ assert mock_request.called
+
+
+@patch('lib.hackernews.http.request')
+def test_search_hackernews_depth_config(mock_request):
+ """Test that depth parameter controls hit count."""
+ mock_request.return_value = {"hits": [], "nbHits": 0}
+
+ # Quick mode should request 15 hits
+ hackernews.search_hackernews("test", "2026-01-01", "2026-01-31", depth="quick")
+
+ call_args = mock_request.call_args[0]
+ url = call_args[1]
+
+ assert "hitsPerPage=15" in url
+
+
+@patch('lib.hackernews.http.request')
+def test_search_hackernews_date_filtering(mock_request):
+ """Test that date range is applied correctly."""
+ mock_request.return_value = {"hits": [], "nbHits": 0}
+
+ hackernews.search_hackernews("test", "2026-01-01", "2026-01-31", depth="quick")
+
+ call_args = mock_request.call_args[0]
+ url = call_args[1]
+
+ # Should have numeric filters for date range
+ assert "numericFilters" in url
+ assert "created_at_i" in url
+
+
+@patch('lib.hackernews.http.request')
+def test_search_hackernews_http_error_handling(mock_request):
+ """Test graceful handling of HTTP errors."""
+ from lib.http import HTTPError
+ mock_request.side_effect = HTTPError("HTTP 429: Too Many Requests")
+
+ result = hackernews.search_hackernews("test", "2026-01-01", "2026-01-31")
+
+ # Should return empty hits with error
+ assert result["hits"] == []
+ assert "error" in result
+
+
+@patch('lib.hackernews.http.request')
+def test_search_hackernews_engagement_filter(mock_request):
+ """Test that low-engagement stories are filtered."""
+ mock_request.return_value = {"hits": [], "nbHits": 0}
+
+ hackernews.search_hackernews("test", "2026-01-01", "2026-01-31")
+
+ call_args = mock_request.call_args[0]
+ url = call_args[1]
+
+ # Should filter for points > 2 (URL-encoded)
+ assert "points" in url and "%3E2" in url
+
+
+# === Tests for parse_hackernews_response() ===
+
+def test_parse_hackernews_response_basic():
+ """Test parsing basic Algolia response."""
+ response = {
+ "hits": [create_mock_hit(
+ object_id="123",
+ title="Test Story",
+ points=100,
+ num_comments=50
+ )]
+ }
+
+ items = hackernews.parse_hackernews_response(response)
+
+ assert len(items) == 1
+ assert items[0]["id"] == "123"
+ assert items[0]["title"] == "Test Story"
+ assert items[0]["engagement"]["points"] == 100
+ assert items[0]["engagement"]["comments"] == 50
+
+
+def test_parse_hackernews_response_hn_url():
+ """Test that HN discussion URL is generated correctly."""
+ response = {
+ "hits": [create_mock_hit(object_id="12345")]
+ }
+
+ items = hackernews.parse_hackernews_response(response)
+
+ assert items[0]["hn_url"] == "https://news.ycombinator.com/item?id=12345"
+
+
+def test_parse_hackernews_response_date_conversion():
+ """Test that Unix timestamp is converted to YYYY-MM-DD."""
+ ts = int(datetime(2026, 1, 15, tzinfo=timezone.utc).timestamp())
+ response = {
+ "hits": [create_mock_hit(created_at_i=ts)]
+ }
+
+ items = hackernews.parse_hackernews_response(response)
+
+ assert items[0]["date"] == "2026-01-15"
+
+
+def test_parse_hackernews_response_missing_fields():
+ """Test handling of hits with missing optional fields."""
+ response = {
+ "hits": [{
+ "objectID": "123",
+ "title": "Test",
+ # Missing points, num_comments, created_at_i
+ }]
+ }
+
+ items = hackernews.parse_hackernews_response(response)
+
+ assert len(items) == 1
+ assert items[0]["engagement"]["points"] == 0
+ assert items[0]["engagement"]["comments"] == 0
+ assert items[0]["date"] is None
+
+
+def test_parse_hackernews_response_relevance_scoring():
+ """Test that relevance scores are calculated."""
+ response = {
+ "hits": [
+ create_mock_hit(object_id="1", points=100),
+ create_mock_hit(object_id="2", points=50),
+ create_mock_hit(object_id="3", points=10),
]
- result = normalize.normalize_hackernews_items(raw_items, "2026-01-01", "2026-03-01")
- self.assertEqual(len(result), 1)
- self.assertIsInstance(result[0], schema.HackerNewsItem)
- self.assertEqual(result[0].id, "HN1")
- self.assertEqual(result[0].title, "Test Story")
- self.assertEqual(result[0].date_confidence, "high")
- self.assertEqual(result[0].engagement.score, 100)
- self.assertEqual(result[0].engagement.num_comments, 50)
+ }
+
+ items = hackernews.parse_hackernews_response(response, query="test")
+
+ # Should have relevance scores
+ for item in items:
+ assert "relevance" in item
+ assert 0 <= item["relevance"] <= 1.0
+
+ # First item should generally have higher relevance (better rank)
+ assert items[0]["relevance"] >= items[2]["relevance"]
- def test_normalize_with_comments(self):
- raw_items = [
- {
- "object_id": "99999",
- "title": "Test",
- "url": "",
- "hn_url": "",
- "author": "user",
- "date": "2026-02-15",
- "engagement": {"points": 10, "num_comments": 5},
- "relevance": 0.5,
- "why_relevant": "Test",
- "top_comments": [
- {"author": "commenter", "text": "Great post!", "points": 5},
- ],
- "comment_insights": ["Great post!"],
- }
+
+def test_parse_hackernews_response_engagement_boost():
+ """Test that high-engagement items get relevance boost."""
+ response = {
+ "hits": [
+ create_mock_hit(object_id="1", points=500, num_comments=200), # High engagement
+ create_mock_hit(object_id="2", points=10, num_comments=5), # Low engagement
]
- result = normalize.normalize_hackernews_items(raw_items, "2026-01-01", "2026-03-01")
- self.assertEqual(len(result[0].top_comments), 1)
- self.assertEqual(result[0].top_comments[0].author, "commenter")
- self.assertEqual(len(result[0].comment_insights), 1)
+ }
+
+ items = hackernews.parse_hackernews_response(response, query="test")
+
+ # Verify engagement is captured
+ assert items[0]["engagement"]["points"] == 500
+ assert items[1]["engagement"]["points"] == 10
-class TestScoreHackernewsItems(unittest.TestCase):
- def test_score_items(self):
- items = [
- schema.HackerNewsItem(
- id="HN1", title="High engagement", url="", hn_url="",
- author="user1", date="2026-02-20",
- engagement=schema.Engagement(score=500, num_comments=200),
- relevance=0.9,
- ),
- schema.HackerNewsItem(
- id="HN2", title="Low engagement", url="", hn_url="",
- author="user2", date="2026-02-18",
- engagement=schema.Engagement(score=10, num_comments=3),
- relevance=0.5,
- ),
+def test_parse_hackernews_response_prefix_filtering():
+ """Test that items matching only HN prefixes are filtered."""
+ response = {
+ "hits": [
+ create_mock_hit(title="Show HN: My AI Project", object_id="1"),
+ create_mock_hit(title="Show HN: Unrelated Project", object_id="2"),
]
- scored = score.score_hackernews_items(items)
- self.assertEqual(len(scored), 2)
- # High engagement + high relevance should score higher
- self.assertGreater(scored[0].score, scored[1].score)
-
- def test_score_empty(self):
- result = score.score_hackernews_items([])
- self.assertEqual(result, [])
-
- def test_engagement_formula(self):
- eng = schema.Engagement(score=100, num_comments=50)
- result = score.compute_hackernews_engagement_raw(eng)
- self.assertIsNotNone(result)
- self.assertGreater(result, 0)
-
- def test_engagement_none(self):
- result = score.compute_hackernews_engagement_raw(None)
- self.assertIsNone(result)
-
- def test_engagement_empty(self):
- eng = schema.Engagement()
- result = score.compute_hackernews_engagement_raw(eng)
- self.assertIsNone(result)
+ }
+
+ # Query for "AI" should keep first, filter second
+ items = hackernews.parse_hackernews_response(response, query="AI")
+
+ assert len(items) == 1
+ assert items[0]["id"] == "1"
-class TestSortItemsWithHN(unittest.TestCase):
- def test_hn_priority_after_youtube(self):
- """HN should sort after YouTube at same score."""
- x_item = schema.XItem(id="X1", text="test", url="", author_handle="user")
- x_item.score = 50
+def test_parse_hackernews_response_empty_response():
+ """Test handling of empty response."""
+ response = {"hits": []}
+
+ items = hackernews.parse_hackernews_response(response)
+
+ assert items == []
- hn_item = schema.HackerNewsItem(id="HN1", title="test", url="", hn_url="", author="user")
- hn_item.score = 50
- yt_item = schema.YouTubeItem(id="YT1", title="test", url="", channel_name="ch")
- yt_item.score = 50
+# === Tests for engagement scoring ===
- sorted_items = score.sort_items([yt_item, hn_item, x_item])
- # Same score, so sorted by source priority: X > YouTube > HN
- self.assertIsInstance(sorted_items[0], schema.XItem)
- self.assertIsInstance(sorted_items[1], schema.YouTubeItem)
- self.assertIsInstance(sorted_items[2], schema.HackerNewsItem)
+def test_engagement_score_calculation():
+ """Test that engagement dict contains points and comments."""
+ response = {
+ "hits": [create_mock_hit(points=150, num_comments=75)]
+ }
+
+ items = hackernews.parse_hackernews_response(response)
+
+ engagement = items[0]["engagement"]
+ assert engagement["points"] == 150
+ assert engagement["comments"] == 75
+
+
+def test_engagement_score_zero_values():
+ """Test handling of zero engagement values."""
+ response = {
+ "hits": [{
+ "objectID": "123",
+ "title": "Test",
+ "points": None,
+ "num_comments": None,
+ }]
+ }
+
+ items = hackernews.parse_hackernews_response(response)
+
+ engagement = items[0]["engagement"]
+ assert engagement["points"] == 0
+ assert engagement["comments"] == 0
if __name__ == "__main__":
- unittest.main()
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_http_v3.py b/tests/test_http_v3.py
new file mode 100644
index 0000000..321f4eb
--- /dev/null
+++ b/tests/test_http_v3.py
@@ -0,0 +1,43 @@
+import sys
+import urllib.error
+import unittest
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import http
+
+
+class Test429RetryLimit(unittest.TestCase):
+ """429 retries must be capped at max_429_retries to avoid wasting latency."""
+
+ @patch("lib.http.urllib.request.urlopen")
+ @patch("lib.http.time.sleep") # Don't actually sleep in tests
+ def test_429_retries_limited_to_2_by_default(self, mock_sleep, mock_urlopen):
+ """With default max_429_retries=2, should attempt 2 times then raise."""
+ error = urllib.error.HTTPError(
+ "http://example.com", 429, "Too Many Requests", {}, None
+ )
+ mock_urlopen.side_effect = error
+
+ with self.assertRaises(http.HTTPError) as ctx:
+ http.request("GET", "http://example.com", retries=5)
+
+ self.assertEqual(ctx.exception.status_code, 429)
+ # Should be called exactly 2 times (initial + 1 retry), not 5
+ self.assertEqual(mock_urlopen.call_count, 2)
+
+ @patch("lib.http.urllib.request.urlopen")
+ @patch("lib.http.time.sleep")
+ def test_non_429_errors_still_use_full_retries(self, mock_sleep, mock_urlopen):
+ """500 errors should still retry up to the full retries count."""
+ error = urllib.error.HTTPError(
+ "http://example.com", 500, "Internal Server Error", {}, None
+ )
+ mock_urlopen.side_effect = error
+
+ with self.assertRaises(http.HTTPError):
+ http.request("GET", "http://example.com", retries=3)
+
+ self.assertEqual(mock_urlopen.call_count, 3)
diff --git a/tests/test_instagram.py b/tests/test_instagram.py
new file mode 100644
index 0000000..23c96b2
--- /dev/null
+++ b/tests/test_instagram.py
@@ -0,0 +1,68 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib.instagram import _parse_items
+
+
+class TestInstagramOwnerTypeSafety(unittest.TestCase):
+ def _make_raw(self, **overrides):
+ base = {
+ "id": "1",
+ "code": "ABC123",
+ "caption": "test caption",
+ "owner": {"username": "testuser"},
+ }
+ base.update(overrides)
+ return base
+
+ def test_owner_as_dict(self):
+ items = _parse_items([self._make_raw()], "test")
+ self.assertEqual("testuser", items[0]["author_name"])
+
+ def test_owner_as_string(self):
+ items = _parse_items([self._make_raw(owner="stringuser")], "test")
+ self.assertEqual("stringuser", items[0]["author_name"])
+
+ def test_owner_missing(self):
+ raw = self._make_raw()
+ del raw["owner"]
+ items = _parse_items([raw], "test")
+ self.assertEqual("", items[0]["author_name"])
+
+ def test_owner_none(self):
+ items = _parse_items([self._make_raw(owner=None)], "test")
+ self.assertEqual("", items[0]["author_name"])
+
+ def test_user_field_fallback(self):
+ raw = self._make_raw()
+ del raw["owner"]
+ raw["user"] = {"username": "fallbackuser"}
+ items = _parse_items([raw], "test")
+ self.assertEqual("fallbackuser", items[0]["author_name"])
+
+
+class TestExpandInstagramQueries(unittest.TestCase):
+ """Tests for expand_instagram_queries() multi-query generation."""
+
+ def test_default_depth_returns_two_plus_queries(self):
+ from lib.instagram import expand_instagram_queries
+ queries = expand_instagram_queries("Kanye West", "default")
+ self.assertGreaterEqual(len(queries), 2)
+ # Breaking_news intent should include reaction/edit variant
+ variant_found = any(
+ "reaction" in q.lower() or "edit" in q.lower()
+ for q in queries
+ )
+ self.assertTrue(variant_found, f"Expected reaction/edit variant: {queries}")
+
+ def test_quick_depth_returns_one_query(self):
+ from lib.instagram import expand_instagram_queries
+ queries = expand_instagram_queries("Kanye West", "quick")
+ self.assertEqual(len(queries), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_internals_v3.py b/tests/test_internals_v3.py
new file mode 100644
index 0000000..4bdd285
--- /dev/null
+++ b/tests/test_internals_v3.py
@@ -0,0 +1,559 @@
+"""Unit tests for untested internal functions across rerank, render, planner, and signals.
+
+These pin the correct behavior of core building blocks that higher-level
+tests exercise transitively but don't assert on directly. A regression in
+any of these functions would silently degrade output quality.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import planner, rerank, render, signals, schema
+
+
+def _item(source: str = "reddit", **kwargs) -> schema.SourceItem:
+ defaults = dict(
+ item_id="t1", source=source, title="Test Title", body="Test body",
+ url="https://example.com", engagement={}, metadata={},
+ )
+ defaults.update(kwargs)
+ return schema.SourceItem(**defaults)
+
+
+def _candidate(source: str = "reddit", **kwargs) -> schema.Candidate:
+ defaults = dict(
+ candidate_id="c1", item_id="t1", source=source, title="Test",
+ url="https://example.com", snippet="snippet", subquery_labels=["primary"],
+ native_ranks={"primary": 1}, local_relevance=0.5, freshness=50,
+ engagement=50, source_quality=0.7, rrf_score=0.01, sources=[source],
+ source_items=[],
+ )
+ defaults.update(kwargs)
+ return schema.Candidate(**defaults)
+
+
+# ---------------------------------------------------------------------------
+# rerank._fallback_tuple
+# ---------------------------------------------------------------------------
+
+class TestFallbackTuple(unittest.TestCase):
+
+ def test_returns_score_and_explanation(self):
+ c = _candidate(local_relevance=0.8, freshness=80, source_quality=0.7)
+ score, explanation = rerank._fallback_tuple(c)
+ self.assertIsInstance(score, float)
+ self.assertEqual(explanation, "fallback-local-score")
+
+ def test_score_clamped_to_0_100(self):
+ c = _candidate(local_relevance=2.0, freshness=200, source_quality=2.0)
+ score, _ = rerank._fallback_tuple(c)
+ self.assertLessEqual(score, 100.0)
+ self.assertGreaterEqual(score, 0.0)
+
+ def test_higher_relevance_gives_higher_score(self):
+ high = _candidate(local_relevance=0.9, freshness=50, source_quality=0.7)
+ low = _candidate(local_relevance=0.1, freshness=50, source_quality=0.7)
+ self.assertGreater(rerank._fallback_tuple(high)[0], rerank._fallback_tuple(low)[0])
+
+
+# ---------------------------------------------------------------------------
+# rerank._normalized_rrf
+# ---------------------------------------------------------------------------
+
+class TestNormalizedRrf(unittest.TestCase):
+
+ def test_zero_input(self):
+ self.assertAlmostEqual(rerank._normalized_rrf(0.0), 0.0)
+
+ def test_positive_input(self):
+ result = rerank._normalized_rrf(0.04)
+ self.assertGreater(result, 0.0)
+ self.assertLessEqual(result, 100.0)
+
+ def test_clamped_at_100(self):
+ result = rerank._normalized_rrf(1.0)
+ self.assertLessEqual(result, 100.0)
+
+
+# ---------------------------------------------------------------------------
+# render._assess_data_freshness
+# ---------------------------------------------------------------------------
+
+class TestAssessDataFreshness(unittest.TestCase):
+
+ def _report(self, items_by_source: dict) -> schema.Report:
+ return schema.Report(
+ topic="test", range_from="2026-02-15", range_to="2026-03-17",
+ generated_at="2026-03-17T00:00:00Z",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="test", planner_model="test", rerank_model="test",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="comparison", freshness_mode="balanced_recent",
+ cluster_mode="debate", raw_topic="test", subqueries=[],
+ source_weights={},
+ ),
+ clusters=[], ranked_candidates=[],
+ items_by_source=items_by_source, errors_by_source={},
+ )
+
+ def test_no_items_returns_warning(self):
+ report = self._report({})
+ result = render._assess_data_freshness(report)
+ self.assertIsNotNone(result)
+ self.assertIn("Limited", result)
+
+ def test_all_old_items_returns_warning(self):
+ items = [_item(published_at="2026-01-01") for _ in range(10)]
+ report = self._report({"reddit": items})
+ result = render._assess_data_freshness(report)
+ self.assertIsNotNone(result)
+
+ def test_many_recent_items_returns_none(self):
+ from datetime import date
+ today = date.today().isoformat()
+ items = [_item(published_at=today) for _ in range(10)]
+ report = self._report({"reddit": items})
+ result = render._assess_data_freshness(report)
+ self.assertIsNone(result)
+
+
+# ---------------------------------------------------------------------------
+# render._format_date
+# ---------------------------------------------------------------------------
+
+class TestFormatDate(unittest.TestCase):
+
+ def test_high_confidence_clean(self):
+ item = _item(published_at="2026-03-10", date_confidence="high")
+ self.assertEqual(render._format_date(item), "2026-03-10")
+
+ def test_low_confidence_tagged(self):
+ item = _item(published_at="2026-03-10", date_confidence="low")
+ self.assertIn("date:low", render._format_date(item))
+
+ def test_none_item(self):
+ self.assertIn("unknown", render._format_date(None).lower())
+
+
+# ---------------------------------------------------------------------------
+# render._format_actor
+# ---------------------------------------------------------------------------
+
+class TestFormatActor(unittest.TestCase):
+
+ def test_reddit_subreddit(self):
+ item = _item(source="reddit", container="python")
+ self.assertEqual(render._format_actor(item), "r/python")
+
+ def test_x_handle(self):
+ item = _item(source="x", author="karpathy")
+ self.assertEqual(render._format_actor(item), "@karpathy")
+
+ def test_youtube_channel(self):
+ item = _item(source="youtube", author="Fireship")
+ self.assertEqual(render._format_actor(item), "Fireship")
+
+
+# ---------------------------------------------------------------------------
+# render._format_engagement
+# ---------------------------------------------------------------------------
+
+class TestFormatEngagement(unittest.TestCase):
+
+ def test_reddit_format(self):
+ item = _item(engagement={"score": 344, "num_comments": 119})
+ result = render._format_engagement(item)
+ self.assertIn("344", result)
+ self.assertIn("pts", result)
+
+ def test_empty_engagement(self):
+ item = _item(engagement={})
+ self.assertIsNone(render._format_engagement(item))
+
+
+# ---------------------------------------------------------------------------
+# render._format_corroboration
+# ---------------------------------------------------------------------------
+
+class TestFormatCorroboration(unittest.TestCase):
+
+ def test_multi_source(self):
+ c = _candidate(sources=["reddit", "x", "hackernews"])
+ result = render._format_corroboration(c)
+ self.assertIn("Also on", result)
+ self.assertIn("X", result)
+
+ def test_single_source_none(self):
+ c = _candidate(sources=["reddit"])
+ self.assertIsNone(render._format_corroboration(c))
+
+
+# ---------------------------------------------------------------------------
+# render._format_explanation
+# ---------------------------------------------------------------------------
+
+class TestFormatExplanation(unittest.TestCase):
+
+ def test_hides_fallback_sentinel(self):
+ c = _candidate(explanation="fallback-local-score")
+ self.assertIsNone(render._format_explanation(c))
+
+ def test_shows_real_explanation(self):
+ c = _candidate(explanation="Directly compares frameworks")
+ self.assertEqual(render._format_explanation(c), "Directly compares frameworks")
+
+
+# ---------------------------------------------------------------------------
+# render._fmt_pairs and _format_number
+# ---------------------------------------------------------------------------
+
+class TestFmtPairs(unittest.TestCase):
+
+ def test_basic(self):
+ self.assertEqual(render._fmt_pairs([(120, "pts"), (48, "cmt")]), "120pts, 48cmt")
+
+ def test_skips_none_and_zero(self):
+ self.assertEqual(render._fmt_pairs([(None, "pts"), (0, "cmt"), (5, "re")]), "5re")
+
+ def test_large_numbers(self):
+ self.assertIn("94,200", render._fmt_pairs([(94200, "views")]))
+
+
+class TestFormatNumber(unittest.TestCase):
+
+ def test_comma_thousands(self):
+ self.assertEqual(render._format_number(94200), "94,200")
+
+ def test_small_integer(self):
+ self.assertEqual(render._format_number(42), "42")
+
+
+# ---------------------------------------------------------------------------
+# render._truncate
+# ---------------------------------------------------------------------------
+
+class TestTruncate(unittest.TestCase):
+
+ def test_short_text(self):
+ self.assertEqual(render._truncate("hello", 100), "hello")
+
+ def test_long_text_has_ellipsis(self):
+ result = render._truncate("a" * 200, 50)
+ self.assertTrue(result.endswith("..."))
+ self.assertEqual(len(result), 50)
+
+
+# ---------------------------------------------------------------------------
+# planner._normalize_subquery_weights
+# ---------------------------------------------------------------------------
+
+class TestNormalizeSubqueryWeights(unittest.TestCase):
+
+ def test_sums_to_one(self):
+ sqs = [
+ schema.SubQuery(label="a", search_query="a", ranking_query="a?", sources=["r"], weight=3.0),
+ schema.SubQuery(label="b", search_query="b", ranking_query="b?", sources=["r"], weight=1.0),
+ ]
+ normed = planner._normalize_subquery_weights(sqs)
+ total = sum(sq.weight for sq in normed)
+ self.assertAlmostEqual(total, 1.0)
+
+ def test_preserves_ratio(self):
+ sqs = [
+ schema.SubQuery(label="a", search_query="a", ranking_query="a?", sources=["r"], weight=4.0),
+ schema.SubQuery(label="b", search_query="b", ranking_query="b?", sources=["r"], weight=1.0),
+ ]
+ normed = planner._normalize_subquery_weights(sqs)
+ self.assertAlmostEqual(normed[0].weight / normed[1].weight, 4.0)
+
+
+# ---------------------------------------------------------------------------
+# planner._normalize_weights
+# ---------------------------------------------------------------------------
+
+class TestNormalizeWeights(unittest.TestCase):
+
+ def test_sums_to_one(self):
+ result = planner._normalize_weights({"a": 3.0, "b": 1.0})
+ self.assertAlmostEqual(sum(result.values()), 1.0)
+
+ def test_negative_clamped_to_zero(self):
+ result = planner._normalize_weights({"a": 2.0, "b": -1.0})
+ self.assertAlmostEqual(result["b"], 0.0)
+
+
+# ---------------------------------------------------------------------------
+# planner._trim_subqueries_for_depth
+# ---------------------------------------------------------------------------
+
+class TestTrimSubqueriesForDepth(unittest.TestCase):
+
+ def _sq(self, label: str = "primary", sources: list[str] = None) -> schema.SubQuery:
+ return schema.SubQuery(
+ label=label, search_query="test", ranking_query="test?",
+ sources=sources or ["reddit", "x", "grounding", "youtube", "hackernews", "polymarket"],
+ weight=1.0,
+ )
+
+ def test_quick_limits_sources(self):
+ sqs = [self._sq()]
+ result = planner._trim_subqueries_for_depth(sqs, "comparison", "quick", ["reddit", "x", "grounding"])
+ self.assertLessEqual(len(result[0].sources), 2)
+
+ def test_default_comparison_expands_via_capabilities(self):
+ available = ["reddit", "x", "grounding", "youtube", "hackernews", "tiktok", "instagram"]
+ sqs = [self._sq(sources=available)]
+ result = planner._trim_subqueries_for_depth(sqs, "comparison", "default", available)
+ # Comparison should use all capability-matched sources, not top-3
+ self.assertGreater(len(result[0].sources), 3)
+
+ def test_deep_expands_via_capabilities(self):
+ available = ["reddit", "x", "youtube", "hackernews", "polymarket"]
+ sqs = [self._sq(sources=available)]
+ result = planner._trim_subqueries_for_depth(sqs, "comparison", "deep", available)
+ # Deep comparison should also use capability expansion, not trim
+ self.assertGreaterEqual(len(result[0].sources), 4)
+
+
+# ---------------------------------------------------------------------------
+# signals.annotate_stream
+# ---------------------------------------------------------------------------
+
+class TestAnnotateStream(unittest.TestCase):
+
+ def test_attaches_metadata(self):
+ items = [
+ _item(engagement={"score": 100, "num_comments": 50, "upvote_ratio": 0.9}),
+ ]
+ annotated = signals.annotate_stream(items, "test query", "balanced_recent")
+ item = annotated[0]
+ self.assertIsNotNone(item.local_relevance)
+ self.assertIsNotNone(item.freshness)
+ self.assertIsNotNone(item.engagement_score)
+ self.assertIsNotNone(item.source_quality)
+ self.assertIsNotNone(item.local_rank_score)
+
+ def test_sorted_by_local_rank_score(self):
+ items = [
+ _item(item_id="low", title="irrelevant stuff", engagement={}),
+ _item(item_id="high", title="test query exact match test query", engagement={"score": 500, "num_comments": 200}),
+ ]
+ annotated = signals.annotate_stream(items, "test query", "balanced_recent")
+ self.assertEqual(annotated[0].item_id, "high")
+
+
+# ---------------------------------------------------------------------------
+# signals.prune_low_relevance
+# ---------------------------------------------------------------------------
+
+class TestPruneLowRelevance(unittest.TestCase):
+
+ def test_removes_low_relevance_items(self):
+ items = [
+ _item(item_id="good"),
+ _item(item_id="bad"),
+ ]
+ items[0].local_relevance = 0.8
+ items[1].local_relevance = 0.01
+ result = signals.prune_low_relevance(items, minimum=0.1)
+ self.assertEqual(len(result), 1)
+ self.assertEqual(result[0].item_id, "good")
+
+ def test_keeps_all_if_all_below_minimum(self):
+ items = [_item(item_id="only")]
+ items[0].local_relevance = 0.05
+ result = signals.prune_low_relevance(items, minimum=0.1)
+ self.assertEqual(len(result), 1) # fallback keeps all
+
+
+# ---------------------------------------------------------------------------
+# Bug fixes found by PR review agents
+# ---------------------------------------------------------------------------
+
+class TestDaysAgoZeroFalsy(unittest.TestCase):
+ """render._assess_data_freshness must not treat days_ago=0 as falsy."""
+
+ def _report_with_items(self, dates_list: list[str]) -> schema.Report:
+ items = [_item(published_at=d) for d in dates_list]
+ return schema.Report(
+ topic="test", range_from="2026-02-15", range_to="2026-03-17",
+ generated_at="2026-03-17T00:00:00Z",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="test", planner_model="test", rerank_model="test",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="comparison", freshness_mode="balanced_recent",
+ cluster_mode="debate", raw_topic="test", subqueries=[],
+ source_weights={},
+ ),
+ clusters=[], ranked_candidates=[],
+ items_by_source={"reddit": items}, errors_by_source={},
+ )
+
+ def test_items_from_today_count_as_recent(self):
+ from datetime import date
+ today = date.today().isoformat()
+ report = self._report_with_items([today] * 5)
+ warning = render._assess_data_freshness(report)
+ self.assertIsNone(warning, f"Items from today should be recent, got warning: {warning}")
+
+
+class TestRerankBoundary(unittest.TestCase):
+ """Rerank demotion must have a clean boundary at exactly 20.0."""
+
+ def test_score_at_exactly_20_is_not_demoted(self):
+ c = _candidate()
+ c.rerank_score = 20.0
+ score_at_20 = rerank._final_score(c)
+ c.rerank_score = 50.0
+ score_at_50 = rerank._final_score(c)
+ self.assertGreater(score_at_20 / score_at_50, 0.3,
+ "Score at 20.0 should not be demoted")
+
+ def test_score_at_19_99_is_demoted(self):
+ c = _candidate()
+ c.rerank_score = 19.99
+ score_demoted = rerank._final_score(c)
+ c.rerank_score = 20.0
+ score_not_demoted = rerank._final_score(c)
+ self.assertLess(score_demoted, score_not_demoted * 0.5,
+ "Score at 19.99 should be heavily demoted vs 20.0")
+
+
+class TestSlashFalsePositives(unittest.TestCase):
+ """Slash regex must not misclassify compound terms as comparisons."""
+
+ def test_ci_cd_is_not_comparison(self):
+ self.assertNotEqual(planner._infer_intent("CI/CD pipeline setup"), "comparison")
+
+ def test_tcp_ip_is_not_comparison(self):
+ self.assertNotEqual(planner._infer_intent("TCP/IP networking guide"), "comparison")
+
+ def test_io_is_not_comparison(self):
+ self.assertNotEqual(planner._infer_intent("I/O performance tuning"), "comparison")
+
+ def test_os_kernel_is_not_comparison(self):
+ self.assertNotEqual(planner._infer_intent("input/output buffering"), "comparison")
+
+ def test_proper_noun_slash_still_works(self):
+ self.assertEqual(planner._infer_intent("React/Vue/Svelte"), "comparison")
+
+
+class TestGenericEngagementFormatter(unittest.TestCase):
+ """Generic formatter must not garble output for unknown sources."""
+
+ def test_xiaohongshu_engagement_not_garbled(self):
+ item = _item(source="xiaohongshu", engagement={"likes": 500, "views": 10000})
+ result = render._format_engagement(item)
+ if result is not None:
+ self.assertNotIn("likes500", result, "Key used as value prefix")
+ self.assertNotIn("views10000", result, "Key used as value prefix")
+ # Should contain numeric values, not dict keys as numbers
+ self.assertIn("500", result)
+
+
+if __name__ == "__main__":
+ unittest.main()
+
+
+class TestDefaultDepthDoesNotCapSources(unittest.TestCase):
+ """Default depth must not aggressively limit sources for any intent.
+
+ E2E testing showed factual/opinion/prediction/concept queries getting
+ 0-1 sources because SOURCE_LIMITS["default"] capped them at 2-3,
+ and those 2-3 sources returned empty. v2.9.5 searched all available
+ sources and let scoring handle quality.
+ """
+
+ ALL_SOURCES = ["reddit", "x", "grounding", "youtube", "hackernews",
+ "tiktok", "instagram", "polymarket"]
+
+ def _plan_sources(self, topic: str) -> list[str]:
+ plan = planner.plan_query(
+ topic=topic,
+ available_sources=self.ALL_SOURCES,
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ return plan.subqueries[0].sources
+
+ def test_factual_gets_more_than_2_sources(self):
+ sources = self._plan_sources("what is quantum computing")
+ self.assertGreater(len(sources), 2,
+ f"Factual query capped at {len(sources)} sources: {sources}")
+
+ def test_opinion_gets_more_than_3_sources(self):
+ sources = self._plan_sources("thoughts on Rust")
+ self.assertGreater(len(sources), 3,
+ f"Opinion query capped at {len(sources)} sources: {sources}")
+
+ def test_prediction_gets_more_than_3_sources(self):
+ sources = self._plan_sources("odds of recession")
+ self.assertGreater(len(sources), 3,
+ f"Prediction query capped at {len(sources)} sources: {sources}")
+
+ def test_breaking_news_gets_more_than_4_sources(self):
+ sources = self._plan_sources("kanye west")
+ self.assertGreater(len(sources), 4,
+ f"Breaking news capped at {len(sources)} sources: {sources}")
+
+ def test_concept_gets_more_than_3_sources(self):
+ sources = self._plan_sources("explain transformer architecture")
+ self.assertGreater(len(sources), 3,
+ f"Concept query capped at {len(sources)} sources: {sources}")
+
+ def test_quick_mode_still_limited(self):
+ """Quick mode should remain tight for latency."""
+ plan = planner.plan_query(
+ topic="what is quantum computing",
+ available_sources=self.ALL_SOURCES,
+ requested_sources=None,
+ depth="quick",
+ provider=None,
+ model=None,
+ )
+ self.assertLessEqual(len(plan.subqueries[0].sources), 3)
+
+
+
+class TestRerankWeightBalance(unittest.TestCase):
+ """Reranker weight must dominate over RRF when candidates have divergent quality."""
+
+ def test_rerank_gap_dominates_with_identical_rrf(self):
+ """Two candidates with identical RRF but rerank_scores of 80 and 40 should have a meaningful final_score gap (rerank still dominates)."""
+ high = _candidate(rrf_score=0.03, freshness=50, source_quality=0.7)
+ high.rerank_score = 80.0
+ high.final_score = rerank._final_score(high)
+
+ low = _candidate(rrf_score=0.03, freshness=50, source_quality=0.7)
+ low.rerank_score = 40.0
+ low.final_score = rerank._final_score(low)
+
+ gap = high.final_score - low.final_score
+ # Rerank weight is 0.60, so gap = 0.60 * 40 = 24 points.
+ # Engagement boost may add a small delta but rerank remains dominant.
+ self.assertGreaterEqual(gap, 23.0,
+ f"Rerank gap should be >= 23 points, got {gap:.1f}")
+
+
+class TestXaiModelDefault(unittest.TestCase):
+ """XAI_DEFAULT must be a model that xAI's API actually accepts."""
+
+ def test_default_is_not_grok_3(self):
+ from lib import providers
+ self.assertNotEqual(providers.XAI_DEFAULT, "grok-3-fast",
+ "grok-3-fast returns HTTP 400 from xAI API")
+ self.assertNotEqual(providers.XAI_DEFAULT, "grok-3-mini-fast",
+ "grok-3-mini-fast returns HTTP 400 from xAI API")
+
+ def test_default_is_grok_4_generation(self):
+ from lib import providers
+ self.assertIn("grok-4", providers.XAI_DEFAULT,
+ f"XAI_DEFAULT should be a grok-4 model, got: {providers.XAI_DEFAULT}")
+
diff --git a/tests/test_models.py b/tests/test_models.py
deleted file mode 100644
index f093f27..0000000
--- a/tests/test_models.py
+++ /dev/null
@@ -1,228 +0,0 @@
-"""Tests for models module."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import models
-
-
-class TestParseVersion(unittest.TestCase):
- def test_simple_version(self):
- result = models.parse_version("gpt-5")
- self.assertEqual(result, (5,))
-
- def test_minor_version(self):
- result = models.parse_version("gpt-5.2")
- self.assertEqual(result, (5, 2))
-
- def test_patch_version(self):
- result = models.parse_version("gpt-5.2.1")
- self.assertEqual(result, (5, 2, 1))
-
- def test_no_version(self):
- result = models.parse_version("custom-model")
- self.assertIsNone(result)
-
-
-class TestIsSearchCapableModel(unittest.TestCase):
- def test_gpt5_is_capable(self):
- """gpt-5 supports web_search when reasoning is not set to 'minimal'.
-
- Per OpenAI docs, gpt-5 with reasoning effort="minimal" does NOT
- support web_search. We never set reasoning params (our usage is
- tool invocation + JSON extraction only), so gpt-5 is safe here.
- """
- self.assertTrue(models.is_search_capable_model("gpt-5"))
-
- def test_gpt52_is_capable(self):
- self.assertTrue(models.is_search_capable_model("gpt-5.2"))
-
- def test_gpt5_mini_is_capable(self):
- self.assertTrue(models.is_search_capable_model("gpt-5-mini"))
-
- def test_gpt41_mini_is_capable(self):
- self.assertTrue(models.is_search_capable_model("gpt-4.1-mini"))
-
- def test_gpt4o_is_capable(self):
- self.assertTrue(models.is_search_capable_model("gpt-4o"))
-
- def test_gpt4o_mini_not_capable(self):
- """gpt-4o-mini does not support web_search with domain filtering."""
- self.assertFalse(models.is_search_capable_model("gpt-4o-mini"))
-
- def test_nano_not_capable(self):
- """nano models don't support web_search."""
- self.assertFalse(models.is_search_capable_model("gpt-4.1-nano"))
- self.assertFalse(models.is_search_capable_model("gpt-5-nano"))
-
- def test_gpt4_not_capable(self):
- self.assertFalse(models.is_search_capable_model("gpt-4"))
-
- def test_codex_not_capable(self):
- self.assertFalse(models.is_search_capable_model("gpt-5.1-codex"))
-
- def test_backward_compat_alias(self):
- """is_mainline_openai_model still works as alias."""
- self.assertTrue(models.is_mainline_openai_model("gpt-5"))
-
-
-class TestSelectOpenAIModel(unittest.TestCase):
- def setUp(self):
- from lib import cache
- cache.MODEL_CACHE_FILE.unlink(missing_ok=True)
-
- def test_pinned_policy(self):
- result = models.select_openai_model(
- "fake-key",
- policy="pinned",
- pin="gpt-5.1"
- )
- self.assertEqual(result, "gpt-5.1")
-
- def test_prefers_mini_over_mainline(self):
- """Mini models should be preferred for cost-efficiency."""
- mock_models = [
- {"id": "gpt-5.2", "created": 1704067200},
- {"id": "gpt-5-mini", "created": 1704067200},
- {"id": "gpt-5.1", "created": 1701388800},
- ]
- result = models.select_openai_model(
- "fake-key",
- policy="auto",
- mock_models=mock_models
- )
- self.assertEqual(result, "gpt-5-mini")
-
- def test_prefers_newer_generation_mini(self):
- """gpt-5-mini should beat gpt-4.1-mini (newer generation)."""
- mock_models = [
- {"id": "gpt-4.1-mini", "created": 1701388800},
- {"id": "gpt-5-mini", "created": 1704067200},
- {"id": "gpt-4.1", "created": 1698710400},
- ]
- result = models.select_openai_model(
- "fake-key",
- policy="auto",
- mock_models=mock_models
- )
- self.assertEqual(result, "gpt-5-mini")
-
- def test_falls_back_to_mainline_when_no_mini(self):
- """Without mini models, mainline models are selected."""
- mock_models = [
- {"id": "gpt-5.2", "created": 1704067200},
- {"id": "gpt-4.1", "created": 1698710400},
- ]
- result = models.select_openai_model(
- "fake-key",
- policy="auto",
- mock_models=mock_models
- )
- self.assertEqual(result, "gpt-5.2")
-
- def test_filters_unsupported_variants(self):
- """Nano, codex, preview models should be excluded."""
- mock_models = [
- {"id": "gpt-5-nano", "created": 1704067200},
- {"id": "gpt-5.1-codex", "created": 1704067200},
- {"id": "gpt-4o-mini", "created": 1704067200},
- {"id": "gpt-4.1-mini", "created": 1698710400},
- ]
- result = models.select_openai_model(
- "fake-key",
- policy="auto",
- mock_models=mock_models
- )
- self.assertEqual(result, "gpt-4.1-mini")
-
-
-class TestSelectOpenAIModelErrorPaths(unittest.TestCase):
- def setUp(self):
- from lib import cache
- cache.MODEL_CACHE_FILE.unlink(missing_ok=True)
-
- def test_http_error_returns_fallback(self):
- """HTTPError during model fetch should return fallback, not crash."""
- from unittest.mock import patch
- from lib import http
- with patch('lib.http.get', side_effect=http.HTTPError("Unauthorized", status_code=401)):
- result = models.select_openai_model("bad-key", policy="auto")
- self.assertEqual(result, models.OPENAI_FALLBACK_MODELS[0])
-
- def test_http_403_returns_fallback(self):
- from unittest.mock import patch
- from lib import http
- with patch('lib.http.get', side_effect=http.HTTPError("Forbidden", status_code=403)):
- result = models.select_openai_model("bad-key", policy="auto")
- self.assertEqual(result, models.OPENAI_FALLBACK_MODELS[0])
-
-
-class TestSelectXAIModel(unittest.TestCase):
- def test_latest_policy(self):
- result = models.select_xai_model(
- "fake-key",
- policy="latest"
- )
- self.assertEqual(result, "grok-4-1-fast-non-reasoning")
-
- def test_stable_policy(self):
- # Clear cache first to avoid interference
- from lib import cache
- cache.MODEL_CACHE_FILE.unlink(missing_ok=True)
- result = models.select_xai_model(
- "fake-key",
- policy="stable"
- )
- self.assertEqual(result, "grok-4-1-fast-non-reasoning")
-
- def test_pinned_policy(self):
- result = models.select_xai_model(
- "fake-key",
- policy="pinned",
- pin="grok-3"
- )
- self.assertEqual(result, "grok-3")
-
-
-class TestGetModels(unittest.TestCase):
- def setUp(self):
- from lib import cache
- cache.MODEL_CACHE_FILE.unlink(missing_ok=True)
-
- def test_no_keys_returns_none(self):
- config = {}
- result = models.get_models(config)
- self.assertIsNone(result["openai"])
- self.assertIsNone(result["xai"])
-
- def test_openai_key_only(self):
- config = {"OPENAI_API_KEY": "sk-test"}
- mock_models = [
- {"id": "gpt-5.2", "created": 1704067200},
- {"id": "gpt-5-mini", "created": 1704067200},
- ]
- result = models.get_models(config, mock_openai_models=mock_models)
- self.assertEqual(result["openai"], "gpt-5-mini")
- self.assertIsNone(result["xai"])
-
- def test_both_keys(self):
- config = {
- "OPENAI_API_KEY": "sk-test",
- "XAI_API_KEY": "xai-test",
- }
- mock_openai = [
- {"id": "gpt-5.2", "created": 1704067200},
- {"id": "gpt-5-mini", "created": 1704067200},
- ]
- mock_xai = [{"id": "grok-4-1-fast-non-reasoning", "created": 1704067200}]
- result = models.get_models(config, mock_openai, mock_xai)
- self.assertEqual(result["openai"], "gpt-5-mini")
- self.assertEqual(result["xai"], "grok-4-1-fast-non-reasoning")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_normalize.py b/tests/test_normalize.py
deleted file mode 100644
index 4ccdd67..0000000
--- a/tests/test_normalize.py
+++ /dev/null
@@ -1,138 +0,0 @@
-"""Tests for normalize module."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import normalize, schema
-
-
-class TestNormalizeRedditItems(unittest.TestCase):
- def test_normalizes_basic_item(self):
- items = [
- {
- "id": "R1",
- "title": "Test Thread",
- "url": "https://reddit.com/r/test/1",
- "subreddit": "test",
- "date": "2026-01-15",
- "why_relevant": "Relevant because...",
- "relevance": 0.85,
- }
- ]
-
- result = normalize.normalize_reddit_items(items, "2026-01-01", "2026-01-31")
-
- self.assertEqual(len(result), 1)
- self.assertIsInstance(result[0], schema.RedditItem)
- self.assertEqual(result[0].id, "R1")
- self.assertEqual(result[0].title, "Test Thread")
- self.assertEqual(result[0].date_confidence, "high")
-
- def test_sets_low_confidence_for_old_date(self):
- items = [
- {
- "id": "R1",
- "title": "Old Thread",
- "url": "https://reddit.com/r/test/1",
- "subreddit": "test",
- "date": "2025-12-01", # Before range
- "relevance": 0.5,
- }
- ]
-
- result = normalize.normalize_reddit_items(items, "2026-01-01", "2026-01-31")
-
- self.assertEqual(result[0].date_confidence, "low")
-
- def test_handles_engagement(self):
- items = [
- {
- "id": "R1",
- "title": "Thread with engagement",
- "url": "https://reddit.com/r/test/1",
- "subreddit": "test",
- "engagement": {
- "score": 100,
- "num_comments": 50,
- "upvote_ratio": 0.9,
- },
- "relevance": 0.5,
- }
- ]
-
- result = normalize.normalize_reddit_items(items, "2026-01-01", "2026-01-31")
-
- self.assertIsNotNone(result[0].engagement)
- self.assertEqual(result[0].engagement.score, 100)
- self.assertEqual(result[0].engagement.num_comments, 50)
-
-
-class TestNormalizeXItems(unittest.TestCase):
- def test_normalizes_basic_item(self):
- items = [
- {
- "id": "X1",
- "text": "Test post content",
- "url": "https://x.com/user/status/123",
- "author_handle": "testuser",
- "date": "2026-01-15",
- "why_relevant": "Relevant because...",
- "relevance": 0.9,
- }
- ]
-
- result = normalize.normalize_x_items(items, "2026-01-01", "2026-01-31")
-
- self.assertEqual(len(result), 1)
- self.assertIsInstance(result[0], schema.XItem)
- self.assertEqual(result[0].id, "X1")
- self.assertEqual(result[0].author_handle, "testuser")
-
- def test_handles_x_engagement(self):
- items = [
- {
- "id": "X1",
- "text": "Post with engagement",
- "url": "https://x.com/user/status/123",
- "author_handle": "user",
- "engagement": {
- "likes": 100,
- "reposts": 25,
- "replies": 15,
- "quotes": 5,
- },
- "relevance": 0.5,
- }
- ]
-
- result = normalize.normalize_x_items(items, "2026-01-01", "2026-01-31")
-
- self.assertIsNotNone(result[0].engagement)
- self.assertEqual(result[0].engagement.likes, 100)
- self.assertEqual(result[0].engagement.reposts, 25)
-
-
-class TestItemsToDicts(unittest.TestCase):
- def test_converts_items(self):
- items = [
- schema.RedditItem(
- id="R1",
- title="Test",
- url="https://reddit.com/r/test/1",
- subreddit="test",
- )
- ]
-
- result = normalize.items_to_dicts(items)
-
- self.assertEqual(len(result), 1)
- self.assertIsInstance(result[0], dict)
- self.assertEqual(result[0]["id"], "R1")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_normalize_v3.py b/tests/test_normalize_v3.py
new file mode 100644
index 0000000..c0bdd63
--- /dev/null
+++ b/tests/test_normalize_v3.py
@@ -0,0 +1,71 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import normalize
+
+
+class NormalizeV3Tests(unittest.TestCase):
+ def test_youtube_evergreen_fallback_keeps_older_items_when_recent_pool_is_empty(self):
+ items = [
+ {
+ "video_id": "vid-1",
+ "title": "Deploy to Fly.io tutorial",
+ "url": "https://youtube.com/watch?v=vid-1",
+ "channel_name": "Example",
+ "date": "2026-01-10",
+ "engagement": {"views": 1000, "likes": 50, "comments": 10},
+ }
+ ]
+ normalized = normalize.normalize_source_items(
+ "youtube",
+ items,
+ "2026-02-15",
+ "2026-03-17",
+ freshness_mode="evergreen_ok",
+ )
+ self.assertEqual(1, len(normalized))
+ self.assertEqual("2026-01-10", normalized[0].published_at)
+
+ def test_grounding_still_drops_older_items_in_evergreen_mode(self):
+ items = [
+ {
+ "id": "g-1",
+ "title": "Fly.io guide",
+ "url": "https://example.com/fly-guide",
+ "date": "2026-01-08",
+ "date_confidence": "high",
+ "snippet": "Step-by-step guide.",
+ }
+ ]
+ normalized = normalize.normalize_source_items(
+ "grounding",
+ items,
+ "2026-02-15",
+ "2026-03-17",
+ freshness_mode="evergreen_ok",
+ )
+ self.assertEqual([], normalized)
+
+ def test_grounding_requires_a_usable_date(self):
+ items = [
+ {
+ "id": "g-1",
+ "title": "Undated result",
+ "url": "https://example.com/undated",
+ "snippet": "No date attached.",
+ }
+ ]
+ normalized = normalize.normalize_source_items(
+ "grounding",
+ items,
+ "2026-02-15",
+ "2026-03-17",
+ )
+ self.assertEqual([], normalized)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_nux_flow.py b/tests/test_nux_flow.py
deleted file mode 100644
index 124d1ad..0000000
--- a/tests/test_nux_flow.py
+++ /dev/null
@@ -1,411 +0,0 @@
-"""End-to-end NUX integration tests.
-
-Verifies that setup wizard, status banner, quality nudge, and SKILL.md
-first-run flow work together coherently:
-- first_run flag emitted / not emitted based on SETUP_COMPLETE
-- setup subcommand runs auto_setup and writes config
-- quality score is consistent with source configuration
-- quality nudge disappears at 100%
-- banner and quality nudge don't contradict each other
-"""
-
-import sys
-import tempfile
-from pathlib import Path
-from unittest.mock import patch, MagicMock
-
-import pytest
-
-# Add scripts dir to path
-SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
-sys.path.insert(0, str(SCRIPTS_DIR))
-
-from lib import setup_wizard, quality_nudge, ui
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _base_config(**overrides):
- """Return a minimal config dict."""
- config = {
- "AUTH_TOKEN": None,
- "CT0": None,
- "XAI_API_KEY": None,
- "SCRAPECREATORS_API_KEY": None,
- }
- config.update(overrides)
- return config
-
-
-def _base_results(**overrides):
- """Return a minimal research_results dict with no errors."""
- results = {
- "x_error": None,
- "youtube_error": None,
- "reddit_error": None,
- }
- results.update(overrides)
- return results
-
-
-def _base_diag(**overrides):
- """Return a minimal diag dict for banner testing."""
- diag = {
- "setup_complete": False,
- "reddit_source": None,
- "x_source": None,
- "x_method": None,
- "youtube": False,
- "tiktok": False,
- "instagram": False,
- "hackernews": True,
- "polymarket": True,
- "bluesky": False,
- "truthsocial": False,
- "xiaohongshu": False,
- "scrapecreators": False,
- "web_search_backend": None,
- }
- diag.update(overrides)
- return diag
-
-
-def _compute(config_overrides=None, result_overrides=None, ytdlp_installed=False):
- """Helper to call compute_quality_score with mocked yt-dlp check."""
- from lib import youtube_yt
-
- config = _base_config(**(config_overrides or {}))
- results = _base_results(**(result_overrides or {}))
-
- with patch.object(youtube_yt, "is_ytdlp_installed", return_value=ytdlp_installed):
- return quality_nudge.compute_quality_score(config, results)
-
-
-# ---------------------------------------------------------------------------
-# First-run flag detection
-# ---------------------------------------------------------------------------
-
-class TestFirstRunDetection:
- """first_run flag is emitted based on SETUP_COMPLETE in config."""
-
- def test_first_run_when_setup_not_complete(self):
- """SETUP_COMPLETE missing -> is_first_run returns True."""
- config = _base_config()
- assert setup_wizard.is_first_run(config) is True
-
- def test_first_run_when_setup_complete_empty(self):
- """SETUP_COMPLETE='' -> is_first_run returns True."""
- config = _base_config(SETUP_COMPLETE="")
- assert setup_wizard.is_first_run(config) is True
-
- def test_not_first_run_when_setup_complete(self):
- """SETUP_COMPLETE=true -> is_first_run returns False."""
- config = _base_config(SETUP_COMPLETE="true")
- assert setup_wizard.is_first_run(config) is False
-
- def test_not_first_run_any_truthy_value(self):
- """SETUP_COMPLETE=1 -> is_first_run returns False."""
- config = _base_config(SETUP_COMPLETE="1")
- assert setup_wizard.is_first_run(config) is False
-
-
-# ---------------------------------------------------------------------------
-# Setup subcommand writes config
-# ---------------------------------------------------------------------------
-
-class TestSetupSubcommandWritesConfig:
- """setup subcommand runs auto_setup and writes SETUP_COMPLETE."""
-
- @patch("lib.cookie_extract.extract_cookies_with_source")
- @patch("shutil.which")
- def test_auto_setup_and_write(self, mock_which, mock_extract):
- """run_auto_setup + write_setup_config creates valid config."""
- mock_extract.return_value = ({"auth_token": "abc", "ct0": "xyz"}, "chrome")
- mock_which.return_value = "/usr/local/bin/yt-dlp"
-
- config = _base_config()
- results = setup_wizard.run_auto_setup(config)
-
- assert results["cookies_found"]["x"] == "chrome"
- assert results["ytdlp_installed"] is True
-
- with tempfile.TemporaryDirectory() as tmpdir:
- env_path = Path(tmpdir) / ".env"
- written = setup_wizard.write_setup_config(env_path)
- assert written is True
-
- content = env_path.read_text()
- assert "SETUP_COMPLETE=true" in content
- assert "FROM_BROWSER=auto" in content
-
- @patch("lib.cookie_extract.extract_cookies_with_source")
- @patch("shutil.which")
- def test_after_setup_not_first_run(self, mock_which, mock_extract):
- """After write_setup_config, is_first_run should return False."""
- mock_extract.return_value = None
- mock_which.return_value = None
-
- with tempfile.TemporaryDirectory() as tmpdir:
- env_path = Path(tmpdir) / ".env"
- setup_wizard.write_setup_config(env_path)
-
- # Simulate reading the config back
- config = {"SETUP_COMPLETE": "true"}
- assert setup_wizard.is_first_run(config) is False
-
-
-# ---------------------------------------------------------------------------
-# Quality score consistency with source config
-# ---------------------------------------------------------------------------
-
-class TestQualityScoreConsistency:
- """Quality score matches the source configuration state."""
-
- def test_zero_config_is_40_pct(self):
- """No X, no yt-dlp, no SC -> 40% (HN + Polymarket only)."""
- q = _compute()
- assert q["score_pct"] == 40
- assert set(q["core_active"]) == {"hn", "polymarket"}
-
- def test_x_cookies_is_60_pct(self):
- """X cookies active -> 60%."""
- q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
- assert q["score_pct"] == 60
- assert "x" in q["core_active"]
-
- def test_x_plus_ytdlp_is_80_pct(self):
- """X cookies + yt-dlp -> 80%."""
- q = _compute(
- config_overrides={"AUTH_TOKEN": "tok123"},
- ytdlp_installed=True,
- )
- assert q["score_pct"] == 80
- assert "x" in q["core_active"]
- assert "youtube" in q["core_active"]
-
- def test_full_config_is_100_pct(self):
- """X + yt-dlp + ScrapeCreators -> 100%."""
- q = _compute(
- config_overrides={
- "AUTH_TOKEN": "tok123",
- "SCRAPECREATORS_API_KEY": "sc_key",
- },
- ytdlp_installed=True,
- )
- assert q["score_pct"] == 100
- assert len(q["core_active"]) == 5
-
- def test_xai_key_also_enables_x(self):
- """XAI_API_KEY activates X source same as cookies."""
- q = _compute(config_overrides={"XAI_API_KEY": "xai_key"})
- assert q["score_pct"] == 60
- assert "x" in q["core_active"]
-
-
-# ---------------------------------------------------------------------------
-# Quality nudge disappears at 100%
-# ---------------------------------------------------------------------------
-
-class TestQualityNudgeDisappears:
- """Quality nudge is None at 100% and present below 100%."""
-
- def test_nudge_present_at_40(self):
- q = _compute()
- assert q["nudge_text"] is not None
- assert len(q["nudge_text"]) > 0
-
- def test_nudge_present_at_60(self):
- q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
- assert q["nudge_text"] is not None
-
- def test_nudge_present_at_80(self):
- q = _compute(
- config_overrides={"AUTH_TOKEN": "tok123"},
- ytdlp_installed=True,
- )
- assert q["nudge_text"] is not None
-
- def test_nudge_none_at_100(self):
- q = _compute(
- config_overrides={
- "AUTH_TOKEN": "tok123",
- "SCRAPECREATORS_API_KEY": "sc_key",
- },
- ytdlp_installed=True,
- )
- assert q["nudge_text"] is None
- assert q["core_missing"] == []
-
-
-# ---------------------------------------------------------------------------
-# Banner and quality nudge consistency
-# ---------------------------------------------------------------------------
-
-class TestBannerQualityNudgeConsistency:
- """Banner source count and quality nudge percentage don't contradict."""
-
- def test_zero_config_banner_3_sources_nudge_40(self):
- """Banner shows 3 sources (Reddit, HN, PM), nudge says 40%."""
- diag = _base_diag()
- banner = "\n".join(ui._build_status_banner(diag))
-
- # Banner shows 3 active sources
- assert "Reddit (threads only)" in banner
- assert "HN" in banner
- assert "Polymarket" in banner
- # X and YouTube should NOT be in the banner
- assert "X (" not in banner
- assert "YouTube" not in banner
-
- # Quality nudge is 40%
- q = _compute()
- assert q["score_pct"] == 40
-
- def test_after_wizard_banner_5_sources_nudge_80(self):
- """After wizard (X cookies + yt-dlp): banner shows 5+ sources, nudge ~80%."""
- diag = _base_diag(
- setup_complete=True,
- x_source="bird",
- x_method="browser-chrome",
- youtube=True,
- )
- banner = "\n".join(ui._build_status_banner(diag))
-
- # Banner shows X and YouTube
- assert "X (Chrome)" in banner
- assert "YouTube" in banner
- assert "Reddit (threads only)" in banner
- assert "HN" in banner
- assert "Polymarket" in banner
-
- # Quality nudge is 80%
- q = _compute(
- config_overrides={"AUTH_TOKEN": "tok123"},
- ytdlp_installed=True,
- )
- assert q["score_pct"] == 80
-
- def test_full_config_banner_all_nudge_gone(self):
- """Full config: banner shows all sources, nudge disappears (100%)."""
- diag = _base_diag(
- setup_complete=True,
- reddit_source="scrapecreators",
- x_source="bird",
- x_method="browser-chrome",
- youtube=True,
- tiktok=True,
- instagram=True,
- scrapecreators=True,
- )
- banner = "\n".join(ui._build_status_banner(diag))
-
- assert "Reddit (with comments)" in banner
- assert "X (Chrome)" in banner
- assert "YouTube" in banner
- assert "TikTok" in banner
- assert "Instagram" in banner
-
- # Quality nudge is 100% / gone
- q = _compute(
- config_overrides={
- "AUTH_TOKEN": "tok123",
- "SCRAPECREATORS_API_KEY": "sc_key",
- },
- ytdlp_installed=True,
- )
- assert q["score_pct"] == 100
- assert q["nudge_text"] is None
-
- def test_banner_first_run_suggests_setup(self):
- """First-run banner suggests running setup wizard."""
- diag = _base_diag(setup_complete=False)
- banner = "\n".join(ui._build_status_banner(diag))
- assert "First Run" in banner
- assert "/last30days setup" in banner
-
- def test_banner_partial_suggests_scrapecreators(self):
- """After setup, missing SC is the only recommendation."""
- diag = _base_diag(
- setup_complete=True,
- x_source="bird",
- x_method="browser-chrome",
- youtube=True,
- scrapecreators=False,
- )
- banner = "\n".join(ui._build_status_banner(diag))
- assert "SCRAPECREATORS_API_KEY" in banner
- assert "100 free calls, no CC" in banner
-
-
-# ---------------------------------------------------------------------------
-# Progressive narrowing of nudge suggestions
-# ---------------------------------------------------------------------------
-
-class TestProgressiveNarrowing:
- """As users configure more, nudge suggestions narrow."""
-
- def test_baseline_mentions_all_three_missing(self):
- q = _compute()
- assert "X/Twitter" in q["nudge_text"]
- assert "YouTube" in q["nudge_text"]
- assert "Reddit with comments" in q["nudge_text"]
-
- def test_with_x_mentions_yt_and_sc(self):
- q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
- assert "X/Twitter" not in q["nudge_text"]
- assert "YouTube" in q["nudge_text"]
- assert "Reddit with comments" in q["nudge_text"]
-
- def test_with_x_and_yt_mentions_sc_only(self):
- q = _compute(
- config_overrides={"AUTH_TOKEN": "tok123"},
- ytdlp_installed=True,
- )
- assert "X/Twitter" not in q["nudge_text"]
- assert "YouTube" not in q["nudge_text"]
- # SC nudge is present
- assert "Reddit" in q["nudge_text"] or "ScrapeCreators" in q["nudge_text"].lower() or "scrapecreators" in q["nudge_text"]
-
- def test_full_coverage_no_nudge(self):
- q = _compute(
- config_overrides={
- "AUTH_TOKEN": "tok123",
- "SCRAPECREATORS_API_KEY": "sc_key",
- },
- ytdlp_installed=True,
- )
- assert q["nudge_text"] is None
-
-
-# ---------------------------------------------------------------------------
-# Render quality nudge integration
-# ---------------------------------------------------------------------------
-
-class TestRenderQualityNudge:
- """render_quality_nudge produces correct output."""
-
- def test_render_at_80_pct(self):
- from lib.render import render_quality_nudge
-
- q = _compute(
- config_overrides={"AUTH_TOKEN": "tok123"},
- ytdlp_installed=True,
- )
- rendered = render_quality_nudge(q)
- assert "80%" in rendered
- assert "Research Coverage" in rendered
-
- def test_render_at_100_pct_is_empty(self):
- from lib.render import render_quality_nudge
-
- q = _compute(
- config_overrides={
- "AUTH_TOKEN": "tok123",
- "SCRAPECREATORS_API_KEY": "sc_key",
- },
- ytdlp_installed=True,
- )
- rendered = render_quality_nudge(q)
- assert rendered == ""
diff --git a/tests/test_openai_reddit.py b/tests/test_openai_reddit.py
deleted file mode 100644
index 74cc4dc..0000000
--- a/tests/test_openai_reddit.py
+++ /dev/null
@@ -1,82 +0,0 @@
-"""Tests for openai_reddit module."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add scripts directory to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import http
-from lib.openai_reddit import _is_model_access_error, MODEL_FALLBACK_ORDER
-
-
-class TestIsModelAccessError(unittest.TestCase):
- """Tests for _is_model_access_error function."""
-
- def test_returns_false_for_non_400_error(self):
- """Non-400 errors should not trigger fallback."""
- error = http.HTTPError("Server error", status_code=500, body="Internal error")
- self.assertFalse(_is_model_access_error(error))
-
- def test_returns_false_for_400_without_body(self):
- """400 without body should not trigger fallback."""
- error = http.HTTPError("Bad request", status_code=400, body=None)
- self.assertFalse(_is_model_access_error(error))
-
- def test_returns_true_for_verification_error(self):
- """Verification error should trigger fallback."""
- error = http.HTTPError(
- "Bad request",
- status_code=400,
- body='{"error": {"message": "Your organization must be verified to use the model \'gpt-5.2\'"}}'
- )
- self.assertTrue(_is_model_access_error(error))
-
- def test_returns_true_for_access_error(self):
- """Access denied error should trigger fallback."""
- error = http.HTTPError(
- "Bad request",
- status_code=400,
- body='{"error": {"message": "Your account does not have access to this model"}}'
- )
- self.assertTrue(_is_model_access_error(error))
-
- def test_returns_true_for_model_not_found(self):
- """Model not found error should trigger fallback."""
- error = http.HTTPError(
- "Bad request",
- status_code=400,
- body='{"error": {"message": "The model gpt-5.2 was not found"}}'
- )
- self.assertTrue(_is_model_access_error(error))
-
- def test_returns_false_for_unrelated_400(self):
- """Unrelated 400 errors should not trigger fallback."""
- error = http.HTTPError(
- "Bad request",
- status_code=400,
- body='{"error": {"message": "Invalid JSON in request body"}}'
- )
- self.assertFalse(_is_model_access_error(error))
-
-
-class TestModelFallbackOrder(unittest.TestCase):
- """Tests for MODEL_FALLBACK_ORDER constant."""
-
- def test_mini_first(self):
- """Mini models should come first (cost-efficient for structured extraction)."""
- self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-5-mini")
-
- def test_contains_mainline_fallbacks(self):
- """Fallback list should include mainline models as last resort."""
- self.assertIn("gpt-4.1", MODEL_FALLBACK_ORDER)
- self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER)
-
- def test_no_gpt4o_mini(self):
- """gpt-4o-mini should NOT be in fallback (no domain filtering support)."""
- self.assertNotIn("gpt-4o-mini", MODEL_FALLBACK_ORDER)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py
new file mode 100644
index 0000000..e1cc799
--- /dev/null
+++ b/tests/test_pipeline_v3.py
@@ -0,0 +1,884 @@
+import sys
+import threading
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import pipeline
+from lib import http
+from lib import schema
+
+
+class PipelineV3Tests(unittest.TestCase):
+ def test_mock_pipeline_report_without_live_credentials(self):
+ report = pipeline.run(
+ topic="test topic",
+ config={"LAST30DAYS_REASONING_PROVIDER": "gemini"},
+ depth="quick",
+ requested_sources=["reddit", "x", "grounding"],
+ mock=True,
+ )
+ self.assertEqual("test topic", report.topic)
+ self.assertTrue(report.ranked_candidates)
+ self.assertTrue(report.clusters)
+ self.assertIn("x", report.items_by_source)
+ # Grounding items now enter the ranked pool (web search backends produce real items)
+ self.assertIn("grounding", report.items_by_source)
+ self.assertEqual("gemini", report.provider_runtime.reasoning_provider)
+
+
+class TestSourceFetchCap(unittest.TestCase):
+ """X source fetch count must be capped by MAX_SOURCE_FETCHES."""
+
+ def test_x_capped_in_max_source_fetches(self):
+ """MAX_SOURCE_FETCHES must cap X at 2 to prevent 429 cascades."""
+ self.assertIn("x", pipeline.MAX_SOURCE_FETCHES)
+ self.assertEqual(pipeline.MAX_SOURCE_FETCHES["x"], 2)
+
+ def test_cap_logic_limits_source_submissions(self):
+ """Verify the cap logic skips submissions beyond the limit."""
+ cap = pipeline.MAX_SOURCE_FETCHES.get("x", float("inf"))
+ subquery_sources = [
+ ["x", "reddit", "youtube"],
+ ["x", "reddit", "youtube"],
+ ["x", "reddit", "youtube"],
+ ["x", "reddit", "youtube"],
+ ]
+ source_fetch_count: dict[str, int] = {}
+ submitted: list[str] = []
+ for sources in subquery_sources:
+ for source in sources:
+ source_cap = pipeline.MAX_SOURCE_FETCHES.get(source)
+ if source_cap is not None:
+ current = source_fetch_count.get(source, 0)
+ if current >= source_cap:
+ continue
+ source_fetch_count[source] = current + 1
+ submitted.append(source)
+
+ x_count = submitted.count("x")
+ reddit_count = submitted.count("reddit")
+ self.assertEqual(x_count, 2, f"X should be capped at 2, got {x_count}")
+ self.assertEqual(reddit_count, 4, f"Reddit should be uncapped, got {reddit_count}")
+
+ @patch("lib.pipeline._retrieve_stream")
+ def test_mock_run_caps_x_fetches(self, mock_retrieve):
+ """Pipeline.run in mock mode should call _retrieve_stream for X at most 2 times."""
+ mock_retrieve.side_effect = lambda **kwargs: pipeline._mock_stream_results(
+ kwargs["source"], kwargs["subquery"]
+ )
+ report = pipeline.run(
+ topic="compare iPhone vs Android vs Pixel vs Samsung",
+ config={"LAST30DAYS_REASONING_PROVIDER": "gemini"},
+ depth="quick",
+ requested_sources=["reddit", "x"],
+ mock=True,
+ )
+ x_calls = [
+ call for call in mock_retrieve.call_args_list
+ if call.kwargs.get("source") == "x"
+ ]
+ self.assertLessEqual(
+ len(x_calls), 2,
+ f"X should be fetched at most 2 times, got {len(x_calls)}",
+ )
+
+
+class TestRateLimitSharing(unittest.TestCase):
+ """429 signals should be shared across subqueries."""
+
+ def test_is_rate_limit_error_detects_429_status(self):
+ exc = http.HTTPError("HTTP 429: Too Many Requests", status_code=429)
+ self.assertTrue(pipeline._is_rate_limit_error(exc))
+
+ def test_is_rate_limit_error_ignores_non_429(self):
+ exc = http.HTTPError("HTTP 400: Bad Request", status_code=400)
+ self.assertFalse(pipeline._is_rate_limit_error(exc))
+
+ def test_is_rate_limit_error_detects_429_in_string(self):
+ exc = RuntimeError("xAI returned 429 rate limit")
+ self.assertTrue(pipeline._is_rate_limit_error(exc))
+
+ def test_is_rate_limit_error_rejects_unrelated_error(self):
+ exc = RuntimeError("Connection refused")
+ self.assertFalse(pipeline._is_rate_limit_error(exc))
+
+ def test_retrieve_stream_skips_rate_limited_source(self):
+ """_retrieve_stream should return empty when source is rate-limited."""
+ from lib import schema
+ rate_limited = {"x"}
+ lock = threading.Lock()
+ subquery = schema.SubQuery(
+ label="test",
+ search_query="test query",
+ ranking_query="test query",
+ sources=["x"],
+ )
+ items, artifact = pipeline._retrieve_stream(
+ topic="test",
+ subquery=subquery,
+ source="x",
+ config={},
+ depth="quick",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=schema.ProviderRuntime(
+ reasoning_provider="mock",
+ planner_model="mock",
+ rerank_model="mock",
+ ),
+ mock=True,
+ rate_limited_sources=rate_limited,
+ rate_limit_lock=lock,
+ )
+ self.assertEqual(items, [])
+ self.assertEqual(artifact, {})
+
+
+class TestThinSourceRetry(unittest.TestCase):
+ @patch("lib.pipeline._retrieve_stream")
+ def test_retry_includes_planned_source_with_zero_initial_items(self, mock_retrieve):
+ mock_retrieve.return_value = (
+ [
+ {
+ "id": "X100",
+ "text": "OpenClaw funding update from an investor",
+ "url": "https://x.com/example/status/100",
+ "author_handle": "example",
+ "date": "2026-03-15",
+ "engagement": {"likes": 25, "reposts": 4, "replies": 2},
+ "relevance": 0.8,
+ "why_relevant": "retry result",
+ }
+ ],
+ {},
+ )
+
+ plan = schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="latest OpenClaw funding updates",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="latest OpenClaw funding updates",
+ ranking_query="What recent evidence matters for OpenClaw funding?",
+ sources=["x", "reddit"],
+ )
+ ],
+ source_weights={"x": 1.0, "reddit": 1.0},
+ )
+ bundle = schema.RetrievalBundle(
+ items_by_source={
+ "reddit": [
+ _make_source_item("reddit", "r1", "https://reddit.com/1"),
+ _make_source_item("reddit", "r2", "https://reddit.com/2"),
+ _make_source_item("reddit", "r3", "https://reddit.com/3"),
+ ]
+ }
+ )
+
+ pipeline._retry_thin_sources(
+ topic="latest OpenClaw funding updates",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ settings=pipeline.DEPTH_SETTINGS["default"],
+ )
+
+ self.assertEqual(["x"], [call.kwargs["source"] for call in mock_retrieve.call_args_list])
+ self.assertIn("x", bundle.items_by_source)
+ self.assertEqual("https://x.com/example/status/100", bundle.items_by_source["x"][0].url)
+
+
+def _make_runtime(x_backend="bird"):
+ return schema.ProviderRuntime(
+ reasoning_provider="mock",
+ planner_model="mock",
+ rerank_model="mock",
+ x_search_backend=x_backend,
+ )
+
+
+def _make_plan(topic="test topic"):
+ return schema.QueryPlan(
+ intent="exploration",
+ freshness_mode="balanced_recent",
+ cluster_mode="topic",
+ raw_topic=topic,
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query=topic,
+ ranking_query=f"What recent evidence matters for {topic}?",
+ sources=["x", "reddit"],
+ )
+ ],
+ source_weights={"x": 1.0, "reddit": 1.0},
+ )
+
+
+def _make_source_item(source, item_id, url, author=None, body="", container=None, metadata=None):
+ return schema.SourceItem(
+ item_id=item_id,
+ source=source,
+ title=f"Item {item_id}",
+ body=body,
+ url=url,
+ author=author,
+ container=container,
+ metadata=metadata or {},
+ )
+
+
+class TestSupplementalSearches(unittest.TestCase):
+ """R1: Phase 2 entity drilling should be wired into the pipeline."""
+
+ def test_run_supplemental_searches_exists(self):
+ """_run_supplemental_searches must be a callable in pipeline module."""
+ self.assertTrue(
+ hasattr(pipeline, "_run_supplemental_searches"),
+ "_run_supplemental_searches function not found in pipeline module",
+ )
+ self.assertTrue(callable(pipeline._run_supplemental_searches))
+
+ @patch("lib.bird_x.search_handles")
+ @patch("lib.entity_extract.extract_entities")
+ def test_entity_extract_called_after_phase1(self, mock_extract, mock_handles):
+ """Phase 2 should call entity_extract on Phase 1 X results, then search_handles."""
+ mock_extract.return_value = {"x_handles": ["analyst1", "reporter2"], "x_hashtags": [], "reddit_subreddits": []}
+ mock_handles.return_value = [
+ {
+ "id": "supp1",
+ "text": "Supplemental tweet from analyst1",
+ "url": "https://x.com/analyst1/status/999",
+ "author_handle": "analyst1",
+ "date": "2026-03-15",
+ "engagement": {"likes": 50},
+ "relevance": 0.8,
+ "why_relevant": "direct handle search",
+ }
+ ]
+
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/analyst1/status/1", author="analyst1", body="Some tweet about AI"),
+ _make_source_item("x", "X2", "https://x.com/reporter2/status/2", author="reporter2", body="AI analysis @expert3"),
+ ]
+
+ plan = _make_plan("AI safety")
+ config = {}
+
+ pipeline._run_supplemental_searches(
+ topic="AI safety",
+ bundle=bundle,
+ plan=plan,
+ config=config,
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ )
+
+ mock_extract.assert_called_once()
+ mock_handles.assert_called_once()
+ # Supplemental items should be merged into bundle
+ x_urls = {item.url for item in bundle.items_by_source.get("x", [])}
+ self.assertIn("https://x.com/analyst1/status/999", x_urls)
+
+ @patch("lib.bird_x.search_handles")
+ @patch("lib.entity_extract.extract_entities")
+ def test_supplemental_items_deduplicated_by_url(self, mock_extract, mock_handles):
+ """Supplemental items with same URL as Phase 1 should not be duplicated."""
+ mock_extract.return_value = {"x_handles": ["analyst1"], "x_hashtags": [], "reddit_subreddits": []}
+ # Return item with same URL as Phase 1
+ mock_handles.return_value = [
+ {
+ "id": "dup1",
+ "text": "Same tweet",
+ "url": "https://x.com/analyst1/status/1",
+ "author_handle": "analyst1",
+ "date": "2026-03-15",
+ "engagement": {"likes": 50},
+ "relevance": 0.8,
+ "why_relevant": "duplicate",
+ }
+ ]
+
+ bundle = schema.RetrievalBundle()
+ original = _make_source_item("x", "X1", "https://x.com/analyst1/status/1", author="analyst1")
+ bundle.items_by_source["x"] = [original]
+
+ plan = _make_plan("AI safety")
+
+ pipeline._run_supplemental_searches(
+ topic="AI safety",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ )
+
+ # Should still have only 1 item (no duplicates)
+ x_items = bundle.items_by_source.get("x", [])
+ urls = [item.url for item in x_items]
+ self.assertEqual(
+ urls.count("https://x.com/analyst1/status/1"), 1,
+ f"Duplicate URL found: {urls}",
+ )
+
+ def test_phase2_skipped_in_quick_mode(self):
+ """_run_supplemental_searches should return immediately when depth='quick'."""
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
+ ]
+
+ # If it tries to import entity_extract, that's fine -- it should return before calling it
+ pipeline._run_supplemental_searches(
+ topic="test",
+ bundle=bundle,
+ plan=_make_plan(),
+ config={},
+ depth="quick",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ )
+ # Bundle should be unchanged (only original item)
+ self.assertEqual(len(bundle.items_by_source["x"]), 1)
+
+ def test_phase2_skipped_in_mock_mode(self):
+ """_run_supplemental_searches should return immediately when mock=True."""
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
+ ]
+
+ pipeline._run_supplemental_searches(
+ topic="test",
+ bundle=bundle,
+ plan=_make_plan(),
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=True,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ )
+ self.assertEqual(len(bundle.items_by_source["x"]), 1)
+
+ def test_phase2_skipped_when_x_rate_limited(self):
+ """_run_supplemental_searches should skip when X is rate-limited."""
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
+ ]
+
+ pipeline._run_supplemental_searches(
+ topic="test",
+ bundle=bundle,
+ plan=_make_plan(),
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=False,
+ rate_limited_sources={"x"},
+ rate_limit_lock=threading.Lock(),
+ )
+ self.assertEqual(len(bundle.items_by_source["x"]), 1)
+
+ def test_phase2_skipped_when_backend_not_bird(self):
+ """_run_supplemental_searches should skip when X backend is not bird."""
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
+ ]
+
+ pipeline._run_supplemental_searches(
+ topic="test",
+ bundle=bundle,
+ plan=_make_plan(),
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("xai"),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ )
+ self.assertEqual(len(bundle.items_by_source["x"]), 1)
+
+
+class TestThinSourceRetry(unittest.TestCase):
+ """R2: Dynamic query refinement on thin results."""
+
+ def test_retry_thin_sources_exists(self):
+ """_retry_thin_sources must be a callable in pipeline module."""
+ self.assertTrue(
+ hasattr(pipeline, "_retry_thin_sources"),
+ "_retry_thin_sources function not found in pipeline module",
+ )
+ self.assertTrue(callable(pipeline._retry_thin_sources))
+
+ @patch("lib.pipeline._retrieve_stream")
+ def test_thin_source_retried_with_core_subject(self, mock_retrieve):
+ """Sources with < 3 items and no errors should be retried."""
+ mock_retrieve.return_value = (
+ [
+ {
+ "id": "retry1",
+ "title": "Retry result",
+ "url": "https://reddit.com/r/test/2",
+ "subreddit": "test",
+ "date": "2026-03-15",
+ "engagement": {"score": 10},
+ "selftext": "Retry content",
+ "relevance": 0.7,
+ "why_relevant": "retry",
+ }
+ ],
+ {},
+ )
+
+ bundle = schema.RetrievalBundle()
+ # Only 1 reddit item (thin)
+ bundle.items_by_source["reddit"] = [
+ _make_source_item("reddit", "R1", "https://reddit.com/r/test/1", container="test"),
+ ]
+ # 5 X items (not thin)
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", f"X{i}", f"https://x.com/a/{i}") for i in range(5)
+ ]
+
+ plan = _make_plan("advanced AI safety techniques")
+ settings = pipeline.DEPTH_SETTINGS["default"]
+
+ pipeline._retry_thin_sources(
+ topic="advanced AI safety techniques",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime(),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ settings=settings,
+ )
+
+ # _retrieve_stream should have been called for reddit (thin source)
+ mock_retrieve.assert_called()
+ call_sources = [c.kwargs.get("source") for c in mock_retrieve.call_args_list]
+ self.assertIn("reddit", call_sources)
+ # X should NOT have been retried
+ self.assertNotIn("x", call_sources)
+
+ def test_sources_with_enough_items_not_retried(self):
+ """Sources with >= 3 items should not be retried."""
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["reddit"] = [
+ _make_source_item("reddit", f"R{i}", f"https://reddit.com/r/test/{i}") for i in range(5)
+ ]
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", f"X{i}", f"https://x.com/a/{i}") for i in range(5)
+ ]
+
+ plan = _make_plan("AI safety")
+ settings = pipeline.DEPTH_SETTINGS["default"]
+
+ with patch("lib.pipeline._retrieve_stream") as mock_retrieve:
+ pipeline._retry_thin_sources(
+ topic="AI safety",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime(),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ settings=settings,
+ )
+ mock_retrieve.assert_not_called()
+
+ def test_errored_sources_not_retried(self):
+ """Sources in errors_by_source should not be retried even if thin.
+ Non-errored thin sources SHOULD still be retried."""
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["reddit"] = [
+ _make_source_item("reddit", "R1", "https://reddit.com/r/test/1"),
+ ]
+ bundle.errors_by_source["reddit"] = "API error"
+
+ plan = _make_plan("AI safety")
+ settings = pipeline.DEPTH_SETTINGS["default"]
+
+ mock_items = [{"id": "X1", "title": "test", "url": "https://x.com/1", "text": "test"}]
+ with patch("lib.pipeline._retrieve_stream", return_value=(mock_items, {})) as mock_retrieve:
+ pipeline._retry_thin_sources(
+ topic="AI safety",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime(),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ settings=settings,
+ )
+ # x (non-errored, thin) should be retried; reddit (errored) should not
+ if mock_retrieve.call_count > 0:
+ retried_sources = [call.kwargs.get("source") or call.args[2] for call in mock_retrieve.call_args_list if hasattr(call, 'kwargs')]
+ self.assertNotIn("reddit", [c.kwargs.get("source") for c in mock_retrieve.call_args_list])
+
+ def test_retry_skipped_in_quick_mode(self):
+ """_retry_thin_sources should return immediately in quick mode."""
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["reddit"] = [
+ _make_source_item("reddit", "R1", "https://reddit.com/r/test/1"),
+ ]
+
+ plan = _make_plan("AI safety")
+ settings = pipeline.DEPTH_SETTINGS["quick"]
+
+ with patch("lib.pipeline._retrieve_stream") as mock_retrieve:
+ pipeline._retry_thin_sources(
+ topic="AI safety",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="quick",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime(),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ settings=settings,
+ )
+ mock_retrieve.assert_not_called()
+
+
+class TestErrorCleanup(unittest.TestCase):
+ """Source errors should be cleared when the source has items from other subqueries."""
+
+ def test_error_cleared_when_source_has_items(self):
+ """A source that 429'd on one subquery but succeeded on another is not errored."""
+ bundle = schema.RetrievalBundle(artifacts={})
+ item = schema.SourceItem(
+ item_id="x1", source="x", title="A tweet", body="content",
+ url="https://x.com/user/status/1",
+ )
+ bundle.items_by_source["x"] = [item]
+ bundle.errors_by_source["x"] = "HTTP 429: Too Many Requests"
+
+ # Simulate the cleanup logic from pipeline.run()
+ for source in list(bundle.errors_by_source):
+ if bundle.items_by_source.get(source):
+ del bundle.errors_by_source[source]
+
+ self.assertNotIn("x", bundle.errors_by_source,
+ "X should not be errored when it has items")
+
+ def test_error_kept_when_source_has_no_items(self):
+ """A source with zero items should remain in errors_by_source."""
+ bundle = schema.RetrievalBundle(artifacts={})
+ bundle.errors_by_source["x"] = "HTTP 429: Too Many Requests"
+
+ for source in list(bundle.errors_by_source):
+ if bundle.items_by_source.get(source):
+ del bundle.errors_by_source[source]
+
+ self.assertIn("x", bundle.errors_by_source,
+ "X should remain errored when it has no items")
+
+
+class TestXHandleFlag(unittest.TestCase):
+ """R3: --x-handle CLI flag and pipeline parameter."""
+
+ def test_cli_accepts_x_handle_flag(self):
+ """build_parser() should accept --x-handle."""
+ import last30days as cli
+
+ parser = cli.build_parser()
+ args = parser.parse_args(["test topic", "--x-handle", "elonmusk"])
+ self.assertEqual(args.x_handle, "elonmusk")
+
+ def test_cli_x_handle_default_is_none(self):
+ """--x-handle should default to None."""
+ import last30days as cli
+
+ parser = cli.build_parser()
+ args = parser.parse_args(["test topic"])
+ self.assertIsNone(args.x_handle)
+
+ def test_pipeline_run_accepts_x_handle(self):
+ """pipeline.run() should accept x_handle keyword argument."""
+ import inspect
+ sig = inspect.signature(pipeline.run)
+ self.assertIn("x_handle", sig.parameters, "pipeline.run() must accept x_handle parameter")
+
+ def test_x_handle_passed_to_supplemental_searches(self):
+ """When x_handle is provided, it should trigger targeted handle search."""
+ # Run pipeline in mock mode with x_handle -- should not raise
+ report = pipeline.run(
+ topic="test topic",
+ config={"LAST30DAYS_REASONING_PROVIDER": "gemini"},
+ depth="quick",
+ requested_sources=["reddit", "x", "grounding"],
+ mock=True,
+ x_handle="testuser",
+ )
+ self.assertEqual("test topic", report.topic)
+
+
+class TestWarnings(unittest.TestCase):
+ def _item(self, source="reddit"):
+ return schema.SourceItem(item_id="1", source=source, title="t", body="b", url="u")
+
+ def _candidate(self, source="reddit", score=50.0):
+ c = schema.Candidate(
+ candidate_id="c1", item_id="1", source=source, title="t", url="u",
+ snippet="s", subquery_labels=["main"], native_ranks={"main:reddit": 1},
+ local_relevance=0.5, freshness=50, engagement=10, source_quality=0.7,
+ rrf_score=0.01, sources=[source],
+ )
+ c.final_score = score
+ return c
+
+ def test_no_candidates_warning(self):
+ w = pipeline._warnings({"reddit": [self._item()]}, [], {})
+ self.assertTrue(any("No candidates" in msg for msg in w))
+
+ def test_thin_evidence_warning(self):
+ candidates = [self._candidate() for _ in range(3)]
+ w = pipeline._warnings({"reddit": [self._item()]}, candidates, {})
+ self.assertTrue(any("thin" in msg.lower() for msg in w))
+
+ def test_single_source_concentration(self):
+ candidates = [self._candidate() for _ in range(5)]
+ w = pipeline._warnings({"reddit": [self._item()]}, candidates, {})
+ self.assertTrue(any("concentrated" in msg.lower() for msg in w))
+
+ def test_source_errors_listed(self):
+ w = pipeline._warnings({}, [self._candidate()], {"x": "timeout"})
+ self.assertTrue(any("x" in msg for msg in w))
+
+ def test_no_items_warning(self):
+ w = pipeline._warnings({}, [], {})
+ self.assertTrue(any("No source returned" in msg for msg in w))
+
+
+class TestXRelatedSupplementalSearch(unittest.TestCase):
+ """Tests for --x-related weighted supplemental search."""
+
+ @patch("lib.bird_x.search_handles")
+ @patch("lib.entity_extract.extract_entities")
+ def test_x_related_triggers_supplemental_related_label(self, mock_extract, mock_handles):
+ """x_related handles should be searched and added with supplemental-related label."""
+ mock_extract.return_value = {"x_handles": [], "x_hashtags": [], "reddit_subreddits": []}
+ mock_handles.return_value = [
+ {
+ "id": "rel1",
+ "text": "Related tweet from biancacensori",
+ "url": "https://x.com/biancacensori/status/555",
+ "author_handle": "biancacensori",
+ "date": "2026-03-15",
+ "engagement": {"likes": 30},
+ "relevance": 0.7,
+ "why_relevant": "related handle search",
+ }
+ ]
+
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/kanyewest/status/1", author="kanyewest"),
+ ]
+
+ plan = _make_plan("Kanye West")
+
+ pipeline._run_supplemental_searches(
+ topic="Kanye West",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ x_related=["biancacensori"],
+ )
+
+ # search_handles should have been called for the related handle
+ mock_handles.assert_called()
+ # The supplemental-related subquery label should exist in the plan
+ labels = [sq.label for sq in plan.subqueries]
+ self.assertIn("supplemental-related", labels)
+ # The supplemental-related subquery should have weight 0.3
+ related_sq = [sq for sq in plan.subqueries if sq.label == "supplemental-related"][0]
+ self.assertAlmostEqual(related_sq.weight, 0.3)
+
+ @patch("lib.bird_x.search_handles")
+ @patch("lib.entity_extract.extract_entities")
+ def test_no_x_related_no_supplemental_related_label(self, mock_extract, mock_handles):
+ """Without x_related, supplemental-related label should not appear."""
+ mock_extract.return_value = {"x_handles": ["analyst1"], "x_hashtags": [], "reddit_subreddits": []}
+ mock_handles.return_value = [
+ {
+ "id": "supp1",
+ "text": "Supplemental tweet",
+ "url": "https://x.com/analyst1/status/999",
+ "author_handle": "analyst1",
+ "date": "2026-03-15",
+ "engagement": {"likes": 50},
+ "relevance": 0.8,
+ "why_relevant": "direct handle search",
+ }
+ ]
+
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/analyst1/status/1", author="analyst1"),
+ ]
+
+ plan = _make_plan("AI safety")
+
+ pipeline._run_supplemental_searches(
+ topic="AI safety",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime("bird"),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ )
+
+ # supplemental-related label should NOT exist (no x_related provided)
+ labels = [sq.label for sq in plan.subqueries]
+ self.assertNotIn("supplemental-related", labels)
+
+
+class TestRetryThinSourcesCoreEqualsTopic(unittest.TestCase):
+ """Test that _retry_thin_sources fires even when core == topic (the fix)."""
+
+ @patch("lib.pipeline._retrieve_stream")
+ def test_retry_fires_when_core_equals_topic(self, mock_retrieve):
+ """Topic 'Kanye West' with 0 YouTube items should trigger retry.
+
+ Previously this was skipped because core 'kanye west' == topic.
+ The fix ensures retry still fires for short topics.
+ """
+ mock_retrieve.return_value = (
+ [
+ {
+ "id": "YT1",
+ "title": "Kanye West new album leak",
+ "url": "https://www.youtube.com/watch?v=abc123",
+ "date": "2026-03-15",
+ "engagement": {"views": 1000},
+ "relevance": 0.8,
+ "why_relevant": "retry result",
+ }
+ ],
+ {},
+ )
+
+ plan = schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="Kanye West",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="Kanye West",
+ ranking_query="What recent evidence matters for Kanye West?",
+ sources=["youtube", "x"],
+ )
+ ],
+ source_weights={"youtube": 1.0, "x": 1.0},
+ )
+ bundle = schema.RetrievalBundle()
+ # YouTube has 0 items (thin), X has enough
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", f"X{i}", f"https://x.com/a/{i}") for i in range(5)
+ ]
+
+ pipeline._retry_thin_sources(
+ topic="Kanye West",
+ bundle=bundle,
+ plan=plan,
+ config={},
+ depth="default",
+ date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime(),
+ mock=False,
+ rate_limited_sources=set(),
+ rate_limit_lock=threading.Lock(),
+ settings=pipeline.DEPTH_SETTINGS["default"],
+ )
+
+ # _retrieve_stream should have been called for youtube
+ mock_retrieve.assert_called()
+ retried_sources = [c.kwargs["source"] for c in mock_retrieve.call_args_list]
+ self.assertIn("youtube", retried_sources)
+ # YouTube should now have items in the bundle
+ self.assertIn("youtube", bundle.items_by_source)
+
+
+class TestZeroKeyPipelineRun(unittest.TestCase):
+ """Pipeline should complete with local fallbacks when no reasoning keys are configured."""
+
+ @patch("lib.pipeline._retrieve_stream")
+ def test_zero_key_run_produces_report(self, mock_retrieve):
+ mock_retrieve.side_effect = lambda **kwargs: pipeline._mock_stream_results(
+ kwargs["source"], kwargs["subquery"]
+ )
+ config = {"LAST30DAYS_REASONING_PROVIDER": "auto"}
+ report = pipeline.run(
+ topic="test zero key topic",
+ config=config,
+ depth="quick",
+ requested_sources=["hackernews"],
+ )
+ self.assertEqual("test zero key topic", report.topic)
+ self.assertEqual("local", report.provider_runtime.reasoning_provider)
+ self.assertEqual("deterministic", report.provider_runtime.planner_model)
+ self.assertTrue(
+ any("fallback" in note for note in report.query_plan.notes),
+ f"Expected fallback plan, got notes: {report.query_plan.notes}",
+ )
+ for candidate in report.ranked_candidates:
+ self.assertEqual("fallback-local-score", candidate.explanation)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_planner_v3.py b/tests/test_planner_v3.py
new file mode 100644
index 0000000..62e4f04
--- /dev/null
+++ b/tests/test_planner_v3.py
@@ -0,0 +1,285 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import planner
+
+
+class PlannerV3Tests(unittest.TestCase):
+ def test_default_how_to_expands_past_llm_narrow_source_weights(self):
+ raw = {
+ "intent": "how_to",
+ "freshness_mode": "balanced_recent",
+ "cluster_mode": "workflow",
+ "source_weights": {"hackernews": 0.7, "reddit": 0.3},
+ "subqueries": [
+ {
+ "label": "primary",
+ "search_query": "deploy app to Fly.io guide",
+ "ranking_query": "How do I deploy an app to Fly.io?",
+ "sources": ["hackernews"],
+ "weight": 1.0,
+ }
+ ],
+ }
+ plan = planner._sanitize_plan(
+ raw,
+ "how to deploy on Fly.io",
+ ["reddit", "x", "youtube", "hackernews"],
+ None,
+ "default",
+ )
+ sources = plan.subqueries[0].sources
+ # how_to capability routing selects video + discussion
+ self.assertIn("reddit", sources)
+ self.assertIn("youtube", sources)
+ self.assertIn("reddit", plan.source_weights)
+ self.assertIn("youtube", plan.source_weights)
+ self.assertEqual("evergreen_ok", plan.freshness_mode)
+
+ def test_comparison_uses_deterministic_plan_and_preserves_entities(self):
+ plan = planner.plan_query(
+ topic="openclaw vs nanoclaw vs ironclaw",
+ available_sources=["reddit", "x", "youtube", "hackernews", "polymarket"],
+ requested_sources=None,
+ depth="default",
+ provider=object(),
+ model="ignored",
+ )
+ self.assertEqual("comparison", plan.intent)
+ self.assertEqual(["deterministic-comparison-plan"], plan.notes)
+ self.assertEqual(4, len(plan.subqueries))
+ joined_queries = "\n".join(subquery.search_query for subquery in plan.subqueries).lower()
+ self.assertIn("openclaw", joined_queries)
+ self.assertIn("nanoclaw", joined_queries)
+ self.assertIn("ironclaw", joined_queries)
+
+ def test_fallback_plan_emits_dual_query_fields(self):
+ plan = planner.plan_query(
+ topic="codex vs claude code",
+ available_sources=["reddit", "x"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("comparison", plan.intent)
+ self.assertGreaterEqual(len(plan.subqueries), 2)
+ for subquery in plan.subqueries:
+ self.assertTrue(subquery.search_query)
+ self.assertTrue(subquery.ranking_query)
+
+ def test_factual_topic_uses_no_cluster_mode(self):
+ plan = planner.plan_query(
+ topic="what is the parameter count of claude code",
+ available_sources=["reddit", "hackernews"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("factual", plan.intent)
+ self.assertEqual("none", plan.cluster_mode)
+
+ def test_quick_mode_collapses_fallback_to_single_subquery(self):
+ plan = planner.plan_query(
+ topic="codex vs claude code",
+ available_sources=["reddit", "x"],
+ requested_sources=None,
+ depth="quick",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("comparison", plan.intent)
+ self.assertEqual(1, len(plan.subqueries))
+ self.assertEqual(["reddit", "x"], plan.subqueries[0].sources)
+
+ def test_default_comparison_uses_all_capable_sources(self):
+ plan = planner.plan_query(
+ topic="codex vs claude code",
+ available_sources=["reddit", "x", "youtube", "hackernews", "polymarket"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("comparison", plan.intent)
+ for subquery in plan.subqueries:
+ # Default depth should not artificially cap sources
+ self.assertGreaterEqual(len(subquery.sources), 4)
+
+ def test_default_how_to_keeps_youtube_in_source_mix(self):
+ plan = planner.plan_query(
+ topic="how to deploy remotion animations for claude code",
+ available_sources=["reddit", "x", "youtube", "hackernews"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("how_to", plan.intent)
+ sources = plan.subqueries[0].sources
+ self.assertIn("youtube", sources)
+ self.assertIn("reddit", sources)
+
+ def test_how_to_sources_includes_capability_matched_extras(self):
+ """how_to routing should include additional sources beyond the core ones."""
+ plan = planner.plan_query(
+ topic="how to deploy on Fly.io",
+ available_sources=["reddit", "tiktok", "instagram", "youtube", "hackernews"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("how_to", plan.intent)
+ sources = plan.subqueries[0].sources
+ self.assertIn("youtube", sources)
+ self.assertIn("reddit", sources)
+ # Additional capability-matched sources should also be included
+ self.assertGreater(len(sources), 2,
+ f"how_to should include >2 sources, got {len(sources)}: {sources}")
+
+ def test_ncaa_tournament_is_breaking_news(self):
+ intent = planner._infer_intent("NCAA tournament brackets")
+ self.assertEqual("breaking_news", intent)
+
+ def test_march_madness_is_breaking_news(self):
+ intent = planner._infer_intent("2026 March Madness")
+ self.assertEqual("breaking_news", intent)
+
+ def test_factual_plan_has_at_most_2_subqueries(self):
+ plan = planner.plan_query(
+ topic="who acquired Wiz",
+ available_sources=["reddit", "x", "hackernews"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("factual", plan.intent)
+ self.assertLessEqual(len(plan.subqueries), 2)
+
+ def test_default_how_to_prefers_longform_video_over_shortform(self):
+ plan = planner.plan_query(
+ topic="how to deploy on Fly.io",
+ available_sources=["reddit", "tiktok", "instagram", "youtube", "hackernews"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("how_to", plan.intent)
+ sources = plan.subqueries[0].sources
+ # how_to routing should include youtube (longform) over tiktok/instagram
+ self.assertIn("youtube", sources)
+ self.assertIn("reddit", sources)
+
+ def test_prediction_includes_tiktok_and_instagram(self):
+ """TikTok and Instagram are no longer excluded from prediction intent."""
+ plan = planner.plan_query(
+ topic="odds of US recession 2026",
+ available_sources=["reddit", "x", "tiktok", "instagram", "youtube", "hackernews", "polymarket"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("prediction", plan.intent)
+ all_sources = set()
+ for subquery in plan.subqueries:
+ all_sources.update(subquery.sources)
+ self.assertIn("tiktok", all_sources)
+ self.assertIn("instagram", all_sources)
+
+ def test_opinion_includes_tiktok_and_instagram(self):
+ """TikTok and Instagram are no longer excluded from opinion intent."""
+ plan = planner.plan_query(
+ topic="thoughts on OpenAI Codex pricing",
+ available_sources=["reddit", "x", "tiktok", "instagram", "youtube", "hackernews"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("opinion", plan.intent)
+ all_sources = set()
+ for subquery in plan.subqueries:
+ all_sources.update(subquery.sources)
+ self.assertIn("tiktok", all_sources)
+ self.assertIn("instagram", all_sources)
+
+ def test_comparison_includes_polymarket(self):
+ """Polymarket should not be excluded from comparison intent plans."""
+ plan = planner.plan_query(
+ topic="Sam Altman vs Dario Amodei",
+ available_sources=["reddit", "x", "youtube", "hackernews", "polymarket"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("comparison", plan.intent)
+ all_sources = set()
+ for subquery in plan.subqueries:
+ all_sources.update(subquery.sources)
+ self.assertIn("polymarket", all_sources)
+
+ def test_polymarket_excluded_from_how_to_and_concept(self):
+ """Polymarket should remain excluded from how_to and concept intents."""
+ for topic, expected_intent in [
+ ("how to deploy on Fly.io", "how_to"),
+ ("explain transformer architecture", "concept"),
+ ]:
+ plan = planner.plan_query(
+ topic=topic,
+ available_sources=["reddit", "x", "youtube", "hackernews", "polymarket"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual(expected_intent, plan.intent)
+ all_sources = set()
+ for subquery in plan.subqueries:
+ all_sources.update(subquery.sources)
+ self.assertNotIn("polymarket", all_sources,
+ f"polymarket should be excluded from {expected_intent}")
+
+ def test_opinion_includes_polymarket(self):
+ """Polymarket should not be excluded from opinion intent plans."""
+ plan = planner.plan_query(
+ topic="thoughts on OpenAI future",
+ available_sources=["reddit", "x", "youtube", "hackernews", "polymarket"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("opinion", plan.intent)
+ all_sources = set()
+ for subquery in plan.subqueries:
+ all_sources.update(subquery.sources)
+ self.assertIn("polymarket", all_sources)
+
+ def test_breaking_news_includes_tiktok_and_instagram(self):
+ plan = planner.plan_query(
+ topic="2026 March Madness",
+ available_sources=["reddit", "x", "tiktok", "instagram", "youtube", "hackernews"],
+ requested_sources=None,
+ depth="default",
+ provider=None,
+ model=None,
+ )
+ self.assertEqual("breaking_news", plan.intent)
+ all_sources = set()
+ for subquery in plan.subqueries:
+ all_sources.update(subquery.sources)
+ self.assertIn("tiktok", all_sources)
+ self.assertIn("instagram", all_sources)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_polymarket.py b/tests/test_polymarket.py
index 7689e9d..f29f99b 100644
--- a/tests/test_polymarket.py
+++ b/tests/test_polymarket.py
@@ -1,833 +1,564 @@
-"""Tests for Polymarket prediction market source module."""
+"""Tests for polymarket.py - Polymarket prediction market search."""
import json
-import math
import sys
-import unittest
from pathlib import Path
+from unittest.mock import Mock, patch
+
+import pytest
-# Add lib to path
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-from lib import polymarket, normalize, schema, score
+from lib import polymarket
-class TestExtractCoreSubject(unittest.TestCase):
- def test_plain_topic(self):
- self.assertEqual(polymarket._extract_core_subject("Arizona Basketball"), "Arizona Basketball")
+# === Helper Functions ===
- def test_strips_last_n_days(self):
- self.assertEqual(polymarket._extract_core_subject("last 7 days Iran"), "Iran")
-
- def test_strips_what_are(self):
- result = polymarket._extract_core_subject("what are people saying about Bitcoin")
- self.assertEqual(result, "Bitcoin")
-
- def test_strips_tell_me_about(self):
- result = polymarket._extract_core_subject("tell me about Ukraine")
- self.assertEqual(result, "Ukraine")
-
- def test_strips_whitespace(self):
- self.assertEqual(polymarket._extract_core_subject(" Iran "), "Iran")
-
- def test_empty_string(self):
- self.assertEqual(polymarket._extract_core_subject(""), "")
+def create_mock_event(
+ event_id="evt-123",
+ title="Test Event",
+ slug="test-event",
+ volume24hr=100000,
+ liquidity=50000,
+ closed=False,
+ markets=None,
+):
+ """Create a mock Polymarket event."""
+ if markets is None:
+ markets = [create_mock_market()]
+
+ return {
+ "id": event_id,
+ "title": title,
+ "slug": slug,
+ "active": True,
+ "closed": closed,
+ "volume24hr": volume24hr,
+ "liquidity": liquidity,
+ "markets": markets,
+ }
-class TestExpandQueries(unittest.TestCase):
- def test_single_word(self):
- queries = polymarket._expand_queries("Iran")
- self.assertIn("Iran", queries)
- # Single word: no split, just the core
- self.assertEqual(len(queries), 1)
-
- def test_multi_word(self):
- queries = polymarket._expand_queries("Arizona Basketball")
- self.assertIn("Arizona Basketball", queries)
- self.assertIn("Arizona", queries)
- self.assertIn("Basketball", queries)
- self.assertEqual(len(queries), 3)
-
- def test_with_prefix_stripped(self):
- queries = polymarket._expand_queries("last 7 days Iran")
- self.assertIn("Iran", queries)
- # Full topic should also be included since it differs from core
- self.assertIn("last 7 days Iran", queries)
-
- def test_deduplication(self):
- queries = polymarket._expand_queries("Iran")
- # Should not have duplicates
- self.assertEqual(len(queries), len(set(q.lower() for q in queries)))
-
- def test_max_6_queries(self):
- queries = polymarket._expand_queries("some really long topic with many words")
- self.assertLessEqual(len(queries), 6)
-
- def test_all_words_included(self):
- queries = polymarket._expand_queries("Iran War")
- self.assertIn("Iran War", queries)
- self.assertIn("Iran", queries)
- self.assertIn("War", queries)
- self.assertEqual(len(queries), 3)
-
- def test_short_words_excluded(self):
- """Single-char words should not become standalone queries."""
- queries = polymarket._expand_queries("A new idea")
- # "A" is single char, should be excluded
- self.assertNotIn("A", queries)
- self.assertIn("new", queries)
- self.assertIn("idea", queries)
-
- def test_low_signal_words_not_expanded_standalone(self):
- queries = polymarket._expand_queries("anthropic odds")
- self.assertIn("anthropic odds", queries)
- self.assertIn("anthropic", queries)
- self.assertNotIn("odds", queries)
+def create_mock_market(
+ market_id="mkt-123",
+ question="Will X happen?",
+ outcomes='["Yes", "No"]',
+ prices='["0.60", "0.40"]',
+ volume="100000",
+ liquidity="50000",
+ closed=False,
+):
+ """Create a mock Polymarket market."""
+ return {
+ "id": market_id,
+ "question": question,
+ "active": True,
+ "closed": closed,
+ "outcomes": outcomes,
+ "outcomePrices": prices,
+ "volume": volume,
+ "liquidity": liquidity,
+ }
-class TestExtractDomainQueries(unittest.TestCase):
- def _make_tag(self, label):
- return {"id": "1", "label": label, "slug": label.lower().replace(" ", "-")}
+# === Tests for _extract_core_subject() ===
- def test_finds_frequent_tags(self):
- tag_ncaa = self._make_tag("NCAA CBB")
- tag_sport = self._make_tag("Sports")
- tag_bball = self._make_tag("Basketball")
- events = [
- {"title": "SEC Champion", "tags": [tag_ncaa, tag_sport, tag_bball]},
- {"title": "ACC Champion", "tags": [tag_ncaa, tag_sport, tag_bball]},
- {"title": "Big 12 Champion", "tags": [tag_ncaa, tag_sport, tag_bball]},
+def test_extract_core_subject_basic():
+ """Test basic subject extraction."""
+ result = polymarket._extract_core_subject("AI frameworks")
+ assert result == "AI frameworks"
+
+
+def test_extract_core_subject_with_time_prefix():
+ """Test stripping time prefixes."""
+ result = polymarket._extract_core_subject("last 7 days AI frameworks")
+ assert result == "AI frameworks"
+
+
+def test_extract_core_subject_with_question_prefix():
+ """Test stripping question prefixes."""
+ result = polymarket._extract_core_subject("what are people saying about Kanye West")
+ assert result == "Kanye West"
+
+
+def test_extract_core_subject_multiple_prefixes():
+ """Test handling multiple prefix patterns."""
+ result = polymarket._extract_core_subject("research AI models")
+ assert result == "AI models"
+
+
+# === Tests for _expand_queries() ===
+
+def test_expand_queries_basic():
+ """Test basic query expansion."""
+ queries = polymarket._expand_queries("AI framework")
+
+ # Should include core + individual words
+ assert "AI framework" in queries or "ai framework" in queries
+ assert len(queries) >= 2
+
+
+def test_expand_queries_single_word():
+ """Test query expansion with single word."""
+ queries = polymarket._expand_queries("AI")
+
+ # Single word, should just return that word
+ assert len(queries) >= 1
+ assert any("ai" in q.lower() for q in queries)
+
+
+def test_expand_queries_filters_noise():
+ """Test that low-signal tokens are filtered."""
+ queries = polymarket._expand_queries("the AI framework")
+
+ # "the" should be filtered from individual word expansions
+ # But may appear in the full phrase
+ assert len(queries) >= 1
+ assert any("ai" in q.lower() or "framework" in q.lower() for q in queries)
+
+
+def test_expand_queries_deduplication():
+ """Test that duplicate queries are removed."""
+ queries = polymarket._expand_queries("test test test")
+
+ # Should dedupe
+ assert len(queries) == len(set(q.lower() for q in queries))
+
+
+def test_expand_queries_cap_at_six():
+ """Test that query list is capped at 6."""
+ long_topic = "one two three four five six seven eight"
+ queries = polymarket._expand_queries(long_topic)
+
+ assert len(queries) <= 6
+
+
+# === Tests for _passes_topic_filter() ===
+
+def test_passes_topic_filter_match():
+ """Test that matching events pass the filter."""
+ assert polymarket._passes_topic_filter("AI safety", "AI Safety Conference 2026") is True
+
+
+def test_passes_topic_filter_no_match():
+ """Test that non-matching events are filtered."""
+ # "West" is a noise word, so "Kanye West" requires "Kanye" to match
+ assert polymarket._passes_topic_filter("Kanye West", "NFC West Championship") is False
+
+
+def test_passes_topic_filter_partial_match():
+ """Test that at least one informative word must match."""
+ # "AI" appears in both topic and title
+ assert polymarket._passes_topic_filter("AI safety", "New AI Safety Conference") is True
+ # "models" doesn't appear, should fail
+ assert polymarket._passes_topic_filter("AI models", "New AI prediction") is False
+
+
+def test_passes_topic_filter_all_noise_words():
+ """Test that all-noise-word topics don't filter anything."""
+ # "the west" has no informative words, so should pass everything
+ result = polymarket._passes_topic_filter("the west", "Any title")
+ assert result is True
+
+
+def test_passes_topic_filter_empty_topic():
+ """Test empty topic always passes."""
+ assert polymarket._passes_topic_filter("", "Any title") is True
+
+
+def test_passes_topic_filter_multi_word_requires_two_matches():
+ """Topics with 3+ informative words require at least 2 to match."""
+ # "Mill food recycler" has 3 informative words. "Meek Mill YC" only matches "mill".
+ assert polymarket._passes_topic_filter(
+ "Mill food recycler", "Meek Mill gets Y Combinator funding"
+ ) is False
+
+
+def test_passes_topic_filter_multi_word_passes_with_two_matches():
+ """Topics with 3+ informative words pass when 2+ match."""
+ assert polymarket._passes_topic_filter(
+ "Sam Altman OpenAI", "Sam Altman CEO OpenAI"
+ ) is True
+
+
+def test_passes_topic_filter_two_word_still_needs_one():
+ """Topics with 2 informative words still only need 1 match (existing behavior)."""
+ assert polymarket._passes_topic_filter(
+ "Kanye West", "Kanye divorce settlement"
+ ) is True
+
+
+def test_passes_topic_filter_multi_word_edge_exactly_three():
+ """Topic with exactly 3 informative words, 1 match -> rejected."""
+ assert polymarket._passes_topic_filter(
+ "Tesla stock price", "Tesla quarterly earnings"
+ ) is False # only "tesla" matches, needs 2
+
+
+# === Tests for _parse_outcome_prices() ===
+
+def test_parse_outcome_prices_basic():
+ """Test basic outcome price parsing."""
+ market = {
+ "outcomes": '["Yes", "No"]',
+ "outcomePrices": '["0.65", "0.35"]',
+ }
+
+ result = polymarket._parse_outcome_prices(market)
+
+ assert len(result) == 2
+ assert result[0] == ("Yes", 0.65)
+ assert result[1] == ("No", 0.35)
+
+
+def test_parse_outcome_prices_already_parsed():
+ """Test handling when outcomes/prices are already lists."""
+ market = {
+ "outcomes": ["Yes", "No"],
+ "outcomePrices": ["0.70", "0.30"],
+ }
+
+ result = polymarket._parse_outcome_prices(market)
+
+ assert len(result) == 2
+ assert result[0][0] == "Yes"
+ assert result[0][1] == 0.70
+
+
+def test_parse_outcome_prices_multi_outcome():
+ """Test multi-outcome markets."""
+ market = {
+ "outcomes": '["Team A", "Team B", "Team C", "Team D"]',
+ "outcomePrices": '["0.40", "0.30", "0.20", "0.10"]',
+ }
+
+ result = polymarket._parse_outcome_prices(market)
+
+ assert len(result) == 4
+ assert result[0] == ("Team A", 0.40)
+ assert result[3] == ("Team D", 0.10)
+
+
+def test_parse_outcome_prices_missing_data():
+ """Test handling of missing outcome prices."""
+ market = {"outcomes": '["Yes"]'}
+
+ result = polymarket._parse_outcome_prices(market)
+
+ assert result == []
+
+
+def test_parse_outcome_prices_invalid_json():
+ """Test handling of invalid JSON."""
+ market = {
+ "outcomes": "not valid json",
+ "outcomePrices": "also not valid",
+ }
+
+ result = polymarket._parse_outcome_prices(market)
+
+ assert result == []
+
+
+# === Tests for _format_price_movement() ===
+
+def test_format_price_movement_one_day():
+ """Test formatting one-day price movement."""
+ market = {
+ "oneDayPriceChange": 0.05, # Up 5%
+ "oneWeekPriceChange": 0.02,
+ "oneMonthPriceChange": 0.01,
+ }
+
+ result = polymarket._format_price_movement(market)
+
+ assert "up" in result
+ assert "5.0%" in result
+ assert "today" in result
+
+
+def test_format_price_movement_negative():
+ """Test formatting negative price movement."""
+ market = {
+ "oneDayPriceChange": -0.15, # Down 15%
+ }
+
+ result = polymarket._format_price_movement(market)
+
+ assert "down" in result
+ assert "15.0%" in result
+
+
+def test_format_price_movement_picks_largest():
+ """Test that largest change is picked."""
+ market = {
+ "oneDayPriceChange": 0.02,
+ "oneWeekPriceChange": 0.10, # Largest
+ "oneMonthPriceChange": 0.05,
+ }
+
+ result = polymarket._format_price_movement(market)
+
+ assert "10.0%" in result
+ assert "this week" in result
+
+
+def test_format_price_movement_below_threshold():
+ """Test that small changes return None."""
+ market = {
+ "oneDayPriceChange": 0.005, # 0.5%, below 1% threshold
+ }
+
+ result = polymarket._format_price_movement(market)
+
+ assert result is None
+
+
+def test_format_price_movement_missing_data():
+ """Test handling of missing price change data."""
+ market = {}
+
+ result = polymarket._format_price_movement(market)
+
+ assert result is None
+
+
+# === Tests for _shorten_question() ===
+
+def test_shorten_question_will_pattern():
+ """Test shortening 'Will X...' questions."""
+ result = polymarket._shorten_question("Will Arizona win the NCAA Tournament?")
+
+ assert result == "Arizona"
+
+
+def test_shorten_question_complex_will():
+ """Test shortening complex Will questions."""
+ result = polymarket._shorten_question("Will Duke be a number 1 seed?")
+
+ assert result == "Duke"
+
+
+def test_shorten_question_no_pattern():
+ """Test questions that don't match patterns."""
+ result = polymarket._shorten_question("Arizona wins championship")
+
+ # Should return original or truncated
+ assert len(result) > 0
+
+
+def test_shorten_question_long():
+ """Test truncation of very long questions."""
+ long_q = "A" * 100
+ result = polymarket._shorten_question(long_q)
+
+ assert len(result) <= 40
+
+
+# === Tests for search_polymarket() ===
+
+def test_search_polymarket_result_cap():
+ """Test that result cap configuration exists."""
+ assert "quick" in polymarket.RESULT_CAP
+ assert "default" in polymarket.RESULT_CAP
+ assert "deep" in polymarket.RESULT_CAP
+
+ # Deep should return more results
+ assert polymarket.RESULT_CAP["deep"] >= polymarket.RESULT_CAP["quick"]
+
+
+def test_search_polymarket_depth_config():
+ """Test that depth configuration exists."""
+ # Verify depth config
+ assert "quick" in polymarket.DEPTH_CONFIG
+ assert "default" in polymarket.DEPTH_CONFIG
+ assert "deep" in polymarket.DEPTH_CONFIG
+
+ # Quick should be least pages
+ assert polymarket.DEPTH_CONFIG["quick"] <= polymarket.DEPTH_CONFIG["default"]
+
+
+def test_search_polymarket_query_expansion():
+ """Test that _expand_queries creates multiple queries."""
+ queries = polymarket._expand_queries("AI framework")
+
+ # Should expand to multiple queries
+ assert len(queries) >= 2
+
+
+@patch('lib.polymarket.http.post')
+def test_search_polymarket_http_error_handling(mock_post):
+ """Test graceful handling of HTTP errors."""
+ from lib.http import HTTPError
+ mock_post.side_effect = HTTPError("HTTP 429: Rate limit")
+
+ result = polymarket.search_polymarket("test", "2026-01-01", "2026-01-31")
+
+ # Should return structure with error
+ assert "events" in result or "error" in result
+
+
+# === Tests for parse_polymarket_response() ===
+
+def test_parse_polymarket_response_basic():
+ """Test basic response parsing."""
+ response = {
+ "events": [create_mock_event(
+ title="Will AI surpass humans?",
+ volume24hr=500000,
+ )]
+ }
+
+ items = polymarket.parse_polymarket_response(response, topic="AI")
+
+ assert len(items) >= 0
+ # Items may be filtered by topic filter
+
+
+def test_parse_polymarket_response_filters_closed():
+ """Test that closed events are filtered."""
+ response = {
+ "events": [
+ create_mock_event(closed=False),
+ create_mock_event(closed=True),
]
- result = polymarket._extract_domain_queries("Arizona Basketball", events)
- self.assertIn("NCAA CBB", result)
-
- def test_skips_generic_tags(self):
- tag_sport = self._make_tag("Sports")
- events = [
- {"title": "Event 1", "tags": [tag_sport]},
- {"title": "Event 2", "tags": [tag_sport]},
- {"title": "Event 3", "tags": [tag_sport]},
- ]
- result = polymarket._extract_domain_queries("test topic", events)
- self.assertNotIn("Sports", result)
-
- def test_skips_topic_word_tags(self):
- tag_bball = self._make_tag("Basketball")
- events = [
- {"title": "Event 1", "tags": [tag_bball]},
- {"title": "Event 2", "tags": [tag_bball]},
- ]
- result = polymarket._extract_domain_queries("Arizona Basketball", events)
- self.assertNotIn("Basketball", result)
-
- def test_requires_minimum_frequency(self):
- tag_a = self._make_tag("Unique Tag A")
- tag_b = self._make_tag("Unique Tag B")
- events = [
- {"title": "Event 1", "tags": [tag_a]},
- {"title": "Event 2", "tags": [tag_b]},
- ]
- result = polymarket._extract_domain_queries("test topic", events)
- self.assertEqual(result, [])
-
- def test_caps_at_two(self):
- tags = [self._make_tag(f"Tag {i}") for i in range(5)]
- events = [{"title": f"Event {i}", "tags": tags} for i in range(3)]
- result = polymarket._extract_domain_queries("test topic", events)
- self.assertLessEqual(len(result), 2)
-
- def test_empty_events(self):
- result = polymarket._extract_domain_queries("Arizona Basketball", [])
- self.assertEqual(result, [])
-
- def test_events_without_tags(self):
- events = [
- {"title": "Event 1"},
- {"title": "Event 2", "tags": None},
- {"title": "Event 3", "tags": []},
- ]
- result = polymarket._extract_domain_queries("test topic", events)
- self.assertEqual(result, [])
-
-
-class TestFormatPriceMovement(unittest.TestCase):
- def test_significant_monthly_change(self):
- market = {
- "oneDayPriceChange": 0.005,
- "oneWeekPriceChange": -0.02,
- "oneMonthPriceChange": -0.117,
- }
- result = polymarket._format_price_movement(market)
- self.assertEqual(result, "down 11.7% this month")
-
- def test_significant_weekly_change(self):
- market = {
- "oneDayPriceChange": 0.01,
- "oneWeekPriceChange": 0.225,
- "oneMonthPriceChange": 0.15,
- }
- result = polymarket._format_price_movement(market)
- self.assertEqual(result, "up 22.5% this week")
-
- def test_significant_daily_change(self):
- market = {
- "oneDayPriceChange": -0.15,
- "oneWeekPriceChange": 0.02,
- "oneMonthPriceChange": 0.03,
- }
- result = polymarket._format_price_movement(market)
- self.assertEqual(result, "down 15.0% today")
-
- def test_no_significant_change(self):
- market = {
- "oneDayPriceChange": 0.005,
- "oneWeekPriceChange": -0.003,
- "oneMonthPriceChange": 0.002,
- }
- result = polymarket._format_price_movement(market)
- self.assertIsNone(result)
-
- def test_missing_fields(self):
- result = polymarket._format_price_movement({})
- self.assertIsNone(result)
-
- def test_none_values(self):
- market = {
- "oneDayPriceChange": None,
- "oneWeekPriceChange": None,
- "oneMonthPriceChange": None,
- }
- result = polymarket._format_price_movement(market)
- self.assertIsNone(result)
-
-
-class TestTextSimilarity(unittest.TestCase):
- def test_short_binary_outcome_does_not_match_substring(self):
- score = polymarket._compute_text_similarity(
- "nano banana pro prompting",
- "NATO x Russia military clash by...?",
- ["No", "Yes"],
- )
- self.assertLess(score, 0.3)
-
- def test_outcome_only_match_is_capped_for_non_prediction_queries(self):
- score = polymarket._compute_text_similarity(
- "kanye west",
- "Top Spotify artist in March?",
- ["Kanye West", "Taylor Swift"],
- )
- self.assertLess(score, 0.3)
-
- def test_direct_title_match_beats_outcome_only_prediction_market(self):
- direct = polymarket._compute_text_similarity(
- "anthropic odds",
- "Will Anthropic or OpenAI IPO first?",
- [],
- )
- generic = polymarket._compute_text_similarity(
- "anthropic odds",
- "Which company will have the best AI model for coding on March 31",
- ["Anthropic", "OpenAI", "Google"],
- )
- self.assertGreater(direct, generic)
-
-
-class TestParseOutcomePrices(unittest.TestCase):
- def test_binary_market_json_strings(self):
- market = {
- "outcomes": '["Yes", "No"]',
- "outcomePrices": '["0.65", "0.35"]',
- }
- result = polymarket._parse_outcome_prices(market)
- self.assertEqual(len(result), 2)
- self.assertEqual(result[0], ("Yes", 0.65))
- self.assertEqual(result[1], ("No", 0.35))
-
- def test_list_inputs(self):
- market = {
- "outcomes": ["Yes", "No"],
- "outcomePrices": ["0.70", "0.30"],
- }
- result = polymarket._parse_outcome_prices(market)
- self.assertEqual(len(result), 2)
- self.assertEqual(result[0], ("Yes", 0.70))
-
- def test_multi_outcome(self):
- market = {
- "outcomes": '["Arizona", "Kansas", "Houston"]',
- "outcomePrices": '["0.35", "0.22", "0.18"]',
- }
- result = polymarket._parse_outcome_prices(market)
- self.assertEqual(len(result), 3)
- self.assertEqual(result[0][0], "Arizona")
- self.assertAlmostEqual(result[0][1], 0.35)
-
- def test_malformed_outcomes_json(self):
- market = {
- "outcomes": "not valid json",
- "outcomePrices": '["0.50", "0.50"]',
- }
- result = polymarket._parse_outcome_prices(market)
- # Should fall back to "Outcome N" names
- self.assertEqual(len(result), 2)
- self.assertEqual(result[0][0], "Outcome 1")
-
- def test_malformed_prices_json(self):
- market = {
- "outcomes": '["Yes", "No"]',
- "outcomePrices": "not valid json",
- }
- result = polymarket._parse_outcome_prices(market)
- self.assertEqual(result, [])
-
- def test_missing_prices(self):
- market = {"outcomes": '["Yes", "No"]'}
- result = polymarket._parse_outcome_prices(market)
- self.assertEqual(result, [])
-
- def test_empty_outcomes(self):
- market = {
- "outcomes": "[]",
- "outcomePrices": '["0.50", "0.50"]',
- }
- result = polymarket._parse_outcome_prices(market)
- self.assertEqual(len(result), 2)
- # Should use fallback names
- self.assertEqual(result[0][0], "Outcome 1")
-
-
-class TestParsePolymarketResponse(unittest.TestCase):
- def setUp(self):
- fixture_path = Path(__file__).parent.parent / "fixtures" / "polymarket_sample.json"
- with open(fixture_path) as f:
- self.sample = json.load(f)
-
- def test_parses_active_events(self):
- items = polymarket.parse_polymarket_response(self.sample)
- # Should include Arizona Big 12, Arizona NCAA, multi-outcome, and malformed
- # Should exclude: closed/resolved event and no-liquidity event
- titles = [item["title"] for item in items]
- self.assertIn("Will Arizona win the Big 12 Championship?", titles)
- self.assertIn("Will Arizona win the NCAA Tournament?", titles)
- self.assertIn("Who will win the Big 12 Tournament?", titles)
-
- def test_filters_closed_events(self):
- items = polymarket.parse_polymarket_response(self.sample)
- titles = [item["title"] for item in items]
- self.assertNotIn("Resolved Event (should be filtered)", titles)
-
- def test_filters_no_liquidity(self):
- items = polymarket.parse_polymarket_response(self.sample)
- titles = [item["title"] for item in items]
- self.assertNotIn("Dead market (no liquidity)", titles)
-
- def test_item_fields(self):
- items = polymarket.parse_polymarket_response(self.sample)
- item = items[0] # Arizona Big 12
- self.assertEqual(item["event_id"], "evt-arizona-big12")
- self.assertEqual(item["url"], "https://polymarket.com/event/arizona-big-12-championship")
- self.assertIsNotNone(item["date"])
- self.assertIsNotNone(item["outcome_prices"])
- self.assertIsNotNone(item["volume24hr"])
- self.assertIsNotNone(item["liquidity"])
-
- def test_outcome_prices_parsed(self):
- items = polymarket.parse_polymarket_response(self.sample)
- item = items[0] # Arizona Big 12 - binary
- self.assertEqual(len(item["outcome_prices"]), 2)
- self.assertEqual(item["outcome_prices"][0][0], "Yes")
- self.assertAlmostEqual(item["outcome_prices"][0][1], 0.64)
-
- def test_multi_outcome_top3(self):
- items = polymarket.parse_polymarket_response(self.sample)
- multi = [i for i in items if i["title"] == "Who will win the Big 12 Tournament?"][0]
- # Top 3 outcomes shown
- self.assertEqual(len(multi["outcome_prices"]), 3)
- # 5 total - 3 shown = 2 remaining
- self.assertEqual(multi["outcomes_remaining"], 2)
-
- def test_price_movement(self):
- items = polymarket.parse_polymarket_response(self.sample)
- item = items[0] # Arizona Big 12
- # Weekly change (22.5%) is the most significant
- self.assertEqual(item["price_movement"], "up 22.5% this week")
-
- def test_date_extraction(self):
- items = polymarket.parse_polymarket_response(self.sample)
- item = items[0]
- self.assertEqual(item["date"], "2026-02-24")
-
- def test_relevance_range(self):
- items = polymarket.parse_polymarket_response(self.sample)
- for item in items:
- self.assertGreaterEqual(item["relevance"], 0.0)
- self.assertLessEqual(item["relevance"], 1.0)
-
- def test_empty_response(self):
- items = polymarket.parse_polymarket_response({"events": []})
- self.assertEqual(items, [])
-
- def test_missing_events_key(self):
- items = polymarket.parse_polymarket_response({})
- self.assertEqual(items, [])
-
- def test_malformed_prices_still_produces_item(self):
- items = polymarket.parse_polymarket_response(self.sample)
- malformed = [i for i in items if i["title"] == "Malformed prices"]
- self.assertEqual(len(malformed), 1)
- # Outcome prices should be empty due to malformed JSON
- self.assertEqual(malformed[0]["outcome_prices"], [])
-
- def test_end_date_extraction(self):
- items = polymarket.parse_polymarket_response(self.sample)
- item = items[0] # Arizona Big 12
- self.assertEqual(item["end_date"], "2026-03-15")
-
-
-class TestNormalizePolymarketItems(unittest.TestCase):
- def test_normalize(self):
- raw_items = [
- {
- "event_id": "evt-1",
- "title": "Test Market",
- "question": "Will test pass?",
- "url": "https://polymarket.com/event/test",
- "outcome_prices": [("Yes", 0.75), ("No", 0.25)],
- "outcomes_remaining": 0,
- "price_movement": "up 5.0% this week",
- "volume24hr": 100000.0,
- "liquidity": 500000.0,
- "date": "2026-02-20",
- "end_date": "2026-03-01",
- "relevance": 0.85,
- "why_relevant": "Prediction market: Test Market",
- }
- ]
- result = normalize.normalize_polymarket_items(raw_items, "2026-01-01", "2026-03-01")
- self.assertEqual(len(result), 1)
- self.assertIsInstance(result[0], schema.PolymarketItem)
- self.assertEqual(result[0].id, "PM1")
- self.assertEqual(result[0].title, "Test Market")
- self.assertEqual(result[0].question, "Will test pass?")
- self.assertEqual(result[0].date_confidence, "high")
- self.assertEqual(result[0].engagement.volume, 100000.0)
- self.assertEqual(result[0].engagement.liquidity, 500000.0)
- self.assertEqual(result[0].price_movement, "up 5.0% this week")
-
- def test_normalize_multiple(self):
- raw_items = [
- {
- "event_id": f"evt-{i}",
- "title": f"Market {i}",
- "question": f"Question {i}?",
- "url": f"https://polymarket.com/event/test-{i}",
- "outcome_prices": [],
- "outcomes_remaining": 0,
- "volume24hr": 0.0,
- "liquidity": 0.0,
- "date": "2026-02-20",
- "relevance": 0.5,
- "why_relevant": f"Market {i}",
- }
- for i in range(3)
- ]
- result = normalize.normalize_polymarket_items(raw_items, "2026-01-01", "2026-03-01")
- self.assertEqual(len(result), 3)
- self.assertEqual(result[0].id, "PM1")
- self.assertEqual(result[1].id, "PM2")
- self.assertEqual(result[2].id, "PM3")
-
-
-class TestScorePolymarketItems(unittest.TestCase):
- def test_score_items(self):
- items = [
- schema.PolymarketItem(
- id="PM1", title="High volume", question="Q1?", url="",
- date="2026-02-20",
- engagement=schema.Engagement(volume=500000.0, liquidity=2000000.0),
- relevance=0.9,
- ),
- schema.PolymarketItem(
- id="PM2", title="Low volume", question="Q2?", url="",
- date="2026-02-18",
- engagement=schema.Engagement(volume=100.0, liquidity=500.0),
- relevance=0.5,
- ),
- ]
- scored = score.score_polymarket_items(items)
- self.assertEqual(len(scored), 2)
- # High engagement + high relevance should score higher
- self.assertGreater(scored[0].score, scored[1].score)
-
- def test_score_empty(self):
- result = score.score_polymarket_items([])
- self.assertEqual(result, [])
-
- def test_engagement_formula(self):
- eng = schema.Engagement(volume=100000.0, liquidity=500000.0)
- result = score.compute_polymarket_engagement_raw(eng)
- self.assertIsNotNone(result)
- self.assertGreater(result, 0)
- # Verify formula: 0.60 * log1p(volume) + 0.40 * log1p(liquidity)
- expected = 0.60 * math.log1p(100000.0) + 0.40 * math.log1p(500000.0)
- self.assertAlmostEqual(result, expected)
-
- def test_engagement_none(self):
- result = score.compute_polymarket_engagement_raw(None)
- self.assertIsNone(result)
-
- def test_engagement_empty(self):
- eng = schema.Engagement()
- result = score.compute_polymarket_engagement_raw(eng)
- self.assertIsNone(result)
-
- def test_zero_volume(self):
- eng = schema.Engagement(volume=0.0, liquidity=0.0)
- result = score.compute_polymarket_engagement_raw(eng)
- self.assertIsNotNone(result)
- self.assertEqual(result, 0.0)
-
-
-class TestSortItemsWithPolymarket(unittest.TestCase):
- def test_pm_priority_after_hn(self):
- """Polymarket should sort after HN at same score."""
- hn_item = schema.HackerNewsItem(id="HN1", title="test", url="", hn_url="", author="user")
- hn_item.score = 50
-
- pm_item = schema.PolymarketItem(id="PM1", title="test", question="test?", url="")
- pm_item.score = 50
-
- web_item = schema.WebSearchItem(id="W1", title="test", url="", source_domain="example.com", snippet="")
- web_item.score = 50
-
- sorted_items = score.sort_items([web_item, pm_item, hn_item])
- # Same score, so sorted by priority: HN(3) > PM(4) > Web(5)
- self.assertIsInstance(sorted_items[0], schema.HackerNewsItem)
- self.assertIsInstance(sorted_items[1], schema.PolymarketItem)
- self.assertIsInstance(sorted_items[2], schema.WebSearchItem)
-
-
-class TestPolymarketSchemaRoundTrip(unittest.TestCase):
- def test_to_dict_and_back(self):
- item = schema.PolymarketItem(
- id="PM1",
- title="Test Event",
- question="Will test pass?",
- url="https://polymarket.com/event/test",
- outcome_prices=[("Yes", 0.75), ("No", 0.25)],
- outcomes_remaining=0,
- price_movement="up 5.0% this week",
- date="2026-02-20",
- engagement=schema.Engagement(volume=100000.0, liquidity=500000.0),
- end_date="2026-03-01",
- relevance=0.85,
- why_relevant="Test",
- cross_refs=["R1", "HN2"],
- )
- d = item.to_dict()
- self.assertEqual(d["id"], "PM1")
- self.assertEqual(d["title"], "Test Event")
- self.assertEqual(d["cross_refs"], ["R1", "HN2"])
-
- def test_empty_crossrefs_omitted(self):
- item = schema.PolymarketItem(id="PM1", title="Test", question="Q?", url="")
- d = item.to_dict()
- self.assertNotIn("cross_refs", d)
-
- def test_report_roundtrip_with_polymarket(self):
- report = schema.Report(
- topic="test", range_from="2026-01-01", range_to="2026-02-01",
- generated_at="2026-02-01T00:00:00Z", mode="both",
- polymarket=[schema.PolymarketItem(
- id="PM1", title="Test", question="Q?", url="",
- outcome_prices=[("Yes", 0.5), ("No", 0.5)],
- cross_refs=["R1"],
- )],
- )
- d = report.to_dict()
- restored = schema.Report.from_dict(d)
- self.assertEqual(len(restored.polymarket), 1)
- self.assertEqual(restored.polymarket[0].id, "PM1")
- self.assertEqual(restored.polymarket[0].cross_refs, ["R1"])
-
- def test_report_backward_compat_no_polymarket(self):
- """Old cached reports without polymarket key should load fine."""
- data = {
- "topic": "test",
- "range": {"from": "2026-01-01", "to": "2026-02-01"},
- "generated_at": "2026-02-01T00:00:00Z",
- "mode": "both",
- "reddit": [],
- "x": [],
- "web": [],
- "youtube": [],
- "hackernews": [],
- # No "polymarket" key
- }
- report = schema.Report.from_dict(data)
- self.assertEqual(report.polymarket, [])
- self.assertIsNone(report.polymarket_error)
-
- def test_engagement_volume_liquidity(self):
- eng = schema.Engagement(volume=342000.0, liquidity=2100000.0)
- d = eng.to_dict()
- self.assertEqual(d["volume"], 342000.0)
- self.assertEqual(d["liquidity"], 2100000.0)
-
-
-class TestDepthConfig(unittest.TestCase):
- def test_quick_pages(self):
- self.assertEqual(polymarket.DEPTH_CONFIG["quick"], 1)
-
- def test_default_pages(self):
- self.assertEqual(polymarket.DEPTH_CONFIG["default"], 3)
-
- def test_deep_pages(self):
- self.assertEqual(polymarket.DEPTH_CONFIG["deep"], 4)
-
- def test_result_cap_quick(self):
- self.assertEqual(polymarket.RESULT_CAP["quick"], 5)
-
- def test_result_cap_default(self):
- self.assertEqual(polymarket.RESULT_CAP["default"], 15)
-
- def test_result_cap_deep(self):
- self.assertEqual(polymarket.RESULT_CAP["deep"], 25)
-
-
-class TestTextSimilarity(unittest.TestCase):
- def test_exact_substring_match(self):
- score = polymarket._compute_text_similarity("Arizona", "Will Arizona win the NCAA Tournament?")
- self.assertEqual(score, 1.0)
-
- def test_full_topic_substring(self):
- score = polymarket._compute_text_similarity("Arizona Basketball", "Arizona Basketball Championship")
- self.assertEqual(score, 1.0)
-
- def test_partial_token_overlap(self):
- score = polymarket._compute_text_similarity("Arizona Basketball", "Will Arizona win?")
- # Partial informative match should stay below exact match.
- self.assertGreater(score, 0.3)
- self.assertLess(score, 0.6)
-
- def test_no_overlap(self):
- score = polymarket._compute_text_similarity("Arizona Basketball", "Will AI regulation pass?")
- self.assertEqual(score, 0.0)
-
- def test_empty_topic(self):
- score = polymarket._compute_text_similarity("", "Will Arizona win?")
- self.assertEqual(score, 0.5)
-
- def test_case_insensitive(self):
- score = polymarket._compute_text_similarity("arizona", "ARIZONA Big 12")
- self.assertEqual(score, 1.0)
-
- def test_prefix_stripped(self):
- score = polymarket._compute_text_similarity("last 7 days Arizona", "Will Arizona win?")
- self.assertEqual(score, 1.0)
-
- def test_outcome_substring_match(self):
- """Prediction queries can still use outcome-only entity matches."""
- score = polymarket._compute_text_similarity(
- "Arizona odds",
- "Who will be the #1 overall seed?",
- outcomes=["Duke", "Arizona", "Houston"],
- )
- self.assertEqual(score, 0.55)
-
- def test_outcome_bidirectional_match(self):
- """Longer prediction topics keep the same moderated outcome-only cap."""
- score = polymarket._compute_text_similarity(
- "Arizona Basketball odds",
- "Who will be the #1 overall seed?",
- outcomes=["Duke", "Arizona", "Houston"],
- )
- self.assertEqual(score, 0.55)
-
- def test_outcome_token_overlap(self):
- """Outcome-only prediction matches stay moderate, not dominant."""
- score = polymarket._compute_text_similarity(
- "Iran War odds",
- "Unrelated geopolitics title",
- outcomes=["War continues", "Peace deal"],
- )
- self.assertGreater(score, 0.3)
- self.assertLess(score, 0.6)
-
- def test_outcome_no_match(self):
- """No outcome match falls through to title token overlap."""
- score = polymarket._compute_text_similarity(
- "Arizona Basketball",
- "Will AI regulation pass in 2026?",
- outcomes=["Yes", "No"],
- )
- self.assertEqual(score, 0.0)
-
- def test_outcome_low_price_filtered_by_caller(self):
- """Outcomes with price <= 1% should be filtered by the caller, not this function."""
- # This function doesn't filter - it trusts the caller to pass only relevant outcomes
- score = polymarket._compute_text_similarity(
- "Arizona odds",
- "Unrelated title",
- outcomes=["Arizona"],
- )
- self.assertEqual(score, 0.55)
-
- def test_generic_only_odds_match_stays_below_threshold(self):
- score = polymarket._compute_text_similarity(
- "Anthropic odds",
- "Republican 2026 House odds",
- outcomes=["Yes", "No"],
- )
- self.assertLess(score, 0.3)
-
- def test_title_match_still_beats_outcome(self):
- """Title substring match (1.0) takes priority over outcome match (0.85)."""
- score = polymarket._compute_text_similarity(
- "Arizona",
- "Will Arizona win the tournament?",
- outcomes=["Arizona", "Duke"],
- )
- self.assertEqual(score, 1.0)
-
- def test_empty_outcomes(self):
- """Empty outcomes list falls through to title token overlap."""
- score = polymarket._compute_text_similarity(
- "Arizona Basketball",
- "Unrelated title",
- outcomes=[],
- )
- self.assertEqual(score, 0.0)
-
- def test_none_outcomes(self):
- """None outcomes falls through to title token overlap."""
- score = polymarket._compute_text_similarity(
- "Arizona Basketball",
- "Unrelated title",
- outcomes=None,
- )
- self.assertEqual(score, 0.0)
-
-
-class TestQualityRanking(unittest.TestCase):
- """Verify quality-signal ranking: high-volume matching events rank above tangential ones."""
-
- def setUp(self):
- fixture_path = Path(__file__).parent.parent / "fixtures" / "polymarket_sample.json"
- with open(fixture_path) as f:
- self.sample = json.load(f)
-
- def test_topic_matching_ranks_above_tangential(self):
- """Arizona markets should rank above AI regulation when topic is 'Arizona Basketball'."""
- items = polymarket.parse_polymarket_response(self.sample, topic="Arizona Basketball")
- titles = [item["title"] for item in items]
- # Arizona events (including outcome-matched ones like NCAA seed) should come before tangential
- arizona_indices = [i for i, t in enumerate(titles) if "Arizona" in t or "Big 12" in t or "NCAA" in t]
- tangential_indices = [i for i, t in enumerate(titles) if "AI regulation" in t]
- if tangential_indices:
- self.assertTrue(max(arizona_indices) < min(tangential_indices),
- f"Arizona markets should rank above tangential. Order: {titles}")
-
- def test_high_volume_ranks_above_low_volume(self):
- """Among title-matched events, higher volume should rank higher."""
- items = polymarket.parse_polymarket_response(self.sample, topic="Arizona Basketball")
- # Arizona Big 12 Championship has $3.5M monthly volume, Arizona NCAA Tournament has $800K
- # Both have "Arizona" in the title (text_score=1.0), so volume breaks the tie
- big12 = [i for i, item in enumerate(items) if "Big 12 Championship" in item["title"]]
- ncaa_win = [i for i, item in enumerate(items) if item["title"] == "Will Arizona win the NCAA Tournament?"]
- if big12 and ncaa_win:
- self.assertLess(big12[0], ncaa_win[0],
- "Higher volume Big 12 Championship should rank above lower volume NCAA Tournament win")
-
- def test_result_cap_applied(self):
- """Parse should respect the _cap from search response."""
- capped_response = dict(self.sample)
- capped_response["_cap"] = 2
- items = polymarket.parse_polymarket_response(capped_response, topic="Arizona")
- self.assertLessEqual(len(items), 2)
-
- def test_no_topic_still_ranks(self):
- """Without a topic, relevance should still be computed from volume/liquidity."""
- items = polymarket.parse_polymarket_response(self.sample)
- self.assertTrue(len(items) > 0)
- for item in items:
- self.assertGreaterEqual(item["relevance"], 0.0)
- self.assertLessEqual(item["relevance"], 1.0)
-
- def test_relevance_sorted_descending(self):
- """Items should be sorted by relevance descending."""
- items = polymarket.parse_polymarket_response(self.sample, topic="Arizona Basketball")
- relevances = [item["relevance"] for item in items]
- self.assertEqual(relevances, sorted(relevances, reverse=True))
-
- def test_ncaa_seed_found_via_outcome_matching(self):
- """NCAA seed market should be found when Arizona is an outcome but not in title."""
- items = polymarket.parse_polymarket_response(self.sample, topic="Arizona Basketball")
- titles = [item["title"] for item in items]
- self.assertIn("Who will be the #1 overall seed in the 2026 NCAA Tournament?", titles)
-
- def test_ncaa_seed_ranks_above_tangential(self):
- """NCAA seed market (outcome match) should rank above AI regulation (no match)."""
- items = polymarket.parse_polymarket_response(self.sample, topic="Arizona Basketball")
- titles = [item["title"] for item in items]
- seed_idx = titles.index("Who will be the #1 overall seed in the 2026 NCAA Tournament?")
- tangential = [i for i, t in enumerate(titles) if "AI regulation" in t]
- if tangential:
- self.assertLess(seed_idx, tangential[0],
- f"NCAA seed should rank above tangential. Order: {titles}")
-
- def test_outcome_reordering_surfaces_topic(self):
- """Arizona should be surfaced to front of outcome_prices when topic matches."""
- items = polymarket.parse_polymarket_response(self.sample, topic="Arizona Basketball")
- seed_market = [i for i in items if "seed" in i["title"].lower()][0]
- # Arizona should be first in outcome_prices (reordered from position 2)
- self.assertEqual(seed_market["outcome_prices"][0][0], "Arizona")
-
-
-class TestNormalizePolymarketVolume1mo(unittest.TestCase):
- """Verify normalization prefers volume1mo over volume24hr for engagement."""
-
- def test_volume1mo_preferred(self):
- raw_items = [
- {
- "event_id": "evt-1",
- "title": "Test",
- "question": "Q?",
- "url": "https://polymarket.com/event/test",
- "outcome_prices": [],
- "outcomes_remaining": 0,
- "volume24hr": 100.0,
- "volume1mo": 5000000.0,
- "liquidity": 1000.0,
- "date": "2026-02-20",
- "relevance": 0.8,
- "why_relevant": "Test",
- }
- ]
- result = normalize.normalize_polymarket_items(raw_items, "2026-01-01", "2026-03-01")
- # Engagement volume should be volume1mo (5M), not volume24hr (100)
- self.assertEqual(result[0].engagement.volume, 5000000.0)
-
- def test_fallback_to_volume24hr(self):
- raw_items = [
- {
- "event_id": "evt-1",
- "title": "Test",
- "question": "Q?",
- "url": "https://polymarket.com/event/test",
- "outcome_prices": [],
- "outcomes_remaining": 0,
- "volume24hr": 50000.0,
- "liquidity": 1000.0,
- "date": "2026-02-20",
- "relevance": 0.8,
- "why_relevant": "Test",
- }
- ]
- result = normalize.normalize_polymarket_items(raw_items, "2026-01-01", "2026-03-01")
- # No volume1mo, should fall back to volume24hr
- self.assertEqual(result[0].engagement.volume, 50000.0)
+ }
+
+ items = polymarket.parse_polymarket_response(response)
+
+ # Closed events should be filtered
+ # (exact count depends on market filtering)
+ assert isinstance(items, list)
+
+
+def test_parse_polymarket_response_empty():
+ """Test handling of empty response."""
+ response = {"events": []}
+
+ items = polymarket.parse_polymarket_response(response)
+
+ assert items == []
+
+
+def test_parse_polymarket_response_market_url():
+ """Test that Polymarket URLs are generated."""
+ response = {
+ "events": [create_mock_event(
+ slug="test-event-slug",
+ title="Test Event"
+ )]
+ }
+
+ items = polymarket.parse_polymarket_response(response, topic="test")
+
+ if items: # If not filtered
+ assert "url" in items[0]
+ assert "polymarket.com" in items[0]["url"]
+
+
+def test_parse_polymarket_response_engagement():
+ """Test that engagement/volume metrics are captured."""
+ response = {
+ "events": [create_mock_event(
+ title="AI Event",
+ volume24hr=250000,
+ liquidity=100000,
+ )]
+ }
+
+ items = polymarket.parse_polymarket_response(response, topic="AI")
+
+ if items: # If not filtered
+ # Check for volume or liquidity fields
+ assert "volume24hr" in items[0] or "liquidity" in items[0] or isinstance(items[0], dict)
+
+
+# === Tests for engagement scoring ===
+
+def test_engagement_with_volume():
+ """Test engagement calculation with volume."""
+ response = {
+ "events": [create_mock_event(
+ title="Test",
+ volume24hr=500000,
+ )]
+ }
+
+ items = polymarket.parse_polymarket_response(response, topic="test")
+
+ if items:
+ engagement = items[0].get("engagement", {})
+ # volume24hr should be captured
+ assert "volume24hr" in engagement or isinstance(engagement, dict)
+
+
+# === Tests for noise-word query skipping ===
+
+def test_expand_queries_skips_noise_words():
+ """Noise words like 'west' should not become standalone queries."""
+ queries = polymarket._expand_queries("kanye west")
+ lowered = [q.lower() for q in queries]
+ assert "kanye west" in lowered # full phrase kept
+ assert "kanye" in lowered # informative word kept
+ assert "west" not in lowered # noise word skipped
+
+
+def test_expand_queries_keeps_informative_words():
+ """Non-noise words should still be expanded as standalone queries."""
+ queries = polymarket._expand_queries("arizona basketball")
+ lowered = [q.lower() for q in queries]
+ assert "arizona" in lowered
+ assert "basketball" in lowered
+
+
+def test_expand_queries_all_noise_words_keeps_phrase():
+ """If all words are noise, the full phrase is still searched."""
+ queries = polymarket._expand_queries("north west")
+ assert len(queries) >= 1
+ assert any("north west" in q.lower() for q in queries)
+ # Neither individual word should be a standalone query
+ lowered = [q.lower() for q in queries]
+ assert "north" not in lowered or "north west" in lowered # only as part of phrase
+
+
+# === Tests for per-item relevance floor ===
+
+def test_per_item_relevance_floor_drops_zero_items():
+ """Items with relevance 0.0 should be dropped even if best item is high."""
+ # Simulate the filtering logic directly
+ items = [
+ {"relevance": 0.85, "title": "Kanye market"},
+ {"relevance": 0.45, "title": "Related market"},
+ {"relevance": 0.0, "title": "Golf noise"},
+ {"relevance": 0.0, "title": "Cycling noise"},
+ ]
+ filtered = [i for i in items if i["relevance"] >= 0.10]
+ assert len(filtered) == 2
+ assert all(i["title"] != "Golf noise" for i in filtered)
+
+
+def test_per_item_relevance_floor_keeps_borderline():
+ """Items at exactly 0.10 should be kept."""
+ items = [
+ {"relevance": 0.85, "title": "Main market"},
+ {"relevance": 0.10, "title": "Borderline market"},
+ {"relevance": 0.09, "title": "Below floor"},
+ ]
+ filtered = [i for i in items if i["relevance"] >= 0.10]
+ assert len(filtered) == 2
+ assert filtered[1]["title"] == "Borderline market"
+
+
+def test_per_item_relevance_floor_no_drops_when_all_high():
+ """Nothing dropped when all items are above the floor."""
+ items = [
+ {"relevance": 0.85, "title": "A"},
+ {"relevance": 0.50, "title": "B"},
+ {"relevance": 0.30, "title": "C"},
+ ]
+ filtered = [i for i in items if i["relevance"] >= 0.10]
+ assert len(filtered) == 3
if __name__ == "__main__":
- unittest.main()
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_providers_v3.py b/tests/test_providers_v3.py
new file mode 100644
index 0000000..c4d61a5
--- /dev/null
+++ b/tests/test_providers_v3.py
@@ -0,0 +1,167 @@
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import providers
+
+
+class ProvidersV3Tests(unittest.TestCase):
+ def test_auto_prefers_gemini_with_google_key(self):
+ runtime, client = providers.resolve_runtime(
+ {"GOOGLE_API_KEY": "test", "LAST30DAYS_REASONING_PROVIDER": "auto"},
+ depth="default",
+ )
+ self.assertEqual("gemini", runtime.reasoning_provider)
+ self.assertEqual("gemini", client.name)
+ self.assertTrue(runtime.planner_model.startswith("gemini-3.1-"))
+
+ def test_auto_falls_back_to_openai(self):
+ runtime, client = providers.resolve_runtime(
+ {
+ "OPENAI_API_KEY": "test-key",
+ "OPENAI_AUTH_STATUS": "ok",
+ "LAST30DAYS_REASONING_PROVIDER": "auto",
+ },
+ depth="default",
+ )
+ self.assertEqual("openai", runtime.reasoning_provider)
+
+ def test_auto_falls_back_to_xai(self):
+ runtime, client = providers.resolve_runtime(
+ {"XAI_API_KEY": "test-key", "LAST30DAYS_REASONING_PROVIDER": "auto"},
+ depth="default",
+ )
+ self.assertEqual("xai", runtime.reasoning_provider)
+
+ def test_auto_returns_local_runtime_when_no_keys(self):
+ runtime, client = providers.resolve_runtime(
+ {"LAST30DAYS_REASONING_PROVIDER": "auto"},
+ depth="default",
+ )
+ self.assertEqual("local", runtime.reasoning_provider)
+ self.assertEqual("deterministic", runtime.planner_model)
+ self.assertEqual("local-score", runtime.rerank_model)
+ self.assertIsNone(client)
+
+ def test_explicit_gemini_without_key_still_raises(self):
+ with self.assertRaises(RuntimeError):
+ providers.resolve_runtime(
+ {"LAST30DAYS_REASONING_PROVIDER": "gemini"},
+ depth="default",
+ )
+
+ def test_explicit_openai_without_key_still_raises(self):
+ with self.assertRaises(RuntimeError):
+ providers.resolve_runtime(
+ {"LAST30DAYS_REASONING_PROVIDER": "openai"},
+ depth="default",
+ )
+
+ def test_explicit_xai_without_key_still_raises(self):
+ with self.assertRaises(RuntimeError):
+ providers.resolve_runtime(
+ {"LAST30DAYS_REASONING_PROVIDER": "xai"},
+ depth="default",
+ )
+
+
+class TestExtractJson(unittest.TestCase):
+ def test_direct_json(self):
+ result = providers.extract_json('{"scores": [1, 2]}')
+ self.assertEqual(result, {"scores": [1, 2]})
+
+ def test_json_in_markdown_fences(self):
+ text = '```json\n{"scores": [1, 2]}\n```'
+ result = providers.extract_json(text)
+ self.assertEqual(result, {"scores": [1, 2]})
+
+ def test_json_with_surrounding_text(self):
+ text = 'Here is the result:\n{"scores": [1]}\nDone.'
+ result = providers.extract_json(text)
+ self.assertEqual(result, {"scores": [1]})
+
+ def test_empty_text_raises(self):
+ with self.assertRaises(ValueError):
+ providers.extract_json("")
+
+ def test_no_json_raises(self):
+ with self.assertRaises(json.JSONDecodeError):
+ providers.extract_json("no json here at all")
+
+
+class TestExtractOpenAIText(unittest.TestCase):
+ def test_output_text_field(self):
+ self.assertEqual("hello", providers.extract_openai_text({"output_text": "hello"}))
+
+ def test_choices_message_content(self):
+ payload = {"choices": [{"message": {"content": "world"}}]}
+ self.assertEqual("world", providers.extract_openai_text(payload))
+
+ def test_output_list_text(self):
+ payload = {"output": [{"text": "foo"}]}
+ self.assertEqual("foo", providers.extract_openai_text(payload))
+
+ def test_output_content_output_text_type(self):
+ payload = {"output": [{"content": [{"type": "output_text", "text": "bar"}]}]}
+ self.assertEqual("bar", providers.extract_openai_text(payload))
+
+ def test_output_string_item(self):
+ payload = {"output": ["direct string"]}
+ self.assertEqual("direct string", providers.extract_openai_text(payload))
+
+ def test_empty_payload_returns_empty(self):
+ self.assertEqual("", providers.extract_openai_text({}))
+
+
+class TestExtractGeminiText(unittest.TestCase):
+ def test_standard_response(self):
+ payload = {"candidates": [{"content": {"parts": [{"text": "gemini says"}]}}]}
+ self.assertEqual("gemini says", providers.extract_gemini_text(payload))
+
+ def test_empty_candidates(self):
+ self.assertEqual("", providers.extract_gemini_text({"candidates": []}))
+
+ def test_empty_payload(self):
+ self.assertEqual("", providers.extract_gemini_text({}))
+
+
+class TestParseSSEChunk(unittest.TestCase):
+ def test_valid_chunk(self):
+ chunk = 'data: {"type": "delta", "text": "hi"}'
+ result = providers._parse_sse_chunk(chunk)
+ self.assertEqual(result, {"type": "delta", "text": "hi"})
+
+ def test_done_sentinel(self):
+ self.assertIsNone(providers._parse_sse_chunk("data: [DONE]"))
+
+ def test_no_data_lines(self):
+ self.assertIsNone(providers._parse_sse_chunk("event: ping"))
+
+ def test_invalid_json(self):
+ self.assertIsNone(providers._parse_sse_chunk("data: {bad json"))
+
+
+class TestParseCodexStream(unittest.TestCase):
+ def test_response_completed_event(self):
+ stream = 'data: {"type": "response.completed", "response": {"output_text": "done"}}\n\n'
+ result = providers._parse_codex_stream(stream)
+ self.assertEqual(result["output_text"], "done")
+
+ def test_delta_text_accumulation(self):
+ stream = 'data: {"delta": "hel"}\n\ndata: {"delta": "lo"}\n\n'
+ result = providers._parse_codex_stream(stream)
+ text = providers.extract_openai_text(result)
+ self.assertEqual(text, "hello")
+
+ def test_empty_stream(self):
+ self.assertEqual({}, providers._parse_codex_stream(""))
+
+ def test_done_only_stream(self):
+ self.assertEqual({}, providers._parse_codex_stream("data: [DONE]\n\n"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_quality_nudge.py b/tests/test_quality_nudge.py
index 519998a..25bad17 100644
--- a/tests/test_quality_nudge.py
+++ b/tests/test_quality_nudge.py
@@ -1,4 +1,9 @@
-"""Tests for post-research quality score and upgrade nudge."""
+"""Tests for post-research quality score and upgrade nudge.
+
+Reddit is always a core source (free public JSON). The 5 core sources are:
+HN, Polymarket, Reddit (always active), X, YouTube.
+ScrapeCreators adds TikTok + Instagram as bonus sources, not core.
+"""
import pytest
from unittest.mock import patch
@@ -48,66 +53,75 @@ def _compute(config_overrides=None, result_overrides=None, ytdlp_installed=False
# ---------------------------------------------------------------------------
class TestBaseline:
- """HN + Polymarket only (no X, no YT, no SC) -> 40%."""
+ """HN + Polymarket + Reddit always active (no X, no YT) -> 60%."""
- def test_score_40(self):
+ def test_score_60(self):
q = _compute()
- assert q["score_pct"] == 40
+ assert q["score_pct"] == 60
def test_active_sources(self):
q = _compute()
assert "hn" in q["core_active"]
assert "polymarket" in q["core_active"]
- assert len(q["core_active"]) == 2
+ assert "reddit" in q["core_active"]
+ assert len(q["core_active"]) == 3
- def test_missing_all_three(self):
+ def test_missing_x_and_youtube(self):
q = _compute()
- assert set(q["core_missing"]) == {"x", "youtube", "reddit_comments"}
+ assert set(q["core_missing"]) == {"x", "youtube"}
- def test_nudge_mentions_all_missing(self):
+ def test_reddit_not_in_missing(self):
+ """Reddit is always active - never appears in missing."""
+ q = _compute()
+ assert "reddit" not in q["core_missing"]
+ assert "reddit_comments" not in q["core_missing"]
+
+ def test_nudge_mentions_x_and_youtube(self):
q = _compute()
assert q["nudge_text"] is not None
assert "X/Twitter" in q["nudge_text"]
assert "YouTube" in q["nudge_text"]
- assert "Reddit with comments" in q["nudge_text"]
+
+ def test_nudge_does_not_mention_reddit(self):
+ """Reddit is free - nudge should not tell user to get SC for it."""
+ q = _compute()
+ assert "Reddit with comments" not in q["nudge_text"]
class TestXCookies:
- """+X cookies -> 60%."""
+ """+X cookies -> 80%."""
- def test_score_60(self):
+ def test_score_80(self):
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
- assert q["score_pct"] == 60
+ assert q["score_pct"] == 80
- def test_nudge_mentions_yt_and_sc(self):
+ def test_nudge_mentions_yt_only(self):
q = _compute(config_overrides={"AUTH_TOKEN": "tok123"})
assert "YouTube" in q["nudge_text"]
- assert "Reddit with comments" in q["nudge_text"]
assert "X/Twitter" not in q["nudge_text"]
class TestXPlusYtdlp:
- """+X + yt-dlp -> 80%."""
+ """+X + yt-dlp -> 100%. No SC needed for full core coverage."""
- def test_score_80(self):
+ def test_score_100(self):
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
- assert q["score_pct"] == 80
+ assert q["score_pct"] == 100
- def test_nudge_mentions_sc_only(self):
+ def test_nudge_is_none(self):
+ """Full core coverage with zero paid keys."""
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
- assert "ScrapeCreators" in q["nudge_text"] or "scrapecreators" in q["nudge_text"]
- assert "YouTube" not in q["nudge_text"]
- assert "X/Twitter" not in q["nudge_text"]
+ assert q["nudge_text"] is None
-class TestFullCoverage:
- """+X + yt-dlp + SC -> 100%, no nudge."""
+class TestFullCoverageWithSC:
+ """+X + yt-dlp + SC -> still 100%, SC adds bonus sources."""
def test_score_100(self):
q = _compute(
@@ -130,10 +144,15 @@ class TestFullCoverage:
assert q["nudge_text"] is None
-class TestSCActiveNoX:
- """SC active but no X -> 80%, nudge suggests browser cookies (free)."""
+class TestSCDoesNotAffectCoreScore:
+ """SC key should not change core score - it only adds bonus sources."""
- def test_score_80(self):
+ def test_sc_alone_still_60(self):
+ """SC key without X or yt-dlp is still 60% (3/5 core)."""
+ q = _compute(config_overrides={"SCRAPECREATORS_API_KEY": "sc_key"})
+ assert q["score_pct"] == 60
+
+ def test_sc_plus_ytdlp_is_80(self):
q = _compute(
config_overrides={"SCRAPECREATORS_API_KEY": "sc_key"},
ytdlp_installed=True,
@@ -146,26 +165,9 @@ class TestSCActiveNoX:
ytdlp_installed=True,
)
assert q["nudge_text"] is not None
- assert "browser" in q["nudge_text"].lower()
assert "x.com" in q["nudge_text"].lower()
-class TestRedditErrored:
- """SC is configured but Reddit errored this run."""
-
- def test_nudge_mentions_error(self):
- q = _compute(
- config_overrides={
- "AUTH_TOKEN": "tok123",
- "SCRAPECREATORS_API_KEY": "sc_key",
- },
- result_overrides={"reddit_error": "ScrapeCreators: 500 Internal Server Error"},
- ytdlp_installed=True,
- )
- assert "reddit_comments" in q["core_errored"]
- assert "errored" in q["nudge_text"].lower()
-
-
class TestDisclaimerAlwaysPresent:
"""Nudge always includes no-affiliate disclaimer when present."""
@@ -179,25 +181,21 @@ class TestDisclaimerAlwaysPresent:
def test_disclaimer_not_present_at_100(self):
q = _compute(
- config_overrides={
- "AUTH_TOKEN": "tok123",
- "SCRAPECREATORS_API_KEY": "sc_key",
- },
+ config_overrides={"AUTH_TOKEN": "tok123"},
ytdlp_installed=True,
)
assert q["nudge_text"] is None
-class TestSCNudgeContent:
- """SC nudge always includes '100 free API calls, no credit card'."""
+class TestRedditNeverInCoreErrored:
+ """Reddit errors don't affect core score since it's always-active via public path."""
- def test_sc_nudge_content(self):
- q = _compute()
- assert "100 free API calls, no credit card" in q["nudge_text"]
-
- def test_sc_nudge_content_when_only_missing_sc(self):
+ def test_reddit_error_does_not_affect_score(self):
q = _compute(
config_overrides={"AUTH_TOKEN": "tok123"},
+ result_overrides={"reddit_error": "429 Too Many Requests"},
ytdlp_installed=True,
)
- assert "100 free API calls, no credit card" in q["nudge_text"]
+ # Reddit is always-active in core (public path), error doesn't demote it
+ assert "reddit" in q["core_active"]
+ assert q["score_pct"] == 100
diff --git a/tests/test_query_type.py b/tests/test_query_type.py
deleted file mode 100644
index f74a30e..0000000
--- a/tests/test_query_type.py
+++ /dev/null
@@ -1,157 +0,0 @@
-"""Tests for query type detection and source tiering."""
-import sys
-import unittest
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib.query_type import (
- detect_query_type,
- is_source_enabled,
- WEBSEARCH_PENALTY_BY_TYPE,
- TIEBREAKER_BY_TYPE,
- SOURCE_TIERS,
-)
-
-
-class TestDetectQueryType(unittest.TestCase):
-
- def test_product_queries(self):
- self.assertEqual(detect_query_type("cursor IDE pricing"), "product")
- self.assertEqual(detect_query_type("is Claude Pro worth the cost"), "product")
- self.assertEqual(detect_query_type("best free tier LLM API"), "product")
- self.assertEqual(detect_query_type("nano banana pro prompting"), "product")
-
- def test_concept_queries(self):
- self.assertEqual(detect_query_type("what is WebTransport"), "concept")
- self.assertEqual(detect_query_type("explain React Server Components"), "concept")
- self.assertEqual(detect_query_type("how does MCP protocol work"), "concept")
-
- def test_opinion_queries(self):
- self.assertEqual(detect_query_type("is cursor worth it"), "opinion")
- self.assertEqual(detect_query_type("thoughts on Claude Code"), "opinion")
- self.assertEqual(detect_query_type("should i switch to Neovim"), "opinion")
-
- def test_howto_queries(self):
- self.assertEqual(detect_query_type("how to deploy on Vercel"), "how_to")
- self.assertEqual(detect_query_type("tutorial for building MCP servers"), "how_to")
- self.assertEqual(detect_query_type("step by step Kubernetes setup"), "how_to")
- self.assertEqual(detect_query_type("nano banana pro prompting"), "how_to")
- self.assertEqual(detect_query_type("remotion animations for Claude Code"), "how_to")
-
- def test_comparison_queries(self):
- self.assertEqual(detect_query_type("cursor vs windsurf"), "comparison")
- self.assertEqual(detect_query_type("Claude compared to GPT-5"), "comparison")
- self.assertEqual(detect_query_type("difference between React and Vue"), "comparison")
-
- def test_breaking_news_queries(self):
- self.assertEqual(detect_query_type("latest AI funding rounds"), "breaking_news")
- self.assertEqual(detect_query_type("OpenAI just announced GPT-6"), "breaking_news")
-
- def test_prediction_queries(self):
- self.assertEqual(detect_query_type("odds of Fed rate cut"), "prediction")
- self.assertEqual(detect_query_type("predict the next recession"), "prediction")
- self.assertEqual(detect_query_type("election outcome 2028"), "prediction")
-
- def test_default_is_breaking_news(self):
- self.assertEqual(detect_query_type("tariffs"), "breaking_news")
- self.assertEqual(detect_query_type("AI agents"), "breaking_news")
-
- def test_comparison_beats_product(self):
- """Comparison is more specific than product."""
- self.assertEqual(detect_query_type("cursor vs windsurf pricing"), "comparison")
-
- def test_howto_beats_concept(self):
- """How-to is more specific than concept."""
- self.assertEqual(detect_query_type("how to explain transformers"), "how_to")
-
- def test_will_alone_not_prediction(self):
- """Bare 'will' should not trigger prediction classification."""
- self.assertNotEqual(detect_query_type("Will React 19 support concurrent mode"), "prediction")
-
- def test_or_for_not_comparison(self):
- """'or X for Y' should not trigger comparison classification."""
- self.assertNotEqual(detect_query_type("best tools or libraries for Python"), "comparison")
-
-
-class TestIsSourceEnabled(unittest.TestCase):
-
- def test_truthsocial_always_opt_in(self):
- for qt in ["product", "concept", "opinion", "breaking_news", "prediction"]:
- self.assertFalse(is_source_enabled("truthsocial", qt))
- self.assertTrue(is_source_enabled("truthsocial", "breaking_news", explicitly_requested=True))
-
- def test_tier1_sources_enabled(self):
- self.assertTrue(is_source_enabled("reddit", "product"))
- self.assertTrue(is_source_enabled("youtube", "how_to"))
- self.assertTrue(is_source_enabled("polymarket", "prediction"))
- self.assertTrue(is_source_enabled("x", "breaking_news"))
-
- def test_tier2_sources_enabled(self):
- self.assertTrue(is_source_enabled("web", "product"))
- self.assertTrue(is_source_enabled("bluesky", "opinion"))
- self.assertTrue(is_source_enabled("x", "how_to"))
- self.assertTrue(is_source_enabled("youtube", "breaking_news"))
- self.assertTrue(is_source_enabled("hn", "prediction"))
-
- def test_tier3_sources_disabled_by_default(self):
- self.assertFalse(is_source_enabled("instagram", "concept"))
- self.assertFalse(is_source_enabled("tiktok", "comparison"))
- self.assertFalse(is_source_enabled("bluesky", "product"))
-
- def test_explicit_request_overrides_tier(self):
- self.assertTrue(is_source_enabled("instagram", "concept", explicitly_requested=True))
- self.assertTrue(is_source_enabled("tiktok", "comparison", explicitly_requested=True))
-
-
-class TestWebSearchPenalty(unittest.TestCase):
-
- def test_concept_has_zero_penalty(self):
- self.assertEqual(WEBSEARCH_PENALTY_BY_TYPE["concept"], 0)
-
- def test_product_has_full_penalty(self):
- self.assertEqual(WEBSEARCH_PENALTY_BY_TYPE["product"], 15)
-
- def test_howto_has_reduced_penalty(self):
- self.assertLess(WEBSEARCH_PENALTY_BY_TYPE["how_to"], WEBSEARCH_PENALTY_BY_TYPE["product"])
-
- def test_all_query_types_have_penalty(self):
- for qt in ["product", "concept", "opinion", "how_to", "comparison", "breaking_news", "prediction"]:
- self.assertIn(qt, WEBSEARCH_PENALTY_BY_TYPE)
-
-
-class TestTiebreakerPriority(unittest.TestCase):
-
- def test_youtube_highest_for_howto(self):
- self.assertEqual(TIEBREAKER_BY_TYPE["how_to"]["youtube"], 0)
-
- def test_x_highest_for_breaking_news(self):
- self.assertEqual(TIEBREAKER_BY_TYPE["breaking_news"]["x"], 0)
-
- def test_polymarket_highest_for_prediction(self):
- self.assertEqual(TIEBREAKER_BY_TYPE["prediction"]["polymarket"], 0)
-
- def test_hn_highest_for_concept(self):
- self.assertEqual(TIEBREAKER_BY_TYPE["concept"]["hn"], 0)
-
- def test_all_query_types_have_tiebreakers(self):
- for qt in ["product", "concept", "opinion", "how_to", "comparison", "breaking_news", "prediction"]:
- self.assertIn(qt, TIEBREAKER_BY_TYPE)
-
-
-class TestSourceTiers(unittest.TestCase):
-
- def test_all_query_types_have_tiers(self):
- for qt in ["product", "concept", "opinion", "how_to", "comparison", "breaking_news", "prediction"]:
- self.assertIn(qt, SOURCE_TIERS)
- self.assertIn("tier1", SOURCE_TIERS[qt])
- self.assertIn("tier2", SOURCE_TIERS[qt])
-
- def test_truthsocial_not_in_any_tier(self):
- for qt, tiers in SOURCE_TIERS.items():
- self.assertNotIn("truthsocial", tiers["tier1"], f"truthsocial in tier1 for {qt}")
- self.assertNotIn("truthsocial", tiers["tier2"], f"truthsocial in tier2 for {qt}")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_query_v3.py b/tests/test_query_v3.py
new file mode 100644
index 0000000..1950272
--- /dev/null
+++ b/tests/test_query_v3.py
@@ -0,0 +1,44 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import query
+
+
+class QueryV3Tests(unittest.TestCase):
+ def test_extract_core_subject_strips_prefix_and_noise(self):
+ result = query.extract_core_subject("What are the best Claude Code skills for startups?")
+ self.assertEqual("claude code startups", result)
+
+ def test_extract_core_subject_supports_suffix_stripping_and_max_words(self):
+ result = query.extract_core_subject(
+ "How to use Claude Code prompting techniques",
+ strip_suffixes=True,
+ max_words=2,
+ )
+ self.assertEqual("claude code", result)
+
+ def test_extract_core_subject_preserves_original_when_noise_removes_everything(self):
+ result = query.extract_core_subject("all tokens", noise=frozenset({"all", "tokens"}))
+ self.assertEqual("all tokens", result)
+
+ def test_extract_core_subject_handles_empty_query_and_custom_noise(self):
+ self.assertEqual("", query.extract_core_subject(" "))
+ result = query.extract_core_subject(
+ "OpenClaw release notes",
+ noise=frozenset({"release"}),
+ max_words=2,
+ )
+ self.assertEqual("openclaw notes", result)
+
+ def test_extract_compound_terms_finds_hyphenated_and_title_cased_phrases(self):
+ terms = query.extract_compound_terms("Best multi-agent patterns in Claude Code and React Native")
+ self.assertIn("multi-agent", terms)
+ self.assertIn("Claude Code", terms)
+ self.assertIn("React Native", terms)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_reddit.py b/tests/test_reddit.py
new file mode 100644
index 0000000..fcecc19
--- /dev/null
+++ b/tests/test_reddit.py
@@ -0,0 +1,269 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib.reddit import (
+ _extract_date,
+ _extract_score,
+ _extract_subreddit_name,
+ _normalize_reddit_id,
+ _total_engagement,
+ enrich_with_comments,
+)
+
+
+class TestExtractSubredditName(unittest.TestCase):
+ def test_from_string(self):
+ self.assertEqual("openclaw", _extract_subreddit_name("openclaw"))
+
+ def test_from_dict_with_name(self):
+ self.assertEqual(
+ "openclaw",
+ _extract_subreddit_name({"id": "t5_ghydwa", "name": "openclaw"}),
+ )
+
+ def test_from_dict_with_display_name(self):
+ self.assertEqual(
+ "LocalLLM",
+ _extract_subreddit_name({"display_name": "LocalLLM"}),
+ )
+
+ def test_from_dict_name_preferred_over_display_name(self):
+ self.assertEqual(
+ "name_wins",
+ _extract_subreddit_name({"name": "name_wins", "display_name": "display"}),
+ )
+
+ def test_empty_string(self):
+ self.assertEqual("", _extract_subreddit_name(""))
+
+ def test_empty_dict(self):
+ self.assertEqual("", _extract_subreddit_name({}))
+
+ def test_strips_whitespace(self):
+ self.assertEqual("test", _extract_subreddit_name(" test "))
+
+
+class TestExtractScore(unittest.TestCase):
+ def test_ups(self):
+ self.assertEqual(42, _extract_score({"ups": 42}))
+
+ def test_score_field(self):
+ self.assertEqual(77, _extract_score({"score": 77}))
+
+ def test_votes(self):
+ self.assertEqual(99, _extract_score({"votes": 99}))
+
+ def test_ups_preferred_over_votes(self):
+ self.assertEqual(10, _extract_score({"ups": 10, "votes": 99}))
+
+ def test_missing(self):
+ self.assertEqual(0, _extract_score({}))
+
+ def test_zero_preserved(self):
+ self.assertEqual(0, _extract_score({"ups": 0}))
+
+ def test_zero_ups_does_not_fall_through(self):
+ # ups=0 should be returned, not fall through to score
+ self.assertEqual(0, _extract_score({"ups": 0, "score": 5}))
+
+
+class TestExtractDate(unittest.TestCase):
+ def test_unix_timestamp(self):
+ self.assertEqual("2024-05-03", _extract_date({"created_utc": 1714694957}))
+
+ def test_iso_string(self):
+ result = _extract_date({"created_at": "2024-05-03T01:09:17.620000+0000"})
+ self.assertEqual("2024-05-03", result)
+
+ def test_iso_with_z_suffix(self):
+ result = _extract_date({"created_at": "2024-05-03T01:09:17Z"})
+ self.assertEqual("2024-05-03", result)
+
+ def test_created_utc_preferred(self):
+ result = _extract_date({"created_utc": 1714694957, "created_at": "2025-01-01T00:00:00Z"})
+ self.assertEqual("2024-05-03", result)
+
+ def test_missing(self):
+ self.assertIsNone(_extract_date({}))
+
+
+class TestNormalizeRedditId(unittest.TestCase):
+ def test_strips_t3_prefix(self):
+ self.assertEqual("abc123", _normalize_reddit_id("t3_abc123"))
+
+ def test_no_prefix(self):
+ self.assertEqual("abc123", _normalize_reddit_id("abc123"))
+
+ def test_empty(self):
+ self.assertEqual("", _normalize_reddit_id(""))
+
+ def test_none(self):
+ self.assertEqual("", _normalize_reddit_id(None))
+
+
+class TestTotalEngagement(unittest.TestCase):
+ def test_score_plus_comments(self):
+ item = {"engagement": {"score": 100, "num_comments": 50}}
+ self.assertEqual(150, _total_engagement(item))
+
+ def test_high_comments_low_score(self):
+ item = {"engagement": {"score": 1, "num_comments": 1387}}
+ self.assertEqual(1388, _total_engagement(item))
+
+ def test_missing_engagement(self):
+ self.assertEqual(0, _total_engagement({}))
+
+ def test_none_values(self):
+ item = {"engagement": {"score": None, "num_comments": None}}
+ self.assertEqual(0, _total_engagement(item))
+
+ def test_score_only(self):
+ item = {"engagement": {"score": 42}}
+ self.assertEqual(42, _total_engagement(item))
+
+
+class TestEnrichSelectsTopEngagement(unittest.TestCase):
+ """Verify enrich_with_comments picks threads by total engagement, not list order."""
+
+ def test_high_comment_thread_enriched_over_low_engagement(self):
+ """A thread with 1387 comments but score:1 should be enriched before
+ a thread with score:5 and 0 comments."""
+ from unittest.mock import patch
+
+ items = [
+ # Low engagement thread (first in list)
+ {
+ "id": "R1",
+ "url": "https://www.reddit.com/r/test/comments/low",
+ "engagement": {"score": 5, "num_comments": 0},
+ },
+ # High engagement thread (second in list)
+ {
+ "id": "R2",
+ "url": "https://www.reddit.com/r/test/comments/high",
+ "engagement": {"score": 1, "num_comments": 1387},
+ },
+ # Medium engagement
+ {
+ "id": "R3",
+ "url": "https://www.reddit.com/r/test/comments/med",
+ "engagement": {"score": 50, "num_comments": 10},
+ },
+ ]
+
+ enriched_urls = []
+
+ def mock_fetch_comments(url, token):
+ enriched_urls.append(url)
+ return [{"body": "Great thread!", "ups": 10, "author": "testuser"}]
+
+ # Only allow 1 enrichment to prove selection order matters
+ with patch("lib.reddit.fetch_post_comments", side_effect=mock_fetch_comments):
+ result = enrich_with_comments(items, token="fake", depth="quick")
+
+ # With quick depth (3 enrichments), all 3 should be enriched.
+ # But the key assertion: the high-comment thread (R2) must be included.
+ self.assertIn(
+ "https://www.reddit.com/r/test/comments/high",
+ enriched_urls,
+ "High-comment thread should always be selected for enrichment",
+ )
+
+ def test_enrichment_order_by_engagement(self):
+ """With a budget of 1, only the highest-engagement thread gets enriched."""
+ from unittest.mock import patch
+
+ items = [
+ {
+ "id": "R1",
+ "url": "https://www.reddit.com/r/test/comments/a",
+ "engagement": {"score": 200, "num_comments": 5},
+ },
+ {
+ "id": "R2",
+ "url": "https://www.reddit.com/r/test/comments/b",
+ "engagement": {"score": 1, "num_comments": 1500},
+ },
+ ]
+
+ enriched_urls = []
+
+ def mock_fetch_comments(url, token):
+ enriched_urls.append(url)
+ return [{"body": "Comment", "ups": 5, "author": "user"}]
+
+ # Override DEPTH_CONFIG to allow only 1 enrichment
+ custom_config = {"comment_enrichments": 1}
+ with patch("lib.reddit.DEPTH_CONFIG", {"test": custom_config, "default": custom_config}), \
+ patch("lib.reddit.fetch_post_comments", side_effect=mock_fetch_comments):
+ enrich_with_comments(items, token="fake", depth="test")
+
+ # R2 has 1501 total engagement vs R1's 205 -- R2 should be picked
+ self.assertEqual(len(enriched_urls), 1)
+ self.assertEqual(
+ enriched_urls[0],
+ "https://www.reddit.com/r/test/comments/b",
+ )
+
+
+class TestEnrichmentBudget(unittest.TestCase):
+ """Tests for the enrichment time budget in enrich_with_comments()."""
+
+ def _make_items(self, n):
+ return [
+ {"url": f"https://reddit.com/r/test/comments/{i}/post", "score": 100 - i, "num_comments": 50,
+ "engagement": {"score": 100 - i, "num_comments": 50}}
+ for i in range(n)
+ ]
+
+ def test_all_complete_within_budget(self):
+ """When enrichment is fast, all items get comments."""
+ from unittest.mock import patch
+ items = self._make_items(3)
+ fast_comments = [{"body": "Great post!", "score": 42, "author": "user1"}]
+
+ with patch("lib.reddit.fetch_post_comments", return_value=fast_comments):
+ result = enrich_with_comments(items, "fake-token", depth="quick", budget_seconds=60)
+
+ enriched = [i for i in result if i.get("top_comments")]
+ self.assertEqual(len(enriched), 3)
+
+ def test_budget_zero_returns_items_unenriched(self):
+ """With budget=0, items are returned without enrichment (not discarded)."""
+ import time as _time
+ from unittest.mock import patch
+
+ items = self._make_items(3)
+
+ def slow_fetch(url, token):
+ _time.sleep(2)
+ return [{"body": "comment", "score": 10, "author": "u"}]
+
+ with patch("lib.reddit.fetch_post_comments", side_effect=slow_fetch):
+ result = enrich_with_comments(items, "fake-token", depth="quick", budget_seconds=0)
+
+ # All 3 items returned (not discarded)
+ self.assertEqual(len(result), 3)
+
+ def test_empty_items_returns_immediately(self):
+ result = enrich_with_comments([], "fake-token", depth="default", budget_seconds=60)
+ self.assertEqual(result, [])
+
+ def test_exceptions_dont_crash(self):
+ """If enrichment raises, items are returned without comments."""
+ from unittest.mock import patch
+ items = self._make_items(3)
+
+ with patch("lib.reddit.fetch_post_comments", side_effect=ConnectionError("boom")):
+ result = enrich_with_comments(items, "fake-token", depth="quick", budget_seconds=60)
+
+ self.assertEqual(len(result), 3)
+ enriched = [i for i in result if i.get("top_comments")]
+ self.assertEqual(len(enriched), 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_reddit_public.py b/tests/test_reddit_public.py
index 586163b..5f55b36 100644
--- a/tests/test_reddit_public.py
+++ b/tests/test_reddit_public.py
@@ -328,3 +328,85 @@ class TestMissingSubreddit:
results = reddit_public.search("test", subreddit="nonexistent")
assert results == []
+
+
+# ---------------------------------------------------------------------------
+# Tests for comment enrichment (Unit 2)
+# ---------------------------------------------------------------------------
+
+class TestEnrichmentIntegration:
+ """search_reddit_public enriches top posts with comments."""
+
+ @mock.patch("lib.reddit_public._enrich_post")
+ @mock.patch("lib.reddit_public.urllib.request.urlopen")
+ def test_search_enriches_top_5_by_default(self, mock_urlopen, mock_enrich):
+ listing = _make_reddit_listing([
+ {"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
+ "score": 100 - i, "created_utc": 1711670400}
+ for i in range(10)
+ ])
+ mock_urlopen.return_value = _mock_urlopen_ok(listing)
+ mock_enrich.side_effect = lambda item, timeout=10: item # pass-through
+
+ results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31")
+
+ assert len(results) == 10
+ # Default depth enriches top 5
+ assert mock_enrich.call_count == 5
+
+ @mock.patch("lib.reddit_public._enrich_post")
+ @mock.patch("lib.reddit_public.urllib.request.urlopen")
+ def test_enrichment_timeout_keeps_posts(self, mock_urlopen, mock_enrich):
+ listing = _make_reddit_listing([
+ {"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
+ "score": 100 - i, "created_utc": 1711670400}
+ for i in range(10)
+ ])
+ mock_urlopen.return_value = _mock_urlopen_ok(listing)
+
+ # Some enrichments raise, some succeed
+ call_count = {"n": 0}
+ def _side_effect(item, timeout=10):
+ call_count["n"] += 1
+ if call_count["n"] % 2 == 0:
+ raise TimeoutError("enrichment timed out")
+ return item
+ mock_enrich.side_effect = _side_effect
+
+ results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31")
+
+ # All 10 posts should still be returned
+ assert len(results) == 10
+
+ @mock.patch("lib.reddit_public._enrich_post")
+ @mock.patch("lib.reddit_public.urllib.request.urlopen")
+ def test_all_enrichment_fails_all_posts_returned(self, mock_urlopen, mock_enrich):
+ listing = _make_reddit_listing([
+ {"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
+ "score": 100 - i, "created_utc": 1711670400}
+ for i in range(10)
+ ])
+ mock_urlopen.return_value = _mock_urlopen_ok(listing)
+ mock_enrich.side_effect = Exception("total failure")
+
+ results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31")
+
+ # All posts returned despite enrichment failure
+ assert len(results) == 10
+
+ @mock.patch("lib.reddit_public._enrich_post")
+ @mock.patch("lib.reddit_public.urllib.request.urlopen")
+ def test_quick_depth_enriches_top_3(self, mock_urlopen, mock_enrich):
+ listing = _make_reddit_listing([
+ {"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
+ "score": 100 - i, "created_utc": 1711670400}
+ for i in range(10)
+ ])
+ mock_urlopen.return_value = _mock_urlopen_ok(listing)
+ mock_enrich.side_effect = lambda item, timeout=10: item
+
+ results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31", depth="quick")
+
+ assert len(results) == 10
+ # Quick depth enriches only top 3
+ assert mock_enrich.call_count == 3
diff --git a/tests/test_regression.py b/tests/test_regression.py
new file mode 100644
index 0000000..c9b4668
--- /dev/null
+++ b/tests/test_regression.py
@@ -0,0 +1,77 @@
+import json
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+def run_mock_json(topic: str) -> dict:
+ result = subprocess.run(
+ [sys.executable, "scripts/last30days.py", topic, "--mock", "--emit=json"],
+ cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ raise AssertionError(f"mock CLI failed for {topic!r}: {result.stderr}")
+ return json.loads(result.stdout)
+
+
+class RegressionTests(unittest.TestCase):
+ def assert_common_shape(self, payload: dict) -> None:
+ self.assertIn("topic", payload)
+ self.assertIn("query_plan", payload)
+ self.assertIn("ranked_candidates", payload)
+ self.assertIn("clusters", payload)
+ self.assertIn("items_by_source", payload)
+
+ def test_openclaw_three_way_comparison_preserves_entities(self):
+ payload = run_mock_json("openclaw vs. nanoclaw vs. ironclaw")
+ self.assert_common_shape(payload)
+ plan = payload["query_plan"]
+ self.assertEqual("comparison", plan["intent"])
+ joined_queries = "\n".join(subquery["search_query"] for subquery in plan["subqueries"]).lower()
+ self.assertIn("openclaw", joined_queries)
+ self.assertIn("nanoclaw", joined_queries)
+ self.assertIn("ironclaw", joined_queries)
+ self.assertNotIn("corsair", joined_queries)
+ self.assertNotIn("mouse", joined_queries)
+ for subquery in plan["subqueries"]:
+ self.assertGreaterEqual(len(subquery["sources"]), 4)
+
+ def test_how_to_keeps_web_video_and_discussion_sources(self):
+ payload = run_mock_json("how to deploy on Fly.io")
+ self.assert_common_shape(payload)
+ plan = payload["query_plan"]
+ self.assertEqual("how_to", plan["intent"])
+ sources = set(plan["subqueries"][0]["sources"])
+ self.assertIn("youtube", sources)
+ self.assertIn("reddit", sources)
+ self.assertGreaterEqual(len(sources), 2)
+
+ def test_breaking_news_query_keeps_expected_shape(self):
+ payload = run_mock_json("latest news about React 20")
+ self.assert_common_shape(payload)
+ plan = payload["query_plan"]
+ self.assertEqual("breaking_news", plan["intent"])
+ joined_queries = "\n".join(subquery["search_query"] for subquery in plan["subqueries"]).lower()
+ self.assertIn("react 20", joined_queries)
+ self.assertGreaterEqual(len(plan["subqueries"][0]["sources"]), 2)
+
+ def test_two_way_comparison_preserves_exact_strings(self):
+ payload = run_mock_json("DeepSeek R1 vs GPT-5")
+ self.assert_common_shape(payload)
+ plan = payload["query_plan"]
+ self.assertEqual("comparison", plan["intent"])
+ joined_queries = "\n".join(subquery["search_query"] for subquery in plan["subqueries"]).lower()
+ self.assertIn("deepseek r1", joined_queries)
+ self.assertIn("gpt-5", joined_queries)
+ self.assertNotIn("corsair", joined_queries)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_relevance_core_v3.py b/tests/test_relevance_core_v3.py
new file mode 100644
index 0000000..edba3a6
--- /dev/null
+++ b/tests/test_relevance_core_v3.py
@@ -0,0 +1,52 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import relevance
+
+
+class RelevanceCoreV3Tests(unittest.TestCase):
+ def test_tokenize_removes_stopwords_and_expands_synonyms(self):
+ tokens = relevance.tokenize("How to use JS for hip hop apps")
+ self.assertIn("js", tokens)
+ self.assertIn("javascript", tokens)
+ self.assertIn("hiphop", tokens)
+ self.assertNotIn("how", tokens)
+
+ def test_token_overlap_relevance_returns_neutral_for_stopword_only_query(self):
+ self.assertEqual(0.5, relevance.token_overlap_relevance("how to", "anything at all"))
+
+ def test_token_overlap_relevance_returns_zero_for_no_overlap(self):
+ self.assertEqual(0.0, relevance.token_overlap_relevance("openclaw", "corsair gaming mouse"))
+
+ def test_token_overlap_relevance_rewards_exact_phrase_matches(self):
+ phrase_score = relevance.token_overlap_relevance(
+ "openclaw nanoclaw",
+ "A detailed openclaw nanoclaw comparison for agents.",
+ )
+ partial_score = relevance.token_overlap_relevance(
+ "openclaw nanoclaw",
+ "A detailed openclaw comparison for agents.",
+ )
+ self.assertGreater(phrase_score, partial_score)
+
+ def test_token_overlap_relevance_caps_generic_only_matches(self):
+ score = relevance.token_overlap_relevance(
+ "anthropic odds",
+ "Latest odds and prediction updates for markets",
+ )
+ self.assertLessEqual(score, 0.24)
+
+ def test_token_overlap_relevance_splits_concatenated_hashtags(self):
+ score = relevance.token_overlap_relevance(
+ "claude code",
+ "Agent workflow discussion",
+ hashtags=["ClaudeCode", "BuildInPublic"],
+ )
+ self.assertGreater(score, 0.0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_render.py b/tests/test_render.py
deleted file mode 100644
index d1f8be9..0000000
--- a/tests/test_render.py
+++ /dev/null
@@ -1,164 +0,0 @@
-"""Tests for render module."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import render, schema
-
-
-class TestRenderCompact(unittest.TestCase):
- def test_renders_basic_report(self):
- report = schema.Report(
- topic="test topic",
- range_from="2026-01-01",
- range_to="2026-01-31",
- generated_at="2026-01-31T12:00:00Z",
- mode="both",
- openai_model_used="gpt-5.2",
- xai_model_used="grok-4-latest",
- )
-
- result = render.render_compact(report)
-
- self.assertIn("test topic", result)
- self.assertIn("2026-01-01", result)
- self.assertIn("both", result)
- self.assertIn("gpt-5.2", result)
-
- def test_renders_reddit_items(self):
- report = schema.Report(
- topic="test",
- range_from="2026-01-01",
- range_to="2026-01-31",
- generated_at="2026-01-31T12:00:00Z",
- mode="reddit-only",
- reddit=[
- schema.RedditItem(
- id="R1",
- title="Test Thread",
- url="https://reddit.com/r/test/1",
- subreddit="test",
- date="2026-01-15",
- date_confidence="high",
- score=85,
- why_relevant="Very relevant",
- )
- ],
- )
-
- result = render.render_compact(report)
-
- self.assertIn("R1", result)
- self.assertIn("Test Thread", result)
- self.assertIn("r/test", result)
-
- def test_shows_coverage_tip_for_reddit_only(self):
- report = schema.Report(
- topic="test",
- range_from="2026-01-01",
- range_to="2026-01-31",
- generated_at="2026-01-31T12:00:00Z",
- mode="reddit-only",
- )
-
- result = render.render_compact(report)
-
- self.assertIn("xAI key", result)
-
-
-class TestRenderContextSnippet(unittest.TestCase):
- def test_renders_snippet(self):
- report = schema.Report(
- topic="Claude Code Skills",
- range_from="2026-01-01",
- range_to="2026-01-31",
- generated_at="2026-01-31T12:00:00Z",
- mode="both",
- )
-
- result = render.render_context_snippet(report)
-
- self.assertIn("Claude Code Skills", result)
- self.assertIn("Last 30 Days", result)
-
-
-class TestRenderFullReport(unittest.TestCase):
- def test_renders_full_report(self):
- report = schema.Report(
- topic="test topic",
- range_from="2026-01-01",
- range_to="2026-01-31",
- generated_at="2026-01-31T12:00:00Z",
- mode="both",
- openai_model_used="gpt-5.2",
- xai_model_used="grok-4-latest",
- )
-
- result = render.render_full_report(report)
-
- self.assertIn("# test topic", result)
- self.assertIn("## Models Used", result)
- self.assertIn("gpt-5.2", result)
-
-
-class TestGetContextPath(unittest.TestCase):
- def test_returns_path_string(self):
- result = render.get_context_path()
- self.assertIsInstance(result, str)
- self.assertIn("last30days.context.md", result)
-
-
-class TestEnsureOutputDir(unittest.TestCase):
- """Tests for ensure_output_dir()."""
-
- def test_creates_directory(self):
- import os
- import tempfile
- test_dir = os.path.join(tempfile.mkdtemp(), "output", "nested")
- os.environ["LAST30DAYS_OUTPUT_DIR"] = test_dir
- try:
- render.ensure_output_dir()
- self.assertTrue(os.path.exists(test_dir))
- finally:
- os.environ.pop("LAST30DAYS_OUTPUT_DIR", None)
-
-
-class TestXrefTag(unittest.TestCase):
- """Tests for _xref_tag()."""
-
- def test_no_refs(self):
- item = schema.RedditItem(id="R1", title="T", url="u", subreddit="s")
- self.assertEqual(render._xref_tag(item), "")
-
- def test_with_refs(self):
- item = schema.RedditItem(
- id="R1", title="T", url="u", subreddit="s",
- cross_refs=["X1", "HN2"],
- )
- tag = render._xref_tag(item)
- self.assertIn("X", tag)
- self.assertIn("HN", tag)
-
-
-class TestRenderEmptyReport(unittest.TestCase):
- """Test render_compact handles empty reports gracefully."""
-
- def test_empty_items_graceful(self):
- report = schema.Report(
- topic="test",
- range_from="2026-02-04",
- range_to="2026-03-06",
- generated_at="2026-03-06T00:00:00+00:00",
- mode="both",
- )
- output = render.render_compact(report)
- self.assertIn("test", output)
- self.assertIsInstance(output, str)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_render_v3.py b/tests/test_render_v3.py
new file mode 100644
index 0000000..105b5ae
--- /dev/null
+++ b/tests/test_render_v3.py
@@ -0,0 +1,372 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import render, schema
+
+
+def sample_report() -> schema.Report:
+ primary_item = schema.SourceItem(
+ item_id="i1",
+ source="grounding",
+ title="Grounded result",
+ body="A grounded body with useful detail.",
+ url="https://example.com",
+ container="example.com",
+ published_at="2026-03-15",
+ date_confidence="high",
+ snippet="A grounded snippet about the topic.",
+ metadata={},
+ )
+ reddit_item = schema.SourceItem(
+ item_id="i2",
+ source="reddit",
+ title="Grounded result",
+ body="Reddit discussion body.",
+ url="https://example.com",
+ container="LocalLLaMA",
+ published_at="2026-03-14",
+ date_confidence="high",
+ engagement={"score": 344, "num_comments": 119, "upvote_ratio": 0.92},
+ metadata={
+ "top_comments": [{"excerpt": "This is the strongest user reaction.", "score": 22}],
+ "comment_insights": ["Users corroborate the main claim."],
+ },
+ )
+ candidate = schema.Candidate(
+ candidate_id="c1",
+ item_id="i2",
+ source="reddit",
+ title="Grounded result",
+ url="https://example.com",
+ snippet="A grounded snippet about the topic.",
+ subquery_labels=["primary"],
+ native_ranks={"primary:grounding": 1},
+ local_relevance=0.9,
+ freshness=90,
+ engagement=88,
+ source_quality=1.0,
+ rrf_score=0.02,
+ rerank_score=92,
+ final_score=90,
+ explanation="high-signal result",
+ sources=["reddit", "grounding"],
+ source_items=[reddit_item, primary_item],
+ )
+ cluster = schema.Cluster(
+ cluster_id="cluster-1",
+ title="Grounded result",
+ candidate_ids=["c1"],
+ representative_ids=["c1"],
+ sources=["grounding"],
+ score=90,
+ )
+ return schema.Report(
+ topic="test topic",
+ range_from="2026-02-14",
+ range_to="2026-03-16",
+ generated_at="2026-03-16T00:00:00+00:00",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="gemini",
+ planner_model="gemini-3.1-flash-lite-preview",
+ rerank_model="gemini-3.1-flash-lite-preview",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test topic",
+ subqueries=[schema.SubQuery(label="primary", search_query="test topic", ranking_query="What happened with test topic?", sources=["grounding"])],
+ source_weights={"grounding": 1.0},
+ ),
+ clusters=[cluster],
+ ranked_candidates=[candidate],
+ items_by_source={"grounding": [primary_item], "reddit": [reddit_item]},
+ errors_by_source={},
+ )
+
+
+class RenderV3Tests(unittest.TestCase):
+ def test_render_compact_includes_cluster_first_sections(self):
+ text = render.render_compact(sample_report())
+ self.assertIn("# last30days-3 v3.0.0-alpha: test topic", text)
+ self.assertIn("## Ranked Evidence Clusters", text)
+ self.assertIn("## Stats", text)
+ self.assertIn("Total evidence: 2 items across 2 sources", text)
+ self.assertIn("Top voices: example.com, r/LocalLLaMA", text)
+ self.assertIn("Web: 1 item | domains: example.com", text)
+ self.assertIn("Reddit: 1 item | 344pts, 119cmt | communities: r/LocalLLaMA", text)
+ self.assertIn("[reddit, grounding] Grounded result", text)
+ self.assertIn("[344pts, 119cmt]", text)
+ self.assertIn("Also on: Web", text)
+ self.assertIn("Comment (22 upvotes): This is the strongest user reaction.", text)
+ self.assertIn("Insight: Users corroborate the main claim.", text)
+ self.assertIn("## Source Coverage", text)
+
+ def test_render_context_includes_top_clusters(self):
+ text = render.render_context(sample_report())
+ self.assertIn("Top clusters:", text)
+ self.assertIn("Grounded result", text)
+
+ def test_render_compact_includes_source_errors_section(self):
+ report = sample_report()
+ report.errors_by_source = {"x": "HTTP 400: Bad Request"}
+ text = render.render_compact(report)
+ self.assertIn("## Source Errors", text)
+ self.assertIn("HTTP 400: Bad Request", text)
+ self.assertIn("X:", text)
+
+
+class RenderTopCommentsTests(unittest.TestCase):
+ """Tests for the top-3 comments rendering in compact cluster view."""
+
+ def _make_report_with_comments(self, source="reddit", top_comments=None, comment_insights=None):
+ """Helper: build a report with a single candidate carrying given comments."""
+ item = schema.SourceItem(
+ item_id="i1",
+ source=source,
+ title="Test post",
+ body="Body text.",
+ url="https://reddit.com/r/test/comments/abc/test/",
+ container="test",
+ published_at="2026-03-15",
+ date_confidence="high",
+ engagement={"score": 100, "num_comments": 50},
+ metadata={
+ "top_comments": top_comments or [],
+ "comment_insights": comment_insights or [],
+ },
+ )
+ candidate = schema.Candidate(
+ candidate_id="c1",
+ item_id="i1",
+ source=source,
+ title="Test post",
+ url="https://reddit.com/r/test/comments/abc/test/",
+ snippet="A test snippet.",
+ subquery_labels=["primary"],
+ native_ranks={"primary:reddit": 1},
+ local_relevance=0.9,
+ freshness=90,
+ engagement=88,
+ source_quality=1.0,
+ rrf_score=0.02,
+ rerank_score=92,
+ final_score=90,
+ sources=[source],
+ source_items=[item],
+ )
+ cluster = schema.Cluster(
+ cluster_id="cluster-1",
+ title="Test cluster",
+ candidate_ids=["c1"],
+ representative_ids=["c1"],
+ sources=[source],
+ score=90,
+ )
+ return schema.Report(
+ topic="test topic",
+ range_from="2026-02-14",
+ range_to="2026-03-16",
+ generated_at="2026-03-16T00:00:00+00:00",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="gemini",
+ planner_model="gemini-3.1-flash-lite-preview",
+ rerank_model="gemini-3.1-flash-lite-preview",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test topic",
+ subqueries=[schema.SubQuery(label="primary", search_query="test", ranking_query="test?", sources=[source])],
+ source_weights={source: 1.0},
+ ),
+ clusters=[cluster],
+ ranked_candidates=[candidate],
+ items_by_source={source: [item]},
+ errors_by_source={},
+ )
+
+ def test_reddit_5_comments_renders_top_3(self):
+ """Reddit candidate with 5 comments (scores 500, 200, 50, 8, 3) renders 3."""
+ comments = [
+ {"score": 500, "excerpt": "Comment with 500 upvotes", "author": "user1"},
+ {"score": 200, "excerpt": "Comment with 200 upvotes", "author": "user2"},
+ {"score": 50, "excerpt": "Comment with 50 upvotes", "author": "user3"},
+ {"score": 8, "excerpt": "Comment with 8 upvotes", "author": "user4"},
+ {"score": 3, "excerpt": "Comment with 3 upvotes", "author": "user5"},
+ ]
+ report = self._make_report_with_comments(top_comments=comments)
+ text = render.render_compact(report)
+ self.assertIn("Comment (500 upvotes):", text)
+ self.assertIn("Comment (200 upvotes):", text)
+ self.assertIn("Comment (50 upvotes):", text)
+ self.assertNotIn("Comment (8 upvotes):", text)
+ self.assertNotIn("Comment (3 upvotes):", text)
+
+ def test_reddit_1_comment_renders_1(self):
+ """Reddit candidate with 1 comment renders 1."""
+ comments = [{"score": 100, "excerpt": "Single comment", "author": "user1"}]
+ report = self._make_report_with_comments(top_comments=comments)
+ text = render.render_compact(report)
+ self.assertIn("Comment (100 upvotes): Single comment", text)
+
+ def test_reddit_0_comments_no_section(self):
+ """Reddit candidate with 0 comments renders no comment section."""
+ report = self._make_report_with_comments(top_comments=[])
+ text = render.render_compact(report)
+ self.assertNotIn("Comment (", text)
+ self.assertNotIn("upvotes)", text)
+
+ def test_non_reddit_no_comments(self):
+ """Non-Reddit candidate doesn't render comments when metadata has none."""
+ report = self._make_report_with_comments(source="grounding", top_comments=[])
+ text = render.render_compact(report)
+ self.assertNotIn("Comment (", text)
+ self.assertIn("Test cluster", text)
+
+ def test_all_comments_below_score_10_no_section(self):
+ """All comments below score 10 renders no comment section."""
+ comments = [
+ {"score": 9, "excerpt": "Low score 1", "author": "user1"},
+ {"score": 5, "excerpt": "Low score 2", "author": "user2"},
+ {"score": 1, "excerpt": "Low score 3", "author": "user3"},
+ ]
+ report = self._make_report_with_comments(top_comments=comments)
+ text = render.render_compact(report)
+ self.assertNotIn("Comment (", text)
+ self.assertNotIn("upvotes)", text)
+
+
+class RenderBestTakesCompactTests(unittest.TestCase):
+ """Tests for Best Takes section in compact output and fun tags on candidates."""
+
+ def _make_candidate(self, cid, fun_score=None, fun_explanation=None, final_score=80):
+ """Helper: build a candidate with a given fun_score."""
+ item = schema.SourceItem(
+ item_id=f"item-{cid}",
+ source="reddit",
+ title=f"Post {cid}",
+ body="Body text.",
+ url=f"https://reddit.com/r/test/comments/{cid}/",
+ container="test",
+ published_at="2026-03-15",
+ date_confidence="high",
+ engagement={"score": 200, "num_comments": 30},
+ metadata={
+ "top_comments": [{"excerpt": "Funny comment", "score": 50, "body": "lmao this is gold"}],
+ },
+ )
+ return schema.Candidate(
+ candidate_id=cid,
+ item_id=f"item-{cid}",
+ source="reddit",
+ title=f"Post {cid}",
+ url=f"https://reddit.com/r/test/comments/{cid}/",
+ snippet="A test snippet.",
+ subquery_labels=["primary"],
+ native_ranks={"primary:reddit": 1},
+ local_relevance=0.9,
+ freshness=90,
+ engagement=88,
+ source_quality=1.0,
+ rrf_score=0.02,
+ rerank_score=92,
+ final_score=final_score,
+ sources=["reddit"],
+ source_items=[item],
+ fun_score=fun_score,
+ fun_explanation=fun_explanation,
+ )
+
+ def _make_report_with_candidates(self, candidates):
+ """Helper: build a report with given candidates."""
+ items = []
+ for c in candidates:
+ items.extend(c.source_items)
+ cluster = schema.Cluster(
+ cluster_id="cluster-1",
+ title="Test cluster",
+ candidate_ids=[c.candidate_id for c in candidates],
+ representative_ids=[c.candidate_id for c in candidates],
+ sources=["reddit"],
+ score=90,
+ )
+ return schema.Report(
+ topic="test topic",
+ range_from="2026-02-14",
+ range_to="2026-03-16",
+ generated_at="2026-03-16T00:00:00+00:00",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="gemini",
+ planner_model="gemini-3.1-flash-lite-preview",
+ rerank_model="gemini-3.1-flash-lite-preview",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test topic",
+ subqueries=[schema.SubQuery(label="primary", search_query="test", ranking_query="test?", sources=["reddit"])],
+ source_weights={"reddit": 1.0},
+ ),
+ clusters=[cluster],
+ ranked_candidates=candidates,
+ items_by_source={"reddit": items},
+ errors_by_source={},
+ )
+
+ def test_compact_includes_best_takes_with_2_high_fun_candidates(self):
+ """Compact output includes Best Takes section when 2+ candidates score >= 70."""
+ candidates = [
+ self._make_candidate("c1", fun_score=85, fun_explanation="hilarious comment"),
+ self._make_candidate("c2", fun_score=75, fun_explanation="witty remark"),
+ self._make_candidate("c3", fun_score=40),
+ ]
+ report = self._make_report_with_candidates(candidates)
+ text = render.render_compact(report)
+ self.assertIn("## Best Takes", text)
+ self.assertIn("(fun:85)", text)
+ self.assertIn("(fun:75)", text)
+
+ def test_candidate_with_fun_score_85_shows_fun_tag(self):
+ """Candidate with fun_score=85 shows 'fun:85' in its detail line."""
+ candidates = [self._make_candidate("c1", fun_score=85)]
+ report = self._make_report_with_candidates(candidates)
+ text = render.render_compact(report)
+ self.assertIn("fun:85", text)
+
+ def test_candidate_with_fun_score_40_no_fun_tag(self):
+ """Candidate with fun_score=40 does NOT show fun tag (below 50 threshold)."""
+ candidates = [self._make_candidate("c1", fun_score=40)]
+ report = self._make_report_with_candidates(candidates)
+ text = render.render_compact(report)
+ self.assertNotIn("fun:40", text)
+ self.assertNotIn("fun:", text)
+
+ def test_no_best_takes_with_0_high_fun_candidates(self):
+ """No Best Takes section when 0 candidates above threshold."""
+ candidates = [
+ self._make_candidate("c1", fun_score=50),
+ self._make_candidate("c2", fun_score=40),
+ ]
+ report = self._make_report_with_candidates(candidates)
+ text = render.render_compact(report)
+ self.assertNotIn("## Best Takes", text)
+
+ def test_no_best_takes_with_1_high_fun_candidate(self):
+ """No Best Takes section when only 1 candidate above threshold."""
+ candidates = [
+ self._make_candidate("c1", fun_score=80),
+ self._make_candidate("c2", fun_score=50),
+ ]
+ report = self._make_report_with_candidates(candidates)
+ text = render.render_compact(report)
+ self.assertNotIn("## Best Takes", text)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_rerank_fun.py b/tests/test_rerank_fun.py
new file mode 100644
index 0000000..b43a917
--- /dev/null
+++ b/tests/test_rerank_fun.py
@@ -0,0 +1,129 @@
+"""Tests for the fun judge heuristic fallback in rerank.py."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from lib import schema
+from lib.rerank import _apply_single_fun_fallback, _extract_comment_text
+
+
+def _make_candidate(
+ title: str = "Some Title",
+ snippet: str = "",
+ engagement: float | None = 0.0,
+ top_comments: list[dict] | None = None,
+) -> schema.Candidate:
+ """Build a minimal Candidate with optional source_items carrying top_comments."""
+ source_items = []
+ if top_comments is not None:
+ source_items.append(
+ schema.SourceItem(
+ item_id="si-1",
+ source="reddit",
+ title=title,
+ body="",
+ url="https://reddit.com/r/test/1",
+ metadata={"top_comments": top_comments},
+ )
+ )
+ return schema.Candidate(
+ candidate_id="c-1",
+ item_id="i-1",
+ source="reddit",
+ title=title,
+ url="https://reddit.com/r/test/1",
+ snippet=snippet,
+ subquery_labels=["q1"],
+ native_ranks={"reddit": 1},
+ local_relevance=0.5,
+ freshness=50,
+ engagement=engagement,
+ source_quality=0.5,
+ rrf_score=0.01,
+ source_items=source_items,
+ )
+
+
+class TestFunFallbackCommentText:
+ """Heuristic fallback reads comment text, not just title+snippet."""
+
+ def test_comment_with_lmao_gets_marker_bonus(self):
+ """A candidate with 'lmao' in a top_comment should get the marker bonus."""
+ candidate = _make_candidate(
+ title="Boring press conference recap",
+ snippet="Coach talked about the game plan.",
+ top_comments=[{"body": "lmao this is gold"}],
+ )
+ _apply_single_fun_fallback(candidate)
+ # marker_bonus = 10, should be reflected in fun_score
+ assert candidate.fun_score is not None
+ assert candidate.fun_score >= 10.0
+ assert candidate.fun_explanation == "heuristic-fallback"
+
+ def test_short_punchy_comment_higher_shortness(self):
+ """A candidate with a short punchy comment should score higher shortness
+ bonus compared to one with a very long title and snippet."""
+ short_candidate = _make_candidate(
+ title="Hot dogs",
+ snippet="",
+ top_comments=[{"body": "bro what"}],
+ )
+ long_candidate = _make_candidate(
+ title="A very long and detailed analysis of the upcoming season with comprehensive breakdown of every roster move and coaching decision that happened over the past thirty days",
+ snippet="This extensive report covers all aspects of the team performance including advanced metrics and historical comparisons going back several decades.",
+ top_comments=[{"body": "bro what"}],
+ )
+ _apply_single_fun_fallback(short_candidate)
+ _apply_single_fun_fallback(long_candidate)
+ # Both get marker bonus from "bro", but short one gets higher shortness bonus
+ assert short_candidate.fun_score > long_candidate.fun_score
+
+ def test_no_comments_falls_back_to_title_snippet(self):
+ """A candidate with no source_items/comments still scores based on title+snippet."""
+ candidate = _make_candidate(
+ title="This is hilarious content",
+ snippet="Very funny stuff",
+ top_comments=None, # no source_items at all
+ )
+ _apply_single_fun_fallback(candidate)
+ assert candidate.fun_score is not None
+ assert candidate.fun_score >= 10.0 # marker bonus from "hilarious"
+ assert candidate.fun_explanation == "heuristic-fallback"
+
+ def test_empty_comment_bodies_no_crash(self):
+ """Candidates with empty comment bodies should not crash."""
+ candidate = _make_candidate(
+ title="Normal title",
+ snippet="Normal snippet",
+ top_comments=[
+ {"body": ""},
+ {"body": None},
+ {},
+ {"body": "actual comment"},
+ ],
+ )
+ _apply_single_fun_fallback(candidate)
+ assert candidate.fun_score is not None
+ assert candidate.fun_explanation == "heuristic-fallback"
+
+
+class TestExtractCommentText:
+ """Verify _extract_comment_text handles edge cases."""
+
+ def test_extracts_from_top_comments(self):
+ candidate = _make_candidate(
+ top_comments=[{"body": "first comment"}, {"body": "second comment"}],
+ )
+ text = _extract_comment_text(candidate)
+ assert "first comment" in text
+ assert "second comment" in text
+
+ def test_empty_source_items(self):
+ candidate = _make_candidate(top_comments=None)
+ text = _extract_comment_text(candidate)
+ assert text == ""
diff --git a/tests/test_rerank_v3.py b/tests/test_rerank_v3.py
new file mode 100644
index 0000000..a29d7dd
--- /dev/null
+++ b/tests/test_rerank_v3.py
@@ -0,0 +1,159 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import rerank, schema
+
+
+def make_candidate(relevance: float) -> schema.Candidate:
+ candidate = schema.Candidate(
+ candidate_id=f"c-{relevance}",
+ item_id="i1",
+ source="reddit",
+ title="Title",
+ url="https://example.com",
+ snippet="Snippet",
+ subquery_labels=["primary"],
+ native_ranks={"primary:reddit": 1},
+ local_relevance=0.8,
+ freshness=80,
+ engagement=50,
+ source_quality=0.7,
+ rrf_score=0.02,
+ )
+ candidate.rerank_score = relevance
+ return candidate
+
+
+def make_plan() -> schema.QueryPlan:
+ return schema.QueryPlan(
+ intent="comparison",
+ freshness_mode="balanced_recent",
+ cluster_mode="debate",
+ raw_topic="openclaw vs nanoclaw",
+ subqueries=[
+ schema.SubQuery(
+ label="primary",
+ search_query="openclaw vs nanoclaw",
+ ranking_query="How does openclaw compare to nanoclaw?",
+ sources=["grounding", "reddit"],
+ )
+ ],
+ source_weights={"grounding": 1.0, "reddit": 0.8},
+ )
+
+
+class FakeProvider:
+ def __init__(self, payload):
+ self.payload = payload
+
+ def generate_json(self, model, prompt):
+ self.model = model
+ self.prompt = prompt
+ return self.payload
+
+
+class RerankV3Tests(unittest.TestCase):
+ def test_low_rerank_score_is_demoted(self):
+ low = make_candidate(4.0)
+ high = make_candidate(40.0)
+ low_score = rerank._final_score(low)
+ high_score = rerank._final_score(high)
+ self.assertLess(low_score, high_score)
+ self.assertLess(low_score, 20.0)
+
+ def test_engagement_boosts_score(self):
+ """Items with engagement score higher than those without."""
+ candidate = make_candidate(80.0)
+ candidate.engagement = None
+ score_without = rerank._final_score(candidate)
+ candidate.engagement = 50
+ score_with = rerank._final_score(candidate)
+ self.assertGreater(score_with, score_without)
+ # Boost is modest, not dominant
+ self.assertLess(score_with - score_without, 10.0)
+
+ def test_build_prompt_includes_source_labels_and_dates(self):
+ candidate = make_candidate(80.0)
+ candidate.sources = ["grounding", "reddit"]
+ candidate.source_items = [
+ schema.SourceItem(
+ item_id="i1",
+ source="grounding",
+ title="Title",
+ body="Body",
+ url="https://example.com",
+ published_at="2026-03-16",
+ )
+ ]
+ prompt = rerank._build_prompt("topic", make_plan(), [candidate])
+ self.assertIn("sources: grounding, reddit", prompt)
+ self.assertIn("date: 2026-03-16", prompt)
+ self.assertIn("How does openclaw compare to nanoclaw?", prompt)
+
+ def test_apply_llm_scores_ignores_invalid_rows_and_clamps_scores(self):
+ candidate = make_candidate(0.0)
+ rerank._apply_llm_scores(
+ [candidate],
+ {
+ "scores": [
+ "bad-row",
+ {"candidate_id": "", "relevance": 99},
+ {"candidate_id": candidate.candidate_id, "relevance": 101, "reason": " best hit "},
+ ]
+ },
+ )
+ self.assertEqual(100.0, candidate.rerank_score)
+ self.assertEqual("best hit", candidate.explanation)
+ self.assertGreater(candidate.final_score, 0.0)
+
+ def test_build_prompt_includes_comparison_intent_hint(self):
+ plan = make_plan() # intent="comparison"
+ candidate = make_candidate(80.0)
+ prompt = rerank._build_prompt("openclaw vs nanoclaw", plan, [candidate])
+ self.assertIn("Intent-specific guidance (comparison)", prompt)
+ self.assertIn("head-to-head", prompt.lower())
+
+ def test_build_prompt_includes_factual_intent_hint(self):
+ plan = make_plan()
+ plan.intent = "factual"
+ candidate = make_candidate(80.0)
+ prompt = rerank._build_prompt("latest GDP numbers", plan, [candidate])
+ self.assertTrue(
+ "facts" in prompt.lower() or "primary sources" in prompt.lower(),
+ "factual intent hint should mention facts or primary sources",
+ )
+
+ def test_build_prompt_no_hint_for_unknown_intent(self):
+ plan = make_plan()
+ plan.intent = "unknown_intent_xyz"
+ candidate = make_candidate(80.0)
+ prompt = rerank._build_prompt("some topic", plan, [candidate])
+ self.assertNotIn("Intent-specific guidance", prompt)
+
+ def test_rerank_candidates_uses_provider_for_shortlist_and_fallback_for_tail(self):
+ first = make_candidate(0.0)
+ second = make_candidate(0.0)
+ second.candidate_id = "tail"
+ provider = FakeProvider(
+ {"scores": [{"candidate_id": first.candidate_id, "relevance": 95, "reason": "high fit"}]}
+ )
+ ranked = rerank.rerank_candidates(
+ topic="openclaw vs nanoclaw",
+ plan=make_plan(),
+ candidates=[first, second],
+ provider=provider,
+ model="gemini-3.1-flash-lite-preview",
+ shortlist_size=1,
+ )
+ self.assertEqual("gemini-3.1-flash-lite-preview", provider.model)
+ self.assertEqual(95.0, first.rerank_score)
+ self.assertEqual("high fit", first.explanation)
+ self.assertEqual("fallback-local-score", second.explanation)
+ self.assertEqual(first.candidate_id, ranked[0].candidate_id)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_resolve.py b/tests/test_resolve.py
new file mode 100644
index 0000000..7690263
--- /dev/null
+++ b/tests/test_resolve.py
@@ -0,0 +1,175 @@
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import resolve
+
+
+class TestHasBackend(unittest.TestCase):
+ def test_no_keys_returns_false(self):
+ self.assertFalse(resolve._has_backend({}))
+
+ def test_brave_key_returns_true(self):
+ self.assertTrue(resolve._has_backend({"BRAVE_API_KEY": "key"}))
+
+ def test_exa_key_returns_true(self):
+ self.assertTrue(resolve._has_backend({"EXA_API_KEY": "key"}))
+
+ def test_serper_key_returns_true(self):
+ self.assertTrue(resolve._has_backend({"SERPER_API_KEY": "key"}))
+
+
+class TestExtractSubreddits(unittest.TestCase):
+ def test_extracts_from_title_and_snippet(self):
+ items = [
+ {"title": "Check out r/MachineLearning", "snippet": "Also r/artificial", "url": ""},
+ {"title": "More at r/datascience", "snippet": "", "url": ""},
+ ]
+ result = resolve._extract_subreddits(items)
+ self.assertEqual(result, ["MachineLearning", "artificial", "datascience"])
+
+ def test_extracts_from_url(self):
+ items = [
+ {"title": "Discussion", "snippet": "", "url": "https://reddit.com/r/python/comments/123"},
+ ]
+ result = resolve._extract_subreddits(items)
+ self.assertEqual(result, ["python"])
+
+ def test_deduplicates_case_insensitive(self):
+ items = [
+ {"title": "r/Python", "snippet": "r/python is great", "url": ""},
+ ]
+ result = resolve._extract_subreddits(items)
+ self.assertEqual(len(result), 1)
+
+ def test_empty_items_returns_empty(self):
+ self.assertEqual(resolve._extract_subreddits([]), [])
+
+ def test_no_subreddits_in_text(self):
+ items = [{"title": "No subreddits here", "snippet": "Just text", "url": ""}]
+ self.assertEqual(resolve._extract_subreddits(items), [])
+
+
+class TestExtractXHandle(unittest.TestCase):
+ def test_extracts_from_url(self):
+ items = [
+ {"title": "OpenAI on X", "snippet": "Updates from @OpenAI", "url": "https://x.com/OpenAI"},
+ ]
+ result = resolve._extract_x_handle(items)
+ self.assertEqual(result, "openai")
+
+ def test_extracts_from_text(self):
+ items = [
+ {"title": "Follow @elonmusk", "snippet": "Also @elonmusk tweeted", "url": ""},
+ ]
+ result = resolve._extract_x_handle(items)
+ self.assertEqual(result, "elonmusk")
+
+ def test_filters_generic_handles(self):
+ items = [
+ {"title": "Go to @twitter", "snippet": "Visit @x", "url": ""},
+ ]
+ result = resolve._extract_x_handle(items)
+ self.assertEqual(result, "")
+
+ def test_empty_items_returns_empty(self):
+ self.assertEqual(resolve._extract_x_handle([]), "")
+
+
+class TestBuildContextSummary(unittest.TestCase):
+ def test_builds_from_snippets(self):
+ items = [
+ {"snippet": "First news item about topic."},
+ {"snippet": "Second news item with details."},
+ {"snippet": "Third item ignored."},
+ ]
+ result = resolve._build_context_summary(items)
+ self.assertIn("First news item", result)
+ self.assertIn("Second news item", result)
+ # Only first 2 snippets used
+ self.assertNotIn("Third item", result)
+
+ def test_truncates_long_text(self):
+ items = [{"snippet": "A" * 200}, {"snippet": "B" * 200}]
+ result = resolve._build_context_summary(items)
+ self.assertLessEqual(len(result), 300)
+ self.assertTrue(result.endswith("..."))
+
+ def test_empty_items_returns_empty(self):
+ self.assertEqual(resolve._build_context_summary([]), "")
+
+ def test_items_with_empty_snippets(self):
+ items = [{"snippet": ""}, {"snippet": ""}]
+ self.assertEqual(resolve._build_context_summary(items), "")
+
+
+class TestAutoResolve(unittest.TestCase):
+ def test_no_backend_returns_empty(self):
+ result = resolve.auto_resolve("test topic", {})
+ self.assertEqual(result["subreddits"], [])
+ self.assertEqual(result["x_handle"], "")
+ self.assertEqual(result["context"], "")
+ self.assertEqual(result["searches_run"], 0)
+
+ @patch("lib.resolve.grounding.web_search")
+ def test_full_resolve(self, mock_search):
+ def side_effect(query, date_range, config):
+ if "subreddit" in query:
+ return [
+ {"title": "r/technology discussion", "snippet": "Also r/gadgets", "url": ""},
+ ], {"label": "brave"}
+ if "news" in query:
+ return [
+ {"snippet": "Major tech breakthrough announced this week."},
+ ], {"label": "brave"}
+ if "handle" in query:
+ return [
+ {"title": "TechCo on X", "snippet": "@TechCo", "url": "https://x.com/TechCo"},
+ ], {"label": "brave"}
+ return [], {}
+
+ mock_search.side_effect = side_effect
+ result = resolve.auto_resolve("tech", {"BRAVE_API_KEY": "fake"})
+
+ self.assertEqual(result["subreddits"], ["technology", "gadgets"])
+ self.assertEqual(result["x_handle"], "techco")
+ self.assertIn("breakthrough", result["context"])
+ self.assertEqual(result["searches_run"], 3)
+ self.assertEqual(mock_search.call_count, 3)
+
+ @patch("lib.resolve.grounding.web_search")
+ def test_search_failure_graceful(self, mock_search):
+ mock_search.side_effect = RuntimeError("API error")
+ result = resolve.auto_resolve("test", {"BRAVE_API_KEY": "fake"})
+ self.assertEqual(result["subreddits"], [])
+ self.assertEqual(result["x_handle"], "")
+ self.assertEqual(result["context"], "")
+ self.assertEqual(result["searches_run"], 0)
+
+ @patch("lib.resolve.grounding.web_search")
+ def test_partial_failure(self, mock_search):
+ call_count = 0
+
+ def side_effect(query, date_range, config):
+ nonlocal call_count
+ call_count += 1
+ if "subreddit" in query:
+ return [{"title": "r/cooking tips", "snippet": "", "url": ""}], {}
+ if "news" in query:
+ raise RuntimeError("Timeout")
+ return [], {}
+
+ mock_search.side_effect = side_effect
+ result = resolve.auto_resolve("cooking", {"EXA_API_KEY": "fake"})
+ self.assertEqual(result["subreddits"], ["cooking"])
+ # News search failed, so context is empty
+ self.assertEqual(result["context"], "")
+ # 2 out of 3 succeeded
+ self.assertEqual(result["searches_run"], 2)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_schema_roundtrip.py b/tests/test_schema_roundtrip.py
deleted file mode 100644
index 25f0b50..0000000
--- a/tests/test_schema_roundtrip.py
+++ /dev/null
@@ -1,142 +0,0 @@
-"""Tests for schema.py — data class serialization roundtrips."""
-
-import sys
-import unittest
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import schema
-
-
-class TestEngagement(unittest.TestCase):
- """Tests for Engagement.to_dict()."""
-
- def test_sparse_fields(self):
- eng = schema.Engagement(score=100, num_comments=50)
- d = eng.to_dict()
- self.assertEqual(d, {"score": 100, "num_comments": 50})
- self.assertNotIn("likes", d)
-
- def test_all_none_returns_none(self):
- eng = schema.Engagement()
- self.assertIsNone(eng.to_dict())
-
- def test_all_fields(self):
- eng = schema.Engagement(
- score=1, num_comments=2, upvote_ratio=0.9,
- likes=3, reposts=4, replies=5, quotes=6,
- views=7, shares=8, volume=9.0, liquidity=10.0,
- )
- d = eng.to_dict()
- self.assertEqual(len(d), 11)
-
-
-class TestComment(unittest.TestCase):
- def test_basic(self):
- c = schema.Comment(score=50, date="2026-03-01", author="user", excerpt="text", url="http://x")
- d = c.to_dict()
- self.assertEqual(d["score"], 50)
- self.assertEqual(d["author"], "user")
- self.assertEqual(len(d), 5)
-
-
-class TestRedditItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.RedditItem(
- id="R1", title="Test", url="http://reddit.com/r/test",
- subreddit="test", date="2026-03-01",
- engagement=schema.Engagement(score=100),
- )
- d = item.to_dict()
- self.assertEqual(d["id"], "R1")
- self.assertEqual(d["subreddit"], "test")
- self.assertEqual(d["engagement"], {"score": 100})
- self.assertNotIn("cross_refs", d)
-
- def test_cross_refs_included_when_present(self):
- item = schema.RedditItem(
- id="R1", title="T", url="u", subreddit="s",
- cross_refs=["X1", "HN2"],
- )
- d = item.to_dict()
- self.assertEqual(d["cross_refs"], ["X1", "HN2"])
-
-
-class TestXItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.XItem(
- id="X1", text="tweet", url="http://x.com/1",
- author_handle="user", date="2026-03-01",
- )
- d = item.to_dict()
- self.assertEqual(d["id"], "X1")
- self.assertEqual(d["author_handle"], "user")
- self.assertNotIn("cross_refs", d)
-
-
-class TestYouTubeItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.YouTubeItem(
- id="YT1", title="Video", url="http://youtube.com/1",
- channel_name="chan",
- )
- d = item.to_dict()
- self.assertEqual(d["channel_name"], "chan")
- self.assertEqual(d["date_confidence"], "high")
-
-
-class TestTikTokItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.TikTokItem(
- id="TK1", text="caption", url="http://tiktok.com/1",
- author_name="creator", hashtags=["ai", "code"],
- )
- d = item.to_dict()
- self.assertEqual(d["hashtags"], ["ai", "code"])
- self.assertEqual(d["author_name"], "creator")
-
-
-class TestInstagramItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.InstagramItem(
- id="IG1", text="caption", url="http://instagram.com/reel/1",
- author_name="creator",
- )
- d = item.to_dict()
- self.assertEqual(d["id"], "IG1")
-
-
-class TestWebSearchItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.WebSearchItem(
- id="W1", title="Article", url="http://example.com",
- source_domain="example.com", snippet="text",
- )
- d = item.to_dict()
- self.assertEqual(d["source_domain"], "example.com")
-
-
-class TestHackerNewsItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.HackerNewsItem(
- id="HN1", title="Show HN", url="http://example.com",
- hn_url="http://news.ycombinator.com/item?id=1", author="pg",
- )
- d = item.to_dict()
- self.assertTrue(d["hn_url"].startswith("http://news.ycombinator.com"))
-
-
-class TestPolymarketItem(unittest.TestCase):
- def test_roundtrip(self):
- item = schema.PolymarketItem(
- id="PM1", title="Election", question="Who wins?",
- url="http://polymarket.com/1",
- )
- d = item.to_dict()
- self.assertEqual(d["question"], "Who wins?")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_schema_v3.py b/tests/test_schema_v3.py
new file mode 100644
index 0000000..e394ba6
--- /dev/null
+++ b/tests/test_schema_v3.py
@@ -0,0 +1,89 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import schema
+
+
+class SchemaV3Tests(unittest.TestCase):
+ def test_report_roundtrip(self):
+ report = schema.Report(
+ topic="test topic",
+ range_from="2026-02-14",
+ range_to="2026-03-16",
+ generated_at="2026-03-16T00:00:00+00:00",
+ provider_runtime=schema.ProviderRuntime(
+ reasoning_provider="gemini",
+ planner_model="gemini-3.1-flash-lite-preview",
+ rerank_model="gemini-3.1-flash-lite-preview",
+ ),
+ query_plan=schema.QueryPlan(
+ intent="breaking_news",
+ freshness_mode="strict_recent",
+ cluster_mode="story",
+ raw_topic="test topic",
+ subqueries=[schema.SubQuery(label="primary", search_query="test topic", ranking_query="What happened with test topic?", sources=["grounding"])],
+ source_weights={"grounding": 1.0},
+ ),
+ clusters=[schema.Cluster(cluster_id="cluster-1", title="Title", candidate_ids=["c1"], representative_ids=["c1"], sources=["grounding"], score=90)],
+ ranked_candidates=[schema.Candidate(
+ candidate_id="c1",
+ item_id="i1",
+ source="grounding",
+ sources=["grounding", "reddit"],
+ title="Title",
+ url="https://example.com",
+ snippet="Snippet",
+ subquery_labels=["primary"],
+ native_ranks={"primary:grounding": 1},
+ local_relevance=0.8,
+ freshness=90,
+ engagement=None,
+ source_quality=1.0,
+ rrf_score=0.02,
+ rerank_score=91,
+ final_score=90,
+ source_items=[
+ schema.SourceItem(item_id="i1", source="grounding", title="Title", body="Body", url="https://example.com", published_at="2026-03-16")
+ ],
+ )],
+ items_by_source={"grounding": [schema.SourceItem(item_id="i1", source="grounding", title="Title", body="Body", url="https://example.com")]},
+ errors_by_source={},
+ warnings=["warning"],
+ artifacts={"grounding": []},
+ )
+ restored = schema.report_from_dict(schema.to_dict(report))
+ self.assertEqual(report.topic, restored.topic)
+ self.assertEqual(report.provider_runtime.planner_model, restored.provider_runtime.planner_model)
+ self.assertEqual(report.ranked_candidates[0].candidate_id, restored.ranked_candidates[0].candidate_id)
+ self.assertEqual(report.ranked_candidates[0].sources, restored.ranked_candidates[0].sources)
+ self.assertEqual(report.items_by_source["grounding"][0].title, restored.items_by_source["grounding"][0].title)
+
+ def test_source_item_from_dict_preserves_zero_valued_signals(self):
+ item = schema.source_item_from_dict(
+ {
+ "item_id": "x1",
+ "source": "x",
+ "title": "Title",
+ "body": "Body",
+ "url": "https://example.com",
+ "relevance_hint": 0.0,
+ "local_relevance": 0.0,
+ "freshness": 0,
+ "engagement_score": 0,
+ "source_quality": 0.0,
+ "local_rank_score": 0.0,
+ }
+ )
+ self.assertEqual(0.0, item.relevance_hint)
+ self.assertEqual(0.0, item.local_relevance)
+ self.assertEqual(0, item.freshness)
+ self.assertEqual(0, item.engagement_score)
+ self.assertEqual(0.0, item.source_quality)
+ self.assertEqual(0.0, item.local_rank_score)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_score.py b/tests/test_score.py
deleted file mode 100644
index 436258e..0000000
--- a/tests/test_score.py
+++ /dev/null
@@ -1,351 +0,0 @@
-"""Tests for score module."""
-
-import sys
-import unittest
-from datetime import datetime, timezone
-from pathlib import Path
-
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-
-from lib import schema, score
-
-
-class TestLog1pSafe(unittest.TestCase):
- def test_positive_value(self):
- result = score.log1p_safe(100)
- self.assertGreater(result, 0)
-
- def test_zero(self):
- result = score.log1p_safe(0)
- self.assertEqual(result, 0)
-
- def test_none(self):
- result = score.log1p_safe(None)
- self.assertEqual(result, 0)
-
- def test_negative(self):
- result = score.log1p_safe(-5)
- self.assertEqual(result, 0)
-
-
-class TestComputeRedditEngagementRaw(unittest.TestCase):
- def test_with_engagement(self):
- eng = schema.Engagement(score=100, num_comments=50, upvote_ratio=0.9)
- result = score.compute_reddit_engagement_raw(eng)
- self.assertIsNotNone(result)
- self.assertGreater(result, 0)
-
- def test_without_engagement(self):
- result = score.compute_reddit_engagement_raw(None)
- self.assertIsNone(result)
-
- def test_empty_engagement(self):
- eng = schema.Engagement()
- result = score.compute_reddit_engagement_raw(eng)
- self.assertIsNone(result)
-
-
-class TestComputeXEngagementRaw(unittest.TestCase):
- def test_with_engagement(self):
- eng = schema.Engagement(likes=100, reposts=25, replies=15, quotes=5)
- result = score.compute_x_engagement_raw(eng)
- self.assertIsNotNone(result)
- self.assertGreater(result, 0)
-
- def test_without_engagement(self):
- result = score.compute_x_engagement_raw(None)
- self.assertIsNone(result)
-
-
-class TestNormalizeTo100(unittest.TestCase):
- def test_normalizes_values(self):
- values = [0, 50, 100]
- result = score.normalize_to_100(values)
- self.assertEqual(result[0], 0)
- self.assertEqual(result[1], 50)
- self.assertEqual(result[2], 100)
-
- def test_handles_none(self):
- values = [0, None, 100]
- result = score.normalize_to_100(values)
- self.assertIsNone(result[1])
-
- def test_single_value(self):
- values = [50]
- result = score.normalize_to_100(values)
- self.assertEqual(result[0], 50)
-
-
-class TestScoreRedditItems(unittest.TestCase):
- def test_scores_items(self):
- today = datetime.now(timezone.utc).date().isoformat()
- items = [
- schema.RedditItem(
- id="R1",
- title="Test",
- url="https://reddit.com/r/test/1",
- subreddit="test",
- date=today,
- date_confidence="high",
- engagement=schema.Engagement(score=100, num_comments=50, upvote_ratio=0.9),
- relevance=0.9,
- ),
- schema.RedditItem(
- id="R2",
- title="Test 2",
- url="https://reddit.com/r/test/2",
- subreddit="test",
- date=today,
- date_confidence="high",
- engagement=schema.Engagement(score=10, num_comments=5, upvote_ratio=0.8),
- relevance=0.5,
- ),
- ]
-
- result = score.score_reddit_items(items)
-
- self.assertEqual(len(result), 2)
- self.assertGreater(result[0].score, 0)
- self.assertGreater(result[1].score, 0)
- # Higher relevance and engagement should score higher
- self.assertGreater(result[0].score, result[1].score)
-
- def test_empty_list(self):
- result = score.score_reddit_items([])
- self.assertEqual(result, [])
-
-
-class TestScoreXItems(unittest.TestCase):
- def test_scores_items(self):
- today = datetime.now(timezone.utc).date().isoformat()
- items = [
- schema.XItem(
- id="X1",
- text="Test post",
- url="https://x.com/user/1",
- author_handle="user1",
- date=today,
- date_confidence="high",
- engagement=schema.Engagement(likes=100, reposts=25, replies=15, quotes=5),
- relevance=0.9,
- ),
- ]
-
- result = score.score_x_items(items)
-
- self.assertEqual(len(result), 1)
- self.assertGreater(result[0].score, 0)
-
-
-class TestSortItems(unittest.TestCase):
- def test_sorts_by_score_descending(self):
- items = [
- schema.RedditItem(id="R1", title="Low", url="", subreddit="", score=30),
- schema.RedditItem(id="R2", title="High", url="", subreddit="", score=90),
- schema.RedditItem(id="R3", title="Mid", url="", subreddit="", score=60),
- ]
-
- result = score.sort_items(items)
-
- self.assertEqual(result[0].id, "R2")
- self.assertEqual(result[1].id, "R3")
- self.assertEqual(result[2].id, "R1")
-
- def test_stable_sort(self):
- items = [
- schema.RedditItem(id="R1", title="A", url="", subreddit="", score=50),
- schema.RedditItem(id="R2", title="B", url="", subreddit="", score=50),
- ]
-
- result = score.sort_items(items)
-
- # Both have same score, should maintain order by title
- self.assertEqual(len(result), 2)
-
-
-class TestCommentQualityWeight(unittest.TestCase):
- """Test that top comment score boosts Reddit engagement."""
-
- def test_comment_boosts_score(self):
- eng = schema.Engagement(score=100, num_comments=50, upvote_ratio=0.9)
- without_comment = score.compute_reddit_engagement_raw(eng, top_comment_score=None)
- with_comment = score.compute_reddit_engagement_raw(eng, top_comment_score=500)
- self.assertGreater(with_comment, without_comment)
-
-
-class TestInstagramEngagement(unittest.TestCase):
- """Tests for compute_instagram_engagement_raw()."""
-
- def test_basic(self):
- eng = schema.Engagement(views=10000, likes=500, num_comments=50)
- raw = score.compute_instagram_engagement_raw(eng)
- self.assertIsNotNone(raw)
- self.assertGreater(raw, 0)
-
- def test_views_dominate(self):
- views_only = schema.Engagement(views=10000)
- likes_only = schema.Engagement(likes=10000)
- self.assertGreater(
- score.compute_instagram_engagement_raw(views_only),
- score.compute_instagram_engagement_raw(likes_only),
- )
-
-
-class TestBlueskyEngagement(unittest.TestCase):
- """Tests for compute_bluesky_engagement_raw()."""
-
- def test_basic(self):
- eng = schema.Engagement(likes=100, reposts=25, replies=15, quotes=5)
- raw = score.compute_bluesky_engagement_raw(eng)
- self.assertIsNotNone(raw)
- self.assertGreater(raw, 0)
-
- def test_likes_dominate(self):
- likes_heavy = schema.Engagement(likes=1000, reposts=0, replies=0, quotes=0)
- reposts_heavy = schema.Engagement(likes=0, reposts=1000, replies=0, quotes=0)
- self.assertGreater(
- score.compute_bluesky_engagement_raw(likes_heavy),
- score.compute_bluesky_engagement_raw(reposts_heavy),
- )
-
- def test_none_engagement(self):
- self.assertIsNone(score.compute_bluesky_engagement_raw(None))
-
- def test_no_likes_no_reposts(self):
- eng = schema.Engagement(replies=10)
- self.assertIsNone(score.compute_bluesky_engagement_raw(eng))
-
-
-class TestTruthSocialEngagement(unittest.TestCase):
- """Tests for compute_truthsocial_engagement_raw()."""
-
- def test_basic(self):
- eng = schema.Engagement(likes=100, reposts=25, replies=15)
- raw = score.compute_truthsocial_engagement_raw(eng)
- self.assertIsNotNone(raw)
- self.assertGreater(raw, 0)
-
- def test_likes_dominate(self):
- likes_heavy = schema.Engagement(likes=1000, reposts=0, replies=0)
- reposts_heavy = schema.Engagement(likes=0, reposts=1000, replies=0)
- self.assertGreater(
- score.compute_truthsocial_engagement_raw(likes_heavy),
- score.compute_truthsocial_engagement_raw(reposts_heavy),
- )
-
- def test_none_engagement(self):
- self.assertIsNone(score.compute_truthsocial_engagement_raw(None))
-
-
-class TestScoreBlueskyItems(unittest.TestCase):
- """Tests for score_bluesky_items()."""
-
- def test_scores_items(self):
- items = [
- schema.BlueskyItem(
- id="bsky1", text="Test", url="https://bsky.app/1",
- author_handle="user.bsky.social", display_name="User",
- engagement=schema.Engagement(likes=50, reposts=10, replies=5, quotes=2),
- relevance=0.8,
- ),
- ]
- result = score.score_bluesky_items(items)
- self.assertEqual(len(result), 1)
- self.assertGreater(result[0].score, 0)
-
- def test_empty_list(self):
- self.assertEqual(score.score_bluesky_items([]), [])
-
-
-class TestScoreTruthSocialItems(unittest.TestCase):
- """Tests for score_truthsocial_items()."""
-
- def test_scores_items(self):
- items = [
- schema.TruthSocialItem(
- id="ts1", text="Test", url="https://truthsocial.com/1",
- author_handle="@user", display_name="User",
- engagement=schema.Engagement(likes=50, reposts=10, replies=5),
- relevance=0.8,
- ),
- ]
- result = score.score_truthsocial_items(items)
- self.assertEqual(len(result), 1)
- self.assertGreater(result[0].score, 0)
-
- def test_empty_list(self):
- self.assertEqual(score.score_truthsocial_items([]), [])
-
-
-class TestSortItemsMixedSources(unittest.TestCase):
- """Test sort_items with Bluesky and TruthSocial items."""
-
- def test_bluesky_item_sorts(self):
- items = [
- schema.RedditItem(id="R1", title="Reddit", url="", subreddit="", score=30),
- schema.BlueskyItem(id="B1", text="Bluesky", url="", author_handle="u.bsky.social", display_name="U", score=90),
- ]
- result = score.sort_items(items)
- self.assertEqual(result[0].id, "B1")
-
- def test_truthsocial_item_sorts(self):
- items = [
- schema.RedditItem(id="R1", title="Reddit", url="", subreddit="", score=30),
- schema.TruthSocialItem(id="T1", text="TS", url="", author_handle="@u", display_name="U", score=90),
- ]
- result = score.sort_items(items)
- self.assertEqual(result[0].id, "T1")
-
-
-class TestRelevanceFilter(unittest.TestCase):
- """Tests for relevance_filter()."""
-
- def _make_items(self, relevances):
- """Helper: create RedditItems with given relevance values."""
- return [
- schema.RedditItem(id=f"R{i}", title=f"Item {i}", url="", subreddit="", relevance=r)
- for i, r in enumerate(relevances)
- ]
-
- def test_filters_below_threshold(self):
- items = self._make_items([0.8, 0.1, 0.5, 0.2])
- result = score.relevance_filter(items, "TEST", threshold=0.3)
- self.assertEqual(len(result), 2)
- self.assertTrue(all(i.relevance >= 0.3 for i in result))
-
- def test_small_list_unchanged(self):
- items = self._make_items([0.1, 0.05, 0.02])
- result = score.relevance_filter(items, "TEST")
- self.assertEqual(len(result), 3)
-
- def test_all_below_threshold_keeps_top_3(self):
- items = self._make_items([0.1, 0.25, 0.05, 0.2, 0.15])
- result = score.relevance_filter(items, "TEST", threshold=0.3)
- self.assertEqual(len(result), 3)
- # Should be sorted by relevance: 0.25, 0.2, 0.15
- self.assertEqual(result[0].relevance, 0.25)
- self.assertEqual(result[1].relevance, 0.2)
-
- def test_empty_list(self):
- result = score.relevance_filter([], "TEST")
- self.assertEqual(result, [])
-
- def test_items_without_relevance_attr_treated_as_zero(self):
- """Objects lacking a relevance attribute get 0.0, failing the filter."""
- class BareItem:
- def __init__(self, id):
- self.id = id
- items = [
- schema.RedditItem(id="R0", title="Has relevance", url="", subreddit="", relevance=0.8),
- BareItem("B1"),
- BareItem("B2"),
- BareItem("B3"),
- ]
- result = score.relevance_filter(items, "TEST", threshold=0.3)
- self.assertEqual(len(result), 1)
- self.assertEqual(result[0].id, "R0")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_scrapecreators_x.py b/tests/test_scrapecreators_x.py
deleted file mode 100644
index 2359c7a..0000000
--- a/tests/test_scrapecreators_x.py
+++ /dev/null
@@ -1,114 +0,0 @@
-"""Tests for scrapecreators_x module."""
-
-import sys
-import unittest
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
-from lib import scrapecreators_x
-from lib.relevance import tokenize as _tokenize
-
-
-class TestTokenize(unittest.TestCase):
- def test_lowercases(self):
- tokens = _tokenize("Claude AI")
- self.assertIn("claude", tokens)
-
- def test_strips_stopwords(self):
- tokens = _tokenize("the best AI tool")
- self.assertNotIn("the", tokens)
- self.assertIn("best", tokens) # 'best' is not a stopword in tokenizer
-
- def test_removes_single_char(self):
- tokens = _tokenize("a b cd ef")
- self.assertNotIn("a", tokens)
- self.assertNotIn("b", tokens)
- self.assertIn("cd", tokens)
-
- def test_expands_synonyms(self):
- tokens = _tokenize("ai research")
- self.assertIn("artificial", tokens)
- self.assertIn("intelligence", tokens)
-
-
-class TestComputeRelevance(unittest.TestCase):
- def test_exact_match_high(self):
- score = scrapecreators_x._compute_relevance("claude code", "claude code is amazing")
- self.assertGreaterEqual(score, 0.8)
-
- def test_no_match_low(self):
- score = scrapecreators_x._compute_relevance("claude code", "pizza recipes today")
- self.assertLessEqual(score, 0.2)
-
- def test_empty_query_returns_neutral(self):
- score = scrapecreators_x._compute_relevance("", "some text")
- self.assertEqual(score, 0.5)
-
- def test_no_match_returns_zero(self):
- score = scrapecreators_x._compute_relevance("abcdef ghijkl", "xyz")
- self.assertEqual(score, 0.0)
-
-
-class TestExtractCoreSubject(unittest.TestCase):
- def test_strips_prefix(self):
- result = scrapecreators_x._extract_core_subject("what are people saying about claude")
- self.assertEqual(result, "claude")
-
- def test_strips_noise(self):
- result = scrapecreators_x._extract_core_subject("latest trending news claude code")
- self.assertNotIn("latest", result)
- self.assertNotIn("trending", result)
- self.assertIn("claude", result)
-
- def test_preserves_core(self):
- result = scrapecreators_x._extract_core_subject("react native")
- self.assertEqual(result, "react native")
-
-
-class TestParseDate(unittest.TestCase):
- def test_twitter_format(self):
- item = {"created_at": "Wed Oct 10 20:19:24 +0000 2018"}
- self.assertEqual(scrapecreators_x._parse_date(item), "2018-10-10")
-
- def test_unix_timestamp(self):
- item = {"timestamp": 1705363200}
- self.assertEqual(scrapecreators_x._parse_date(item), "2024-01-16")
-
- def test_iso_format(self):
- item = {"created_at": "2024-06-15T12:00:00Z"}
- self.assertEqual(scrapecreators_x._parse_date(item), "2024-06-15")
-
- def test_none_returns_none(self):
- self.assertIsNone(scrapecreators_x._parse_date({}))
-
-
-class TestSearchX(unittest.TestCase):
- def test_no_token_returns_error(self):
- result = scrapecreators_x.search_x("test", "2024-01-01", "2024-12-31")
- self.assertEqual(result["items"], [])
- self.assertIn("No SCRAPECREATORS_API_KEY", result["error"])
-
- def test_parse_x_response(self):
- response = {"items": [{"id": "1", "text": "hello"}]}
- items = scrapecreators_x.parse_x_response(response)
- self.assertEqual(len(items), 1)
- self.assertEqual(items[0]["text"], "hello")
-
- def test_parse_empty_response(self):
- items = scrapecreators_x.parse_x_response({})
- self.assertEqual(items, [])
-
-
-class TestDepthConfig(unittest.TestCase):
- def test_all_depths_exist(self):
- for depth in ("quick", "default", "deep"):
- self.assertIn(depth, scrapecreators_x.DEPTH_CONFIG)
-
- def test_deep_has_more_results(self):
- quick = scrapecreators_x.DEPTH_CONFIG["quick"]["results_per_page"]
- deep = scrapecreators_x.DEPTH_CONFIG["deep"]["results_per_page"]
- self.assertGreater(deep, quick)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_setup_openclaw.py b/tests/test_setup_openclaw.py
new file mode 100644
index 0000000..4f1a2cb
--- /dev/null
+++ b/tests/test_setup_openclaw.py
@@ -0,0 +1,562 @@
+"""Tests for OpenClaw setup and device auth functions."""
+
+import json
+import sys
+import time
+from pathlib import Path
+from unittest.mock import patch, MagicMock, call
+
+import pytest
+
+# Add scripts dir to path
+SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from lib import setup_wizard
+
+
+class TestRunOpenclawSetup:
+ """Tests for run_openclaw_setup()."""
+
+ @patch("shutil.which")
+ def test_all_tools_present_no_keys(self, mock_which):
+ """All CLI tools found, no API keys configured."""
+ mock_which.side_effect = lambda cmd: f"/usr/bin/{cmd}"
+ config = {}
+
+ result = setup_wizard.run_openclaw_setup(config)
+
+ assert result["yt_dlp"] is True
+ assert result["node"] is True
+ assert result["python3"] is True
+ assert all(v is False for v in result["keys"].values())
+ assert result["x_method"] is None
+
+ @patch("shutil.which")
+ def test_missing_tools(self, mock_which):
+ """Some CLI tools missing."""
+ def which_side(cmd):
+ if cmd == "node":
+ return None
+ return f"/usr/bin/{cmd}"
+ mock_which.side_effect = which_side
+ config = {}
+
+ result = setup_wizard.run_openclaw_setup(config)
+
+ assert result["yt_dlp"] is True
+ assert result["node"] is False
+ assert result["python3"] is True
+
+ @patch("shutil.which")
+ def test_keys_detected(self, mock_which):
+ """API keys in config are reported as present."""
+ mock_which.return_value = None
+ config = {
+ "XAI_API_KEY": "xai-abc123",
+ "BRAVE_API_KEY": "brav-xyz",
+ "SCRAPECREATORS_API_KEY": "", # empty = falsy
+ }
+
+ result = setup_wizard.run_openclaw_setup(config)
+
+ assert result["keys"]["xai"] is True
+ assert result["keys"]["brave"] is True
+ assert result["keys"]["scrapecreators"] is False
+
+ @patch("shutil.which")
+ def test_x_method_xai(self, mock_which):
+ """x_method is 'xai' when XAI_API_KEY is set."""
+ mock_which.return_value = None
+ config = {"XAI_API_KEY": "xai-key"}
+
+ result = setup_wizard.run_openclaw_setup(config)
+
+ assert result["x_method"] == "xai"
+
+ @patch("shutil.which")
+ def test_x_method_cookies(self, mock_which):
+ """x_method is 'cookies' when AUTH_TOKEN + CT0 are set."""
+ mock_which.return_value = None
+ config = {"AUTH_TOKEN": "tok", "CT0": "ct0val"}
+
+ result = setup_wizard.run_openclaw_setup(config)
+
+ assert result["x_method"] == "cookies"
+
+ @patch("shutil.which")
+ def test_x_method_xai_over_cookies(self, mock_which):
+ """XAI takes priority over cookies for x_method."""
+ mock_which.return_value = None
+ config = {"XAI_API_KEY": "xai-key", "AUTH_TOKEN": "tok", "CT0": "ct0val"}
+
+ result = setup_wizard.run_openclaw_setup(config)
+
+ assert result["x_method"] == "xai"
+
+ @patch("shutil.which")
+ def test_x_method_null_when_nothing(self, mock_which):
+ """x_method is None when no X access configured."""
+ mock_which.return_value = None
+ config = {}
+
+ result = setup_wizard.run_openclaw_setup(config)
+
+ assert result["x_method"] is None
+
+ @patch("shutil.which")
+ def test_output_is_json_serializable(self, mock_which):
+ """Result can be serialized to JSON without errors."""
+ mock_which.return_value = "/usr/bin/something"
+ config = {"XAI_API_KEY": "k", "OPENAI_API_KEY": "ok"}
+
+ result = setup_wizard.run_openclaw_setup(config)
+ serialized = json.dumps(result)
+ parsed = json.loads(serialized)
+
+ assert parsed["yt_dlp"] is True
+ assert parsed["keys"]["xai"] is True
+
+
+class TestRunDeviceAuth:
+ """Tests for run_device_auth()."""
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_success(self, mock_urlopen):
+ """Successful device code request returns tuple."""
+ resp_data = {
+ "device_code": "dc-123",
+ "user_code": "ABCD-1234",
+ "verification_uri": "https://github.com/login/device",
+ "interval": 5,
+ }
+ mock_resp = MagicMock()
+ mock_resp.read.return_value = json.dumps(resp_data).encode()
+ mock_resp.__enter__ = lambda s: s
+ mock_resp.__exit__ = MagicMock(return_value=False)
+ mock_urlopen.return_value = mock_resp
+
+ result = setup_wizard.run_device_auth()
+
+ assert result is not None
+ device_code, user_code, verification_uri, interval = result
+ assert device_code == "dc-123"
+ assert user_code == "ABCD-1234"
+ assert verification_uri == "https://github.com/login/device"
+ assert interval == 5
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_http_error_returns_none(self, mock_urlopen):
+ """HTTP error during code request returns None."""
+ from urllib.error import HTTPError
+ mock_urlopen.side_effect = HTTPError(
+ "https://example.com", 500, "Server Error", {}, None
+ )
+
+ result = setup_wizard.run_device_auth()
+ assert result is None
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_missing_device_code_returns_none(self, mock_urlopen):
+ """Incomplete response (no device_code) returns None."""
+ resp_data = {"user_code": "ABCD-1234"}
+ mock_resp = MagicMock()
+ mock_resp.read.return_value = json.dumps(resp_data).encode()
+ mock_resp.__enter__ = lambda s: s
+ mock_resp.__exit__ = MagicMock(return_value=False)
+ mock_urlopen.return_value = mock_resp
+
+ result = setup_wizard.run_device_auth()
+ assert result is None
+
+
+class TestPollDeviceAuth:
+ """Tests for poll_device_auth()."""
+
+ @patch("lib.setup_wizard.time")
+ @patch("lib.setup_wizard.urlopen")
+ def test_success_on_second_poll(self, mock_urlopen, mock_time):
+ """Returns access_token after initial pending then success."""
+ # First call: time check (within deadline), second: after sleep, etc.
+ mock_time.time = MagicMock(side_effect=[0, 0, 0, 0])
+ mock_time.sleep = MagicMock()
+
+ pending_resp = MagicMock()
+ pending_resp.read.return_value = json.dumps({"error": "authorization_pending"}).encode()
+ pending_resp.__enter__ = lambda s: s
+ pending_resp.__exit__ = MagicMock(return_value=False)
+
+ success_resp = MagicMock()
+ success_resp.read.return_value = json.dumps({"access_token": "gho_abc123"}).encode()
+ success_resp.__enter__ = lambda s: s
+ success_resp.__exit__ = MagicMock(return_value=False)
+
+ mock_urlopen.side_effect = [pending_resp, success_resp]
+
+ result = setup_wizard.poll_device_auth("dc-123", interval=1, timeout=300)
+ assert result == "gho_abc123"
+
+ @patch("lib.setup_wizard.time")
+ @patch("lib.setup_wizard.urlopen")
+ def test_timeout_returns_none(self, mock_urlopen, mock_time):
+ """Returns None when timeout is exceeded."""
+ # Simulate time passing beyond deadline
+ mock_time.time = MagicMock(side_effect=[0, 301])
+ mock_time.sleep = MagicMock()
+
+ result = setup_wizard.poll_device_auth("dc-123", interval=5, timeout=300)
+ assert result is None
+
+ @patch("lib.setup_wizard.time")
+ @patch("lib.setup_wizard.urlopen")
+ def test_expired_token_returns_none(self, mock_urlopen, mock_time):
+ """Returns None on expired_token error."""
+ mock_time.time = MagicMock(side_effect=[0, 0])
+ mock_time.sleep = MagicMock()
+
+ expired_resp = MagicMock()
+ expired_resp.read.return_value = json.dumps({"error": "expired_token"}).encode()
+ expired_resp.__enter__ = lambda s: s
+ expired_resp.__exit__ = MagicMock(return_value=False)
+
+ mock_urlopen.return_value = expired_resp
+
+ result = setup_wizard.poll_device_auth("dc-123", interval=1, timeout=300)
+ assert result is None
+
+ @patch("lib.setup_wizard.time")
+ @patch("lib.setup_wizard.urlopen")
+ def test_http_400_continues_polling(self, mock_urlopen, mock_time):
+ """HTTP 400 during polling continues (authorization pending)."""
+ from urllib.error import HTTPError
+
+ mock_time.time = MagicMock(side_effect=[0, 0, 0])
+ mock_time.sleep = MagicMock()
+
+ success_resp = MagicMock()
+ success_resp.read.return_value = json.dumps({"access_token": "gho_ok"}).encode()
+ success_resp.__enter__ = lambda s: s
+ success_resp.__exit__ = MagicMock(return_value=False)
+
+ mock_urlopen.side_effect = [
+ HTTPError("url", 400, "Bad Request", {}, None),
+ success_resp,
+ ]
+
+ result = setup_wizard.poll_device_auth("dc-123", interval=1, timeout=300)
+ assert result == "gho_ok"
+
+
+class TestFetchApiKey:
+ """Tests for fetch_api_key()."""
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_success(self, mock_urlopen):
+ """Returns api_key from profile response."""
+ resp_data = {"api_key": "sc-key-abc123", "username": "testuser"}
+ mock_resp = MagicMock()
+ mock_resp.read.return_value = json.dumps(resp_data).encode()
+ mock_resp.__enter__ = lambda s: s
+ mock_resp.__exit__ = MagicMock(return_value=False)
+ mock_urlopen.return_value = mock_resp
+
+ result = setup_wizard.fetch_api_key("gho_token")
+ assert result == "sc-key-abc123"
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_no_api_key_in_response(self, mock_urlopen):
+ """Returns None when api_key is not in the response."""
+ resp_data = {"username": "testuser"}
+ mock_resp = MagicMock()
+ mock_resp.read.return_value = json.dumps(resp_data).encode()
+ mock_resp.__enter__ = lambda s: s
+ mock_resp.__exit__ = MagicMock(return_value=False)
+ mock_urlopen.return_value = mock_resp
+
+ result = setup_wizard.fetch_api_key("gho_token")
+ assert result is None
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_http_error_returns_none(self, mock_urlopen):
+ """HTTP error returns None."""
+ from urllib.error import HTTPError
+ mock_urlopen.side_effect = HTTPError(
+ "https://example.com", 401, "Unauthorized", {}, None
+ )
+
+ result = setup_wizard.fetch_api_key("bad_token")
+ assert result is None
+
+
+class TestRunFullDeviceAuth:
+ """Tests for run_full_device_auth()."""
+
+ @patch("lib.setup_wizard.fetch_api_key")
+ @patch("lib.setup_wizard.poll_device_auth")
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("webbrowser.open")
+ def test_happy_path(self, mock_browser, mock_start, mock_poll, mock_fetch):
+ """Full flow succeeds: start -> poll -> fetch -> return api_key."""
+ mock_start.return_value = ("dev123", "ABCD-1234", "https://example.com/device", 5)
+ mock_poll.return_value = "access_tok"
+ mock_fetch.return_value = "sc_live_abc123"
+
+ result = setup_wizard.run_full_device_auth(timeout=10)
+
+ assert result["status"] == "success"
+ assert result["api_key"] == "sc_live_abc123"
+ assert result["user_code"] == "ABCD-1234"
+ mock_browser.assert_called_once_with("https://example.com/device")
+
+ @patch("lib.setup_wizard.run_device_auth")
+ def test_start_fails(self, mock_start):
+ """Device code request fails -> error status."""
+ mock_start.return_value = None
+
+ result = setup_wizard.run_full_device_auth()
+
+ assert result["status"] == "error"
+ assert "Failed to start" in result["message"]
+
+ @patch("lib.setup_wizard.poll_device_auth")
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("webbrowser.open")
+ def test_poll_timeout(self, mock_browser, mock_start, mock_poll):
+ """Poll times out -> timeout status with user_code."""
+ mock_start.return_value = ("dev123", "WXYZ-5678", "https://example.com/device", 5)
+ mock_poll.return_value = None
+
+ result = setup_wizard.run_full_device_auth(timeout=10)
+
+ assert result["status"] == "timeout"
+ assert result["user_code"] == "WXYZ-5678"
+
+ @patch("lib.setup_wizard.fetch_api_key")
+ @patch("lib.setup_wizard.poll_device_auth")
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("webbrowser.open")
+ def test_fetch_fails_after_auth(self, mock_browser, mock_start, mock_poll, mock_fetch):
+ """Auth succeeds but profile fetch fails -> error status."""
+ mock_start.return_value = ("dev123", "CODE-1111", "https://example.com/device", 5)
+ mock_poll.return_value = "access_tok"
+ mock_fetch.return_value = None
+
+ result = setup_wizard.run_full_device_auth(timeout=10)
+
+ assert result["status"] == "error"
+ assert "failed to fetch" in result["message"].lower()
+
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("webbrowser.open")
+ def test_browser_open_fails_gracefully(self, mock_browser, mock_start):
+ """webbrowser.open raises -> flow continues without crashing."""
+ mock_start.return_value = ("dev123", "CODE-2222", "https://example.com/device", 5)
+ mock_browser.side_effect = Exception("no display")
+
+ with patch("lib.setup_wizard.poll_device_auth", return_value=None):
+ result = setup_wizard.run_full_device_auth(timeout=1)
+
+ # Should not crash, just timeout
+ assert result["status"] == "timeout"
+
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("webbrowser.open")
+ def test_no_verification_uri_skips_browser(self, mock_browser, mock_start):
+ """Empty verification_uri -> browser not opened."""
+ mock_start.return_value = ("dev123", "CODE-3333", "", 5)
+
+ with patch("lib.setup_wizard.poll_device_auth", return_value=None):
+ setup_wizard.run_full_device_auth(timeout=1)
+
+ mock_browser.assert_not_called()
+
+
+class TestAuthWithPat:
+ """Tests for auth_with_pat()."""
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_success(self, mock_urlopen):
+ """Valid PAT -> returns dict with api_key."""
+ resp = MagicMock()
+ resp.read.return_value = json.dumps({
+ "api_key": "sc_live_test123",
+ "github_username": "testuser",
+ "credits_remaining": 100,
+ }).encode()
+ resp.__enter__ = lambda s: s
+ resp.__exit__ = MagicMock(return_value=False)
+ mock_urlopen.return_value = resp
+
+ result = setup_wizard.auth_with_pat("gho_validtoken")
+
+ assert result is not None
+ assert result["api_key"] == "sc_live_test123"
+ assert result["github_username"] == "testuser"
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_invalid_token_returns_none(self, mock_urlopen):
+ """HTTP 401 (invalid token) -> returns None."""
+ from urllib.error import HTTPError
+ mock_urlopen.side_effect = HTTPError(
+ url="https://api.scrapecreators.com/v1/github/pat/auth",
+ code=401, msg="Unauthorized", hdrs={}, fp=None,
+ )
+
+ result = setup_wizard.auth_with_pat("gho_badtoken")
+ assert result is None
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_insufficient_scope_returns_none(self, mock_urlopen):
+ """HTTP 422 (insufficient scope) -> returns None."""
+ from urllib.error import HTTPError
+ mock_urlopen.side_effect = HTTPError(
+ url="https://api.scrapecreators.com/v1/github/pat/auth",
+ code=422, msg="Unprocessable", hdrs={}, fp=None,
+ )
+
+ result = setup_wizard.auth_with_pat("gho_noscope")
+ assert result is None
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_network_error_returns_none(self, mock_urlopen):
+ """URLError -> returns None."""
+ from urllib.error import URLError
+ mock_urlopen.side_effect = URLError("Connection refused")
+
+ result = setup_wizard.auth_with_pat("gho_anytoken")
+ assert result is None
+
+ @patch("lib.setup_wizard.urlopen")
+ def test_no_api_key_in_response(self, mock_urlopen):
+ """Response without api_key -> returns None."""
+ resp = MagicMock()
+ resp.read.return_value = json.dumps({"error": "something"}).encode()
+ resp.__enter__ = lambda s: s
+ resp.__exit__ = MagicMock(return_value=False)
+ mock_urlopen.return_value = resp
+
+ result = setup_wizard.auth_with_pat("gho_validtoken")
+ assert result is None
+
+
+class TestClipboardDeviceAuth:
+ """Tests for clipboard-first behavior in run_full_device_auth()."""
+
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("lib.setup_wizard.poll_device_auth", return_value=None)
+ @patch("webbrowser.open")
+ @patch("subprocess.run")
+ def test_pbcopy_called_on_macos(self, mock_subproc, mock_browser, mock_poll, mock_start):
+ """On macOS, pbcopy is called with the user code before browser opens."""
+ mock_start.return_value = ("dev123", "CLIP-CODE", "https://github.com/login/device", 5)
+
+ with patch("sys.platform", "darwin"):
+ setup_wizard.run_full_device_auth(timeout=1)
+
+ mock_subproc.assert_called_once()
+ call_args = mock_subproc.call_args
+ assert call_args[0][0] == ["pbcopy"]
+ assert call_args[1]["input"] == b"CLIP-CODE"
+
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("lib.setup_wizard.poll_device_auth", return_value=None)
+ @patch("webbrowser.open")
+ @patch("subprocess.run")
+ def test_no_pbcopy_on_linux(self, mock_subproc, mock_browser, mock_poll, mock_start):
+ """On Linux, subprocess.run (pbcopy) is not called."""
+ mock_start.return_value = ("dev123", "CLIP-CODE", "https://github.com/login/device", 5)
+
+ with patch("sys.platform", "linux"):
+ setup_wizard.run_full_device_auth(timeout=1)
+
+ mock_subproc.assert_not_called()
+
+ @patch("lib.setup_wizard.run_device_auth")
+ @patch("lib.setup_wizard.poll_device_auth", return_value=None)
+ @patch("webbrowser.open")
+ @patch("subprocess.run", side_effect=Exception("pbcopy not found"))
+ def test_pbcopy_failure_continues(self, mock_subproc, mock_browser, mock_poll, mock_start):
+ """pbcopy failing -> flow continues, browser still opens."""
+ mock_start.return_value = ("dev123", "CLIP-CODE", "https://github.com/login/device", 5)
+
+ with patch("sys.platform", "darwin"):
+ result = setup_wizard.run_full_device_auth(timeout=1)
+
+ # Should not crash, browser still called
+ mock_browser.assert_called_once()
+ assert result["status"] == "timeout"
+
+
+class TestRunGithubAuth:
+ """Tests for run_github_auth() — PAT-first with device fallback."""
+
+ @patch("lib.setup_wizard.auth_with_pat")
+ @patch("subprocess.run")
+ @patch("shutil.which", return_value="/usr/local/bin/gh")
+ def test_pat_success(self, mock_which, mock_subproc, mock_pat):
+ """gh found + valid token + PAT endpoint success -> pat method."""
+ mock_subproc.return_value = MagicMock(
+ returncode=0, stdout="gho_testtoken123\n",
+ )
+ mock_pat.return_value = {
+ "api_key": "sc_live_fromPAT",
+ "github_username": "testuser",
+ }
+
+ result = setup_wizard.run_github_auth(timeout=10)
+
+ assert result["status"] == "success"
+ assert result["method"] == "pat"
+ assert result["api_key"] == "sc_live_fromPAT"
+
+ @patch("lib.setup_wizard.run_full_device_auth")
+ @patch("lib.setup_wizard.auth_with_pat", return_value=None)
+ @patch("subprocess.run")
+ @patch("shutil.which", return_value="/usr/local/bin/gh")
+ def test_pat_fails_falls_to_device(self, mock_which, mock_subproc, mock_pat, mock_device):
+ """gh found + PAT endpoint fails -> falls through to device flow."""
+ mock_subproc.return_value = MagicMock(
+ returncode=0, stdout="gho_badtoken\n",
+ )
+ mock_device.return_value = {
+ "status": "success", "method": "device",
+ "api_key": "sc_live_fromDevice",
+ }
+
+ result = setup_wizard.run_github_auth(timeout=10)
+
+ assert result["status"] == "success"
+ assert result["method"] == "device"
+ mock_device.assert_called_once()
+
+ @patch("lib.setup_wizard.run_full_device_auth")
+ @patch("shutil.which", return_value=None)
+ def test_no_gh_goes_to_device(self, mock_which, mock_device):
+ """gh not installed -> straight to device flow."""
+ mock_device.return_value = {
+ "status": "success", "method": "device",
+ "api_key": "sc_live_deviceOnly",
+ }
+
+ result = setup_wizard.run_github_auth(timeout=10)
+
+ assert result["status"] == "success"
+ assert result["method"] == "device"
+
+ @patch("lib.setup_wizard.run_full_device_auth")
+ @patch("subprocess.run")
+ @patch("shutil.which", return_value="/usr/local/bin/gh")
+ def test_gh_not_logged_in_falls_to_device(self, mock_which, mock_subproc, mock_device):
+ """gh exists but not logged in (exit code 1) -> device flow."""
+ mock_subproc.return_value = MagicMock(
+ returncode=1, stdout="", stderr="not logged in",
+ )
+ mock_device.return_value = {
+ "status": "success", "method": "device",
+ "api_key": "sc_live_fallback",
+ }
+
+ result = setup_wizard.run_github_auth(timeout=10)
+
+ assert result["status"] == "success"
+ assert result["method"] == "device"
diff --git a/tests/test_signals_v3.py b/tests/test_signals_v3.py
new file mode 100644
index 0000000..6029200
--- /dev/null
+++ b/tests/test_signals_v3.py
@@ -0,0 +1,621 @@
+import math
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import schema, signals
+from lib.hackernews import parse_hackernews_response
+
+
+class SignalsV3Tests(unittest.TestCase):
+ def test_reddit_engagement_uses_source_specific_formula(self):
+ item = schema.SourceItem(
+ item_id="r1",
+ source="reddit",
+ title="Title",
+ body="Body",
+ url="https://example.com",
+ engagement={"score": 99, "num_comments": 20, "upvote_ratio": 0.8},
+ metadata={"top_comments": [{"score": 10}]},
+ )
+ expected = (
+ 0.50 * math.log1p(99)
+ + 0.35 * math.log1p(20)
+ + 0.05 * (0.8 * 10.0)
+ + 0.10 * math.log1p(10)
+ )
+ self.assertAlmostEqual(expected, signals.engagement_raw(item))
+
+ def test_polymarket_engagement_uses_market_fields(self):
+ item = schema.SourceItem(
+ item_id="pm1",
+ source="polymarket",
+ title="Title",
+ body="Body",
+ url="https://example.com",
+ engagement={"volume": 1000, "liquidity": 250},
+ )
+ expected = (0.60 * math.log1p(1000)) + (0.40 * math.log1p(250))
+ self.assertAlmostEqual(expected, signals.engagement_raw(item))
+
+ def test_grounding_uses_generic_fallback(self):
+ item = schema.SourceItem(
+ item_id="g1",
+ source="grounding",
+ title="Title",
+ body="Body",
+ url="https://example.com",
+ engagement={"shares": 10, "reads": 100},
+ )
+ expected = (math.log1p(10) + math.log1p(100)) / 2
+ self.assertAlmostEqual(expected, signals.engagement_raw(item))
+
+ def test_annotate_stream_sorts_by_source_specific_reddit_engagement(self):
+ higher = schema.SourceItem(
+ item_id="r-high",
+ source="reddit",
+ title="High signal",
+ body="claude code skill",
+ url="https://example.com/high",
+ published_at="2026-03-15",
+ engagement={"score": 120, "num_comments": 40, "upvote_ratio": 0.9},
+ metadata={"top_comments": [{"score": 15}]},
+ )
+ lower = schema.SourceItem(
+ item_id="r-low",
+ source="reddit",
+ title="Lower signal",
+ body="claude code skill",
+ url="https://example.com/low",
+ published_at="2026-03-15",
+ engagement={"score": 4, "num_comments": 1, "upvote_ratio": 0.5},
+ metadata={"top_comments": [{"score": 1}]},
+ )
+ ranked = signals.annotate_stream(
+ [lower, higher],
+ ranking_query="What recent evidence matters for claude code skill?",
+ freshness_mode="balanced_recent",
+ )
+ self.assertEqual(["r-high", "r-low"], [item.item_id for item in ranked])
+
+ def test_local_relevance_dominates_over_high_engagement_noise(self):
+ relevant = schema.SourceItem(
+ item_id="relevant",
+ source="reddit",
+ title="Deploy to Fly.io with MCP in 60 seconds",
+ body="Deploy to Fly.io guide with concrete steps.",
+ url="https://example.com/relevant",
+ published_at="2026-03-15",
+ engagement={"score": 2, "num_comments": 0, "upvote_ratio": 0.8},
+ metadata={"top_comments": []},
+ )
+ noisy = schema.SourceItem(
+ item_id="noisy",
+ source="reddit",
+ title="BATTLEFIELD 6 GAME UPDATE 1.2.2.0",
+ body="Patch notes and gameplay discussion.",
+ url="https://example.com/noisy",
+ published_at="2026-03-15",
+ engagement={"score": 5000, "num_comments": 1200, "upvote_ratio": 0.95},
+ metadata={"top_comments": [{"score": 400}]},
+ )
+ ranked = signals.annotate_stream(
+ [noisy, relevant],
+ ranking_query="How do I deploy on Fly.io?",
+ freshness_mode="evergreen_ok",
+ )
+ self.assertEqual("relevant", ranked[0].item_id)
+
+ def test_prune_low_relevance_keeps_stronger_matches(self):
+ strong = schema.SourceItem(
+ item_id="strong",
+ source="reddit",
+ title="Deploy to Fly.io",
+ body="Step-by-step Fly.io deploy guide.",
+ url="https://example.com/strong",
+ local_relevance=0.3,
+ )
+ weak = schema.SourceItem(
+ item_id="weak",
+ source="reddit",
+ title="Battlefield update",
+ body="Patch notes.",
+ url="https://example.com/weak",
+ local_relevance=0.0,
+ )
+ pruned = signals.prune_low_relevance([strong, weak], minimum=0.1)
+ self.assertEqual(["strong"], [item.item_id for item in pruned])
+
+ def test_prune_low_relevance_falls_back_when_all_are_weak(self):
+ weak = schema.SourceItem(
+ item_id="weak",
+ source="reddit",
+ title="Generic post",
+ body="Generic body.",
+ url="https://example.com/weak",
+ metadata={"local_relevance": 0.02},
+ )
+ pruned = signals.prune_low_relevance([weak], minimum=0.1)
+ self.assertEqual(["weak"], [item.item_id for item in pruned])
+
+
+ # -- Iteration 1: HN engagement bug --
+
+ def test_hackernews_parse_emits_comments_key(self):
+ """parse_hackernews_response must emit 'comments' (not 'num_comments')."""
+ response = {
+ "hits": [
+ {
+ "objectID": "123",
+ "title": "Show HN: Something Cool",
+ "url": "https://example.com",
+ "author": "pg",
+ "points": 150,
+ "num_comments": 45,
+ "created_at_i": 1710720000,
+ },
+ ],
+ }
+ items = parse_hackernews_response(response, query="something cool")
+ self.assertIn("comments", items[0]["engagement"])
+ self.assertNotIn("num_comments", items[0]["engagement"])
+ self.assertEqual(items[0]["engagement"]["comments"], 45)
+
+ def test_hackernews_engagement_raw_uses_both_fields(self):
+ """engagement_raw for HN must weight both points and comments."""
+ item = schema.SourceItem(
+ item_id="hn1",
+ source="hackernews",
+ title="Show HN: Something",
+ body="Description",
+ url="https://example.com",
+ engagement={"points": 150, "comments": 45},
+ )
+ expected = 0.55 * math.log1p(150) + 0.45 * math.log1p(45)
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ self.assertAlmostEqual(expected, result)
+ # Verify comments actually contributed (not just points)
+ points_only = 0.55 * math.log1p(150)
+ self.assertGreater(result, points_only)
+
+ # -- Iteration 4: Missing engagement formula tests --
+
+ def test_x_engagement_dominant_weight(self):
+ """X: likes at 0.55 should dominate over quotes at 0.05."""
+ item = schema.SourceItem(
+ item_id="x1", source="x", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 100, "reposts": 100, "replies": 100, "quotes": 100},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = (
+ 0.55 * math.log1p(100)
+ + 0.25 * math.log1p(100)
+ + 0.15 * math.log1p(100)
+ + 0.05 * math.log1p(100)
+ )
+ self.assertAlmostEqual(expected, result)
+
+ def test_x_engagement_all_zero_returns_none(self):
+ item = schema.SourceItem(
+ item_id="x2", source="x", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 0, "reposts": 0, "replies": 0, "quotes": 0},
+ )
+ self.assertIsNone(signals.engagement_raw(item))
+
+ def test_x_engagement_missing_fields(self):
+ """Missing fields default to 0, no crash."""
+ item = schema.SourceItem(
+ item_id="x3", source="x", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 50},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = 0.55 * math.log1p(50)
+ self.assertAlmostEqual(expected, result)
+
+ def test_youtube_engagement_dominant_weight(self):
+ """YouTube: views at 0.50 should dominate over comments at 0.15."""
+ item = schema.SourceItem(
+ item_id="yt1", source="youtube", title="T", body="B",
+ url="https://example.com",
+ engagement={"views": 10000, "likes": 500, "comments": 80},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = (
+ 0.50 * math.log1p(10000)
+ + 0.35 * math.log1p(500)
+ + 0.15 * math.log1p(80)
+ )
+ self.assertAlmostEqual(expected, result)
+
+ def test_youtube_engagement_all_zero_returns_none(self):
+ item = schema.SourceItem(
+ item_id="yt2", source="youtube", title="T", body="B",
+ url="https://example.com",
+ engagement={"views": 0, "likes": 0, "comments": 0},
+ )
+ self.assertIsNone(signals.engagement_raw(item))
+
+ def test_youtube_engagement_missing_fields(self):
+ item = schema.SourceItem(
+ item_id="yt3", source="youtube", title="T", body="B",
+ url="https://example.com",
+ engagement={"views": 5000},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = 0.50 * math.log1p(5000)
+ self.assertAlmostEqual(expected, result)
+
+ def test_tiktok_engagement_dominant_weight(self):
+ item = schema.SourceItem(
+ item_id="tt1", source="tiktok", title="T", body="B",
+ url="https://example.com",
+ engagement={"views": 50000, "likes": 3000, "comments": 200},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = (
+ 0.50 * math.log1p(50000)
+ + 0.30 * math.log1p(3000)
+ + 0.20 * math.log1p(200)
+ )
+ self.assertAlmostEqual(expected, result)
+
+ def test_tiktok_engagement_all_zero_returns_none(self):
+ item = schema.SourceItem(
+ item_id="tt2", source="tiktok", title="T", body="B",
+ url="https://example.com",
+ engagement={"views": 0, "likes": 0, "comments": 0},
+ )
+ self.assertIsNone(signals.engagement_raw(item))
+
+ def test_tiktok_engagement_missing_fields(self):
+ item = schema.SourceItem(
+ item_id="tt3", source="tiktok", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 1000},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = 0.30 * math.log1p(1000)
+ self.assertAlmostEqual(expected, result)
+
+ def test_instagram_engagement_dominant_weight(self):
+ item = schema.SourceItem(
+ item_id="ig1", source="instagram", title="T", body="B",
+ url="https://example.com",
+ engagement={"views": 8000, "likes": 1500, "comments": 100},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = (
+ 0.50 * math.log1p(8000)
+ + 0.30 * math.log1p(1500)
+ + 0.20 * math.log1p(100)
+ )
+ self.assertAlmostEqual(expected, result)
+
+ def test_instagram_engagement_all_zero_returns_none(self):
+ item = schema.SourceItem(
+ item_id="ig2", source="instagram", title="T", body="B",
+ url="https://example.com",
+ engagement={"views": 0, "likes": 0, "comments": 0},
+ )
+ self.assertIsNone(signals.engagement_raw(item))
+
+ def test_instagram_engagement_missing_fields(self):
+ item = schema.SourceItem(
+ item_id="ig3", source="instagram", title="T", body="B",
+ url="https://example.com",
+ engagement={"comments": 50},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = 0.20 * math.log1p(50)
+ self.assertAlmostEqual(expected, result)
+
+ def test_hackernews_engagement_all_zero_returns_none(self):
+ item = schema.SourceItem(
+ item_id="hn2", source="hackernews", title="T", body="B",
+ url="https://example.com",
+ engagement={"points": 0, "comments": 0},
+ )
+ self.assertIsNone(signals.engagement_raw(item))
+
+ def test_hackernews_engagement_missing_fields(self):
+ item = schema.SourceItem(
+ item_id="hn3", source="hackernews", title="T", body="B",
+ url="https://example.com",
+ engagement={"points": 75},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = 0.55 * math.log1p(75)
+ self.assertAlmostEqual(expected, result)
+
+ def test_bluesky_engagement_dominant_weight(self):
+ """Bluesky: likes at 0.40 should dominate over quotes at 0.10."""
+ item = schema.SourceItem(
+ item_id="bs1", source="bluesky", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 200, "reposts": 50, "replies": 30, "quotes": 10},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = (
+ 0.40 * math.log1p(200)
+ + 0.30 * math.log1p(50)
+ + 0.20 * math.log1p(30)
+ + 0.10 * math.log1p(10)
+ )
+ self.assertAlmostEqual(expected, result)
+
+ def test_bluesky_engagement_all_zero_returns_none(self):
+ item = schema.SourceItem(
+ item_id="bs2", source="bluesky", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 0, "reposts": 0, "replies": 0, "quotes": 0},
+ )
+ self.assertIsNone(signals.engagement_raw(item))
+
+ def test_bluesky_engagement_missing_fields(self):
+ item = schema.SourceItem(
+ item_id="bs3", source="bluesky", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 100, "replies": 20},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = 0.40 * math.log1p(100) + 0.20 * math.log1p(20)
+ self.assertAlmostEqual(expected, result)
+
+ def test_truthsocial_engagement_dominant_weight(self):
+ """Truth Social: likes at 0.45 should dominate over replies at 0.25."""
+ item = schema.SourceItem(
+ item_id="ts1", source="truthsocial", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 500, "reposts": 100, "replies": 50},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = (
+ 0.45 * math.log1p(500)
+ + 0.30 * math.log1p(100)
+ + 0.25 * math.log1p(50)
+ )
+ self.assertAlmostEqual(expected, result)
+
+ def test_truthsocial_engagement_all_zero_returns_none(self):
+ item = schema.SourceItem(
+ item_id="ts2", source="truthsocial", title="T", body="B",
+ url="https://example.com",
+ engagement={"likes": 0, "reposts": 0, "replies": 0},
+ )
+ self.assertIsNone(signals.engagement_raw(item))
+
+ def test_truthsocial_engagement_missing_fields(self):
+ item = schema.SourceItem(
+ item_id="ts3", source="truthsocial", title="T", body="B",
+ url="https://example.com",
+ engagement={"reposts": 80},
+ )
+ result = signals.engagement_raw(item)
+ self.assertIsNotNone(result)
+ expected = 0.30 * math.log1p(80)
+ self.assertAlmostEqual(expected, result)
+
+ # -- Fix 5: Rebalance engagement weight --
+
+ def test_engagement_weight_meaningful_for_social_ranking(self):
+ """Engagement must have enough weight to differentiate otherwise-equal items."""
+ high_engagement = schema.SourceItem(
+ item_id="viral",
+ source="x",
+ title="Trending topic discussion",
+ body="Popular social post",
+ url="https://example.com/viral",
+ published_at="2026-03-15",
+ engagement={"likes": 50000, "reposts": 5000, "replies": 2000, "quotes": 500},
+ )
+ low_engagement = schema.SourceItem(
+ item_id="quiet",
+ source="x",
+ title="Trending topic discussion",
+ body="Popular social post",
+ url="https://example.com/quiet",
+ published_at="2026-03-15",
+ engagement={"likes": 10, "reposts": 1, "replies": 0, "quotes": 0},
+ )
+ ranked = signals.annotate_stream(
+ [low_engagement, high_engagement],
+ ranking_query="trending topic discussion",
+ freshness_mode="balanced_recent",
+ )
+ high_score = ranked[0].local_rank_score
+ low_score = ranked[1].local_rank_score
+ gap = high_score - low_score
+ # With 10% engagement weight, the gap should be >= 0.06
+ # With 5% weight, gap would be ~0.04
+ self.assertGreaterEqual(gap, 0.06,
+ f"Engagement gap should be >= 0.06 with 10% weight, got {gap:.4f}")
+
+ # -- Fix 4: Lower prune threshold for social media --
+
+ def test_prune_keeps_social_items_above_003(self):
+ """Social media items with low but non-trivial relevance should survive pruning."""
+ social = schema.SourceItem(
+ item_id="social",
+ source="x",
+ title="Viral tweet about topic",
+ body="Short social post",
+ url="https://example.com/social",
+ metadata={"local_relevance": 0.05},
+ )
+ strong = schema.SourceItem(
+ item_id="strong",
+ source="grounding",
+ title="Detailed article about topic",
+ body="In-depth analysis",
+ url="https://example.com/strong",
+ metadata={"local_relevance": 0.4},
+ )
+ pruned = signals.prune_low_relevance([strong, social])
+ ids = [item.item_id for item in pruned]
+ self.assertIn("social", ids, "Item with relevance 0.05 should survive pruning")
+ self.assertIn("strong", ids)
+
+
+ # -- Unit 3: YouTube high-engagement relevance floor --
+
+ def test_youtube_high_engagement_gets_relevance_floor(self):
+ """YouTube video with >100K views gets at least 0.3 relevance even with low text overlap."""
+ item = schema.SourceItem(
+ item_id="yt-official",
+ source="youtube",
+ title="YE - FATHER (feat. TRAVIS SCOTT)",
+ body="Official music video",
+ url="https://youtube.com/watch?v=abc",
+ engagement={"views": 8_000_000, "likes": 422_000, "comments": 5000},
+ )
+ rel = signals.local_relevance(item, "kanye west")
+ self.assertGreaterEqual(rel, 0.3, f"High-engagement YouTube should get >= 0.3 relevance, got {rel}")
+
+ def test_youtube_low_engagement_no_floor(self):
+ """YouTube video with <100K views does NOT get the relevance floor."""
+ item = schema.SourceItem(
+ item_id="yt-small",
+ source="youtube",
+ title="Random unrelated video title",
+ body="Nothing relevant here",
+ url="https://youtube.com/watch?v=xyz",
+ engagement={"views": 500, "likes": 10, "comments": 1},
+ )
+ rel = signals.local_relevance(item, "kanye west")
+ self.assertLess(rel, 0.3, f"Low-engagement YouTube should not get floor, got {rel}")
+
+ def test_non_youtube_high_engagement_no_floor(self):
+ """Non-YouTube items with high engagement do NOT get the YouTube floor."""
+ item = schema.SourceItem(
+ item_id="reddit-viral",
+ source="reddit",
+ title="Completely unrelated post",
+ body="Nothing about the topic",
+ url="https://reddit.com/r/test",
+ engagement={"score": 50000, "num_comments": 3000},
+ )
+ rel = signals.local_relevance(item, "kanye west")
+ self.assertLess(rel, 0.3, f"Non-YouTube item should not get YouTube floor, got {rel}")
+
+
+ # -- Unit 8: Engagement floor for TikTok/Instagram --
+
+ def test_tiktok_below_1000_views_pruned(self):
+ """TikTok items with <1000 views should be pruned when other sources exist."""
+ spam = schema.SourceItem(
+ item_id="tt-spam", source="tiktok", title="AI news clip", body="Generic",
+ url="https://tiktok.com/spam",
+ local_relevance=0.4, engagement={"views": 500, "likes": 10, "comments": 1},
+ )
+ good = schema.SourceItem(
+ item_id="r-good", source="reddit", title="Good discussion", body="Quality",
+ url="https://reddit.com/good",
+ local_relevance=0.5, engagement_score=50,
+ )
+ pruned = signals.prune_low_relevance([good, spam])
+ ids = [item.item_id for item in pruned]
+ self.assertNotIn("tt-spam", ids, "TikTok with 500 views should be pruned")
+ self.assertIn("r-good", ids)
+
+ def test_instagram_below_1000_views_pruned(self):
+ """Instagram items with <1000 views should be pruned when other sources exist."""
+ spam = schema.SourceItem(
+ item_id="ig-spam", source="instagram", title="Repost clip", body="Generic",
+ url="https://instagram.com/spam",
+ local_relevance=0.4, engagement={"views": 200, "likes": 5, "comments": 0},
+ )
+ good = schema.SourceItem(
+ item_id="x-good", source="x", title="Good tweet", body="Quality",
+ url="https://x.com/good",
+ local_relevance=0.5, engagement_score=50,
+ )
+ pruned = signals.prune_low_relevance([good, spam])
+ ids = [item.item_id for item in pruned]
+ self.assertNotIn("ig-spam", ids, "Instagram with 200 views should be pruned")
+
+ def test_tiktok_above_1000_views_kept(self):
+ """TikTok items with >=1000 views should survive pruning."""
+ good_tt = schema.SourceItem(
+ item_id="tt-good", source="tiktok", title="Popular clip", body="Relevant",
+ url="https://tiktok.com/good",
+ local_relevance=0.4, engagement={"views": 5000, "likes": 200, "comments": 30},
+ )
+ other = schema.SourceItem(
+ item_id="r-other", source="reddit", title="Reddit post", body="Relevant",
+ url="https://reddit.com/other",
+ local_relevance=0.5, engagement_score=50,
+ )
+ pruned = signals.prune_low_relevance([other, good_tt])
+ ids = [item.item_id for item in pruned]
+ self.assertIn("tt-good", ids, "TikTok with 5000 views should be kept")
+
+ def test_tiktok_sole_source_not_pruned(self):
+ """When TikTok is the only source, low-view items should NOT be pruned."""
+ items = [
+ schema.SourceItem(
+ item_id=f"tt-{i}", source="tiktok", title=f"Clip {i}", body="Content",
+ url=f"https://tiktok.com/{i}",
+ local_relevance=0.4, engagement={"views": 300, "likes": 5, "comments": 0},
+ )
+ for i in range(3)
+ ]
+ pruned = signals.prune_low_relevance(items)
+ self.assertEqual(len(pruned), 3, "Sole-source TikTok items should all survive")
+
+ def test_non_video_sources_unaffected_by_floor(self):
+ """Reddit/X items should not be affected by the video engagement floor."""
+ low_eng_x = schema.SourceItem(
+ item_id="x-low", source="x", title="Tweet", body="Topic discussion",
+ url="https://x.com/low",
+ local_relevance=0.5, engagement={"likes": 2, "reposts": 0},
+ engagement_score=5,
+ )
+ other = schema.SourceItem(
+ item_id="r-other", source="reddit", title="Post", body="Topic",
+ url="https://reddit.com/other",
+ local_relevance=0.5, engagement_score=50,
+ )
+ pruned = signals.prune_low_relevance([other, low_eng_x])
+ ids = [item.item_id for item in pruned]
+ self.assertIn("x-low", ids, "X items should not be affected by video floor")
+
+ def test_aspiresnippets_scenario(self):
+ """@aspiresnippets scenario: 5 TikTok items with 200-700 views all pruned."""
+ spam_items = [
+ schema.SourceItem(
+ item_id=f"aspire-{i}", source="tiktok", title=f"AI news {i}", body="Generic clip",
+ url=f"https://tiktok.com/aspire/{i}",
+ local_relevance=0.3, engagement={"views": 200 + i * 100, "likes": 5, "comments": 0},
+ )
+ for i in range(5)
+ ]
+ good = schema.SourceItem(
+ item_id="good-yt", source="youtube", title="In-depth analysis", body="Quality content",
+ url="https://youtube.com/good",
+ local_relevance=0.6, engagement_score=70,
+ )
+ pruned = signals.prune_low_relevance([good] + spam_items)
+ aspire_ids = [item.item_id for item in pruned if item.item_id.startswith("aspire")]
+ self.assertEqual(len(aspire_ids), 0, f"All @aspiresnippets items should be pruned, got {aspire_ids}")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_smoke.py b/tests/test_smoke.py
deleted file mode 100644
index 26c4c8d..0000000
--- a/tests/test_smoke.py
+++ /dev/null
@@ -1,99 +0,0 @@
-"""End-to-end smoke tests — run the actual script as subprocess."""
-
-import json
-import subprocess
-import sys
-import unittest
-from pathlib import Path
-
-SCRIPT = str(Path(__file__).parent.parent / "scripts" / "last30days.py")
-
-
-def _run(args, timeout=30):
- """Run last30days.py with args, return (returncode, stdout, stderr)."""
- result = subprocess.run(
- [sys.executable, SCRIPT] + args,
- capture_output=True, text=True, timeout=timeout,
- )
- return result.returncode, result.stdout, result.stderr
-
-
-class TestDiagnose(unittest.TestCase):
- """Tests for --diagnose flag."""
-
- def test_exits_zero(self):
- rc, stdout, stderr = _run(["--diagnose"])
- self.assertEqual(rc, 0, f"--diagnose failed: {stderr}")
-
- def test_returns_valid_json(self):
- rc, stdout, stderr = _run(["--diagnose"])
- data = json.loads(stdout)
- self.assertIsInstance(data, dict)
-
- def test_has_expected_keys(self):
- rc, stdout, stderr = _run(["--diagnose"])
- data = json.loads(stdout)
- for key in ("openai", "xai", "youtube", "tiktok", "instagram", "hackernews", "polymarket"):
- self.assertIn(key, data, f"Missing key: {key}")
-
- def test_boolean_values(self):
- rc, stdout, stderr = _run(["--diagnose"])
- data = json.loads(stdout)
- for key in ("openai", "xai", "youtube", "tiktok", "instagram", "hackernews", "polymarket"):
- self.assertIsInstance(data[key], bool, f"{key} should be boolean")
-
- def test_hackernews_always_true(self):
- data = json.loads(_run(["--diagnose"])[1])
- self.assertTrue(data["hackernews"])
-
- def test_polymarket_always_true(self):
- data = json.loads(_run(["--diagnose"])[1])
- self.assertTrue(data["polymarket"])
-
-
-class TestHelp(unittest.TestCase):
- """Tests for --help flag."""
-
- def test_exits_zero(self):
- rc, stdout, stderr = _run(["--help"])
- self.assertEqual(rc, 0)
-
- def test_shows_usage(self):
- rc, stdout, stderr = _run(["--help"])
- self.assertTrue(
- "topic" in stdout.lower() or "usage" in stdout.lower(),
- "Expected usage info in help output"
- )
-
-
-class TestNoTopic(unittest.TestCase):
- """Tests for missing topic."""
-
- def test_exits_nonzero_without_topic(self):
- rc, stdout, stderr = _run([])
- self.assertNotEqual(rc, 0)
-
- def test_error_message(self):
- rc, stdout, stderr = _run([])
- self.assertTrue(
- "topic" in stderr.lower() or "error" in stderr.lower(),
- "Expected error about missing topic"
- )
-
-
-class TestMockMode(unittest.TestCase):
- """Tests for --mock mode (fixture-based, no API calls)."""
-
- def test_mock_json_exits_zero(self):
- rc, stdout, stderr = _run(["--mock", "--emit", "json", "test topic"], timeout=120)
- self.assertEqual(rc, 0, f"--mock failed: {stderr}")
-
- def test_mock_json_valid(self):
- rc, stdout, stderr = _run(["--mock", "--emit", "json", "test topic"], timeout=120)
- data = json.loads(stdout)
- self.assertIn("topic", data)
- self.assertEqual(data["topic"], "test topic")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_snippet_v3.py b/tests/test_snippet_v3.py
new file mode 100644
index 0000000..9bb4439
--- /dev/null
+++ b/tests/test_snippet_v3.py
@@ -0,0 +1,63 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import schema, snippet
+
+
+def make_item(**overrides):
+ payload = {
+ "item_id": "i1",
+ "source": "grounding",
+ "title": "OpenClaw comparison guide",
+ "body": "",
+ "url": "https://example.com",
+ "snippet": "",
+ }
+ payload.update(overrides)
+ return schema.SourceItem(**payload)
+
+
+class SnippetV3Tests(unittest.TestCase):
+ def test_truncate_words_preserves_short_text_and_truncates_long_text(self):
+ self.assertEqual("short text", snippet._truncate_words("short text", 5))
+ self.assertEqual("one two three...", snippet._truncate_words("one two three four", 3))
+
+ def test_windows_handles_empty_short_and_overlapping_inputs(self):
+ self.assertEqual([], snippet._windows([], size=5, overlap=2))
+ self.assertEqual(["one two"], snippet._windows(["one", "two"], size=5, overlap=2))
+ self.assertEqual(
+ ["one two three", "two three four", "three four five", "four five", "five"],
+ snippet._windows(["one", "two", "three", "four", "five"], size=3, overlap=2),
+ )
+
+ def test_extract_best_snippet_prefers_existing_snippet(self):
+ item = make_item(snippet="existing evidence window " * 20)
+ result = snippet.extract_best_snippet(item, "ignored", max_words=5)
+ self.assertEqual("existing evidence window existing evidence...", result)
+
+ def test_extract_best_snippet_falls_back_to_title_when_body_missing(self):
+ item = make_item(title="OpenClaw vs NanoClaw", body="")
+ self.assertEqual("OpenClaw vs NanoClaw", snippet.extract_best_snippet(item, "openclaw"))
+
+ def test_extract_best_snippet_selects_best_matching_body_window(self):
+ body = " ".join(
+ [
+ "generic filler words" for _ in range(40)
+ ]
+ + [
+ "openclaw nanoclaw ironclaw comparison details" for _ in range(15)
+ ]
+ + [
+ "more generic filler words" for _ in range(40)
+ ]
+ )
+ item = make_item(body=body)
+ result = snippet.extract_best_snippet(item, "openclaw nanoclaw ironclaw", max_words=20)
+ self.assertIn("openclaw nanoclaw ironclaw", result)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_source_priority.py b/tests/test_source_priority.py
deleted file mode 100644
index 1f75e66..0000000
--- a/tests/test_source_priority.py
+++ /dev/null
@@ -1,737 +0,0 @@
-"""Tests for source resolution priority hierarchy (Unit 4).
-
-Validates the free-first priority chain:
- env AUTH_TOKEN/CT0 -> browser cookies -> XAI_API_KEY -> None
-"""
-
-import os
-from unittest.mock import patch, MagicMock
-
-import pytest
-
-from scripts.lib import env
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _base_config(**overrides):
- """Return a minimal config dict with typical defaults."""
- cfg = {
- "AUTH_TOKEN": None,
- "CT0": None,
- "XAI_API_KEY": None,
- "SCRAPECREATORS_API_KEY": None,
- "OPENAI_API_KEY": None,
- "OPENAI_AUTH_STATUS": "missing",
- "OPENROUTER_API_KEY": None,
- "PARALLEL_API_KEY": None,
- "BRAVE_API_KEY": None,
- "BSKY_HANDLE": None,
- "BSKY_APP_PASSWORD": None,
- "TRUTHSOCIAL_TOKEN": None,
- "FROM_BROWSER": None,
- "SETUP_COMPLETE": None,
- "_AUTH_TOKEN_SOURCE": None,
- }
- cfg.update(overrides)
- return cfg
-
-
-def _mock_bird_installed(installed=True):
- """Patch bird_x.is_bird_installed to return the given value."""
- return patch("scripts.lib.bird_x.is_bird_installed", return_value=installed)
-
-
-def _mock_bird_authenticated(username=None):
- """Patch bird_x.is_bird_authenticated to return the given value."""
- return patch("scripts.lib.bird_x.is_bird_authenticated", return_value=username)
-
-
-def _mock_bird_status(installed=True, authenticated=True, username="env AUTH_TOKEN"):
- """Patch bird_x.get_bird_status to return a status dict."""
- return patch("scripts.lib.bird_x.get_bird_status", return_value={
- "installed": installed,
- "authenticated": authenticated,
- "username": username,
- "can_install": True,
- })
-
-
-# ---------------------------------------------------------------------------
-# Tests: X source resolution priority
-# ---------------------------------------------------------------------------
-
-class TestXSourcePriority:
- """Test the X source priority chain: env -> browser cookies -> xAI -> None."""
-
- def test_no_env_cookies_found_resolves_bird_browser(self):
- """No env vars, SETUP_COMPLETE=true, cookies found -> Bird with browser method."""
- config = _base_config(
- AUTH_TOKEN="cookie_tok",
- CT0="cookie_ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-firefox",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "bird"
- assert method == "browser-firefox"
-
- def test_env_auth_token_plus_cookies_env_wins(self):
- """AUTH_TOKEN in .env + cookies available -> env wins (method='env')."""
- config = _base_config(
- AUTH_TOKEN="explicit_token",
- CT0="explicit_ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="env",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "bird"
- assert method == "env"
-
- def test_no_env_no_cookies_xai_key_resolves_xai(self):
- """No env vars, no cookies, XAI_API_KEY set -> xAI with method 'api'."""
- config = _base_config(XAI_API_KEY="xai-key-123")
-
- with _mock_bird_installed(True), _mock_bird_authenticated(None):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "xai"
- assert method == "api"
-
- def test_no_env_no_cookies_no_api_keys_none(self):
- """No env vars, no cookies, no API keys -> X not available."""
- config = _base_config()
-
- with _mock_bird_installed(True), _mock_bird_authenticated(None):
- source, method = env.get_x_source_with_method(config)
-
- assert source is None
- assert method is None
-
- def test_bird_not_installed_falls_to_xai(self):
- """Bird not installed, XAI_API_KEY set -> xAI."""
- config = _base_config(XAI_API_KEY="xai-key")
-
- with _mock_bird_installed(False), _mock_bird_authenticated(None):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "xai"
- assert method == "api"
-
- def test_bird_not_installed_no_xai_none(self):
- """Bird not installed, no XAI_API_KEY -> None."""
- config = _base_config()
-
- with _mock_bird_installed(False), _mock_bird_authenticated(None):
- source, method = env.get_x_source_with_method(config)
-
- assert source is None
- assert method is None
-
- def test_browser_chrome_method_tracked(self):
- """Cookies from Chrome -> method is 'browser-chrome'."""
- config = _base_config(
- AUTH_TOKEN="chrome_tok",
- CT0="chrome_ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-chrome",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "bird"
- assert method == "browser-chrome"
-
- def test_browser_safari_method_tracked(self):
- """Cookies from Safari -> method is 'browser-safari'."""
- config = _base_config(
- AUTH_TOKEN="safari_tok",
- CT0="safari_ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-safari",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "bird"
- assert method == "browser-safari"
-
-
-# ---------------------------------------------------------------------------
-# Tests: get_x_source() backward compat
-# ---------------------------------------------------------------------------
-
-class TestGetXSourceBackwardCompat:
- """Ensure get_x_source() returns the same string as before."""
-
- def test_bird_returns_bird(self):
- config = _base_config(AUTH_TOKEN="tok", CT0="ct0", _AUTH_TOKEN_SOURCE="env")
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- assert env.get_x_source(config) == "bird"
-
- def test_xai_returns_xai(self):
- config = _base_config(XAI_API_KEY="key")
- with _mock_bird_installed(True), _mock_bird_authenticated(None):
- assert env.get_x_source(config) == "xai"
-
- def test_none_returns_none(self):
- config = _base_config()
- with _mock_bird_installed(False):
- assert env.get_x_source(config) is None
-
-
-# ---------------------------------------------------------------------------
-# Tests: get_x_source_status() method field
-# ---------------------------------------------------------------------------
-
-class TestGetXSourceStatusMethod:
- """Test that get_x_source_status() includes the method field."""
-
- def test_bird_env_method(self):
- config = _base_config(AUTH_TOKEN="tok", CT0="ct0", _AUTH_TOKEN_SOURCE="env")
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"), \
- _mock_bird_status(installed=True, authenticated=True, username="env AUTH_TOKEN"):
- status = env.get_x_source_status(config)
-
- assert status["source"] == "bird"
- assert status["method"] == "env"
- assert "bird_installed" in status
- assert "xai_available" in status
-
- def test_bird_browser_method(self):
- config = _base_config(
- AUTH_TOKEN="tok", CT0="ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-firefox",
- )
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"), \
- _mock_bird_status(installed=True, authenticated=True):
- status = env.get_x_source_status(config)
-
- assert status["source"] == "bird"
- assert status["method"] == "browser-firefox"
-
- def test_xai_api_method(self):
- config = _base_config(XAI_API_KEY="key")
- with _mock_bird_installed(True), _mock_bird_authenticated(None), \
- _mock_bird_status(installed=True, authenticated=False, username=None):
- status = env.get_x_source_status(config)
-
- assert status["source"] == "xai"
- assert status["method"] == "api"
-
- def test_no_source_method_none(self):
- config = _base_config()
- with _mock_bird_installed(False), \
- _mock_bird_status(installed=False, authenticated=False, username=None):
- status = env.get_x_source_status(config)
-
- assert status["source"] is None
- assert status["method"] is None
-
-
-# ---------------------------------------------------------------------------
-# Tests: get_available_sources() with various configs
-# ---------------------------------------------------------------------------
-
-class TestGetAvailableSources:
- """Test get_available_sources() returns correct strings."""
-
- def test_no_x_no_web_reddit_only(self):
- """No X source, no web keys -> 'reddit' (Reddit always available)."""
- config = _base_config()
- with _mock_bird_installed(False):
- result = env.get_available_sources(config)
- assert result == "reddit"
-
- def test_xai_key_no_web(self):
- """XAI_API_KEY set, no web keys -> 'both'."""
- config = _base_config(XAI_API_KEY="key")
- with _mock_bird_installed(True), _mock_bird_authenticated(None):
- result = env.get_available_sources(config)
- assert result == "both"
-
- def test_bird_auth_no_web(self):
- """Bird authenticated (cookies), SETUP_COMPLETE=true, no web keys -> 'both'."""
- config = _base_config(
- AUTH_TOKEN="tok", CT0="ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-firefox",
- )
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- result = env.get_available_sources(config)
- assert result == "both"
-
- def test_bird_auth_with_web(self):
- """Bird authenticated + web keys -> 'all'."""
- config = _base_config(
- AUTH_TOKEN="tok", CT0="ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-firefox",
- BRAVE_API_KEY="brave-key",
- )
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- result = env.get_available_sources(config)
- assert result == "all"
-
- def test_no_x_with_web(self):
- """No X source, web keys -> 'reddit-web'."""
- config = _base_config(BRAVE_API_KEY="brave-key")
- with _mock_bird_installed(False):
- result = env.get_available_sources(config)
- assert result == "reddit-web"
-
- def test_reddit_hn_polymarket_always_available(self):
- """Reddit, HN, and Polymarket are always available regardless of config."""
- config = _base_config()
- # These functions don't depend on config
- assert env.is_hackernews_available() is True
- assert env.is_polymarket_available() is True
- # Reddit: get_available_sources always includes it
- with _mock_bird_installed(False):
- result = env.get_available_sources(config)
- assert result in ("reddit", "reddit-web") # never 'none' when Reddit is always True
-
-
-# ---------------------------------------------------------------------------
-# Tests: Full resolution with mixed config
-# ---------------------------------------------------------------------------
-
-class TestFullResolution:
- """Test that each source resolves independently with mixed config."""
-
- def test_mixed_config_all_sources(self):
- """Bird for X, public Reddit, web keys, YouTube/TikTok available."""
- config = _base_config(
- AUTH_TOKEN="tok", CT0="ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-chrome",
- BRAVE_API_KEY="brave-key",
- SCRAPECREATORS_API_KEY="sc-key",
- BSKY_HANDLE="user.bsky.social",
- BSKY_APP_PASSWORD="app-pw",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- x_source, x_method = env.get_x_source_with_method(config)
- available = env.get_available_sources(config)
-
- assert x_source == "bird"
- assert x_method == "browser-chrome"
- assert available == "all" # reddit + x + web
- assert env.is_bluesky_available(config) is True
- assert env.is_tiktok_available(config) is True
- assert env.is_hackernews_available() is True
- assert env.is_polymarket_available() is True
-
- def test_no_config_minimal_sources(self):
- """No API keys, no cookies -> Reddit + HN + Polymarket only."""
- config = _base_config()
-
- with _mock_bird_installed(False):
- x_source = env.get_x_source(config)
- available = env.get_available_sources(config)
-
- assert x_source is None
- assert available == "reddit"
- assert env.is_hackernews_available() is True
- assert env.is_polymarket_available() is True
- assert env.is_bluesky_available(config) is False
- assert env.is_tiktok_available(config) is False
-
-
-# ---------------------------------------------------------------------------
-# Tests: extract_browser_credentials tracks browser source
-# ---------------------------------------------------------------------------
-
-class TestExtractBrowserCredentialsSource:
- """Test that extract_browser_credentials tracks __X_BROWSER."""
-
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- def test_tracks_firefox_source(self, mock_extract):
- """Cookies from Firefox -> __X_BROWSER set to 'firefox'."""
- mock_extract.return_value = (
- {"auth_token": "tok", "ct0": "ct0val"},
- "firefox",
- )
-
- config = {
- "AUTH_TOKEN": None,
- "CT0": None,
- "TRUTHSOCIAL_TOKEN": None,
- "FROM_BROWSER": "auto",
- "SETUP_COMPLETE": "true",
- }
- result = env.extract_browser_credentials(config)
-
- assert result["AUTH_TOKEN"] == "tok"
- assert result["CT0"] == "ct0val"
- assert result["__X_BROWSER"] == "firefox"
-
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- def test_tracks_chrome_source(self, mock_extract):
- """Cookies from Chrome -> __X_BROWSER set to 'chrome'."""
- mock_extract.return_value = (
- {"auth_token": "tok", "ct0": "ct0val"},
- "chrome",
- )
-
- config = {
- "AUTH_TOKEN": None,
- "CT0": None,
- "TRUTHSOCIAL_TOKEN": None,
- "FROM_BROWSER": "chrome",
- "SETUP_COMPLETE": "true",
- }
- result = env.extract_browser_credentials(config)
-
- assert result["__X_BROWSER"] == "chrome"
-
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- def test_no_cookies_no_browser_key(self, mock_extract):
- """No cookies found -> no __X_BROWSER key."""
- mock_extract.return_value = None
-
- config = {
- "AUTH_TOKEN": None,
- "CT0": None,
- "TRUTHSOCIAL_TOKEN": None,
- "FROM_BROWSER": "auto",
- "SETUP_COMPLETE": "true",
- }
- result = env.extract_browser_credentials(config)
-
- assert "__X_BROWSER" not in result
-
-
-# ---------------------------------------------------------------------------
-# Tests: get_config() _AUTH_TOKEN_SOURCE tracking
-# ---------------------------------------------------------------------------
-
-class TestGetConfigAuthTokenSource:
- """Test that get_config() sets _AUTH_TOKEN_SOURCE correctly."""
-
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- @patch("scripts.lib.env._find_project_env", return_value=None)
- @patch("scripts.lib.env.load_env_file", return_value={})
- @patch("scripts.lib.env.get_openai_auth")
- def test_env_var_auth_token_source_env(
- self, mock_openai, mock_load, mock_proj, mock_extract
- ):
- """AUTH_TOKEN from env var -> _AUTH_TOKEN_SOURCE='env'."""
- from scripts.lib.env import get_config, OpenAIAuth
-
- mock_openai.return_value = OpenAIAuth(
- token=None, source="none", status="missing",
- account_id=None, codex_auth_file="/fake",
- )
- mock_extract.return_value = None
-
- env_patch = {
- "AUTH_TOKEN": "env_token",
- "CT0": "env_ct0",
- "LAST30DAYS_CONFIG_DIR": "",
- }
- with patch.dict(os.environ, env_patch, clear=False):
- config = get_config()
-
- assert config["_AUTH_TOKEN_SOURCE"] == "env"
-
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- @patch("scripts.lib.env._find_project_env", return_value=None)
- @patch("scripts.lib.env.load_env_file", return_value={})
- @patch("scripts.lib.env.get_openai_auth")
- def test_browser_cookies_auth_token_source_browser(
- self, mock_openai, mock_load, mock_proj, mock_extract
- ):
- """AUTH_TOKEN from cookies -> _AUTH_TOKEN_SOURCE='browser-firefox'."""
- from scripts.lib.env import get_config, OpenAIAuth
-
- mock_openai.return_value = OpenAIAuth(
- token=None, source="none", status="missing",
- account_id=None, codex_auth_file="/fake",
- )
- mock_extract.return_value = (
- {"auth_token": "cookie_tok", "ct0": "cookie_ct0"},
- "firefox",
- )
-
- env_patch = {
- "SETUP_COMPLETE": "true",
- "FROM_BROWSER": "auto",
- "LAST30DAYS_CONFIG_DIR": "",
- }
- # Ensure AUTH_TOKEN is NOT in env
- clean_env = {k: v for k, v in os.environ.items()
- if k not in ("AUTH_TOKEN", "CT0")}
- clean_env.update(env_patch)
- with patch.dict(os.environ, clean_env, clear=True):
- config = get_config()
-
- assert config["AUTH_TOKEN"] == "cookie_tok"
- assert config["_AUTH_TOKEN_SOURCE"] == "browser-firefox"
-
- @patch("scripts.lib.cookie_extract.extract_cookies_with_source")
- @patch("scripts.lib.env._find_project_env", return_value=None)
- @patch("scripts.lib.env.load_env_file", return_value={})
- @patch("scripts.lib.env.get_openai_auth")
- def test_no_auth_token_source_none(
- self, mock_openai, mock_load, mock_proj, mock_extract
- ):
- """No AUTH_TOKEN at all -> _AUTH_TOKEN_SOURCE=None."""
- from scripts.lib.env import get_config, OpenAIAuth
-
- mock_openai.return_value = OpenAIAuth(
- token=None, source="none", status="missing",
- account_id=None, codex_auth_file="/fake",
- )
- mock_extract.return_value = None
-
- clean_env = {k: v for k, v in os.environ.items()
- if k not in ("AUTH_TOKEN", "CT0")}
- clean_env["LAST30DAYS_CONFIG_DIR"] = ""
- with patch.dict(os.environ, clean_env, clear=True):
- config = get_config()
-
- assert config["_AUTH_TOKEN_SOURCE"] is None
-
-
-# ---------------------------------------------------------------------------
-# Tests: SETUP_COMPLETE gate — Bird cookie probing blocked before consent
-# ---------------------------------------------------------------------------
-
-class TestSetupCompleteGate:
- """Test that Bird cookie probing is gated behind SETUP_COMPLETE consent."""
-
- def test_no_setup_complete_bird_has_cookies_returns_none(self):
- """SETUP_COMPLETE not set, Bird has browser cookies -> returns None (not 'bird').
-
- Bird's is_bird_authenticated() should NOT be called at all because
- there is no consent yet. Cookie-sourced AUTH_TOKEN must be ignored.
- """
- config = _base_config(
- AUTH_TOKEN="cookie_tok",
- CT0="cookie_ct0",
- SETUP_COMPLETE=None, # not set — first run
- _AUTH_TOKEN_SOURCE="browser-chrome",
- )
-
- with _mock_bird_installed(True), \
- _mock_bird_authenticated("Chrome") as mock_auth:
- source = env.get_x_source(config)
-
- assert source is None
- # is_bird_authenticated should NOT have been called (no cookie probing)
- mock_auth.assert_not_called()
-
- def test_setup_complete_bird_has_cookies_returns_bird(self):
- """SETUP_COMPLETE=true, Bird has browser cookies -> returns 'bird'.
-
- After user consent, cookie probing should work normally.
- """
- config = _base_config(
- AUTH_TOKEN="cookie_tok",
- CT0="cookie_ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-chrome",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("Chrome"):
- source = env.get_x_source(config)
-
- assert source == "bird"
-
- def test_no_setup_complete_explicit_auth_token_returns_bird(self):
- """SETUP_COMPLETE not set, AUTH_TOKEN from env var -> returns 'bird' with method 'env'.
-
- Explicit env var credentials must ALWAYS work regardless of SETUP_COMPLETE.
- """
- config = _base_config(
- AUTH_TOKEN="explicit_token",
- CT0="explicit_ct0",
- SETUP_COMPLETE=None, # not set
- _AUTH_TOKEN_SOURCE="env",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("env AUTH_TOKEN"):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "bird"
- assert method == "env"
-
- def test_no_setup_complete_xai_key_returns_xai(self):
- """SETUP_COMPLETE not set, XAI_API_KEY configured -> returns 'xai'.
-
- API keys always work without setup consent.
- """
- config = _base_config(
- XAI_API_KEY="xai-key-123",
- SETUP_COMPLETE=None, # not set
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated(None):
- source, method = env.get_x_source_with_method(config)
-
- assert source == "xai"
- assert method == "api"
-
- def test_no_setup_complete_status_reports_not_configured(self):
- """First-run status banner should show X as not configured when only cookies exist."""
- config = _base_config(
- AUTH_TOKEN="cookie_tok",
- CT0="cookie_ct0",
- SETUP_COMPLETE=None,
- _AUTH_TOKEN_SOURCE="browser-firefox",
- )
-
- with _mock_bird_installed(True):
- status = env.get_x_source_status(config)
-
- assert status["source"] is None
- assert status["method"] is None
- assert status["bird_authenticated"] is False
-
- def test_setup_complete_status_reports_bird(self):
- """After consent, status banner should show Bird as configured."""
- config = _base_config(
- AUTH_TOKEN="cookie_tok",
- CT0="cookie_ct0",
- SETUP_COMPLETE="true",
- _AUTH_TOKEN_SOURCE="browser-firefox",
- )
-
- with _mock_bird_installed(True), _mock_bird_authenticated("Firefox"), \
- _mock_bird_status(installed=True, authenticated=True, username="Firefox"):
- status = env.get_x_source_status(config)
-
- assert status["source"] == "bird"
- assert status["method"] == "browser-firefox"
- assert status["bird_authenticated"] is True
-
-
-# ---------------------------------------------------------------------------
-# Tests: BIRD_DISABLE_BROWSER_COOKIES env var on first run
-# ---------------------------------------------------------------------------
-
-class TestBirdDisableBrowserCookiesEnvVar:
- """Test that BIRD_DISABLE_BROWSER_COOKIES is set to block Bird's Node.js
- sweet-cookie scanner on first run before user consent."""
-
- def _run_main_flow_env_setup(self, config, first_run):
- """Simulate the env-var-setting logic from last30days.py main flow.
-
- Mirrors the block right after first_run detection in main().
- """
- if first_run and config.get('_AUTH_TOKEN_SOURCE') != 'env':
- os.environ['BIRD_DISABLE_BROWSER_COOKIES'] = '1'
- else:
- os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
-
- def test_first_run_no_auth_token_sets_env_var(self):
- """first_run=True, no AUTH_TOKEN -> BIRD_DISABLE_BROWSER_COOKIES is set."""
- config = _base_config(SETUP_COMPLETE=None, _AUTH_TOKEN_SOURCE=None)
- try:
- self._run_main_flow_env_setup(config, first_run=True)
- assert os.environ.get('BIRD_DISABLE_BROWSER_COOKIES') == '1'
- finally:
- os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
-
- def test_not_first_run_no_env_var(self):
- """first_run=False -> BIRD_DISABLE_BROWSER_COOKIES is NOT set."""
- config = _base_config(SETUP_COMPLETE="true", _AUTH_TOKEN_SOURCE="browser-firefox")
- try:
- self._run_main_flow_env_setup(config, first_run=False)
- assert 'BIRD_DISABLE_BROWSER_COOKIES' not in os.environ
- finally:
- os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
-
- def test_first_run_explicit_auth_token_no_env_var(self):
- """first_run=True, AUTH_TOKEN explicitly set -> BIRD_DISABLE_BROWSER_COOKIES is NOT set."""
- config = _base_config(
- AUTH_TOKEN="explicit_token",
- CT0="explicit_ct0",
- SETUP_COMPLETE=None,
- _AUTH_TOKEN_SOURCE="env",
- )
- try:
- self._run_main_flow_env_setup(config, first_run=True)
- assert 'BIRD_DISABLE_BROWSER_COOKIES' not in os.environ
- finally:
- os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None)
-
-
-# ---------------------------------------------------------------------------
-# Tests: INCLUDE_SOURCES config override
-# ---------------------------------------------------------------------------
-
-class TestIncludeSourcesOverride:
- """Test that INCLUDE_SOURCES forces sources on regardless of tier."""
-
- def _simulate_source_decisions(self, config, query_type="breaking_news"):
- """Simulate the source decision logic from last30days.py main flow.
-
- Returns (search_run_tiktok, search_run_instagram) after tier + override.
- """
- from scripts.lib import query_type as qt
-
- has_tiktok = env.is_tiktok_available(config)
- has_instagram = env.is_instagram_available(config)
-
- # Tier system decision
- search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type)
- search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type)
-
- # INCLUDE_SOURCES override (mirrors last30days.py logic)
- _include_sources = {
- s.strip().lower()
- for s in config.get('INCLUDE_SOURCES', '').split(',')
- if s.strip()
- }
- if _include_sources:
- if 'tiktok' in _include_sources and has_tiktok:
- if not search_run_tiktok:
- search_run_tiktok = True
- if 'instagram' in _include_sources and has_instagram:
- if not search_run_instagram:
- search_run_instagram = True
-
- return search_run_tiktok, search_run_instagram
-
- def test_include_sources_forces_tiktok_and_instagram_on(self):
- """INCLUDE_SOURCES=tiktok,instagram + SC key + GENERAL query -> both forced on."""
- config = _base_config(
- SCRAPECREATORS_API_KEY="sc-key",
- INCLUDE_SOURCES="tiktok,instagram",
- )
- run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
- assert run_tiktok is True
- assert run_instagram is True
-
- def test_no_include_sources_tier_controls(self):
- """No INCLUDE_SOURCES + SC key + GENERAL query -> tier system controls (both off)."""
- config = _base_config(
- SCRAPECREATORS_API_KEY="sc-key",
- )
- run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
- # breaking_news tier doesn't include tiktok or instagram
- assert run_tiktok is False
- assert run_instagram is False
-
- def test_include_sources_no_sc_key_still_off(self):
- """INCLUDE_SOURCES=tiktok but no SC key -> TikTok still off (no backend)."""
- config = _base_config(
- INCLUDE_SOURCES="tiktok",
- )
- run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news")
- assert run_tiktok is False
- assert run_instagram is False
diff --git a/tests/test_status_banner.py b/tests/test_status_banner.py
deleted file mode 100644
index 852699b..0000000
--- a/tests/test_status_banner.py
+++ /dev/null
@@ -1,283 +0,0 @@
-"""Tests for the redesigned status banner (free-first design)."""
-
-import pytest
-
-from scripts.lib.ui import _build_status_banner
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _base_diag(**overrides):
- """Return a minimal diag dict with common defaults."""
- diag = {
- "setup_complete": False,
- "reddit_source": None, # None = public fallback
- "x_source": None,
- "x_method": None,
- "youtube": False,
- "tiktok": False,
- "instagram": False,
- "hackernews": True,
- "polymarket": True,
- "bluesky": False,
- "truthsocial": False,
- "xiaohongshu": False,
- "scrapecreators": False,
- "web_search_backend": None,
- }
- diag.update(overrides)
- return diag
-
-
-def _banner_text(diag):
- """Return full banner as a single string."""
- return "\n".join(_build_status_banner(diag))
-
-
-# ---------------------------------------------------------------------------
-# Tests
-# ---------------------------------------------------------------------------
-
-class TestZeroConfig:
- """Zero-config state: SETUP_COMPLETE not set, wizard hasn't run."""
-
- def test_shows_first_run_title(self):
- banner = _banner_text(_base_diag())
- assert "First Run" in banner
-
- def test_shows_three_free_sources(self):
- banner = _banner_text(_base_diag())
- assert "Reddit (threads only)" in banner
- assert "HN" in banner
- assert "Polymarket" in banner
-
- def test_shows_setup_prompt(self):
- banner = _banner_text(_base_diag())
- assert "/last30days setup" in banner
-
- def test_does_not_show_source_status_title(self):
- banner = _banner_text(_base_diag())
- assert "Source Status" not in banner
-
-
-class TestFullConfig:
- """Fully configured: X + yt-dlp + ScrapeCreators = everything active."""
-
- def _full_diag(self):
- return _base_diag(
- setup_complete=True,
- reddit_source="scrapecreators",
- x_source="bird",
- x_method="browser-chrome",
- youtube=True,
- tiktok=True,
- instagram=True,
- hackernews=True,
- polymarket=True,
- bluesky=True,
- truthsocial=True,
- xiaohongshu=True,
- scrapecreators=True,
- web_search_backend="parallel",
- )
-
- def test_shows_source_status_title(self):
- banner = _banner_text(self._full_diag())
- assert "Source Status" in banner
-
- def test_shows_all_sources(self):
- banner = _banner_text(self._full_diag())
- assert "Reddit (with comments)" in banner
- assert "X (Chrome)" in banner
- assert "YouTube" in banner
- assert "HN" in banner
- assert "Polymarket" in banner
- assert "TikTok" in banner
- assert "Instagram" in banner
- assert "Bluesky" in banner
- assert "Truth Social" in banner
- assert "Xiaohongshu" in banner
-
- def test_no_recommendations(self):
- banner = _banner_text(self._full_diag())
- assert "⭐" not in banner
- assert "scrapecreators.com" not in banner
- assert "/last30days setup" not in banner
-
-
-class TestPartialConfig:
- """Partially configured: X + yt-dlp but no ScrapeCreators."""
-
- def _partial_diag(self):
- return _base_diag(
- setup_complete=True,
- reddit_source=None, # public fallback
- x_source="bird",
- x_method="browser-chrome",
- youtube=True,
- scrapecreators=False,
- )
-
- def test_shows_active_sources(self):
- banner = _banner_text(self._partial_diag())
- assert "Reddit (threads only)" in banner
- assert "X (Chrome)" in banner
- assert "YouTube" in banner
- assert "HN" in banner
- assert "Polymarket" in banner
-
- def test_recommends_scrapecreators(self):
- banner = _banner_text(self._partial_diag())
- assert "SCRAPECREATORS_API_KEY" in banner
-
- def test_scrapecreators_free_calls_copy(self):
- banner = _banner_text(self._partial_diag())
- assert "100 free calls, no CC" in banner
- assert "scrapecreators.com" in banner
-
- def test_shows_tiktok_instagram_unlock(self):
- banner = _banner_text(self._partial_diag())
- assert "TikTok" in banner
- assert "Instagram" in banner
-
-
-class TestScrapeCreatorsRecommendation:
- """ScrapeCreators recommendation always includes key copy."""
-
- def test_always_includes_free_calls_no_cc(self):
- """Any config missing SC should show the free-calls copy."""
- # Zero config
- banner_zero = _banner_text(_base_diag())
- # Zero config doesn't recommend SC directly (recommends setup wizard)
- # But after setup, missing SC should always include the copy
- banner_partial = _banner_text(_base_diag(
- setup_complete=True,
- scrapecreators=False,
- ))
- assert "100 free calls, no CC" in banner_partial
-
- def test_present_when_x_available_but_no_sc(self):
- banner = _banner_text(_base_diag(
- setup_complete=True,
- x_source="xai",
- x_method="api",
- scrapecreators=False,
- ))
- assert "100 free calls, no CC" in banner
- assert "scrapecreators.com" in banner
-
-
-class TestRedditLabelDisplay:
- """Reddit label reflects comment availability, not implementation details."""
-
- def test_no_scrapecreators_shows_threads_only(self):
- """Without SC, Reddit label should say 'threads only' regardless of OpenAI auth."""
- banner = _banner_text(_base_diag(
- setup_complete=True,
- reddit_source="openai",
- scrapecreators=False,
- ))
- assert "Reddit (threads only)" in banner
- assert "OpenAI" not in banner
- assert "Codex" not in banner
-
- def test_no_scrapecreators_public_shows_threads_only(self):
- """Public fallback also shows 'threads only'."""
- banner = _banner_text(_base_diag(
- setup_complete=True,
- reddit_source=None,
- scrapecreators=False,
- ))
- assert "Reddit (threads only)" in banner
-
- def test_scrapecreators_shows_with_comments(self):
- """With SC configured, Reddit label should say 'with comments'."""
- banner = _banner_text(_base_diag(
- setup_complete=True,
- reddit_source="scrapecreators",
- scrapecreators=True,
- ))
- assert "Reddit (with comments)" in banner
-
- def test_no_openai_or_codex_in_banner(self):
- """Banner should never mention OpenAI or Codex — those are implementation details."""
- for source in [None, "openai", "scrapecreators"]:
- banner = _banner_text(_base_diag(
- setup_complete=True,
- reddit_source=source,
- scrapecreators=(source == "scrapecreators"),
- ))
- assert "OpenAI" not in banner, f"Found 'OpenAI' with reddit_source={source}"
- assert "Codex" not in banner, f"Found 'Codex' with reddit_source={source}"
-
-
-class TestXMethodDisplay:
- """X source shows auth method in parens."""
-
- def test_browser_chrome(self):
- banner = _banner_text(_base_diag(
- setup_complete=True,
- x_source="bird",
- x_method="browser-chrome",
- ))
- assert "X (Chrome)" in banner
-
- def test_browser_firefox(self):
- banner = _banner_text(_base_diag(
- setup_complete=True,
- x_source="bird",
- x_method="browser-firefox",
- ))
- assert "X (Firefox)" in banner
-
- def test_env_method(self):
- banner = _banner_text(_base_diag(
- setup_complete=True,
- x_source="bird",
- x_method="env",
- ))
- assert "X (env)" in banner
-
- def test_xai_api(self):
- banner = _banner_text(_base_diag(
- setup_complete=True,
- x_source="xai",
- x_method="api",
- ))
- assert "X (xAI)" in banner
-
-
-class TestBannerStructure:
- """Banner formatting and structure tests."""
-
- def test_has_box_drawing(self):
- lines = _build_status_banner(_base_diag())
- assert lines[0].startswith("┌")
- assert lines[-1].startswith("└")
-
- def test_shows_config_path(self):
- banner = _banner_text(_base_diag())
- assert "~/.config/last30days/.env" in banner
-
- def test_max_10_inner_lines(self):
- """Banner should be compact — max 10 lines inside the box."""
- # Full config (most lines)
- diag = _base_diag(
- setup_complete=True,
- reddit_source="scrapecreators",
- x_source="bird",
- x_method="browser-chrome",
- youtube=True,
- tiktok=True,
- instagram=True,
- bluesky=True,
- truthsocial=True,
- xiaohongshu=True,
- scrapecreators=True,
- )
- lines = _build_status_banner(diag)
- # Subtract top and bottom border
- inner_lines = [l for l in lines if l.startswith("│")]
- assert len(inner_lines) <= 10, f"Banner has {len(inner_lines)} inner lines, max is 10"
diff --git a/tests/test_store.py b/tests/test_store.py
new file mode 100644
index 0000000..f1c3d1a
--- /dev/null
+++ b/tests/test_store.py
@@ -0,0 +1,495 @@
+"""Tests for store.py - SQLite research accumulator and watchlist storage."""
+
+import json
+import sqlite3
+import tempfile
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pytest
+
+# Import the module under test
+import sys
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+import store
+from lib import schema
+
+
+@pytest.fixture
+def temp_db():
+ """Create a temporary database for testing."""
+ with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
+ db_path = Path(f.name)
+
+ # Override the database path
+ original_override = store._db_override
+ store._db_override = db_path
+
+ # Initialize fresh database
+ store.init_db()
+
+ yield db_path
+
+ # Cleanup
+ store._db_override = original_override
+ if db_path.exists():
+ db_path.unlink()
+
+
+@pytest.fixture
+def sample_report():
+ """Create a sample Report with multiple sources including HN and Polymarket."""
+ return schema.report_from_dict({
+ "topic": "Test Topic",
+ "range_from": "2026-01-01",
+ "range_to": "2026-04-03",
+ "generated_at": "2026-04-03T00:00:00Z",
+ "provider_runtime": {
+ "reasoning_provider": "gemini",
+ "planner_model": "gemini-2.0-flash-exp",
+ "rerank_model": "gemini-2.0-flash-exp",
+ },
+ "query_plan": {
+ "intent": "test",
+ "freshness_mode": "recent",
+ "cluster_mode": "standard",
+ "raw_topic": "test",
+ "subqueries": [],
+ "source_weights": {},
+ },
+ "clusters": [],
+ "ranked_candidates": [],
+ "items_by_source": {
+ "reddit": [
+ {
+ "item_id": "R1",
+ "source": "reddit",
+ "title": "Test Reddit Post",
+ "body": "Reddit discussion content",
+ "url": "https://reddit.com/r/test/1",
+ "author": "testuser",
+ "engagement_score": 50.0,
+ "local_relevance": 0.8,
+ "snippet": "Reddit snippet",
+ }
+ ],
+ "x": [
+ {
+ "item_id": "X1",
+ "source": "x",
+ "title": "Test X Post",
+ "body": "X post content",
+ "url": "https://x.com/test/status/1",
+ "author": "xuser",
+ "engagement_score": 75.0,
+ "local_relevance": 0.85,
+ "snippet": "X snippet",
+ }
+ ],
+ "hackernews": [
+ {
+ "item_id": "HN1",
+ "source": "hackernews",
+ "title": "Test HN Story",
+ "body": "HN story content with comments",
+ "url": "https://news.ycombinator.com/item?id=12345",
+ "author": "hnuser",
+ "engagement_score": 120.0,
+ "local_relevance": 0.9,
+ "snippet": "HN snippet",
+ }
+ ],
+ "polymarket": [
+ {
+ "item_id": "PM1",
+ "source": "polymarket",
+ "title": "Will event happen?",
+ "body": "Yes: 64% / No: 36%",
+ "url": "https://polymarket.com/event/test-event",
+ "author": None,
+ "engagement_score": 342000.0,
+ "local_relevance": 0.7,
+ "snippet": "Prediction market",
+ }
+ ],
+ },
+ "errors_by_source": {},
+ "warnings": [],
+ })
+
+
+# === Tests for findings_from_report() ===
+
+def test_findings_from_report_processes_all_sources(sample_report):
+ """Test that findings_from_report extracts items from all sources in items_by_source."""
+ findings = store.findings_from_report(sample_report)
+
+ # Should have 4 findings (reddit + x + hackernews + polymarket)
+ assert len(findings) == 4
+
+ # Check all sources are present
+ sources = {f["source"] for f in findings}
+ assert sources == {"reddit", "x", "hackernews", "polymarket"}
+
+
+def test_findings_from_report_includes_hackernews(sample_report):
+ """Test that HN items are extracted correctly (PR #85 feature)."""
+ findings = store.findings_from_report(sample_report)
+
+ hn_findings = [f for f in findings if f["source"] == "hackernews"]
+ assert len(hn_findings) == 1
+
+ hn = hn_findings[0]
+ assert hn["source_url"] == "https://news.ycombinator.com/item?id=12345"
+ assert hn["source_title"] == "Test HN Story"
+ assert hn["engagement_score"] == 120.0
+ assert hn["relevance_score"] == 0.9
+ assert "HN story content" in hn["content"]
+
+
+def test_findings_from_report_includes_polymarket(sample_report):
+ """Test that Polymarket items are extracted correctly (PR #85 feature)."""
+ findings = store.findings_from_report(sample_report)
+
+ pm_findings = [f for f in findings if f["source"] == "polymarket"]
+ assert len(pm_findings) == 1
+
+ pm = pm_findings[0]
+ assert pm["source_url"] == "https://polymarket.com/event/test-event"
+ assert pm["source_title"] == "Will event happen?"
+ assert pm["engagement_score"] == 342000.0
+ assert pm["relevance_score"] == 0.7
+ assert "Yes: 64%" in pm["content"]
+
+
+def test_findings_from_report_respects_limit(sample_report):
+ """Test that limit parameter works correctly."""
+ findings = store.findings_from_report(sample_report, limit=2)
+
+ # Should have at most 2 items per source
+ source_counts = {}
+ for f in findings:
+ source = f["source"]
+ source_counts[source] = source_counts.get(source, 0) + 1
+
+ for count in source_counts.values():
+ assert count <= 2
+
+
+def test_findings_from_report_handles_empty_sources():
+ """Test that empty sources in items_by_source don't cause issues."""
+ report = schema.report_from_dict({
+ "topic": "Test",
+ "range_from": "2026-01-01",
+ "range_to": "2026-04-03",
+ "generated_at": "2026-04-03T00:00:00Z",
+ "provider_runtime": {
+ "reasoning_provider": "gemini",
+ "planner_model": "gemini-2.0-flash-exp",
+ "rerank_model": "gemini-2.0-flash-exp",
+ },
+ "query_plan": {
+ "intent": "test",
+ "freshness_mode": "recent",
+ "cluster_mode": "standard",
+ "raw_topic": "test",
+ "subqueries": [],
+ "source_weights": {},
+ },
+ "clusters": [],
+ "ranked_candidates": [],
+ "items_by_source": {
+ "reddit": [],
+ "x": [],
+ "hackernews": [],
+ "polymarket": [],
+ },
+ "errors_by_source": {},
+ "warnings": [],
+ })
+
+ findings = store.findings_from_report(report)
+ assert len(findings) == 0
+
+
+def test_findings_from_report_handles_missing_fields():
+ """Test that missing optional fields (author, snippet) are handled gracefully."""
+ report = schema.report_from_dict({
+ "topic": "Test",
+ "range_from": "2026-01-01",
+ "range_to": "2026-04-03",
+ "generated_at": "2026-04-03T00:00:00Z",
+ "provider_runtime": {
+ "reasoning_provider": "gemini",
+ "planner_model": "gemini-2.0-flash-exp",
+ "rerank_model": "gemini-2.0-flash-exp",
+ },
+ "query_plan": {
+ "intent": "test",
+ "freshness_mode": "recent",
+ "cluster_mode": "standard",
+ "raw_topic": "test",
+ "subqueries": [],
+ "source_weights": {},
+ },
+ "clusters": [],
+ "ranked_candidates": [],
+ "items_by_source": {
+ "reddit": [
+ {
+ "item_id": "R1",
+ "source": "reddit",
+ "title": "Test",
+ "body": "Content",
+ "url": "https://reddit.com/1",
+ "author": None, # Missing author
+ "engagement_score": None, # Missing engagement
+ "local_relevance": None, # Missing relevance
+ "snippet": None, # Missing snippet
+ }
+ ],
+ },
+ "errors_by_source": {},
+ "warnings": [],
+ })
+
+ findings = store.findings_from_report(report)
+ assert len(findings) == 1
+
+ f = findings[0]
+ assert f["author"] == ""
+ assert f["engagement_score"] == 0.0
+ assert f["relevance_score"] == 0.5
+ assert f["summary"] == "Content" # Falls back to body
+
+
+# === Tests for store_findings() ===
+
+def test_store_findings_basic(temp_db, sample_report):
+ """Test basic storage of findings."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+
+ findings = store.findings_from_report(sample_report)
+ counts = store.store_findings(run_id, topic["id"], findings)
+
+ assert counts["new"] == 4
+ assert counts["updated"] == 0
+
+ # Verify in database
+ conn = sqlite3.connect(str(temp_db))
+ total = conn.execute("SELECT COUNT(*) FROM findings").fetchone()[0]
+ assert total == 4
+ conn.close()
+
+
+def test_store_findings_deduplicates_by_url(temp_db, sample_report):
+ """Test that duplicate URLs are detected and updated, not duplicated."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+
+ findings = store.findings_from_report(sample_report)
+
+ # Store once
+ counts1 = store.store_findings(run_id, topic["id"], findings)
+ assert counts1["new"] == 4
+ assert counts1["updated"] == 0
+
+ # Store again (same URLs)
+ counts2 = store.store_findings(run_id, topic["id"], findings)
+ assert counts2["new"] == 0
+ assert counts2["updated"] == 4
+
+ # Verify total count didn't double
+ conn = sqlite3.connect(str(temp_db))
+ total = conn.execute("SELECT COUNT(*) FROM findings").fetchone()[0]
+ assert total == 4
+ conn.close()
+
+
+def test_store_findings_updates_engagement_on_resighting(temp_db, sample_report):
+ """Test that re-sighting a finding updates engagement score if higher."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+
+ findings = store.findings_from_report(sample_report)
+
+ # Store with initial engagement
+ store.store_findings(run_id, topic["id"], findings)
+
+ # Modify engagement score for HN finding
+ hn_finding = next(f for f in findings if f["source"] == "hackernews")
+ hn_finding["engagement_score"] = 200.0 # Higher than original 120.0
+
+ # Store again
+ store.store_findings(run_id, topic["id"], [hn_finding])
+
+ # Verify engagement was updated
+ conn = sqlite3.connect(str(temp_db))
+ score = conn.execute(
+ "SELECT engagement_score FROM findings WHERE source='hackernews'"
+ ).fetchone()[0]
+ assert score == 200.0
+ conn.close()
+
+
+def test_store_findings_increments_sighting_count(temp_db, sample_report):
+ """Test that re-sighting increments sighting_count."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+
+ findings = store.findings_from_report(sample_report)
+
+ # Store once
+ store.store_findings(run_id, topic["id"], findings)
+
+ # Store again
+ store.store_findings(run_id, topic["id"], findings)
+
+ # Verify sighting_count
+ conn = sqlite3.connect(str(temp_db))
+ counts = conn.execute(
+ "SELECT sighting_count FROM findings"
+ ).fetchall()
+
+ for (count,) in counts:
+ assert count == 2 # Seen twice
+ conn.close()
+
+
+def test_store_findings_skips_items_without_url(temp_db):
+ """Test that findings without URLs are skipped."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+
+ findings = [
+ {
+ "source": "reddit",
+ "source_url": None, # Missing URL
+ "source_title": "Test",
+ "content": "Content",
+ },
+ {
+ "source": "reddit",
+ "source_url": "https://reddit.com/1", # Has URL
+ "source_title": "Test 2",
+ "content": "Content 2",
+ },
+ ]
+
+ counts = store.store_findings(run_id, topic["id"], findings)
+
+ # Only the one with URL should be stored
+ assert counts["new"] == 1
+
+
+# === Tests for topic management ===
+
+def test_add_topic(temp_db):
+ """Test adding a topic."""
+ topic = store.add_topic("Test Topic", schedule="0 8 * * *")
+
+ assert topic["name"] == "Test Topic"
+ assert topic["schedule"] == "0 8 * * *"
+ assert topic["enabled"] == 1
+
+
+def test_add_topic_with_search_queries(temp_db):
+ """Test adding a topic with custom search queries."""
+ topic = store.add_topic(
+ "Test Topic",
+ search_queries=["query1", "query2"],
+ schedule="0 8 * * *",
+ )
+
+ assert topic["name"] == "Test Topic"
+ assert json.loads(topic["search_queries"]) == ["query1", "query2"]
+
+
+def test_remove_topic_cascades_findings(temp_db, sample_report):
+ """Test that removing a topic deletes its findings and runs."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+ findings = store.findings_from_report(sample_report)
+ store.store_findings(run_id, topic["id"], findings)
+
+ # Verify data exists
+ conn = sqlite3.connect(str(temp_db))
+ finding_count = conn.execute("SELECT COUNT(*) FROM findings").fetchone()[0]
+ run_count = conn.execute("SELECT COUNT(*) FROM research_runs").fetchone()[0]
+ assert finding_count == 4
+ assert run_count == 1
+ conn.close()
+
+ # Remove topic
+ removed = store.remove_topic("Test Topic")
+ assert removed is True
+
+ # Verify cascade delete
+ conn = sqlite3.connect(str(temp_db))
+ finding_count = conn.execute("SELECT COUNT(*) FROM findings").fetchone()[0]
+ run_count = conn.execute("SELECT COUNT(*) FROM research_runs").fetchone()[0]
+ assert finding_count == 0
+ assert run_count == 0
+ conn.close()
+
+
+def test_list_topics(temp_db):
+ """Test listing topics with stats."""
+ # Add multiple topics
+ store.add_topic("Topic 1")
+ store.add_topic("Topic 2")
+ store.add_topic("Topic 3")
+
+ topics = store.list_topics()
+
+ assert len(topics) == 3
+ assert {t["name"] for t in topics} == {"Topic 1", "Topic 2", "Topic 3"}
+
+ # Check that stats fields are present
+ for topic in topics:
+ assert "finding_count" in topic
+ assert "last_run" in topic
+ assert "last_status" in topic
+
+
+# === Tests for get_new_findings() ===
+
+def test_get_new_findings(temp_db, sample_report):
+ """Test retrieving new findings for a topic."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+ findings = store.findings_from_report(sample_report)
+ store.store_findings(run_id, topic["id"], findings)
+
+ new_findings = store.get_new_findings(topic["id"])
+
+ assert len(new_findings) == 4
+ sources = {f["source"] for f in new_findings}
+ assert "hackernews" in sources
+ assert "polymarket" in sources
+
+
+def test_get_new_findings_filters_by_date(temp_db, sample_report):
+ """Test that since parameter filters findings correctly."""
+ topic = store.add_topic("Test Topic")
+ run_id = store.record_run(topic["id"], source_mode="v3")
+ findings = store.findings_from_report(sample_report)
+ store.store_findings(run_id, topic["id"], findings)
+
+ # Get findings since tomorrow (should be empty)
+ tomorrow = (datetime.now() + timedelta(days=1)).strftime("%Y-%m-%d")
+ new_findings = store.get_new_findings(topic["id"], since=tomorrow)
+
+ assert len(new_findings) == 0
+
+ # Get findings since yesterday (should have all)
+ yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
+ new_findings = store.get_new_findings(topic["id"], since=yesterday)
+
+ assert len(new_findings) == 4
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_tiktok.py b/tests/test_tiktok.py
index 91e68be..2dea29d 100644
--- a/tests/test_tiktok.py
+++ b/tests/test_tiktok.py
@@ -1,267 +1,108 @@
-"""Tests for TikTok module (search, normalize, score, dedupe, render)."""
-
-import json
import sys
import unittest
from pathlib import Path
-# Add lib to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
-from lib import schema, score, normalize, dedupe, render
-from lib import tiktok
+from lib.tiktok import _parse_items
-class TestTikTokRelevance(unittest.TestCase):
- """Test relevance scoring for TikTok items."""
+class TestTikTokAuthorTypeSafety(unittest.TestCase):
+ def _make_raw(self, **overrides):
+ base = {
+ "aweme_id": "1",
+ "desc": "test video",
+ "share_url": "https://www.tiktok.com/@u/video/1",
+ "author": {"unique_id": "testuser"},
+ "statistics": {"play_count": 100, "digg_count": 50, "comment_count": 10, "share_count": 5},
+ }
+ base.update(overrides)
+ return base
- def test_exact_match(self):
- rel = tiktok._compute_relevance("claude code", "Claude Code tricks and tips")
- self.assertGreaterEqual(rel, 0.8)
+ def test_author_as_dict(self):
+ items = _parse_items([self._make_raw()], "test")
+ self.assertEqual("testuser", items[0]["author_name"])
- def test_partial_match(self):
- rel = tiktok._compute_relevance("claude code tips", "Best AI tools for coding")
- self.assertLess(rel, 0.5)
+ def test_author_as_string(self):
+ items = _parse_items([self._make_raw(author="stringuser")], "test")
+ self.assertEqual("stringuser", items[0]["author_name"])
- def test_hashtag_boost(self):
- """Hashtags should boost relevance."""
- rel_no_hash = tiktok._compute_relevance("claude code", "random video about stuff")
- rel_with_hash = tiktok._compute_relevance("claude code", "random video about stuff", ["claudecode", "ai"])
- self.assertGreater(rel_with_hash, rel_no_hash)
+ def test_author_missing(self):
+ raw = self._make_raw()
+ del raw["author"]
+ items = _parse_items([raw], "test")
+ self.assertEqual("", items[0]["author_name"])
- def test_empty_query(self):
- rel = tiktok._compute_relevance("", "Some video title")
- self.assertEqual(rel, 0.5)
-
- def test_no_match_returns_zero(self):
- rel = tiktok._compute_relevance("quantum physics", "cat dancing video")
- self.assertEqual(rel, 0.0)
+ def test_author_none(self):
+ items = _parse_items([self._make_raw(author=None)], "test")
+ self.assertEqual("", items[0]["author_name"])
-class TestExtractCoreSubject(unittest.TestCase):
- """Test core subject extraction for TikTok search."""
+class TestTikTokStatsZeroPreserved(unittest.TestCase):
+ def test_zero_play_count(self):
+ raw = {
+ "aweme_id": "1",
+ "desc": "test",
+ "share_url": "https://www.tiktok.com/@u/video/1",
+ "author": {"unique_id": "u"},
+ "statistics": {"play_count": 0, "digg_count": 0, "comment_count": 0, "share_count": 0},
+ }
+ items = _parse_items([raw], "test")
+ self.assertEqual(0, items[0]["engagement"]["views"])
+ self.assertEqual(0, items[0]["engagement"]["likes"])
+ self.assertEqual(0, items[0]["engagement"]["comments"])
+ self.assertEqual(0, items[0]["engagement"]["shares"])
- def test_strips_prefix(self):
- result = tiktok._extract_core_subject("what are the best claude code tips")
- self.assertNotIn("what are the best", result)
- self.assertIn("claude", result)
+ def test_stats_missing(self):
+ raw = {
+ "aweme_id": "1",
+ "desc": "test",
+ "share_url": "https://www.tiktok.com/@u/video/1",
+ "author": {"unique_id": "u"},
+ }
+ items = _parse_items([raw], "test")
+ self.assertEqual(0, items[0]["engagement"]["views"])
- def test_strips_noise(self):
- result = tiktok._extract_core_subject("latest trending updates on React")
- self.assertNotIn("latest", result)
- self.assertNotIn("trending", result)
- self.assertIn("react", result.lower())
-
- def test_preserves_core(self):
- result = tiktok._extract_core_subject("Claude Code")
- self.assertEqual(result, "claude code")
+ def test_stats_as_non_dict(self):
+ raw = {
+ "aweme_id": "1",
+ "desc": "test",
+ "share_url": "https://www.tiktok.com/@u/video/1",
+ "author": {"unique_id": "u"},
+ "statistics": "invalid",
+ }
+ items = _parse_items([raw], "test")
+ self.assertEqual(0, items[0]["engagement"]["views"])
-class TestParseDate(unittest.TestCase):
- """Test date parsing from ScrapeCreators items."""
+class TestExpandTikTokQueries(unittest.TestCase):
+ """Tests for expand_tiktok_queries() multi-query generation."""
- def test_unix_timestamp(self):
- item = {"create_time": 1756403075}
- result = tiktok._parse_date(item)
- self.assertIsNotNone(result)
- self.assertRegex(result, r"\d{4}-\d{2}-\d{2}")
-
- def test_no_date(self):
- item = {}
- self.assertIsNone(tiktok._parse_date(item))
-
-
-class TestCleanWebVTT(unittest.TestCase):
- """Test WebVTT transcript cleaning."""
-
- def test_strips_timestamps(self):
- raw = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello world\n\n00:00:02.000 --> 00:00:04.000\nGoodbye"
- result = tiktok._clean_webvtt(raw)
- self.assertEqual(result, "Hello world Goodbye")
-
- def test_empty_input(self):
- self.assertEqual(tiktok._clean_webvtt(""), "")
- self.assertEqual(tiktok._clean_webvtt(None), "")
-
-
-class TestNormalizeTikTokItems(unittest.TestCase):
- """Test TikTok normalization."""
-
- def setUp(self):
- self.fixtures_dir = Path(__file__).parent.parent / "fixtures"
- with open(self.fixtures_dir / "tiktok_search.json") as f:
- data = json.load(f)
- self.raw_items = data["items"]
-
- def test_normalizes_items(self):
- items = normalize.normalize_tiktok_items(self.raw_items, "2026-02-01", "2026-03-03")
- self.assertEqual(len(items), 3)
- self.assertIsInstance(items[0], schema.TikTokItem)
-
- def test_ids_are_sequential(self):
- items = normalize.normalize_tiktok_items(self.raw_items, "2026-02-01", "2026-03-03")
- self.assertEqual(items[0].id, "TK1")
- self.assertEqual(items[1].id, "TK2")
- self.assertEqual(items[2].id, "TK3")
-
- def test_engagement_parsed(self):
- items = normalize.normalize_tiktok_items(self.raw_items, "2026-02-01", "2026-03-03")
- eng = items[0].engagement
- self.assertIsNotNone(eng)
- self.assertEqual(eng.views, 2100000)
- self.assertEqual(eng.likes, 45000)
- self.assertEqual(eng.shares, 8400)
-
- def test_hashtags_preserved(self):
- items = normalize.normalize_tiktok_items(self.raw_items, "2026-02-01", "2026-03-03")
- self.assertEqual(items[0].hashtags, ["claudecode", "ai", "coding"])
-
- def test_caption_snippet_preserved(self):
- items = normalize.normalize_tiktok_items(self.raw_items, "2026-02-01", "2026-03-03")
- self.assertIn("slash commands", items[0].caption_snippet)
-
-
-class TestScoreTikTokItems(unittest.TestCase):
- """Test TikTok scoring."""
-
- def test_engagement_scoring(self):
- eng = schema.Engagement(views=1000000, likes=50000, num_comments=2000)
- raw = score.compute_tiktok_engagement_raw(eng)
- self.assertIsNotNone(raw)
- self.assertGreater(raw, 0)
-
- def test_none_engagement(self):
- raw = score.compute_tiktok_engagement_raw(None)
- self.assertIsNone(raw)
-
- def test_empty_engagement(self):
- eng = schema.Engagement()
- raw = score.compute_tiktok_engagement_raw(eng)
- self.assertIsNone(raw)
-
- def test_scoring_pipeline(self):
- items = [
- schema.TikTokItem(
- id="TK1", text="High views video", url="https://tiktok.com/1",
- author_name="creator1", date="2026-03-01",
- engagement=schema.Engagement(views=2000000, likes=50000, num_comments=1000),
- relevance=0.9,
- ),
- schema.TikTokItem(
- id="TK2", text="Low views video", url="https://tiktok.com/2",
- author_name="creator2", date="2026-02-20",
- engagement=schema.Engagement(views=1000, likes=50, num_comments=5),
- relevance=0.5,
- ),
- ]
- scored = score.score_tiktok_items(items)
- self.assertEqual(len(scored), 2)
- self.assertGreater(scored[0].score, 0)
- self.assertGreater(scored[0].score, scored[1].score)
-
-
-class TestDedupeTikTok(unittest.TestCase):
- """Test TikTok deduplication."""
-
- def test_no_dupes(self):
- items = [
- schema.TikTokItem(id="TK1", text="Totally different video A",
- url="https://tiktok.com/1", author_name="a", score=80),
- schema.TikTokItem(id="TK2", text="Completely unique video B",
- url="https://tiktok.com/2", author_name="b", score=70),
- ]
- result = dedupe.dedupe_tiktok(items)
- self.assertEqual(len(result), 2)
-
- def test_removes_dupes(self):
- items = [
- schema.TikTokItem(id="TK1", text="Claude Code is amazing for AI coding",
- url="https://tiktok.com/1", author_name="a", score=80),
- schema.TikTokItem(id="TK2", text="Claude Code is amazing for AI coding wow",
- url="https://tiktok.com/2", author_name="a", score=60),
- ]
- result = dedupe.dedupe_tiktok(items)
- self.assertEqual(len(result), 1)
- self.assertEqual(result[0].id, "TK1") # Higher score kept
-
-
-class TestRenderTikTok(unittest.TestCase):
- """Test TikTok rendering in reports."""
-
- def test_renders_tiktok_section(self):
- report = schema.Report(
- topic="test", range_from="2026-02-01", range_to="2026-03-03",
- generated_at="2026-03-03T00:00:00Z", mode="all",
- tiktok=[
- schema.TikTokItem(
- id="TK1", text="Video caption here", url="https://tiktok.com/1",
- author_name="creator", date="2026-03-01", score=85,
- engagement=schema.Engagement(views=1000000, likes=50000),
- hashtags=["ai", "coding"],
- why_relevant="TikTok: Video caption here",
- ),
- ],
+ def test_default_depth_returns_two_plus_queries(self):
+ from lib.tiktok import expand_tiktok_queries
+ queries = expand_tiktok_queries("Kanye West", "default")
+ self.assertGreaterEqual(len(queries), 2)
+ # Breaking_news intent should include reaction/edit variant
+ variant_found = any(
+ "reaction" in q.lower() or "edit" in q.lower() or "trend" in q.lower()
+ for q in queries
)
- output = render.render_compact(report)
- self.assertIn("### TikTok Videos", output)
- self.assertIn("TK1", output)
- self.assertIn("@creator", output)
- self.assertIn("1,000,000 views", output)
+ self.assertTrue(variant_found, f"Expected reaction/edit/trend variant: {queries}")
- def test_renders_source_status(self):
- report = schema.Report(
- topic="test", range_from="2026-02-01", range_to="2026-03-03",
- generated_at="2026-03-03T00:00:00Z", mode="all",
- tiktok=[
- schema.TikTokItem(
- id="TK1", text="test", url="https://tiktok.com/1",
- author_name="creator", caption_snippet="some caption",
- ),
- ],
+ def test_product_intent_includes_review_variant(self):
+ from lib.tiktok import expand_tiktok_queries
+ # "best laptop for coding" triggers the product intent (best .* for pattern)
+ queries = expand_tiktok_queries("best laptop for coding", "deep")
+ variant_found = any(
+ "review" in q.lower() or "haul" in q.lower() or "unboxing" in q.lower()
+ for q in queries
)
- status = render.render_source_status(report)
- self.assertIn("TikTok", status)
- self.assertIn("1 videos", status)
+ self.assertTrue(variant_found, f"Expected review/haul/unboxing variant: {queries}")
- def test_xref_tag_tiktok(self):
- """Test that TK prefix is recognized in cross-ref tags."""
- item = schema.RedditItem(id="R1", title="test", url="test", subreddit="test",
- cross_refs=["TK1"])
- tag = render._xref_tag(item)
- self.assertIn("TikTok", tag)
-
-
-class TestSchemaRoundtrip(unittest.TestCase):
- """Test TikTokItem serialization round-trip via Report."""
-
- def test_to_dict_and_back(self):
- original = schema.TikTokItem(
- id="TK1", text="Test caption", url="https://tiktok.com/1",
- author_name="creator", date="2026-03-01",
- date_confidence="high",
- engagement=schema.Engagement(views=100, likes=10, num_comments=5, shares=3),
- caption_snippet="spoken words",
- hashtags=["test", "ai"],
- relevance=0.8, why_relevant="TikTok: Test",
- subs=schema.SubScores(relevance=80, recency=90, engagement=70),
- score=80, cross_refs=["R1"],
- )
- report = schema.Report(
- topic="test", range_from="2026-02-01", range_to="2026-03-03",
- generated_at="2026-03-03T00:00:00Z", mode="all",
- tiktok=[original],
- )
- d = report.to_dict()
- restored = schema.Report.from_dict(d)
- self.assertEqual(len(restored.tiktok), 1)
- tk = restored.tiktok[0]
- self.assertEqual(tk.id, "TK1")
- self.assertEqual(tk.author_name, "creator")
- self.assertEqual(tk.hashtags, ["test", "ai"])
- self.assertEqual(tk.engagement.views, 100)
- self.assertEqual(tk.engagement.shares, 3)
- self.assertEqual(tk.caption_snippet, "spoken words")
- self.assertEqual(tk.cross_refs, ["R1"])
+ def test_quick_depth_returns_one_query(self):
+ from lib.tiktok import expand_tiktok_queries
+ queries = expand_tiktok_queries("Kanye West", "quick")
+ self.assertEqual(len(queries), 1)
if __name__ == "__main__":
diff --git a/tests/test_ui_v3.py b/tests/test_ui_v3.py
new file mode 100644
index 0000000..125e7dd
--- /dev/null
+++ b/tests/test_ui_v3.py
@@ -0,0 +1,77 @@
+# ruff: noqa: E402
+import io
+import sys
+import unittest
+from contextlib import redirect_stderr
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib import ui
+
+
+class UiV3Tests(unittest.TestCase):
+ def test_show_diagnostic_banner_uses_v3_source_model(self):
+ diag = {
+ "available_sources": ["grounding", "youtube"],
+ "providers": {"google": True, "openai": False, "xai": False},
+ "x_backend": None,
+ "bird_installed": True,
+ "bird_authenticated": False,
+ "bird_username": None,
+ "native_web_backend": "brave",
+ }
+ with mock.patch.object(ui, "IS_TTY", False):
+ stderr = io.StringIO()
+ with redirect_stderr(stderr):
+ ui.show_diagnostic_banner(diag)
+ output = stderr.getvalue()
+ self.assertIn("Reddit", output)
+ self.assertIn("unavailable", output)
+ self.assertIn("Add AUTH_TOKEN/CT0 or XAI_API_KEY", output)
+ self.assertIn("brave API available", output)
+
+ def test_build_nux_message_mentions_v3_unlock_paths(self):
+ text = ui._build_nux_message(
+ {"available_sources": ["reddit", "youtube", "grounding"]}
+ )
+ self.assertIn("Reddit ✓, X ✗, YouTube ✓, Web ✓", text)
+ self.assertIn("works fine as-is", text)
+ self.assertIn("all free", text)
+
+ def test_show_complete_uses_actual_sources_for_source_restricted_runs(self):
+ with mock.patch.object(ui, "IS_TTY", False):
+ stderr = io.StringIO()
+ with redirect_stderr(stderr):
+ progress = ui.ProgressDisplay("test topic", show_banner=False)
+ progress.show_complete(
+ source_counts={"grounding": 2},
+ display_sources=["grounding"],
+ )
+ output = stderr.getvalue()
+ self.assertIn("Web: 2 results", output)
+ self.assertNotIn("Reddit:", output)
+ self.assertNotIn("X:", output)
+
+ def test_show_complete_supports_newer_sources(self):
+ with mock.patch.object(ui, "IS_TTY", False):
+ stderr = io.StringIO()
+ with redirect_stderr(stderr):
+ progress = ui.ProgressDisplay("test topic", show_banner=False)
+ progress.show_complete(
+ source_counts={
+ "bluesky": 3,
+ "truthsocial": 1,
+ "xiaohongshu": 4,
+ },
+ display_sources=["bluesky", "truthsocial", "xiaohongshu"],
+ )
+ output = stderr.getvalue()
+ self.assertIn("Bluesky: 3 posts", output)
+ self.assertIn("Truth Social: 1 post", output)
+ self.assertIn("Xiaohongshu: 4 posts", output)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_verify_v3.py b/tests/test_verify_v3.py
new file mode 100644
index 0000000..f985ea2
--- /dev/null
+++ b/tests/test_verify_v3.py
@@ -0,0 +1,27 @@
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+def load_verify_module():
+ path = Path(__file__).resolve().parents[1] / "scripts" / "verify_v3.py"
+ spec = importlib.util.spec_from_file_location("verify_v3_module", path)
+ module = importlib.util.module_from_spec(spec)
+ assert spec and spec.loader
+ spec.loader.exec_module(module)
+ return module
+
+
+class VerifyV3Tests(unittest.TestCase):
+ def test_parser_defaults(self):
+ module = load_verify_module()
+ parser = module.build_parser()
+ args = parser.parse_args([])
+ self.assertEqual(args.baseline, "HEAD~1")
+ self.assertEqual(args.candidate, "WORKTREE")
+ self.assertFalse(args.skip_eval)
+ self.assertFalse(args.skip_latency)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_watchlist_commands.py b/tests/test_watchlist_commands.py
new file mode 100644
index 0000000..a162e11
--- /dev/null
+++ b/tests/test_watchlist_commands.py
@@ -0,0 +1,487 @@
+"""Tests for watchlist.py command functions."""
+
+import json
+import sqlite3
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+import store
+import watchlist
+from lib import schema
+
+
+@pytest.fixture
+def temp_db():
+ """Create a temporary database for testing."""
+ with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
+ db_path = Path(f.name)
+
+ # Override the database path
+ original_override = store._db_override
+ store._db_override = db_path
+
+ # Initialize fresh database
+ store.init_db()
+
+ yield db_path
+
+ # Cleanup
+ store._db_override = original_override
+ if db_path.exists():
+ db_path.unlink()
+
+
+# === Tests for cmd_add() ===
+
+def test_cmd_add_basic(temp_db, capsys):
+ """Test adding a topic with default schedule."""
+ args = Mock()
+ args.topic = "Test Topic"
+ args.weekly = False
+ args.schedule = None
+ args.queries = None
+
+ watchlist.cmd_add(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert output["action"] == "added"
+ assert output["topic"] == "Test Topic"
+ assert "daily" in output["schedule"]
+
+
+def test_cmd_add_with_custom_schedule(temp_db, capsys):
+ """Test adding a topic with custom schedule."""
+ args = Mock()
+ args.topic = "Test Topic"
+ args.weekly = False
+ args.schedule = "0 12 * * *"
+ args.queries = None
+
+ watchlist.cmd_add(args)
+
+ # Verify in database
+ topic = store.get_topic("Test Topic")
+ assert topic["schedule"] == "0 12 * * *"
+
+
+def test_cmd_add_weekly(temp_db, capsys):
+ """Test adding a topic with weekly schedule."""
+ args = Mock()
+ args.topic = "Test Topic"
+ args.weekly = True
+ args.schedule = None
+ args.queries = None
+
+ watchlist.cmd_add(args)
+
+ # Verify weekly schedule
+ topic = store.get_topic("Test Topic")
+ assert topic["schedule"] == "0 8 * * 1" # Monday 8am
+
+
+def test_cmd_add_with_search_queries(temp_db, capsys):
+ """Test adding a topic with custom search queries."""
+ args = Mock()
+ args.topic = "Test Topic"
+ args.weekly = False
+ args.schedule = None
+ args.queries = "query1, query2, query3"
+
+ watchlist.cmd_add(args)
+
+ # Verify queries stored
+ topic = store.get_topic("Test Topic")
+ queries = json.loads(topic["search_queries"])
+ assert queries == ["query1", "query2", "query3"]
+
+
+# === Tests for cmd_remove() ===
+
+def test_cmd_remove_existing_topic(temp_db, capsys):
+ """Test removing an existing topic."""
+ # Add a topic first
+ store.add_topic("Test Topic")
+
+ args = Mock()
+ args.topic = "Test Topic"
+
+ watchlist.cmd_remove(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert output["action"] == "removed"
+ assert output["topic"] == "Test Topic"
+
+
+def test_cmd_remove_nonexistent_topic(temp_db, capsys):
+ """Test removing a topic that doesn't exist."""
+ args = Mock()
+ args.topic = "Nonexistent Topic"
+
+ watchlist.cmd_remove(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert output["action"] == "not_found"
+ assert output["topic"] == "Nonexistent Topic"
+
+
+# === Tests for cmd_list() ===
+
+def test_cmd_list_empty(temp_db, capsys):
+ """Test listing when no topics exist."""
+ args = Mock()
+
+ watchlist.cmd_list(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert output["topics"] == []
+ assert output["budget_used"] == 0.0
+ assert output["budget_limit"] == 5.0
+
+
+def test_cmd_list_with_topics(temp_db, capsys):
+ """Test listing with multiple topics."""
+ # Add topics
+ store.add_topic("Topic 1")
+ store.add_topic("Topic 2")
+ store.add_topic("Topic 3")
+
+ args = Mock()
+
+ watchlist.cmd_list(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert len(output["topics"]) == 3
+ topic_names = {t["name"] for t in output["topics"]}
+ assert topic_names == {"Topic 1", "Topic 2", "Topic 3"}
+
+
+# === Tests for cmd_config() ===
+
+def test_cmd_config_delivery(temp_db, capsys):
+ """Test configuring delivery channel."""
+ args = Mock()
+ args.key = "delivery"
+ args.value = "https://hooks.slack.com/services/TEST"
+
+ watchlist.cmd_config(args)
+
+ # Verify setting stored
+ channel = store.get_setting("delivery_channel")
+ assert channel == "https://hooks.slack.com/services/TEST"
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert output["action"] == "config"
+ assert output["key"] == "delivery_channel"
+
+
+def test_cmd_config_budget(temp_db, capsys):
+ """Test configuring daily budget."""
+ args = Mock()
+ args.key = "budget"
+ args.value = 10.0
+
+ watchlist.cmd_config(args)
+
+ # Verify setting stored
+ budget = store.get_setting("daily_budget")
+ assert budget == "10.0"
+
+
+def test_cmd_config_unknown_key(temp_db):
+ """Test that unknown config key raises error."""
+ args = Mock()
+ args.key = "unknown_key"
+ args.value = "value"
+
+ with pytest.raises(SystemExit):
+ watchlist.cmd_config(args)
+
+
+# === Tests for _run_topic() ===
+
+@patch('watchlist.subprocess.run')
+def test_run_topic_success(mock_subprocess, temp_db):
+ """Test successful topic run."""
+ topic = store.add_topic("Test Topic")
+
+ # Mock successful subprocess call
+ mock_result = Mock()
+ mock_result.returncode = 0
+ mock_result.stdout = json.dumps({
+ "topic": "Test Topic",
+ "range_from": "2026-01-01",
+ "range_to": "2026-04-03",
+ "generated_at": "2026-04-03T00:00:00Z",
+ "provider_runtime": {
+ "reasoning_provider": "gemini",
+ "planner_model": "gemini-2.0-flash-exp",
+ "rerank_model": "gemini-2.0-flash-exp",
+ },
+ "query_plan": {
+ "intent": "test",
+ "freshness_mode": "recent",
+ "cluster_mode": "standard",
+ "raw_topic": "test",
+ "subqueries": [],
+ "source_weights": {},
+ },
+ "clusters": [],
+ "ranked_candidates": [],
+ "items_by_source": {
+ "reddit": [
+ {
+ "item_id": "R1",
+ "source": "reddit",
+ "title": "Test",
+ "body": "Content",
+ "url": "https://reddit.com/1",
+ "author": "user",
+ "engagement_score": 50.0,
+ "local_relevance": 0.8,
+ "snippet": "Snippet",
+ }
+ ],
+ },
+ "errors_by_source": {},
+ "warnings": [],
+ })
+ mock_subprocess.return_value = mock_result
+
+ result = watchlist._run_topic(topic)
+
+ assert result["status"] == "completed"
+ assert result["new"] == 1
+ assert result["topic"] == "Test Topic"
+
+
+@patch('watchlist.subprocess.run')
+def test_run_topic_failure(mock_subprocess, temp_db):
+ """Test topic run failure."""
+ topic = store.add_topic("Test Topic")
+
+ # Mock failed subprocess call
+ mock_result = Mock()
+ mock_result.returncode = 1
+ mock_result.stderr = "Error message"
+ mock_subprocess.return_value = mock_result
+
+ result = watchlist._run_topic(topic)
+
+ assert result["status"] == "failed"
+ assert "Error message" in result["error"]
+
+
+@patch('watchlist.subprocess.run')
+def test_run_topic_timeout(mock_subprocess, temp_db):
+ """Test topic run timeout."""
+ topic = store.add_topic("Test Topic")
+
+ # Mock timeout
+ mock_subprocess.side_effect = subprocess.TimeoutExpired("cmd", 300)
+
+ result = watchlist._run_topic(topic)
+
+ assert result["status"] == "failed"
+ assert result["error"] == "timeout"
+
+
+@patch('watchlist.subprocess.run')
+@patch('watchlist._deliver_findings')
+def test_run_topic_calls_delivery(mock_deliver, mock_subprocess, temp_db):
+ """Test that successful run calls delivery."""
+ topic = store.add_topic("Test Topic")
+
+ # Mock successful subprocess call with findings
+ mock_result = Mock()
+ mock_result.returncode = 0
+ mock_result.stdout = json.dumps({
+ "topic": "Test Topic",
+ "range_from": "2026-01-01",
+ "range_to": "2026-04-03",
+ "generated_at": "2026-04-03T00:00:00Z",
+ "provider_runtime": {
+ "reasoning_provider": "gemini",
+ "planner_model": "gemini-2.0-flash-exp",
+ "rerank_model": "gemini-2.0-flash-exp",
+ },
+ "query_plan": {
+ "intent": "test",
+ "freshness_mode": "recent",
+ "cluster_mode": "standard",
+ "raw_topic": "test",
+ "subqueries": [],
+ "source_weights": {},
+ },
+ "clusters": [],
+ "ranked_candidates": [],
+ "items_by_source": {
+ "reddit": [
+ {
+ "item_id": "R1",
+ "source": "reddit",
+ "title": "Test",
+ "body": "Content",
+ "url": "https://reddit.com/1",
+ "author": "user",
+ "engagement_score": 50.0,
+ "local_relevance": 0.8,
+ "snippet": "Snippet",
+ }
+ ],
+ },
+ "errors_by_source": {},
+ "warnings": [],
+ })
+ mock_subprocess.return_value = mock_result
+
+ watchlist._run_topic(topic)
+
+ # Verify delivery was called
+ assert mock_deliver.called
+ call_args = mock_deliver.call_args[0]
+ assert call_args[0] == "Test Topic"
+ assert call_args[1]["new"] == 1
+
+
+# === Tests for cmd_run_one() ===
+
+@patch('watchlist._run_topic')
+def test_cmd_run_one(mock_run, temp_db, capsys):
+ """Test running a single topic."""
+ topic = store.add_topic("Test Topic")
+
+ mock_run.return_value = {
+ "topic": "Test Topic",
+ "status": "completed",
+ "new": 5,
+ "updated": 2,
+ "duration": 60.0,
+ }
+
+ args = Mock()
+ args.topic = "Test Topic"
+
+ watchlist.cmd_run_one(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert output["status"] == "completed"
+ assert output["new"] == 5
+
+
+def test_cmd_run_one_nonexistent_topic(temp_db, capsys):
+ """Test running a nonexistent topic."""
+ args = Mock()
+ args.topic = "Nonexistent Topic"
+
+ with pytest.raises(SystemExit):
+ watchlist.cmd_run_one(args)
+
+
+# === Tests for cmd_run_all() ===
+
+@patch('watchlist._run_topic')
+def test_cmd_run_all_no_topics(mock_run, temp_db, capsys):
+ """Test running all topics when none exist."""
+ args = Mock()
+
+ watchlist.cmd_run_all(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert "No enabled topics" in output["message"]
+
+
+@patch('watchlist._run_topic')
+def test_cmd_run_all_multiple_topics(mock_run, temp_db, capsys):
+ """Test running multiple topics."""
+ # Add topics
+ store.add_topic("Topic 1")
+ store.add_topic("Topic 2")
+
+ mock_run.return_value = {
+ "topic": "Test",
+ "status": "completed",
+ "new": 5,
+ "updated": 2,
+ "duration": 60.0,
+ }
+
+ args = Mock()
+
+ watchlist.cmd_run_all(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ assert output["action"] == "run_all"
+ assert len(output["results"]) == 2
+
+
+@patch('watchlist._run_topic')
+@patch('watchlist.store.get_daily_cost')
+def test_cmd_run_all_respects_budget(mock_cost, mock_run, temp_db, capsys):
+ """Test that run-all respects daily budget."""
+ # Add topics
+ store.add_topic("Topic 1")
+ store.add_topic("Topic 2")
+ store.add_topic("Topic 3")
+
+ # Mock budget exceeded (budget limit is 5.0)
+ mock_cost.return_value = 6.0 # Over budget
+
+ mock_run.return_value = {
+ "topic": "Test",
+ "status": "completed",
+ "new": 5,
+ "updated": 2,
+ "duration": 60.0,
+ }
+
+ args = Mock()
+
+ watchlist.cmd_run_all(args)
+
+ # Verify output
+ captured = capsys.readouterr()
+ output = json.loads(captured.out)
+
+ # All topics should be skipped due to budget
+ results = output["results"]
+ skipped = [r for r in results if r["status"] == "skipped"]
+
+ assert len(skipped) == 3 # All 3 topics skipped
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_watchlist_delivery.py b/tests/test_watchlist_delivery.py
new file mode 100644
index 0000000..7cf7486
--- /dev/null
+++ b/tests/test_watchlist_delivery.py
@@ -0,0 +1,291 @@
+"""Tests for watchlist.py delivery functions (PR #86 feature)."""
+
+import sys
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+
+import watchlist
+
+
+# === Tests for _format_delivery_message() ===
+
+def test_format_message_announce_mode():
+ """Test announce mode formatting (default mode with emoji)."""
+ message = watchlist._format_delivery_message(
+ "Test Topic", {"new": 5, "updated": 2}, "announce"
+ )
+
+ assert "📰" in message
+ assert "Test Topic" in message
+ assert "5 new" in message
+ assert "2 updated" in message
+
+
+def test_format_message_silent_mode():
+ """Test silent mode formatting (no emoji)."""
+ message = watchlist._format_delivery_message(
+ "Test Topic", {"new": 5, "updated": 2}, "silent"
+ )
+
+ assert "📰" not in message
+ assert "Test Topic" in message
+ assert "5 new" in message
+
+
+def test_format_message_default_mode():
+ """Test default mode formatting."""
+ message = watchlist._format_delivery_message(
+ "Test Topic", {"new": 5, "updated": 2}, "default"
+ )
+
+ assert "complete" in message.lower()
+ assert "Test Topic" in message
+
+
+def test_format_message_handles_zero_counts():
+ """Test formatting with zero counts."""
+ message = watchlist._format_delivery_message(
+ "Test Topic", {"new": 0, "updated": 0}, "announce"
+ )
+
+ assert "0 new" in message
+ assert "0 updated" in message
+
+
+# === Tests for _send_slack_webhook() ===
+
+@patch('watchlist.requests')
+def test_send_slack_webhook_format(mock_requests):
+ """Test that Slack webhook uses correct format."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_requests.post.return_value = mock_response
+
+ watchlist._send_slack_webhook(
+ "https://hooks.slack.com/services/TEST",
+ "Test message"
+ )
+
+ # Verify POST was called with correct format
+ assert mock_requests.post.called
+ call_args = mock_requests.post.call_args
+
+ assert call_args[0][0] == "https://hooks.slack.com/services/TEST"
+ assert call_args[1]["json"] == {"text": "Test message"}
+ assert call_args[1]["headers"]["Content-Type"] == "application/json"
+ assert call_args[1]["timeout"] == 10
+
+
+@patch('watchlist.requests')
+def test_send_slack_webhook_raises_on_error(mock_requests):
+ """Test that Slack webhook raises on HTTP error."""
+ mock_response = Mock()
+ mock_response.status_code = 400
+ mock_response.raise_for_status.side_effect = Exception("HTTP 400")
+ mock_requests.post.return_value = mock_response
+
+ with pytest.raises(Exception, match="HTTP 400"):
+ watchlist._send_slack_webhook(
+ "https://hooks.slack.com/services/TEST",
+ "Test message"
+ )
+
+
+# === Tests for _send_generic_webhook() ===
+
+@patch('watchlist.requests')
+def test_send_generic_webhook_format(mock_requests):
+ """Test that generic webhook uses correct format."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_requests.post.return_value = mock_response
+
+ watchlist._send_generic_webhook(
+ "https://webhook.example.com/hook",
+ "Test message"
+ )
+
+ # Verify POST was called with correct format
+ assert mock_requests.post.called
+ call_args = mock_requests.post.call_args
+
+ assert call_args[0][0] == "https://webhook.example.com/hook"
+
+ json_data = call_args[1]["json"]
+ assert json_data["message"] == "Test message"
+ assert json_data["source"] == "last30days"
+ assert "timestamp" in json_data
+ assert isinstance(json_data["timestamp"], float)
+
+
+@patch('watchlist.requests')
+def test_send_generic_webhook_raises_on_error(mock_requests):
+ """Test that generic webhook raises on HTTP error."""
+ mock_response = Mock()
+ mock_response.status_code = 500
+ mock_response.raise_for_status.side_effect = Exception("HTTP 500")
+ mock_requests.post.return_value = mock_response
+
+ with pytest.raises(Exception, match="HTTP 500"):
+ watchlist._send_generic_webhook(
+ "https://webhook.example.com/hook",
+ "Test message"
+ )
+
+
+# === Tests for _deliver_findings() ===
+
+@patch('watchlist.store.get_setting')
+@patch('watchlist.requests')
+def test_deliver_findings_sends_when_new_greater_than_zero(mock_requests, mock_get_setting):
+ """Test that delivery fires when new > 0."""
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "https://webhook.example.com/test",
+ "delivery_mode": "announce",
+ }.get(key, default)
+
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_requests.post.return_value = mock_response
+
+ watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
+
+ # Verify webhook was called
+ assert mock_requests.post.called
+
+
+@patch('watchlist.store.get_setting')
+@patch('watchlist.requests')
+def test_deliver_findings_skips_when_new_is_zero(mock_requests, mock_get_setting):
+ """Test that delivery is skipped when new=0."""
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "https://webhook.example.com/test",
+ "delivery_mode": "announce",
+ }.get(key, default)
+
+ watchlist._deliver_findings("Test Topic", {"new": 0, "updated": 5})
+
+ # Verify webhook was NOT called
+ assert not mock_requests.post.called
+
+
+@patch('watchlist.store.get_setting')
+@patch('watchlist.requests')
+def test_deliver_findings_skips_when_channel_empty(mock_requests, mock_get_setting):
+ """Test that delivery is skipped when delivery_channel is empty."""
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "",
+ "delivery_mode": "announce",
+ }.get(key, default)
+
+ watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
+
+ # Verify webhook was NOT called
+ assert not mock_requests.post.called
+
+
+@patch('watchlist.store.get_setting')
+@patch('watchlist.requests')
+def test_deliver_findings_uses_slack_format_for_slack_urls(mock_requests, mock_get_setting):
+ """Test that Slack URLs trigger Slack-specific format."""
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "https://hooks.slack.com/services/TEST",
+ "delivery_mode": "announce",
+ }.get(key, default)
+
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_requests.post.return_value = mock_response
+
+ watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
+
+ # Verify Slack format was used
+ call_args = mock_requests.post.call_args
+ json_data = call_args[1]["json"]
+ assert "text" in json_data
+ assert "Test Topic" in json_data["text"]
+
+
+@patch('watchlist.store.get_setting')
+@patch('watchlist.requests')
+def test_deliver_findings_uses_generic_format_for_other_urls(mock_requests, mock_get_setting):
+ """Test that non-Slack URLs trigger generic format."""
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "https://webhook.example.com/test",
+ "delivery_mode": "announce",
+ }.get(key, default)
+
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_requests.post.return_value = mock_response
+
+ watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
+
+ # Verify generic format was used
+ call_args = mock_requests.post.call_args
+ json_data = call_args[1]["json"]
+ assert "message" in json_data
+ assert "source" in json_data
+ assert "timestamp" in json_data
+
+
+@patch('watchlist.store.get_setting')
+@patch('watchlist.requests')
+def test_deliver_findings_handles_failure_gracefully(mock_requests, mock_get_setting, capsys):
+ """Test that delivery failures don't crash the process."""
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "https://webhook.example.com/test",
+ "delivery_mode": "announce",
+ }.get(key, default)
+
+ # Simulate HTTP error
+ mock_response = Mock()
+ mock_response.status_code = 500
+ mock_response.raise_for_status.side_effect = Exception("HTTP 500")
+ mock_requests.post.return_value = mock_response
+
+ # Should not raise, just log to stderr
+ watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
+
+ # Verify error was logged
+ captured = capsys.readouterr()
+ assert "Delivery failed" in captured.err
+
+
+@patch('watchlist.store.get_setting')
+@patch('watchlist.requests')
+def test_deliver_findings_respects_delivery_mode(mock_requests, mock_get_setting):
+ """Test that different delivery modes produce different messages."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_requests.post.return_value = mock_response
+
+ # Test announce mode
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "https://webhook.example.com/test",
+ "delivery_mode": "announce",
+ }.get(key, default)
+
+ watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
+
+ announce_message = mock_requests.post.call_args[1]["json"]["message"]
+ assert "📰" in announce_message
+
+ # Test silent mode
+ mock_get_setting.side_effect = lambda key, default="": {
+ "delivery_channel": "https://webhook.example.com/test",
+ "delivery_mode": "silent",
+ }.get(key, default)
+
+ watchlist._deliver_findings("Test Topic", {"new": 5, "updated": 2})
+
+ silent_message = mock_requests.post.call_args[1]["json"]["message"]
+ assert "📰" not in silent_message
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_xai_x.py b/tests/test_xai_x.py
new file mode 100644
index 0000000..eaa5ca0
--- /dev/null
+++ b/tests/test_xai_x.py
@@ -0,0 +1,68 @@
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from lib.xai_x import parse_x_response
+
+
+def _wrap_items(items):
+ """Wrap items list in the xAI response envelope."""
+ payload = json.dumps({"items": items})
+ return {
+ "output": [
+ {
+ "type": "message",
+ "content": [{"type": "output_text", "text": payload}],
+ }
+ ]
+ }
+
+
+class TestXaiXEngagementZero(unittest.TestCase):
+ def test_zero_likes_preserved(self):
+ raw_items = [
+ {
+ "text": "test",
+ "url": "https://x.com/u/status/1",
+ "engagement": {"likes": 0, "reposts": 5},
+ "relevance": 0.8,
+ }
+ ]
+ items = parse_x_response(_wrap_items(raw_items))
+ self.assertEqual(0, items[0]["engagement"]["likes"])
+ self.assertEqual(5, items[0]["engagement"]["reposts"])
+
+ def test_none_engagement_when_missing(self):
+ raw_items = [
+ {
+ "text": "test",
+ "url": "https://x.com/u/status/1",
+ "engagement": {},
+ "relevance": 0.8,
+ }
+ ]
+ items = parse_x_response(_wrap_items(raw_items))
+ self.assertIsNone(items[0]["engagement"]["likes"])
+
+ def test_nonzero_engagement(self):
+ raw_items = [
+ {
+ "text": "test",
+ "url": "https://x.com/u/status/1",
+ "engagement": {"likes": 42, "reposts": 3, "replies": 1, "quotes": 0},
+ "relevance": 0.9,
+ }
+ ]
+ items = parse_x_response(_wrap_items(raw_items))
+ eng = items[0]["engagement"]
+ self.assertEqual(42, eng["likes"])
+ self.assertEqual(3, eng["reposts"])
+ self.assertEqual(1, eng["replies"])
+ self.assertEqual(0, eng["quotes"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py
index a274982..504ef9a 100644
--- a/tests/test_youtube_yt.py
+++ b/tests/test_youtube_yt.py
@@ -1,4 +1,4 @@
-"""Tests for yt-dlp invocation safety flags."""
+"""Tests for YouTube transcript highlights and yt-dlp safety flags."""
import json
import sys
@@ -25,6 +25,43 @@ class _DummyProc:
return 0
+class TestYouTubeEngagementZero(unittest.TestCase):
+ """Verify that 0 engagement counts are preserved (not coerced to fallback)."""
+
+ def test_zero_view_count_preserved(self):
+ """video.get('view_count') == 0 must stay 0, not become the fallback."""
+ import json
+ import tempfile
+ import os
+
+ video = {
+ "id": "abc123",
+ "title": "Test",
+ "view_count": 0,
+ "like_count": 0,
+ "comment_count": 0,
+ "upload_date": "20260301",
+ "description": "desc",
+ }
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f:
+ f.write(json.dumps(video) + "\n")
+ f.flush()
+ with open(f.name) as rf:
+ lines = rf.readlines()
+
+ # Re-parse as the search function would
+ parsed = json.loads(lines[0])
+ view_count = parsed.get("view_count") if parsed.get("view_count") is not None else 0
+ like_count = parsed.get("like_count") if parsed.get("like_count") is not None else 0
+ comment_count = parsed.get("comment_count") if parsed.get("comment_count") is not None else 0
+
+ os.unlink(f.name)
+
+ self.assertEqual(0, view_count)
+ self.assertEqual(0, like_count)
+ self.assertEqual(0, comment_count)
+
+
class TestYtDlpFlags(unittest.TestCase):
def test_search_ignores_global_config_and_browser_cookies(self):
proc = _DummyProc()
@@ -61,7 +98,6 @@ class TestExtractTranscriptHighlights(unittest.TestCase):
)
highlights = youtube_yt.extract_transcript_highlights(transcript, "Lego")
self.assertTrue(len(highlights) > 0)
- # Should pick the sentences with numbers and topic relevance, not filler
joined = " ".join(highlights)
self.assertIn("13,438", joined)
self.assertNotIn("subscribe", joined)
@@ -78,6 +114,17 @@ class TestExtractTranscriptHighlights(unittest.TestCase):
highlights = youtube_yt.extract_transcript_highlights(sentences, "model", limit=3)
self.assertEqual(len(highlights), 3)
+ def test_punctuation_free_transcript_produces_highlights(self):
+ # Auto-generated YouTube captions often lack sentence-ending punctuation
+ words = (
+ "the new Tesla Model Y has 350 miles of range and costs about 45000 dollars "
+ "which makes it one of the most affordable electric vehicles on the market today "
+ "compared to the BMW iX which starts at 87000 the value proposition is pretty clear "
+ "and with the 7500 dollar tax credit you can get it for under 40000"
+ )
+ highlights = youtube_yt.extract_transcript_highlights(words, "Tesla Model Y")
+ self.assertTrue(len(highlights) > 0, "Should produce highlights from punctuation-free text")
+
class TestFetchTranscriptDirect(unittest.TestCase):
"""Tests for _fetch_transcript_direct() — direct HTTP transcript fetching."""
@@ -214,5 +261,149 @@ class TestFetchTranscriptFallback(unittest.TestCase):
self.assertIsNone(result)
+class TestExpandYouTubeQueries(unittest.TestCase):
+ """Tests for expand_youtube_queries() multi-query generation."""
+
+ def test_default_depth_returns_two_plus_queries(self):
+ queries = youtube_yt.expand_youtube_queries("Kanye West", "default")
+ self.assertGreaterEqual(len(queries), 2)
+ # First query is the core subject
+ self.assertEqual(queries[0].lower(), "kanye west")
+
+ def test_how_to_intent_includes_tutorial_variant(self):
+ # Use deep depth so the intent variant isn't capped out by core + original
+ queries = youtube_yt.expand_youtube_queries("how to use Docker", "deep")
+ variant_found = any(
+ "tutorial" in q.lower() or "guide" in q.lower() or "explained" in q.lower()
+ for q in queries
+ )
+ self.assertTrue(
+ variant_found,
+ f"Expected tutorial/guide/explained in queries: {queries}",
+ )
+
+ def test_product_intent_includes_review_variant(self):
+ # Use deep depth so the intent variant isn't capped out
+ queries = youtube_yt.expand_youtube_queries("best running shoes", "deep")
+ variant_found = any("review" in q.lower() for q in queries)
+ self.assertTrue(variant_found, f"Expected 'review' in queries: {queries}")
+
+ def test_comparison_intent_includes_vs_variant(self):
+ queries = youtube_yt.expand_youtube_queries("Claude vs Gemini", "default")
+ variant_found = any("vs" in q.lower() or "compared" in q.lower() for q in queries)
+ self.assertTrue(variant_found, f"Expected 'vs' or 'compared' in queries: {queries}")
+
+ def test_quick_depth_returns_one_query(self):
+ queries = youtube_yt.expand_youtube_queries("Kanye West", "quick")
+ self.assertEqual(len(queries), 1)
+
+ def test_deep_depth_returns_three_queries(self):
+ queries = youtube_yt.expand_youtube_queries("Kanye West", "deep")
+ self.assertEqual(len(queries), 3)
+
+ def test_single_word_returns_at_least_one(self):
+ queries = youtube_yt.expand_youtube_queries("React", "default")
+ self.assertGreaterEqual(len(queries), 1)
+
+ def test_temporal_words_stripped_from_core(self):
+ queries = youtube_yt.expand_youtube_queries("kanye west last 30 days", "default")
+ core = queries[0].lower()
+ self.assertNotIn("last", core)
+ self.assertNotIn("days", core)
+ self.assertIn("kanye", core)
+ self.assertIn("west", core)
+
+
+class TestInferQueryIntent(unittest.TestCase):
+ """Tests for _infer_query_intent() classification."""
+
+ def test_comparison_intent(self):
+ self.assertEqual(youtube_yt._infer_query_intent("Claude vs Gemini"), "comparison")
+
+ def test_how_to_intent(self):
+ self.assertEqual(youtube_yt._infer_query_intent("how to deploy Kubernetes"), "how_to")
+
+ def test_opinion_intent(self):
+ self.assertEqual(youtube_yt._infer_query_intent("thoughts on Claude Code"), "opinion")
+
+ def test_product_intent(self):
+ self.assertEqual(youtube_yt._infer_query_intent("best laptop for programming"), "product")
+
+ def test_breaking_news_default(self):
+ self.assertEqual(youtube_yt._infer_query_intent("Kanye West"), "breaking_news")
+
+
+class TestSearchAndTranscribe(unittest.TestCase):
+ """Tests for search_and_transcribe() end-to-end flow."""
+
+ def _make_item(self, video_id, views):
+ return {
+ "video_id": video_id,
+ "title": f"Video {video_id}",
+ "url": f"https://www.youtube.com/watch?v={video_id}",
+ "channel_name": "TestChannel",
+ "date": "2026-03-15",
+ "engagement": {"views": views, "likes": 10, "comments": 5},
+ "relevance": 0.8,
+ "why_relevant": "test",
+ "description": "test desc",
+ "duration": 600,
+ }
+
+ def test_transcripts_attached_when_top_videos_lack_captions(self):
+ """When top-viewed videos have no captions, lower-ranked ones still get transcripts."""
+ items = [
+ self._make_item("music1", 1_000_000), # no captions (music video)
+ self._make_item("music2", 500_000), # no captions (music video)
+ self._make_item("talk1", 50_000), # has captions
+ self._make_item("talk2", 25_000), # has captions
+ ]
+
+ # fetch_transcripts_parallel returns None for music videos, text for talks
+ def fake_parallel(video_ids, max_workers=5):
+ result = {}
+ for vid in video_ids:
+ if vid.startswith("talk"):
+ result[vid] = "This is a detailed discussion about the topic with 100 data points."
+ else:
+ result[vid] = None
+ return result
+
+ with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": items}), \
+ mock.patch.object(youtube_yt, "fetch_transcripts_parallel", side_effect=fake_parallel) as ft_mock:
+ result = youtube_yt.search_and_transcribe("test topic", "2026-03-01", "2026-03-31", depth="default")
+
+ # Should have attempted more than just the top 2 (transcript_limit=2)
+ called_ids = ft_mock.call_args[0][0]
+ self.assertGreater(len(called_ids), 2, "Should attempt more than transcript_limit candidates")
+ self.assertIn("talk1", called_ids)
+ self.assertIn("talk2", called_ids)
+
+ # talk1 and talk2 should have transcripts
+ items_by_id = {i["video_id"]: i for i in result["items"]}
+ self.assertTrue(items_by_id["talk1"]["transcript_snippet"])
+ self.assertTrue(items_by_id["talk1"]["transcript_highlights"])
+ # music videos should have empty transcripts
+ self.assertFalse(items_by_id["music1"]["transcript_snippet"])
+
+ def test_transcript_limit_zero_skips_fetch(self):
+ """When transcript_limit is 0 (quick depth), no transcripts are fetched."""
+ items = [self._make_item("vid1", 1000)]
+ with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": items}), \
+ mock.patch.object(youtube_yt, "fetch_transcripts_parallel") as ft_mock:
+ result = youtube_yt.search_and_transcribe("test", "2026-03-01", "2026-03-31", depth="quick")
+
+ ft_mock.assert_not_called()
+ self.assertEqual(result["items"][0]["transcript_snippet"], "")
+
+ def test_no_items_returns_early(self):
+ """When search returns no items, returns without fetching transcripts."""
+ with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": []}), \
+ mock.patch.object(youtube_yt, "fetch_transcripts_parallel") as ft_mock:
+ result = youtube_yt.search_and_transcribe("nothing", "2026-03-01", "2026-03-31")
+
+ ft_mock.assert_not_called()
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/uv.lock b/uv.lock
new file mode 100644
index 0000000..1280b15
--- /dev/null
+++ b/uv.lock
@@ -0,0 +1,300 @@
+version = 1
+revision = 3
+requires-python = ">=3.12"
+
+[[package]]
+name = "certifi"
+version = "2026.2.25"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" },
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7b/60/e3bec1881450851b087e301bedc3daa9377a4d45f1c26aa90b0b235e38aa/charset_normalizer-3.4.6.tar.gz", hash = "sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6", size = 143363, upload-time = "2026-03-15T18:53:25.478Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e5/62/c0815c992c9545347aeea7859b50dc9044d147e2e7278329c6e02ac9a616/charset_normalizer-3.4.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab", size = 295154, upload-time = "2026-03-15T18:50:50.88Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/37/bdca6613c2e3c58c7421891d80cc3efa1d32e882f7c4a7ee6039c3fc951a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21", size = 199191, upload-time = "2026-03-15T18:50:52.658Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/92/9934d1bbd69f7f398b38c5dae1cbf9cc672e7c34a4adf7b17c0a9c17d15d/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2", size = 218674, upload-time = "2026-03-15T18:50:54.102Z" },
+ { url = "https://files.pythonhosted.org/packages/af/90/25f6ab406659286be929fd89ab0e78e38aa183fc374e03aa3c12d730af8a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff", size = 215259, upload-time = "2026-03-15T18:50:55.616Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/ef/79a463eb0fff7f96afa04c1d4c51f8fc85426f918db467854bfb6a569ce3/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5", size = 207276, upload-time = "2026-03-15T18:50:57.054Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/72/d0426afec4b71dc159fa6b4e68f868cd5a3ecd918fec5813a15d292a7d10/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0", size = 195161, upload-time = "2026-03-15T18:50:58.686Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/18/c82b06a68bfcb6ce55e508225d210c7e6a4ea122bfc0748892f3dc4e8e11/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a", size = 203452, upload-time = "2026-03-15T18:51:00.196Z" },
+ { url = "https://files.pythonhosted.org/packages/44/d6/0c25979b92f8adafdbb946160348d8d44aa60ce99afdc27df524379875cb/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2", size = 202272, upload-time = "2026-03-15T18:51:01.703Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/3d/7fea3e8fe84136bebbac715dd1221cc25c173c57a699c030ab9b8900cbb7/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5", size = 195622, upload-time = "2026-03-15T18:51:03.526Z" },
+ { url = "https://files.pythonhosted.org/packages/57/8a/d6f7fd5cb96c58ef2f681424fbca01264461336d2a7fc875e4446b1f1346/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6", size = 220056, upload-time = "2026-03-15T18:51:05.269Z" },
+ { url = "https://files.pythonhosted.org/packages/16/50/478cdda782c8c9c3fb5da3cc72dd7f331f031e7f1363a893cdd6ca0f8de0/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d", size = 203751, upload-time = "2026-03-15T18:51:06.858Z" },
+ { url = "https://files.pythonhosted.org/packages/75/fc/cc2fcac943939c8e4d8791abfa139f685e5150cae9f94b60f12520feaa9b/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2", size = 216563, upload-time = "2026-03-15T18:51:08.564Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/b7/a4add1d9a5f68f3d037261aecca83abdb0ab15960a3591d340e829b37298/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923", size = 209265, upload-time = "2026-03-15T18:51:10.312Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/18/c094561b5d64a24277707698e54b7f67bd17a4f857bbfbb1072bba07c8bf/charset_normalizer-3.4.6-cp312-cp312-win32.whl", hash = "sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4", size = 144229, upload-time = "2026-03-15T18:51:11.694Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/20/0567efb3a8fd481b8f34f739ebddc098ed062a59fed41a8d193a61939e8f/charset_normalizer-3.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb", size = 154277, upload-time = "2026-03-15T18:51:13.004Z" },
+ { url = "https://files.pythonhosted.org/packages/15/57/28d79b44b51933119e21f65479d0864a8d5893e494cf5daab15df0247c17/charset_normalizer-3.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4", size = 142817, upload-time = "2026-03-15T18:51:14.408Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/1d/4fdabeef4e231153b6ed7567602f3b68265ec4e5b76d6024cf647d43d981/charset_normalizer-3.4.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f", size = 294823, upload-time = "2026-03-15T18:51:15.755Z" },
+ { url = "https://files.pythonhosted.org/packages/47/7b/20e809b89c69d37be748d98e84dce6820bf663cf19cf6b942c951a3e8f41/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843", size = 198527, upload-time = "2026-03-15T18:51:17.177Z" },
+ { url = "https://files.pythonhosted.org/packages/37/a6/4f8d27527d59c039dce6f7622593cdcd3d70a8504d87d09eb11e9fdc6062/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf", size = 218388, upload-time = "2026-03-15T18:51:18.934Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/9b/4770ccb3e491a9bacf1c46cc8b812214fe367c86a96353ccc6daf87b01ec/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8", size = 214563, upload-time = "2026-03-15T18:51:20.374Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/58/a199d245894b12db0b957d627516c78e055adc3a0d978bc7f65ddaf7c399/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9", size = 206587, upload-time = "2026-03-15T18:51:21.807Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/70/3def227f1ec56f5c69dfc8392b8bd63b11a18ca8178d9211d7cc5e5e4f27/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88", size = 194724, upload-time = "2026-03-15T18:51:23.508Z" },
+ { url = "https://files.pythonhosted.org/packages/58/ab/9318352e220c05efd31c2779a23b50969dc94b985a2efa643ed9077bfca5/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84", size = 202956, upload-time = "2026-03-15T18:51:25.239Z" },
+ { url = "https://files.pythonhosted.org/packages/75/13/f3550a3ac25b70f87ac98c40d3199a8503676c2f1620efbf8d42095cfc40/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd", size = 201923, upload-time = "2026-03-15T18:51:26.682Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/db/c5c643b912740b45e8eec21de1bbab8e7fc085944d37e1e709d3dcd9d72f/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c", size = 195366, upload-time = "2026-03-15T18:51:28.129Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/67/3b1c62744f9b2448443e0eb160d8b001c849ec3fef591e012eda6484787c/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194", size = 219752, upload-time = "2026-03-15T18:51:29.556Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/98/32ffbaf7f0366ffb0445930b87d103f6b406bc2c271563644bde8a2b1093/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc", size = 203296, upload-time = "2026-03-15T18:51:30.921Z" },
+ { url = "https://files.pythonhosted.org/packages/41/12/5d308c1bbe60cabb0c5ef511574a647067e2a1f631bc8634fcafaccd8293/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f", size = 215956, upload-time = "2026-03-15T18:51:32.399Z" },
+ { url = "https://files.pythonhosted.org/packages/53/e9/5f85f6c5e20669dbe56b165c67b0260547dea97dba7e187938833d791687/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2", size = 208652, upload-time = "2026-03-15T18:51:34.214Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/11/897052ea6af56df3eef3ca94edafee410ca699ca0c7b87960ad19932c55e/charset_normalizer-3.4.6-cp313-cp313-win32.whl", hash = "sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d", size = 143940, upload-time = "2026-03-15T18:51:36.15Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/5c/724b6b363603e419829f561c854b87ed7c7e31231a7908708ac086cdf3e2/charset_normalizer-3.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389", size = 154101, upload-time = "2026-03-15T18:51:37.876Z" },
+ { url = "https://files.pythonhosted.org/packages/01/a5/7abf15b4c0968e47020f9ca0935fb3274deb87cb288cd187cad92e8cdffd/charset_normalizer-3.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f", size = 143109, upload-time = "2026-03-15T18:51:39.565Z" },
+ { url = "https://files.pythonhosted.org/packages/25/6f/ffe1e1259f384594063ea1869bfb6be5cdb8bc81020fc36c3636bc8302a1/charset_normalizer-3.4.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8", size = 294458, upload-time = "2026-03-15T18:51:41.134Z" },
+ { url = "https://files.pythonhosted.org/packages/56/60/09bb6c13a8c1016c2ed5c6a6488e4ffef506461aa5161662bd7636936fb1/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421", size = 199277, upload-time = "2026-03-15T18:51:42.953Z" },
+ { url = "https://files.pythonhosted.org/packages/00/50/dcfbb72a5138bbefdc3332e8d81a23494bf67998b4b100703fd15fa52d81/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2", size = 218758, upload-time = "2026-03-15T18:51:44.339Z" },
+ { url = "https://files.pythonhosted.org/packages/03/b3/d79a9a191bb75f5aa81f3aaaa387ef29ce7cb7a9e5074ba8ea095cc073c2/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30", size = 215299, upload-time = "2026-03-15T18:51:45.871Z" },
+ { url = "https://files.pythonhosted.org/packages/76/7e/bc8911719f7084f72fd545f647601ea3532363927f807d296a8c88a62c0d/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db", size = 206811, upload-time = "2026-03-15T18:51:47.308Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/40/c430b969d41dda0c465aa36cc7c2c068afb67177bef50905ac371b28ccc7/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8", size = 193706, upload-time = "2026-03-15T18:51:48.849Z" },
+ { url = "https://files.pythonhosted.org/packages/48/15/e35e0590af254f7df984de1323640ef375df5761f615b6225ba8deb9799a/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815", size = 202706, upload-time = "2026-03-15T18:51:50.257Z" },
+ { url = "https://files.pythonhosted.org/packages/5e/bd/f736f7b9cc5e93a18b794a50346bb16fbfd6b37f99e8f306f7951d27c17c/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a", size = 202497, upload-time = "2026-03-15T18:51:52.012Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/ba/2cc9e3e7dfdf7760a6ed8da7446d22536f3d0ce114ac63dee2a5a3599e62/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43", size = 193511, upload-time = "2026-03-15T18:51:53.723Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/cb/5be49b5f776e5613be07298c80e1b02a2d900f7a7de807230595c85a8b2e/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0", size = 220133, upload-time = "2026-03-15T18:51:55.333Z" },
+ { url = "https://files.pythonhosted.org/packages/83/43/99f1b5dad345accb322c80c7821071554f791a95ee50c1c90041c157ae99/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1", size = 203035, upload-time = "2026-03-15T18:51:56.736Z" },
+ { url = "https://files.pythonhosted.org/packages/87/9a/62c2cb6a531483b55dddff1a68b3d891a8b498f3ca555fbcf2978e804d9d/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f", size = 216321, upload-time = "2026-03-15T18:51:58.17Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/79/94a010ff81e3aec7c293eb82c28f930918e517bc144c9906a060844462eb/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815", size = 208973, upload-time = "2026-03-15T18:51:59.998Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/57/4ecff6d4ec8585342f0c71bc03efaa99cb7468f7c91a57b105bcd561cea8/charset_normalizer-3.4.6-cp314-cp314-win32.whl", hash = "sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d", size = 144610, upload-time = "2026-03-15T18:52:02.213Z" },
+ { url = "https://files.pythonhosted.org/packages/80/94/8434a02d9d7f168c25767c64671fead8d599744a05d6a6c877144c754246/charset_normalizer-3.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f", size = 154962, upload-time = "2026-03-15T18:52:03.658Z" },
+ { url = "https://files.pythonhosted.org/packages/46/4c/48f2cdbfd923026503dfd67ccea45c94fd8fe988d9056b468579c66ed62b/charset_normalizer-3.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e", size = 143595, upload-time = "2026-03-15T18:52:05.123Z" },
+ { url = "https://files.pythonhosted.org/packages/31/93/8878be7569f87b14f1d52032946131bcb6ebbd8af3e20446bc04053dc3f1/charset_normalizer-3.4.6-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866", size = 314828, upload-time = "2026-03-15T18:52:06.831Z" },
+ { url = "https://files.pythonhosted.org/packages/06/b6/fae511ca98aac69ecc35cde828b0a3d146325dd03d99655ad38fc2cc3293/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc", size = 208138, upload-time = "2026-03-15T18:52:08.239Z" },
+ { url = "https://files.pythonhosted.org/packages/54/57/64caf6e1bf07274a1e0b7c160a55ee9e8c9ec32c46846ce59b9c333f7008/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e", size = 224679, upload-time = "2026-03-15T18:52:10.043Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/cb/9ff5a25b9273ef160861b41f6937f86fae18b0792fe0a8e75e06acb08f1d/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077", size = 223475, upload-time = "2026-03-15T18:52:11.854Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/97/440635fc093b8d7347502a377031f9605a1039c958f3cd18dcacffb37743/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f", size = 215230, upload-time = "2026-03-15T18:52:13.325Z" },
+ { url = "https://files.pythonhosted.org/packages/cd/24/afff630feb571a13f07c8539fbb502d2ab494019492aaffc78ef41f1d1d0/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e", size = 199045, upload-time = "2026-03-15T18:52:14.752Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/17/d1399ecdaf7e0498c327433e7eefdd862b41236a7e484355b8e0e5ebd64b/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484", size = 211658, upload-time = "2026-03-15T18:52:16.278Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/38/16baa0affb957b3d880e5ac2144caf3f9d7de7bc4a91842e447fbb5e8b67/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7", size = 210769, upload-time = "2026-03-15T18:52:17.782Z" },
+ { url = "https://files.pythonhosted.org/packages/05/34/c531bc6ac4c21da9ddfddb3107be2287188b3ea4b53b70fc58f2a77ac8d8/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff", size = 201328, upload-time = "2026-03-15T18:52:19.553Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/73/a5a1e9ca5f234519c1953608a03fe109c306b97fdfb25f09182babad51a7/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e", size = 225302, upload-time = "2026-03-15T18:52:21.043Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/f6/cd782923d112d296294dea4bcc7af5a7ae0f86ab79f8fefbda5526b6cfc0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659", size = 211127, upload-time = "2026-03-15T18:52:22.491Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/c5/0b6898950627af7d6103a449b22320372c24c6feda91aa24e201a478d161/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602", size = 222840, upload-time = "2026-03-15T18:52:24.113Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/25/c4bba773bef442cbdc06111d40daa3de5050a676fa26e85090fc54dd12f0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407", size = 216890, upload-time = "2026-03-15T18:52:25.541Z" },
+ { url = "https://files.pythonhosted.org/packages/35/1a/05dacadb0978da72ee287b0143097db12f2e7e8d3ffc4647da07a383b0b7/charset_normalizer-3.4.6-cp314-cp314t-win32.whl", hash = "sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579", size = 155379, upload-time = "2026-03-15T18:52:27.05Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/7a/d269d834cb3a76291651256f3b9a5945e81d0a49ab9f4a498964e83c0416/charset_normalizer-3.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4", size = 169043, upload-time = "2026-03-15T18:52:28.502Z" },
+ { url = "https://files.pythonhosted.org/packages/23/06/28b29fba521a37a8932c6a84192175c34d49f84a6d4773fa63d05f9aff22/charset_normalizer-3.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c", size = 148523, upload-time = "2026-03-15T18:52:29.956Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" },
+]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
+[[package]]
+name = "coverage"
+version = "7.13.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" },
+ { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" },
+ { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" },
+ { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" },
+ { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" },
+ { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" },
+ { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" },
+ { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" },
+ { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" },
+ { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" },
+ { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" },
+ { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" },
+ { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" },
+ { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" },
+ { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" },
+ { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" },
+ { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" },
+ { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" },
+ { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" },
+ { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" },
+ { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" },
+ { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" },
+ { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" },
+ { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" },
+ { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" },
+ { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" },
+ { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" },
+ { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" },
+ { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" },
+]
+
+[[package]]
+name = "idna"
+version = "3.11"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" },
+]
+
+[[package]]
+name = "iniconfig"
+version = "2.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
+]
+
+[[package]]
+name = "last30days-skill"
+version = "3.0.0"
+source = { virtual = "." }
+dependencies = [
+ { name = "requests" },
+]
+
+[package.dev-dependencies]
+dev = [
+ { name = "pytest" },
+ { name = "pytest-cov" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "requests", specifier = ">=2.32,<3" }]
+
+[package.metadata.requires-dev]
+dev = [
+ { name = "pytest", specifier = ">=9,<10" },
+ { name = "pytest-cov", specifier = ">=7,<8" },
+]
+
+[[package]]
+name = "packaging"
+version = "26.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
+]
+
+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.20.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
+]
+
+[[package]]
+name = "pytest"
+version = "9.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+ { name = "iniconfig" },
+ { name = "packaging" },
+ { name = "pluggy" },
+ { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" },
+]
+
+[[package]]
+name = "pytest-cov"
+version = "7.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "coverage" },
+ { name = "pluggy" },
+ { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" },
+]
+
+[[package]]
+name = "requests"
+version = "2.33.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "certifi" },
+ { name = "charset-normalizer" },
+ { name = "idna" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" },
+]
+
+[[package]]
+name = "urllib3"
+version = "2.6.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" },
+]
diff --git a/variants/open/SKILL.md b/variants/open/SKILL.md
deleted file mode 100644
index c43381a..0000000
--- a/variants/open/SKILL.md
+++ /dev/null
@@ -1,80 +0,0 @@
----
-name: last30days
-version: "2.1-open"
-description: "Research topics, manage watchlists, get briefings, query history. Also triggered by 'last30'. Sources: Reddit, X, YouTube, web."
-argument-hint: 'last30 AI video tools, last30 watch my competitor every week, last30 give me my briefing'
-allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch
----
-
-# last30days (open variant): Research + Watchlist + Briefings
-
-Multi-mode research skill with persistent knowledge accumulation.
-
-## Command Routing
-
-Parse the user's first argument to determine the mode:
-
-| First word | Mode | Reference |
-|---|---|---|
-| `watch` | Watchlist management | `references/watchlist.md` |
-| `briefing` | Morning briefing | `references/briefing.md` |
-| `history` | Query accumulated knowledge | `references/history.md` |
-| *(anything else)* | One-shot research | `references/research.md` |
-
-## Setup: Find Skill Root
-
-```bash
-for dir in \
- "." \
- "${CLAUDE_PLUGIN_ROOT:-}" \
- "${GEMINI_EXTENSION_DIR:-}" \
- "$HOME/.gemini/extensions/last30days-skill" \
- "$HOME/.gemini/extensions/last30days" \
- "$HOME/.claude/skills/last30days" \
- "$HOME/.agents/skills/last30days" \
- "$HOME/.codex/skills/last30days"; do
- [ -n "$dir" ] && [ -f "$dir/scripts/last30days.py" ] && SKILL_ROOT="$dir" && break
-done
-
-if [ -z "${SKILL_ROOT:-}" ]; then
- echo "ERROR: Could not find scripts/last30days.py" >&2
- exit 1
-fi
-```
-
-Use `$SKILL_ROOT` for all script and reference file paths.
-
-## Load Context
-
-At session start, read `${SKILL_ROOT}/variants/open/context.md` for user preferences and source quality notes. Update it after interactions.
-
-## Shared Configuration
-
-- **Database**: `~/.local/share/last30days/research.db` (SQLite, WAL mode)
-- **Briefings**: `~/.local/share/last30days/briefs/`
-- **API keys**: `~/.config/last30days/.env` or environment variables
-- **Key priority**: env vars > config file
-
-### API Keys
-
-| Key | Required | Purpose |
-|---|---|---|
-| `OPENAI_API_KEY` | For Reddit | Reddit search via OpenAI responses API |
-| `XAI_API_KEY` | For X (fallback) | X search via xAI Grok API |
-| `PARALLEL_API_KEY` | Optional | Web search via Parallel AI |
-| `BRAVE_API_KEY` | Optional | Web search via Brave Search |
-| `OPENROUTER_API_KEY` | Optional | Web search via Perplexity Sonar Pro |
-
-Bird CLI provides free X search if installed. YouTube search uses yt-dlp (free).
-
-Run `python3 "${SKILL_ROOT}/scripts/last30days.py" --diagnose` to check source availability.
-
-## Routing Logic
-
-After determining the mode, **read the corresponding reference file** using the Read tool:
-
-```
-Read: ${SKILL_ROOT}/variants/open/references/{mode}.md
-```
-
-Then follow the instructions in that reference file exactly.
diff --git a/variants/open/context.md b/variants/open/context.md
deleted file mode 100644
index 86221ae..0000000
--- a/variants/open/context.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# last30days Context
-
-Agent memory for improving research quality over time.
-
-## User Preferences
-
-
-
-## Source Quality Notes
-
-
-
-
-## Interaction History
-
-
diff --git a/variants/open/references/briefing.md b/variants/open/references/briefing.md
deleted file mode 100644
index d9f3ef2..0000000
--- a/variants/open/references/briefing.md
+++ /dev/null
@@ -1,82 +0,0 @@
-# Morning Briefing
-
-Synthesize accumulated findings into a formatted briefing.
-
-## Commands
-
-| Command | Action |
-|---|---|
-| `briefing` | Generate today's briefing |
-| `briefing --weekly` | Weekly digest with trends |
-| `briefing --since YYYY-MM-DD` | Briefing since specific date |
-
-## Generate Briefing
-
-```bash
-python3 "${SKILL_ROOT}/scripts/briefing.py" generate [--weekly] [--since DATE]
-```
-
-The script returns JSON with per-topic findings, staleness info, and cost data.
-
-## Staleness Check
-
-Before synthesizing, check each topic's freshness:
-- **Fresh** (< 12h): show normally
-- **Aging** (12-36h): note when last run was
-- **Stale** (> 36h): warn user, suggest running `watch run-one "topic"`
-
-## Daily Briefing Format
-
-```
-Good morning! Here's your research briefing for [DATE].
-
-TL;DR: [One sentence about the top finding across all topics]
-
----
-
-**[Topic 1]** (N new findings)
-Top signal: [Highest engagement finding with source]
-Also trending: [2nd finding], [3rd finding]
-
-**[Topic 2]** (N new findings)
-Top signal: [Highest engagement finding]
-Also trending: [2nd finding]
-
----
-Cost: $X.XX / $Y.YY budget | N topics active | N findings today
-```
-
-## Weekly Digest Format
-
-```
-Weekly digest for week of [DATE]:
-
-**[Topic 1]**
-This week: N findings (up/down X% from last week)
-Trending up: [engagement increasing]
-Key voices: @handle1, r/sub1
-
-**[Topic 2]**
-This week: N findings
-Trending down: [engagement decreasing]
-```
-
-## Synthesis Rules
-
-- Lead with people, not publications
-- 3-5 topics max per briefing
-- 2-3 findings per topic
-- Include cost/budget footer
-- Note any failed or stale topics
-
-## No Data Handling
-
-If no topics or no findings:
-```
-No briefing data available.
-
-To get started:
-1. Add a topic: /last30days watch add "your topic"
-2. Run research: /last30days watch run-all
-3. Generate briefing: /last30days briefing
-```
diff --git a/variants/open/references/history.md b/variants/open/references/history.md
deleted file mode 100644
index 73688bc..0000000
--- a/variants/open/references/history.md
+++ /dev/null
@@ -1,102 +0,0 @@
-# History & Knowledge Query
-
-Query the accumulated findings database.
-
-## Commands
-
-| Command | Action |
-|---|---|
-| `history "topic"` | Show findings for a topic |
-| `history "topic" --since=7d` | Findings from last N days |
-| `history --search "query"` | Full-text search across all findings |
-| `history --trending` | Topics with most recent activity |
-| `history --stats` | Watchlist health dashboard |
-
-## Topic History
-
-```bash
-python3 "${SKILL_ROOT}/scripts/store.py" query "TOPIC" [--since DAYS]
-```
-
-Display findings grouped by date (newest first):
-```
-**[Topic Name]** — N findings since [date]
-
-[DATE]
-- [Reddit] Title (score pts, N comments) — r/subreddit
-- [X] Tweet text... (N likes) — @handle
-- [YouTube] Video title (N views) — channel
-
-[EARLIER DATE]
-- ...
-```
-
-Mark updated findings (engagement changed since first seen).
-
-## Full-Text Search
-
-```bash
-python3 "${SKILL_ROOT}/scripts/store.py" search "QUERY"
-```
-
-Uses FTS5 with BM25 ranking. Show results across all topics:
-```
-Search: "QUERY" — N results
-
-1. [Reddit] Title — r/subreddit (topic: AI video)
- ...snippet with **highlighted** matches...
-
-2. [X] Tweet text — @handle (topic: NVIDIA)
- ...snippet...
-```
-
-## Trending Topics
-
-```bash
-python3 "${SKILL_ROOT}/scripts/store.py" trending
-```
-
-Show topics ranked by recent activity:
-```
-Trending topics (last 7 days):
-
-1. AI video tools — 12 new findings, engagement up 45%
-2. NVIDIA news — 8 new findings, engagement steady
-3. Claude Code — 3 new findings, engagement down 20%
-```
-
-## Stats Dashboard
-
-```bash
-python3 "${SKILL_ROOT}/scripts/store.py" stats
-```
-
-Display as a health dashboard:
-```
-Watchlist Health
-- Active topics: N
-- Total findings: N
-- Database size: N KB
-
-Research Runs (7 days)
-- Successful: N
-- Failed: N
-- Cost: $X.XX
-
-Source Breakdown
-- Reddit: N findings
-- X: N findings
-- YouTube: N findings
-- Web: N findings
-```
-
-## No Data Handling
-
-If no findings exist:
-```
-No research history yet.
-
-To start building knowledge:
-1. Run research: /last30days "your topic"
-2. Or add a watchlist topic: /last30days watch add "topic"
-```
diff --git a/variants/open/references/research.md b/variants/open/references/research.md
deleted file mode 100644
index c7a5eca..0000000
--- a/variants/open/references/research.md
+++ /dev/null
@@ -1,153 +0,0 @@
-# One-Shot Research Mode
-
-Research ANY topic across Reddit, X, YouTube, Polymarket, and the web. Surface what people are actually discussing, recommending, betting on, and debating right now.
-
-## Parse User Intent
-
-Before doing anything, parse the user's input for:
-
-1. **TOPIC**: What they want to learn about
-2. **TARGET TOOL** (if specified): Where they'll use the prompts
-3. **QUERY TYPE**:
- - **PROMPTING** — "X prompts", "prompting for X" → copy-paste prompts
- - **RECOMMENDATIONS** — "best X", "top X" → list of specific things
- - **NEWS** — "what's happening with X" → current events
- - **GENERAL** — anything else → broad understanding
-
-**Do NOT ask about target tool before research.** Run research first, ask after.
-
-**Display your parsing** before calling tools:
-
-```
-I'll research {TOPIC} across Reddit, X, YouTube, Polymarket, and the web.
-
-Parsed intent:
-- TOPIC = {TOPIC}
-- TARGET_TOOL = {TARGET_TOOL or "unknown"}
-- QUERY_TYPE = {QUERY_TYPE}
-
-Research typically takes 2-8 minutes. Starting now.
-```
-
----
-
-## Step 0.5: Resolve X Handle (if topic could have an X account)
-
-If TOPIC looks like it could have its own X account - **people, creators, brands, products, tools, companies** (e.g., "Dor Brothers", "Nano Banana Pro", "Seedance"), do ONE quick WebSearch:
-
-```
-WebSearch("{TOPIC} X twitter handle site:x.com")
-```
-
-Extract their X handle from results (look for `x.com/{handle}` URLs or "@handle" mentions). **Verify it's the real account, not a parody/fan page** - check for verified status, official website links, consistent naming. If found, pass as `--x-handle={handle}` (no @). Skip if TOPIC is a generic concept (not an entity), already has @, uses `--quick`, or no official account exists.
-
----
-
-## Research Execution
-
-**Step 1: Run the research script (FOREGROUND)**
-
-```bash
-python3 "${SKILL_ROOT}/scripts/last30days.py" "$ARGUMENTS" --emit=compact --store 2>&1 # Add --x-handle=HANDLE if resolved
-```
-
-Use a **timeout of 300000** (5 minutes). The `--store` flag persists findings for watchlist/briefing integration.
-
-The script auto-detects: API keys, Bird CLI, yt-dlp, web search backends.
-
-**Read the ENTIRE output.** It contains Reddit, X, YouTube, AND web sections.
-
----
-
-**Step 2: WebSearch (supplement)**
-
-After the script finishes, use your WebSearch tool for additional coverage.
-
-Choose queries based on QUERY_TYPE:
-
-- **RECOMMENDATIONS**: `best {TOPIC} recommendations`, `{TOPIC} list examples`
-- **NEWS**: `{TOPIC} news 2026`, `{TOPIC} announcement update`
-- **PROMPTING**: `{TOPIC} prompts examples 2026`, `{TOPIC} techniques tips`
-- **GENERAL**: `{TOPIC} 2026`, `{TOPIC} discussion`
-
-Rules:
-- **USE THE USER'S EXACT TERMINOLOGY**
-- EXCLUDE reddit.com, x.com, twitter.com (covered by script)
-- Do NOT output "Sources:" list
-
----
-
-## Synthesis
-
-**Judge Agent rules:**
-1. Weight Reddit/X HIGHER (engagement signals)
-2. Weight YouTube HIGH (views + transcript content)
-3. Weight web LOWER (no engagement data)
-4. Identify cross-source patterns (strongest signals)
-5. Extract top 3-5 actionable insights
-6. **Prediction markets are high-signal when relevant** - real money on outcomes cuts through opinion. Prefer structural/long-term markets (championship > regular season, regime change > near-term deadline). When the topic is an outcome in a multi-outcome market, call out that specific outcome's odds and movement. Weave odds into narrative: "Polymarket has X at Y% (up/down Z%)"
-
-**Ground synthesis in ACTUAL research, not pre-existing knowledge.**
-
-### Citation Rules
-
-- Cite sparingly: 1-2 sources per topic
-- Priority: @handles > r/subreddits > YouTube channels > web sources
-- Use publication names, never raw URLs
-- Lead with people, not publications
-
----
-
-## Display Results
-
-**1. "What I learned"** (format depends on QUERY_TYPE)
-
-**If RECOMMENDATIONS** — show specific items with sources:
-```
-Most mentioned:
-
-[Name] - {n}x mentions
-Use Case: [what it does]
-Sources: @handle1, r/sub, blog.com
-```
-
-**If PROMPTING/NEWS/GENERAL** — show synthesis:
-```
-What I learned:
-
-**{Topic 1}** — [1-2 sentences, per @handle or r/sub]
-
-KEY PATTERNS:
-1. [Pattern] — per @handle
-2. [Pattern] — per r/sub
-```
-
-**2. Stats box** (calculate from actual output):
-```
----
-All agents reported back!
-|- Reddit: {N} threads | {N} upvotes | {N} comments
-|- X: {N} posts | {N} likes | {N} reposts
-|- YouTube: {N} videos | {N} views | {N} with transcripts
-|- Polymarket: {N} markets | {summary of up to 5 most relevant market odds}
-|- Web: {N} pages (supplementary)
-|- Top voices: @{handle1}, @{handle2} | r/{sub1}, r/{sub2}
----
-```
-
-**CRITICAL: Omit any source line that returned 0 results.** Do NOT show "0 threads", "0 stories", "0 markets", or "(no results this cycle)". If a source found nothing, DELETE that line entirely.
-
-**3. Invitation** with 2-3 specific follow-up suggestions based on research.
-
----
-
-## Follow-Up
-
-After research, you are an **EXPERT** on this topic.
-
-- **QUESTION** → Answer from research (no new searches)
-- **GO DEEPER** → Elaborate from findings
-- **CREATE/PROMPT** → Write ONE prompt using research insights
-- **Different topic** → Run new research
-
-When writing prompts, match the FORMAT the research recommends (JSON, structured, etc.).
diff --git a/variants/open/references/watchlist.md b/variants/open/references/watchlist.md
deleted file mode 100644
index 272b1b1..0000000
--- a/variants/open/references/watchlist.md
+++ /dev/null
@@ -1,107 +0,0 @@
-# Watchlist Management
-
-Manage topics you want to track continuously. Findings accumulate in the SQLite database for briefings and history queries.
-
-## Commands
-
-| Command | Action |
-|---|---|
-| `watch add "topic"` | Add a topic (daily schedule) |
-| `watch add "topic" --weekly` | Add with weekly schedule |
-| `watch "topic"` | Shorthand for `watch add` |
-| `watch remove "topic"` | Remove a topic |
-| `watch list` | Show all topics with status |
-| `watch config delivery [channel]` | Set delivery channel |
-| `watch config budget AMOUNT` | Set daily cost budget |
-| `watch run-all` | Run research for all topics now |
-| `watch run-one "topic"` | Run research for one topic now |
-
-## Adding a Topic
-
-```bash
-python3 "${SKILL_ROOT}/scripts/watchlist.py" add "TOPIC_NAME" [--weekly] [--queries "q1,q2"]
-```
-
-The script auto-bootstraps the SQLite database on first add.
-
-**Default schedule**: Daily at 8am (`0 8 * * *`).
-**Weekly**: Mondays at 8am (`0 8 * * 1`).
-
-**After adding**, confirm to the user:
-```
-Added "TOPIC_NAME" to watchlist.
-Schedule: daily at 8am (or weekly on Mondays)
-
-To run research now: /last30days watch run-one "TOPIC_NAME"
-To set up automated runs: add a cron/launchd job for `python3 ${SKILL_ROOT}/scripts/watchlist.py run-all`
-```
-
-## Removing a Topic
-
-```bash
-python3 "${SKILL_ROOT}/scripts/watchlist.py" remove "TOPIC_NAME"
-```
-
-Show confirmation or "not found" message.
-
-## Listing Topics
-
-```bash
-python3 "${SKILL_ROOT}/scripts/watchlist.py" list
-```
-
-Display as a formatted table:
-```
-Topic | Schedule | Last Run | Findings | Status
---------------+--------------+--------------+----------+--------
-AI video | daily 8am | 2h ago | 47 | ok
-NVIDIA news | weekly Mon | 3d ago | 23 | ok
-
-Budget: $0.42 / $5.00 today
-```
-
-## Running Research
-
-```bash
-# All enabled topics (with budget guard)
-python3 "${SKILL_ROOT}/scripts/watchlist.py" run-all
-
-# Single topic
-python3 "${SKILL_ROOT}/scripts/watchlist.py" run-one "TOPIC_NAME"
-```
-
-Show results: new findings count, updated findings, duration, and any errors.
-
-## Configuration
-
-```bash
-# Set delivery channel (for future notification support)
-python3 "${SKILL_ROOT}/scripts/watchlist.py" config delivery telegram
-
-# Set daily budget limit
-python3 "${SKILL_ROOT}/scripts/watchlist.py" config budget 10.00
-```
-
-## Scheduling
-
-The watchlist doesn't auto-schedule. To automate, set up a system job:
-
-**macOS (launchd)**:
-```bash
-# Run daily at 8am
-crontab -e
-# Add: 0 8 * * * python3 /path/to/scripts/watchlist.py run-all
-```
-
-**Linux (cron)**:
-```bash
-crontab -e
-# Add: 0 8 * * * python3 /path/to/scripts/watchlist.py run-all
-```
-
-## Error Handling
-
-- Duplicate topic: update the existing schedule
-- Topic not found on remove: show "not found" message
-- Budget exceeded: skip remaining topics, show which were skipped
-- Research failure: record error, continue to next topic
+
+ 
-
- →
- 
-
-