sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources

Browse Source 
v3 rewrites the search engine from the ground up:

- Intelligent pre-research: resolves X handles, GitHub repos, subreddits,
  TikTok hashtags, and YouTube channels before searching
- GitHub person-mode: PR velocity, top repos by stars, release notes
- GitHub project-mode: live star counts, README, releases, top issues
- ELI5 mode: plain language synthesis, no jargon
- 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket,
  GitHub, Threads, Pinterest, Perplexity, Bluesky, Web
- Free Reddit comments via public JSON (no API key needed)
- Fun judge v2: humor scoring baked into narrative
- Cookie consent before browser scanning
- 10,000 free ScrapeCreators calls
- 1,012 tests

Thank you to the community contributors whose issues and PRs shaped v3:
@uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116),
@thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78),
@zl190 (#115), @hnshah (#84, #85, #86)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-04-08 10:52:23 -07:00
parent 61904b31e3
commit 0a9ff16dfc
397 changed files with 21427 additions and 53106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lib/bird_x.py
+37 -35
View File
@@ -1,7 +1,8 @@
"""Bird X search client - vendored Twitter GraphQL search for /last30days v2.1.
"""Bird X search client for the v3.0.0 last30days pipeline.
Uses a vendored subset of @steipete/bird v0.8.0 (MIT License) to search X
via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js 22+.
via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js.
See scripts/lib/vendor/bird-search/package.json for authoritative version.
"""
import json
@@ -11,11 +12,21 @@ import shutil
import subprocess
import sys
from pathlib import Path
from . import http, log
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
from .relevance import token_overlap_relevance as _compute_relevance
def _first_of(*values):
"""Return first value that is not None."""
for v in values:
if v is not None:
return v
return None
# Path to the vendored bird-search wrapper
_BIRD_SEARCH_MJS = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
@@ -43,21 +54,23 @@ def _has_injected_credentials() -> bool:
return bool(_credentials.get('AUTH_TOKEN') and _credentials.get('CT0'))
def _has_process_credentials() -> bool:
"""Return True when AUTH_TOKEN/CT0 are present in process env."""
return bool(os.environ.get("AUTH_TOKEN") and os.environ.get("CT0"))
def _subprocess_env() -> Dict[str, str]:
"""Build env dict for Node subprocesses, merging injected credentials."""
env = os.environ.copy()
env.update(_credentials)
# When repo config already provides cookies, disable browser-cookie fallback
# so vendored Bird never hits Safari/Chrome keychain during automation.
if _has_injected_credentials():
env.setdefault("BIRD_DISABLE_BROWSER_COOKIES", "1")
# Hard-disable browser-cookie fallback so normal pipeline runs never hit
# Safari/Chrome Keychain prompts during source detection or search.
env["BIRD_DISABLE_BROWSER_COOKIES"] = "1"
return env
def _log(msg: str):
"""Log to stderr."""
sys.stderr.write(f"[Bird] {msg}\n")
sys.stderr.flush()
log.source_log("Bird", msg, tty_only=False)
def _extract_core_subject(topic: str) -> str:
@@ -75,7 +88,7 @@ def is_bird_installed() -> bool:
"""Check if vendored Bird search module is available.
Returns:
True if bird-search.mjs exists and Node.js 22+ is in PATH.
True if bird-search.mjs exists and Node.js is in PATH.
"""
if not _BIRD_SEARCH_MJS.exists():
return False
@@ -83,7 +96,7 @@ def is_bird_installed() -> bool:
def is_bird_authenticated() -> Optional[str]:
"""Check if X credentials are available (env vars or browser cookies).
"""Check if explicit X credentials are available.
Returns:
Auth source string if authenticated, None otherwise.
@@ -93,20 +106,9 @@ def is_bird_authenticated() -> Optional[str]:
if _has_injected_credentials():
return "env AUTH_TOKEN"
try:
result = subprocess.run(
["node", str(_BIRD_SEARCH_MJS), "--whoami"],
capture_output=True,
text=True,
timeout=15,
env=_subprocess_env(),
)
if result.returncode == 0 and result.stdout.strip():
return result.stdout.strip().split('\n')[0]
return None
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
return None
if _has_process_credentials():
return "env AUTH_TOKEN"
return None
def check_npm_available() -> bool:
@@ -119,13 +121,13 @@ def check_npm_available() -> bool:
def install_bird() -> Tuple[bool, str]:
"""No-op - Bird search is vendored in v2.1, no installation needed.
"""No-op. Bird search is vendored in v3.0.0, no installation needed.
Returns:
Tuple of (success, message).
"""
if is_bird_installed():
return True, "Bird search is bundled with /last30days v2.1 - no installation needed."
return True, "Bird search is bundled with /last30days v3.0.0 - no installation needed."
if not shutil.which("node"):
return False, "Node.js 22+ is required for X search. Install Node.js first."
return False, f"Vendored bird-search.mjs not found at {_BIRD_SEARCH_MJS}"
@@ -144,7 +146,7 @@ def get_bird_status() -> Dict[str, Any]:
"installed": installed,
"authenticated": auth_source is not None,
"username": auth_source, # Now returns auth source (e.g., "Safari", "env AUTH_TOKEN")
"can_install": True, # Always vendored in v2.1
"can_install": True, # Always vendored in v3.0.0
}
@@ -200,7 +202,7 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
try:
from last30days import unregister_child_pid
unregister_child_pid(proc.pid)
except (ImportError, Exception):
except Exception:
pass
if proc.returncode != 0:
@@ -361,7 +363,7 @@ def search_handles(
except json.JSONDecodeError:
_log(f"Invalid JSON from handle search for @{handle}")
except Exception as e:
except (OSError, subprocess.SubprocessError) as e:
_log(f"Handle search error for @{handle}: {e}")
return all_items
@@ -428,10 +430,10 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
# Build engagement dict (Bird uses camelCase: likeCount, retweetCount, etc.)
engagement = {
"likes": tweet.get("likeCount") or tweet.get("like_count") or tweet.get("favorite_count"),
"reposts": tweet.get("retweetCount") or tweet.get("retweet_count"),
"replies": tweet.get("replyCount") or tweet.get("reply_count"),
"quotes": tweet.get("quoteCount") or tweet.get("quote_count"),
"likes": _first_of(tweet.get("likeCount"), tweet.get("like_count"), tweet.get("favorite_count")),
"reposts": _first_of(tweet.get("retweetCount"), tweet.get("retweet_count")),
"replies": _first_of(tweet.get("replyCount"), tweet.get("reply_count")),
"quotes": _first_of(tweet.get("quoteCount"), tweet.get("quote_count")),
}
# Convert to int where possible
for key in engagement:
@@ -448,7 +450,7 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
"url": url,
"author_handle": author_handle.lstrip("@"),
"date": date,
"engagement": engagement if any(v is not None for v in engagement.values()) else None,
"engagement": engagement,
"why_relevant": "", # Bird doesn't provide relevance explanations
"relevance": _compute_relevance(query, str(tweet.get("text", ""))) if query else 0.7,
}
scripts/lib/bluesky.py
+2 -5
View File
@@ -10,7 +10,7 @@ import sys
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from . import http
from . import http, log
BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
@@ -27,10 +27,7 @@ _session_error: Optional[str] = None
def _log(msg: str):
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
if sys.stderr.isatty():
sys.stderr.write(f"[Bluesky] {msg}\n")
sys.stderr.flush()
log.source_log("Bluesky", msg)
def _create_session(handle: str, app_password: str) -> Optional[str]:
scripts/lib/brave_search.py
-329
View File
@@ -1,329 +0,0 @@
"""Brave Search web search for last30days skill.
Uses the Brave Search API as a web search backend.
Requires a paid Brave Search subscription.
Two modes:
- Standard: /res/v1/web/search — returns URLs + snippets (default)
- LLM Context: /res/v1/llm/context — returns pre-extracted text chunks
optimized for LLM consumption. Enable with BRAVE_LLM_CONTEXT=1 env var.
API docs: https://api-dashboard.search.brave.com/app/documentation/web-search/get-started
"""
import html
import re
import sys
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from urllib.parse import urlencode, urlparse
from . import http
ENDPOINT = "https://api.search.brave.com/res/v1/web/search"
LLM_CONTEXT_ENDPOINT = "https://api.search.brave.com/res/v1/llm/context"
# Freshness codes: pd=24h, pw=7d, pm=31d
FRESHNESS_MAP = {1: "pd", 7: "pw", 31: "pm"}
# Domains to exclude (handled by Reddit/X search)
EXCLUDED_DOMAINS = {
"reddit.com", "www.reddit.com", "old.reddit.com",
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
}
def search_web(
topic: str,
from_date: str,
to_date: str,
api_key: str,
depth: str = "default",
use_llm_context: bool = False,
) -> List[Dict[str, Any]]:
"""Search the web via Brave Search API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
api_key: Brave Search API key
depth: 'quick', 'default', or 'deep'
use_llm_context: Use LLM Context endpoint for pre-extracted content
Returns:
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
Raises:
http.HTTPError: On API errors
"""
if use_llm_context:
return _search_llm_context(topic, from_date, to_date, api_key, depth)
count = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
# Calculate days for freshness filter
days = _days_between(from_date, to_date)
freshness = _brave_freshness(days)
params = {
"q": topic,
"result_filter": "web,news",
"count": count,
"safesearch": "strict",
"text_decorations": 0,
"spellcheck": 0,
}
if freshness:
params["freshness"] = freshness
url = f"{ENDPOINT}?{urlencode(params)}"
sys.stderr.write(f"[Web] Searching Brave for: {topic}\n")
sys.stderr.flush()
response = http.request(
"GET",
url,
headers={"X-Subscription-Token": api_key},
timeout=15,
)
return _normalize_results(response, from_date, to_date)
def _search_llm_context(
topic: str,
from_date: str,
to_date: str,
api_key: str,
depth: str = "default",
) -> List[Dict[str, Any]]:
"""Search via Brave LLM Context endpoint for pre-extracted web content.
Returns results in the same schema as search_web() for downstream compatibility.
Snippets contain actual page content instead of short descriptions.
"""
count = {"quick": 5, "default": 20, "deep": 50}.get(depth, 20)
max_tokens = {"quick": 2048, "default": 8192, "deep": 16384}.get(depth, 8192)
days = _days_between(from_date, to_date)
freshness = _brave_freshness(days)
params = {
"q": topic,
"count": count,
"maximum_number_of_tokens": max_tokens,
"context_threshold_mode": "balanced",
}
if freshness:
params["freshness"] = freshness
url = f"{LLM_CONTEXT_ENDPOINT}?{urlencode(params)}"
sys.stderr.write(f"[Web] Searching Brave LLM Context for: {topic}\n")
sys.stderr.flush()
response = http.request(
"GET",
url,
headers={"X-Subscription-Token": api_key},
timeout=30,
)
return _normalize_llm_context(response)
def _days_between(from_date: str, to_date: str) -> int:
"""Calculate days between two YYYY-MM-DD dates."""
try:
d1 = datetime.strptime(from_date, "%Y-%m-%d")
d2 = datetime.strptime(to_date, "%Y-%m-%d")
return max(1, (d2 - d1).days)
except (ValueError, TypeError):
return 30
def _brave_freshness(days: Optional[int]) -> Optional[str]:
"""Convert days to Brave freshness parameter.
Uses canned codes for <=31d, explicit date range for longer periods.
"""
if days is None:
return None
code = next((v for d, v in sorted(FRESHNESS_MAP.items()) if days <= d), None)
if code:
return code
start = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
end = datetime.now(timezone.utc).strftime("%Y-%m-%d")
return f"{start}to{end}"
def _normalize_results(
response: Dict[str, Any],
from_date: str,
to_date: str,
) -> List[Dict[str, Any]]:
"""Convert Brave Search response to websearch item schema.
Merges news + web results, cleans HTML entities, filters excluded domains.
"""
items = []
# Merge news results (tend to be more recent) with web results
raw_results = (
response.get("news", {}).get("results", []) +
response.get("web", {}).get("results", [])
)
for i, result in enumerate(raw_results):
if not isinstance(result, dict):
continue
url = result.get("url", "")
if not url:
continue
# Skip excluded domains
try:
domain = urlparse(url).netloc.lower()
if domain in EXCLUDED_DOMAINS:
continue
if domain.startswith("www."):
domain = domain[4:]
except (ValueError, TypeError):
domain = ""
title = _clean_html(str(result.get("title", "")).strip())
snippet = _clean_html(str(result.get("description", "")).strip())
if not title and not snippet:
continue
# Parse date from Brave's 'age' field or 'page_age'
date = _parse_brave_date(result.get("age"), result.get("page_age"))
date_confidence = "med" if date else "low"
items.append({
"id": f"W{i+1}",
"title": title[:200],
"url": url,
"source_domain": domain,
"snippet": snippet[:500],
"date": date,
"date_confidence": date_confidence,
"relevance": 0.6, # Brave doesn't provide relevance scores
"why_relevant": "",
})
sys.stderr.write(f"[Web] Brave: {len(items)} results\n")
sys.stderr.flush()
return items
def _normalize_llm_context(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Convert Brave LLM Context response to websearch item schema.
LLM Context returns grounding.generic[] with url, title, snippets[].
Sources metadata provides hostname and age for each URL.
"""
items = []
grounding = response.get("grounding", {})
sources = response.get("sources", {})
for i, result in enumerate(grounding.get("generic", [])):
if not isinstance(result, dict):
continue
url = result.get("url", "")
if not url:
continue
# Skip excluded domains
try:
domain = urlparse(url).netloc.lower()
if domain in EXCLUDED_DOMAINS:
continue
if domain.startswith("www."):
domain = domain[4:]
except (ValueError, TypeError):
domain = ""
title = str(result.get("title", "")).strip()
snippets = result.get("snippets", [])
snippet = "\n".join(str(s).strip() for s in snippets if s)
if not title and not snippet:
continue
# Parse date from sources metadata
source_meta = sources.get(url, {})
age_list = source_meta.get("age") or []
date = None
for age_str in age_list:
date = _parse_brave_date(age_str, None)
if date:
break
date_confidence = "med" if date else "low"
items.append({
"id": f"W{i+1}",
"title": title[:200],
"url": url,
"source_domain": source_meta.get("hostname", domain),
"snippet": snippet[:1500], # LLM Context returns richer content
"date": date,
"date_confidence": date_confidence,
"relevance": 0.7, # LLM Context pre-filters for relevance
"why_relevant": "",
})
sys.stderr.write(f"[Web] Brave LLM Context: {len(items)} results\n")
sys.stderr.flush()
return items
def _clean_html(text: str) -> str:
"""Remove HTML tags and decode entities."""
text = re.sub(r"<[^>]*>", "", text)
text = html.unescape(text)
return text
def _parse_brave_date(age: Optional[str], page_age: Optional[str]) -> Optional[str]:
"""Parse Brave's age/page_age fields to YYYY-MM-DD.
Brave returns dates like "3 hours ago", "2 days ago", "January 24, 2026".
"""
text = age or page_age
if not text:
return None
text_lower = text.lower().strip()
now = datetime.now()
# "X hours ago" -> today
if re.search(r'\d+\s*hours?\s*ago', text_lower):
return now.strftime("%Y-%m-%d")
# "X days ago"
match = re.search(r'(\d+)\s*days?\s*ago', text_lower)
if match:
days = int(match.group(1))
if days <= 60:
return (now - timedelta(days=days)).strftime("%Y-%m-%d")
# "X weeks ago"
match = re.search(r'(\d+)\s*weeks?\s*ago', text_lower)
if match:
weeks = int(match.group(1))
return (now - timedelta(weeks=weeks)).strftime("%Y-%m-%d")
# ISO format: 2026-01-24T...
match = re.search(r'(\d{4}-\d{2}-\d{2})', text)
if match:
return match.group(1)
return None
scripts/lib/cache.py
-165
View File
@@ -1,165 +0,0 @@
"""Caching utilities for last30days skill."""
import hashlib
import json
import os
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional
CACHE_DIR = Path.home() / ".cache" / "last30days"
DEFAULT_TTL_HOURS = 24
MODEL_CACHE_TTL_DAYS = 7
MODEL_CACHE_FILE = CACHE_DIR / "model_selection.json"
def ensure_cache_dir():
"""Ensure cache directory exists. Supports env override and sandbox fallback."""
global CACHE_DIR, MODEL_CACHE_FILE
env_dir = os.environ.get("LAST30DAYS_CACHE_DIR")
if env_dir:
CACHE_DIR = Path(env_dir)
MODEL_CACHE_FILE = CACHE_DIR / "model_selection.json"
try:
CACHE_DIR.mkdir(parents=True, exist_ok=True)
except PermissionError:
CACHE_DIR = Path(tempfile.gettempdir()) / "last30days" / "cache"
MODEL_CACHE_FILE = CACHE_DIR / "model_selection.json"
CACHE_DIR.mkdir(parents=True, exist_ok=True)
def get_cache_key(topic: str, from_date: str, to_date: str, sources: str) -> str:
"""Generate a cache key from query parameters."""
key_data = f"{topic}|{from_date}|{to_date}|{sources}"
return hashlib.sha256(key_data.encode()).hexdigest()[:16]
def get_cache_path(cache_key: str) -> Path:
"""Get path to cache file."""
return CACHE_DIR / f"{cache_key}.json"
def is_cache_valid(cache_path: Path, ttl_hours: int = DEFAULT_TTL_HOURS) -> bool:
"""Check if cache file exists and is within TTL."""
if not cache_path.exists():
return False
try:
stat = cache_path.stat()
mtime = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
now = datetime.now(timezone.utc)
age_hours = (now - mtime).total_seconds() / 3600
return age_hours < ttl_hours
except OSError:
return False
def load_cache(cache_key: str, ttl_hours: int = DEFAULT_TTL_HOURS) -> Optional[dict]:
"""Load data from cache if valid."""
cache_path = get_cache_path(cache_key)
if not is_cache_valid(cache_path, ttl_hours):
return None
try:
with open(cache_path, 'r') as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
return None
def get_cache_age_hours(cache_path: Path) -> Optional[float]:
"""Get age of cache file in hours."""
if not cache_path.exists():
return None
try:
stat = cache_path.stat()
mtime = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
now = datetime.now(timezone.utc)
return (now - mtime).total_seconds() / 3600
except OSError:
return None
def load_cache_with_age(cache_key: str, ttl_hours: int = DEFAULT_TTL_HOURS) -> tuple:
"""Load data from cache with age info.
Returns:
Tuple of (data, age_hours) or (None, None) if invalid
"""
cache_path = get_cache_path(cache_key)
if not is_cache_valid(cache_path, ttl_hours):
return None, None
age = get_cache_age_hours(cache_path)
try:
with open(cache_path, 'r') as f:
return json.load(f), age
except (json.JSONDecodeError, OSError):
return None, None
def save_cache(cache_key: str, data: dict):
"""Save data to cache."""
ensure_cache_dir()
cache_path = get_cache_path(cache_key)
try:
with open(cache_path, 'w') as f:
json.dump(data, f)
except OSError:
pass # Silently fail on cache write errors
def clear_cache():
"""Clear all cache files."""
if CACHE_DIR.exists():
for f in CACHE_DIR.glob("*.json"):
try:
f.unlink()
except OSError:
pass
# Model selection cache (longer TTL) — MODEL_CACHE_FILE is set at module level
# and updated by ensure_cache_dir() if env override or fallback is needed.
def load_model_cache() -> dict:
"""Load model selection cache."""
if not is_cache_valid(MODEL_CACHE_FILE, MODEL_CACHE_TTL_DAYS * 24):
return {}
try:
with open(MODEL_CACHE_FILE, 'r') as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
return {}
def save_model_cache(data: dict):
"""Save model selection cache."""
ensure_cache_dir()
try:
with open(MODEL_CACHE_FILE, 'w') as f:
json.dump(data, f)
except OSError:
pass
def get_cached_model(provider: str) -> Optional[str]:
"""Get cached model selection for a provider."""
cache = load_model_cache()
return cache.get(provider)
def set_cached_model(provider: str, model: str):
"""Cache model selection for a provider."""
cache = load_model_cache()
cache[provider] = model
cache['updated_at'] = datetime.now(timezone.utc).isoformat()
save_model_cache(cache)
scripts/lib/cluster.py
+255
View File
@@ -0,0 +1,255 @@
"""Candidate clustering and representative selection."""
from __future__ import annotations
import re
from . import dedupe, schema
CLUSTERABLE_INTENTS = {"breaking_news", "opinion", "comparison", "prediction"}
# Words too common to signal shared topic between clusters.
_ENTITY_STOPWORDS = frozenset({
"the", "a", "an", "to", "for", "how", "is", "in", "of", "on", "and",
"with", "from", "by", "at", "this", "that", "it", "what", "are", "do",
"can", "his", "her", "he", "she", "its", "was", "has", "new", "just",
"says", "said", "will", "about", "after", "now", "all", "been", "here",
"not", "out", "up", "more", "also", "but", "who", "year", "first",
"make", "being", "making", "over", "into", "than", "they", "their",
"would", "could", "get", "got", "some", "like", "back", "going",
"breaking", "https", "http", "www", "com",
})
def _candidate_text(candidate: schema.Candidate) -> str:
return " ".join(part for part in [candidate.title, candidate.snippet] if part).strip()
def _extract_entities(text: str) -> set[str]:
"""Extract significant words (proper nouns, numbers, capitalized words) from text.
Used for cross-source cluster merging where phrasing differs but entities overlap.
"""
# Normalize but preserve word boundaries
words = re.sub(r"[^\w\s]", " ", text).split()
entities = set()
for word in words:
lower = word.lower()
if lower in _ENTITY_STOPWORDS or len(word) <= 2:
continue
# Keep words that are: capitalized, ALL CAPS, contain digits, or 4+ chars
if word[0].isupper() or word.isupper() or any(c.isdigit() for c in word) or len(word) >= 4:
entities.add(lower)
return entities
def _entity_overlap(entities_a: set[str], entities_b: set[str]) -> float:
"""Jaccard-style overlap on extracted entities."""
if not entities_a or not entities_b:
return 0.0
intersection = entities_a & entities_b
smaller = min(len(entities_a), len(entities_b))
# Use overlap coefficient (intersection / min) instead of Jaccard,
# because a short tweet about the same event as a long Reddit post
# will have fewer total entities but high overlap with the larger set.
return len(intersection) / smaller if smaller > 0 else 0.0
def _mmr_representatives(
candidates: list[schema.Candidate],
limit: int = 3,
diversity_lambda: float = 0.75,
) -> list[str]:
selected: list[schema.Candidate] = []
remaining = list(candidates)
while remaining and len(selected) < limit:
if not selected:
best = max(remaining, key=lambda candidate: candidate.final_score)
selected.append(best)
remaining.remove(best)
continue
def score(candidate: schema.Candidate) -> float:
diversity_penalty = max(
dedupe.hybrid_similarity(_candidate_text(candidate), _candidate_text(existing))
for existing in selected
)
return (diversity_lambda * candidate.final_score) - ((1 - diversity_lambda) * diversity_penalty * 100)
best = max(remaining, key=score)
selected.append(best)
remaining.remove(best)
return [candidate.candidate_id for candidate in selected]
def cluster_candidates(
candidates: list[schema.Candidate],
plan: schema.QueryPlan,
) -> list[schema.Cluster]:
"""Greedy clustering around high-ranked leaders."""
if plan.intent not in CLUSTERABLE_INTENTS or plan.cluster_mode == "none":
clusters = []
for index, candidate in enumerate(candidates, start=1):
cluster_id = f"cluster-{index}"
candidate.cluster_id = cluster_id
clusters.append(
schema.Cluster(
cluster_id=cluster_id,
title=candidate.title,
candidate_ids=[candidate.candidate_id],
representative_ids=[candidate.candidate_id],
sources=sorted(schema.candidate_sources(candidate)),
score=candidate.final_score,
uncertainty=None,
)
)
return clusters
groups: list[list[schema.Candidate]] = []
# Lower threshold for breaking_news: related articles share fewer exact
# words but cover the same event.
threshold = 0.42 if plan.intent == "breaking_news" else 0.48
for candidate in candidates:
assigned = False
for group in groups:
leader = group[0]
similarity = dedupe.hybrid_similarity(_candidate_text(candidate), _candidate_text(leader))
if similarity >= threshold:
group.append(candidate)
assigned = True
break
if not assigned:
groups.append([candidate])
clusters: list[schema.Cluster] = []
for index, group in enumerate(groups, start=1):
group.sort(key=lambda candidate: candidate.final_score, reverse=True)
cluster_id = f"cluster-{index}"
representatives = _mmr_representatives(group)
for candidate in group:
candidate.cluster_id = cluster_id
clusters.append(
schema.Cluster(
cluster_id=cluster_id,
title=group[0].title,
candidate_ids=[candidate.candidate_id for candidate in group],
representative_ids=representatives,
sources=sorted({source for candidate in group for source in schema.candidate_sources(candidate)}),
score=max(candidate.final_score for candidate in group),
uncertainty=_cluster_uncertainty(group),
)
)
# Second pass: merge small clusters that share entities across sources.
clusters = _merge_entity_clusters(clusters, candidates)
return sorted(clusters, key=lambda cluster: cluster.score, reverse=True)
def _merge_entity_clusters(
clusters: list[schema.Cluster],
all_candidates: list[schema.Candidate],
) -> list[schema.Cluster]:
"""Merge small clusters that cover the same story across different sources.
The initial greedy pass uses text similarity which misses cross-source
matches where phrasing differs. This second pass looks at entity overlap
(proper nouns, names, numbers) to catch cases like:
- Reddit: "Kanye West to headline all three nights of Wireless Festival 2026"
- X: "BREAKING: Kanye West (Ye) is making his massive UK comeback!"
"""
if len(clusters) < 2:
return clusters
candidate_map = {c.candidate_id: c for c in all_candidates}
# Build entity sets per cluster
cluster_entities: list[set[str]] = []
for cl in clusters:
entities: set[str] = set()
for cid in cl.candidate_ids:
cand = candidate_map.get(cid)
if cand:
entities |= _extract_entities(_candidate_text(cand))
cluster_entities.append(entities)
# Only merge clusters with <= 3 items (don't merge already-large clusters)
merged_into: dict[int, int] = {} # index -> merge target index
for i in range(len(clusters)):
if i in merged_into or len(clusters[i].candidate_ids) > 3:
continue
for j in range(i + 1, len(clusters)):
if j in merged_into or len(clusters[j].candidate_ids) > 3:
continue
# Require different sources to merge (same-source should already be grouped)
sources_i = set(clusters[i].sources)
sources_j = set(clusters[j].sources)
if sources_i == sources_j and len(sources_i) == 1:
continue
# Prevent Polymarket clusters from merging with non-Polymarket
# clusters. Prediction markets about "Sam Altman equity" should not
# merge into a news cluster about "Sam Altman rivalry" just because
# both mention the same entity.
poly_i = "polymarket" in sources_i
poly_j = "polymarket" in sources_j
if poly_i != poly_j:
continue
overlap = _entity_overlap(cluster_entities[i], cluster_entities[j])
if overlap >= 0.45:
merged_into[j] = i
if not merged_into:
return clusters
# Build merged cluster list
result: list[schema.Cluster] = []
for i, cl in enumerate(clusters):
if i in merged_into:
continue
# Collect all clusters merged into this one
merge_sources = [i] + [j for j, target in merged_into.items() if target == i]
if len(merge_sources) == 1:
result.append(cl)
continue
# Combine candidates from all merged clusters
combined_cids: list[str] = []
combined_sources: set[str] = set()
best_score = 0.0
for idx in merge_sources:
combined_cids.extend(clusters[idx].candidate_ids)
combined_sources.update(clusters[idx].sources)
best_score = max(best_score, clusters[idx].score)
# Pick representatives from combined pool
combined_candidates = [candidate_map[cid] for cid in combined_cids if cid in candidate_map]
combined_candidates.sort(key=lambda c: c.final_score, reverse=True)
reps = _mmr_representatives(combined_candidates)
cluster_id = cl.cluster_id
for cid in combined_cids:
cand = candidate_map.get(cid)
if cand:
cand.cluster_id = cluster_id
result.append(schema.Cluster(
cluster_id=cluster_id,
title=combined_candidates[0].title if combined_candidates else cl.title,
candidate_ids=combined_cids,
representative_ids=reps,
sources=sorted(combined_sources),
score=best_score,
uncertainty=_cluster_uncertainty(combined_candidates),
))
return result
def _cluster_uncertainty(group: list[schema.Candidate]) -> str | None:
sources = {source for candidate in group for source in schema.candidate_sources(candidate)}
if len(sources) == 1:
return "single-source"
if max(candidate.final_score for candidate in group) < 55:
return "thin-evidence"
return None
scripts/lib/dates.py
+5 -9
View File
@@ -41,7 +41,10 @@ def parse_date(date_str: Optional[str]) -> Optional[datetime]:
for fmt in formats:
try:
return datetime.strptime(date_str, fmt).replace(tzinfo=timezone.utc)
dt = datetime.strptime(date_str, fmt)
if dt.tzinfo is not None:
return dt.astimezone(timezone.utc)
return dt.replace(tzinfo=timezone.utc)
except ValueError:
continue
@@ -78,14 +81,7 @@ def get_date_confidence(date_str: Optional[str], from_date: str, to_date: str) -
start = datetime.strptime(from_date, "%Y-%m-%d").date()
end = datetime.strptime(to_date, "%Y-%m-%d").date()
if start <= dt <= end:
return 'high'
elif dt < start:
# Older than range
return 'low'
else:
# Future date (suspicious)
return 'low'
return 'high' if start <= dt <= end else 'low'
except ValueError:
return 'low'
scripts/lib/dedupe.py
+75 -266
View File
@@ -1,290 +1,99 @@
"""Near-duplicate detection for last30days skill."""
"""Within-source near-duplicate detection."""
from __future__ import annotations
import re
from typing import List, Set, Tuple, Union
from . import schema
# Stopwords for token-based Jaccard (cross-source linking)
STOPWORDS = frozenset({
'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
'all', 'just', 'get', 'has', 'have', 'was', 'will', 'show', 'hn',
})
STOPWORDS = frozenset(
{
"the",
"a",
"an",
"to",
"for",
"how",
"is",
"in",
"of",
"on",
"and",
"with",
"from",
"by",
"at",
"this",
"that",
"it",
"what",
"are",
"do",
"can",
}
)
def normalize_text(text: str) -> str:
"""Normalize text for comparison.
- Lowercase
- Remove punctuation
- Collapse whitespace
"""
text = text.lower()
text = re.sub(r'[^\w\s]', ' ', text)
text = re.sub(r'\s+', ' ', text)
return text.strip()
text = re.sub(r"[^\w\s]", " ", text.lower())
return re.sub(r"\s+", " ", text).strip()
def get_ngrams(text: str, n: int = 3) -> Set[str]:
"""Get character n-grams from text."""
def get_ngrams(text: str, n: int = 3) -> set[str]:
text = normalize_text(text)
if len(text) < n:
return {text}
return {text[i:i+n] for i in range(len(text) - n + 1)}
return {text} if text else set()
return {text[index:index + n] for index in range(len(text) - n + 1)}
def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
"""Compute Jaccard similarity between two sets."""
if not set1 or not set2:
def jaccard_similarity(left: set[str], right: set[str]) -> float:
if not left or not right:
return 0.0
intersection = len(set1 & set2)
union = len(set1 | set2)
return intersection / union if union > 0 else 0.0
AnyItem = Union[schema.RedditItem, schema.XItem, schema.YouTubeItem, schema.TikTokItem,
schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem, schema.WebSearchItem]
def get_item_text(item: AnyItem) -> str:
"""Get comparable text from an item."""
if isinstance(item, schema.RedditItem):
return item.title
elif isinstance(item, schema.HackerNewsItem):
return item.title
elif isinstance(item, schema.YouTubeItem):
return f"{item.title} {item.channel_name}"
elif isinstance(item, schema.TikTokItem):
return f"{item.text} {item.author_name}"
elif isinstance(item, schema.InstagramItem):
return f"{item.text} {item.author_name}"
elif isinstance(item, schema.PolymarketItem):
return f"{item.title} {item.question}"
elif isinstance(item, schema.WebSearchItem):
return item.title
else:
return item.text
def _get_cross_source_text(item: AnyItem) -> str:
"""Get text for cross-source comparison.
Same as get_item_text() but truncates X posts to 100 chars
to level the playing field against short Reddit/HN titles.
Strips 'Show HN:' prefix from HN titles for fairer matching.
"""
if isinstance(item, schema.XItem):
return item.text[:100]
if isinstance(item, schema.TikTokItem):
return item.text[:100]
if isinstance(item, schema.InstagramItem):
return item.text[:100]
if isinstance(item, schema.HackerNewsItem):
title = item.title
if title.startswith("Show HN:"):
title = title[8:].strip()
elif title.startswith("Ask HN:"):
title = title[7:].strip()
return title
if isinstance(item, schema.PolymarketItem):
return item.title
return get_item_text(item)
def _tokenize_for_xref(text: str) -> Set[str]:
"""Tokenize text for cross-source token Jaccard comparison."""
words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
return {w for w in words if w not in STOPWORDS and len(w) > 1}
def _token_jaccard(text_a: str, text_b: str) -> float:
"""Token-level Jaccard similarity (word overlap)."""
tokens_a = _tokenize_for_xref(text_a)
tokens_b = _tokenize_for_xref(text_b)
if not tokens_a or not tokens_b:
union = left | right
if not union:
return 0.0
intersection = len(tokens_a & tokens_b)
union = len(tokens_a | tokens_b)
return intersection / union if union else 0.0
return len(left & right) / len(union)
def _hybrid_similarity(text_a: str, text_b: str) -> float:
"""Hybrid similarity: max of char-trigram Jaccard and token Jaccard."""
trigram_sim = jaccard_similarity(get_ngrams(text_a), get_ngrams(text_b))
token_sim = _token_jaccard(text_a, text_b)
return max(trigram_sim, token_sim)
def token_jaccard(text_a: str, text_b: str) -> float:
tokens_a = {
token
for token in normalize_text(text_a).split()
if len(token) > 1 and token not in STOPWORDS
}
tokens_b = {
token
for token in normalize_text(text_b).split()
if len(token) > 1 and token not in STOPWORDS
}
return jaccard_similarity(tokens_a, tokens_b)
def find_duplicates(
items: List[Union[schema.RedditItem, schema.XItem]],
threshold: float = 0.7,
) -> List[Tuple[int, int]]:
"""Find near-duplicate pairs in items.
Args:
items: List of items to check
threshold: Similarity threshold (0-1)
Returns:
List of (i, j) index pairs where i < j and items are similar
"""
duplicates = []
# Pre-compute n-grams
ngrams = [get_ngrams(get_item_text(item)) for item in items]
for i in range(len(items)):
for j in range(i + 1, len(items)):
similarity = jaccard_similarity(ngrams[i], ngrams[j])
if similarity >= threshold:
duplicates.append((i, j))
return duplicates
def hybrid_similarity(text_a: str, text_b: str) -> float:
return max(
jaccard_similarity(get_ngrams(text_a), get_ngrams(text_b)),
token_jaccard(text_a, text_b),
)
def dedupe_items(
items: List[Union[schema.RedditItem, schema.XItem]],
threshold: float = 0.7,
) -> List[Union[schema.RedditItem, schema.XItem]]:
"""Remove near-duplicates, keeping highest-scored item.
Args:
items: List of items (should be pre-sorted by score descending)
threshold: Similarity threshold
Returns:
Deduplicated items
"""
if len(items) <= 1:
return items
# Find duplicate pairs
dup_pairs = find_duplicates(items, threshold)
# Mark indices to remove (always remove the lower-scored one)
# Since items are pre-sorted by score, the second index is always lower
to_remove = set()
for i, j in dup_pairs:
# Keep the higher-scored one (lower index in sorted list)
if items[i].score >= items[j].score:
to_remove.add(j)
else:
to_remove.add(i)
# Return items not marked for removal
return [item for idx, item in enumerate(items) if idx not in to_remove]
def item_text(item: schema.SourceItem) -> str:
parts = [item.title, item.body, item.author or "", item.container or ""]
return " ".join(part for part in parts if part).strip()
def dedupe_reddit(
items: List[schema.RedditItem],
threshold: float = 0.7,
) -> List[schema.RedditItem]:
"""Dedupe Reddit items."""
return dedupe_items(items, threshold)
def dedupe_x(
items: List[schema.XItem],
threshold: float = 0.7,
) -> List[schema.XItem]:
"""Dedupe X items."""
return dedupe_items(items, threshold)
def dedupe_youtube(
items: List[schema.YouTubeItem],
threshold: float = 0.7,
) -> List[schema.YouTubeItem]:
"""Dedupe YouTube items."""
return dedupe_items(items, threshold)
def dedupe_tiktok(
items: List[schema.TikTokItem],
threshold: float = 0.7,
) -> List[schema.TikTokItem]:
"""Dedupe TikTok items."""
return dedupe_items(items, threshold)
def dedupe_instagram(
items: List[schema.InstagramItem],
threshold: float = 0.7,
) -> List[schema.InstagramItem]:
"""Dedupe Instagram items."""
return dedupe_items(items, threshold)
def dedupe_hackernews(
items: List[schema.HackerNewsItem],
threshold: float = 0.7,
) -> List[schema.HackerNewsItem]:
"""Dedupe Hacker News items."""
return dedupe_items(items, threshold)
def dedupe_bluesky(
items: List[schema.BlueskyItem],
threshold: float = 0.7,
) -> List[schema.BlueskyItem]:
"""Dedupe Bluesky items."""
return dedupe_items(items, threshold)
def dedupe_truthsocial(
items: List[schema.TruthSocialItem],
threshold: float = 0.7,
) -> List[schema.TruthSocialItem]:
"""Dedupe Truth Social items."""
return dedupe_items(items, threshold)
def dedupe_polymarket(
items: List[schema.PolymarketItem],
threshold: float = 0.7,
) -> List[schema.PolymarketItem]:
"""Dedupe Polymarket items."""
return dedupe_items(items, threshold)
def cross_source_link(
*source_lists: List[AnyItem],
threshold: float = 0.40,
) -> None:
"""Annotate items with cross-source references.
Compares items across different source types using hybrid similarity
(max of char-trigram Jaccard and token Jaccard). When similarity exceeds
threshold, adds bidirectional cross_refs with the related item's ID.
Modifies items in-place.
Args:
*source_lists: Variable number of per-source item lists
threshold: Similarity threshold for cross-linking (default 0.40)
"""
all_items = []
for source_list in source_lists:
all_items.extend(source_list)
if len(all_items) <= 1:
return
# Pre-compute cross-source text for each item
texts = [_get_cross_source_text(item) for item in all_items]
for i in range(len(all_items)):
for j in range(i + 1, len(all_items)):
# Skip same-source comparisons (handled by per-source dedupe)
if type(all_items[i]) is type(all_items[j]):
continue
similarity = _hybrid_similarity(texts[i], texts[j])
if similarity >= threshold:
# Bidirectional cross-reference
if all_items[j].id not in all_items[i].cross_refs:
all_items[i].cross_refs.append(all_items[j].id)
if all_items[i].id not in all_items[j].cross_refs:
all_items[j].cross_refs.append(all_items[i].id)
def dedupe_items(items: list[schema.SourceItem], threshold: float = 0.7) -> list[schema.SourceItem]:
"""Remove near-duplicates while keeping earlier, better-scored items."""
kept: list[schema.SourceItem] = []
for item in items:
text = item_text(item)
if not text:
kept.append(item)
continue
is_duplicate = False
for existing in kept:
if hybrid_similarity(text, item_text(existing)) >= threshold:
is_duplicate = True
break
if not is_duplicate:
kept.append(item)
return kept
scripts/lib/entity_extract.py
+1 -1
View File
@@ -1,4 +1,4 @@
"""Entity extraction from Phase 1 search results for supplemental searches."""
"""Entity extraction from initial search results for supplemental searches."""
import re
from collections import Counter
scripts/lib/env.py
+221 -413
View File
@@ -1,39 +1,16 @@
"""Environment and API key management for last30days skill."""
from __future__ import annotations
import base64
import binascii
import json
import logging
import os
import sys
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Optional, Dict, Any, List, Literal
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Cookie domain registry: maps source names to browser cookie extraction params.
# Each entry: (domain, cookie_names, config_key_mapping)
# config_key_mapping: {cookie_name: config_key} so we know which config key
# each extracted cookie should populate.
# ---------------------------------------------------------------------------
COOKIE_DOMAINS: Dict[str, Dict[str, Any]] = {
"x": {
"domain": ".x.com",
"cookies": ["auth_token", "ct0"],
"mapping": {
"auth_token": "AUTH_TOKEN",
"ct0": "CT0",
},
},
"truthsocial": {
"domain": ".truthsocial.com",
"cookies": ["_session_id"],
"mapping": {
"_session_id": "TRUTHSOCIAL_TOKEN",
},
},
}
from typing import Any, Literal
# Allow override via environment variable for testing
# Set LAST30DAYS_CONFIG_DIR="" for clean/no-config mode
@@ -67,10 +44,10 @@ AUTH_STATUS_MISSING_ACCOUNT_ID: AuthStatus = "missing_account_id"
@dataclass(frozen=True)
class OpenAIAuth:
token: Optional[str]
token: str | None
source: AuthSource
status: AuthStatus
account_id: Optional[str]
account_id: str | None
codex_auth_file: str
@@ -80,17 +57,17 @@ def _check_file_permissions(path: Path) -> None:
mode = path.stat().st_mode
# Check if group or other can read (bits 0o044)
if mode & 0o044:
import sys
sys.stderr.write(
f"[last30days] WARNING: {path} is readable by other users. "
f"Run: chmod 600 {path}\n"
)
sys.stderr.flush()
except OSError:
pass
except OSError as exc:
sys.stderr.write(f"[last30days] WARNING: could not stat {path}: {exc}\n")
sys.stderr.flush()
def load_env_file(path: Path) -> Dict[str, str]:
def load_env_file(path: Path) -> dict[str, str]:
"""Load environment variables from a file."""
env = {}
if not path or not path.exists():
@@ -114,7 +91,7 @@ def load_env_file(path: Path) -> Dict[str, str]:
return env
def _decode_jwt_payload(token: str) -> Optional[Dict[str, Any]]:
def _decode_jwt_payload(token: str) -> dict[str, Any] | None:
"""Decode JWT payload without verification."""
try:
parts = token.split(".")
@@ -124,7 +101,9 @@ def _decode_jwt_payload(token: str) -> Optional[Dict[str, Any]]:
pad = "=" * (-len(payload_b64) % 4)
decoded = base64.urlsafe_b64decode(payload_b64 + pad)
return json.loads(decoded.decode("utf-8"))
except Exception:
except (json.JSONDecodeError, UnicodeDecodeError, binascii.Error, IndexError) as exc:
sys.stderr.write(f"[last30days] WARNING: malformed JWT token: {exc}\n")
sys.stderr.flush()
return None
@@ -139,7 +118,7 @@ def _token_expired(token: str, leeway_seconds: int = 60) -> bool:
return exp <= (time.time() + leeway_seconds)
def extract_chatgpt_account_id(access_token: str) -> Optional[str]:
def extract_chatgpt_account_id(access_token: str) -> str | None:
"""Extract chatgpt_account_id from JWT token."""
payload = _decode_jwt_payload(access_token)
if not payload:
@@ -150,18 +129,22 @@ def extract_chatgpt_account_id(access_token: str) -> Optional[str]:
return None
def load_codex_auth(path: Path = CODEX_AUTH_FILE) -> Dict[str, Any]:
def load_codex_auth(path: Path = CODEX_AUTH_FILE) -> dict[str, Any]:
"""Load Codex auth JSON."""
if not path.exists():
return {}
try:
with open(path, "r") as f:
return json.load(f)
except Exception:
except json.JSONDecodeError:
sys.stderr.write(
f"[last30days] WARNING: {path} exists but contains invalid JSON -- ignoring\n"
)
sys.stderr.flush()
return {}
def get_codex_access_token() -> tuple[Optional[str], str]:
def get_codex_access_token() -> tuple[str | None, str]:
"""Get Codex access token from auth.json.
Returns:
@@ -182,7 +165,7 @@ def get_codex_access_token() -> tuple[Optional[str], str]:
return token, AUTH_STATUS_OK
def get_openai_auth(file_env: Dict[str, str]) -> OpenAIAuth:
def get_openai_auth(file_env: dict[str, str]) -> OpenAIAuth:
"""Resolve OpenAI auth from API key or Codex login."""
api_key = os.environ.get('OPENAI_API_KEY') or file_env.get('OPENAI_API_KEY')
if api_key:
@@ -194,35 +177,20 @@ def get_openai_auth(file_env: Dict[str, str]) -> OpenAIAuth:
codex_auth_file=str(CODEX_AUTH_FILE),
)
codex_token, codex_status = get_codex_access_token()
if codex_token:
account_id = extract_chatgpt_account_id(codex_token)
if account_id:
return OpenAIAuth(
token=codex_token,
source=AUTH_SOURCE_CODEX,
status=AUTH_STATUS_OK,
account_id=account_id,
codex_auth_file=str(CODEX_AUTH_FILE),
)
return OpenAIAuth(
token=None,
source=AUTH_SOURCE_CODEX,
status=AUTH_STATUS_MISSING_ACCOUNT_ID,
account_id=None,
codex_auth_file=str(CODEX_AUTH_FILE),
)
# Codex auth (chatgpt.com backend) intentionally skipped.
# The endpoint is unstable and causes crashes when the token expires.
# Users who want OpenAI should set OPENAI_API_KEY explicitly.
return OpenAIAuth(
token=None,
source=AUTH_SOURCE_NONE,
status=codex_status,
status=AUTH_STATUS_MISSING,
account_id=None,
codex_auth_file=str(CODEX_AUTH_FILE),
)
def _find_project_env() -> Optional[Path]:
def _find_project_env() -> Path | None:
"""Find per-project .env by walking up from cwd.
Searches for .claude/last30days.env in each parent directory,
@@ -239,95 +207,13 @@ def _find_project_env() -> Optional[Path]:
return None
def extract_browser_credentials(config: Dict[str, Any]) -> Dict[str, str]:
"""Extract credentials from browser cookies for sources that need them.
Checks the FROM_BROWSER config key to decide whether/how to extract:
- 'auto': try browsers in platform order
- 'firefox', 'chrome', 'safari': try only that browser
- 'off': skip extraction entirely
If SETUP_COMPLETE is not set AND FROM_BROWSER is not explicitly set,
defaults to 'off' (wizard hasn't run yet — no extraction without consent).
If SETUP_COMPLETE is set and FROM_BROWSER is not set, defaults to 'auto'.
Explicit env var/config values always take priority over extracted cookies.
Returns:
Dict of {config_key: value} for credentials discovered from cookies.
"""
setup_complete = config.get("SETUP_COMPLETE")
from_browser = config.get("FROM_BROWSER")
# Determine effective browser setting
if from_browser is None:
if setup_complete:
from_browser = "auto"
else:
from_browser = "off"
from_browser = from_browser.lower().strip() if isinstance(from_browser, str) else "off"
if from_browser == "off":
return {}
# Lazy import to avoid loading cookie_extract at module level
try:
from . import cookie_extract
except Exception:
logger.debug("cookie_extract module not available")
return {}
credentials: Dict[str, str] = {}
for source_name, spec in COOKIE_DOMAINS.items():
domain = spec["domain"]
cookie_names: List[str] = spec["cookies"]
mapping: Dict[str, str] = spec["mapping"]
# Skip if ALL mapped config keys already have values
all_present = all(config.get(config_key) for config_key in mapping.values())
if all_present:
logger.debug(
"Skipping cookie extraction for %s: credentials already set",
source_name,
)
continue
try:
result = cookie_extract.extract_cookies_with_source(from_browser, domain, cookie_names)
except Exception as exc:
logger.debug(
"Cookie extraction failed for %s: %s", source_name, exc
)
continue
if result is None:
continue
cookies, browser_name = result
filled_any = False
for cookie_name, config_key in mapping.items():
# Only fill in keys not already present
if not config.get(config_key) and cookie_name in cookies:
credentials[config_key] = cookies[cookie_name]
filled_any = True
# Track which browser provided the credentials for this source
if filled_any:
credentials[f"__{source_name.upper()}_BROWSER"] = browser_name
return credentials
def get_config() -> Dict[str, Any]:
def get_config() -> dict[str, Any]:
"""Load configuration from multiple sources.
Priority (highest wins):
1. Environment variables (os.environ)
2. .claude/last30days.env (per-project config)
3. ~/.config/last30days/.env (global config)
4. Browser cookies (only fills in missing keys)
"""
# Load from global config file
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
@@ -355,15 +241,13 @@ def get_config() -> Dict[str, Any]:
('GOOGLE_API_KEY', None),
('GEMINI_API_KEY', None),
('GOOGLE_GENAI_API_KEY', None),
('OPENROUTER_API_KEY', None),
('PARALLEL_API_KEY', None),
('BRAVE_API_KEY', None),
('EXA_API_KEY', None),
('XIAOHONGSHU_API_BASE', None),
('GEMINI_MODEL', None),
('OPENAI_MODEL_POLICY', 'auto'),
('LAST30DAYS_REASONING_PROVIDER', 'auto'),
('LAST30DAYS_PLANNER_MODEL', None),
('LAST30DAYS_RERANK_MODEL', None),
('LAST30DAYS_X_MODEL', None),
('LAST30DAYS_X_BACKEND', None),
('OPENAI_MODEL_PIN', None),
('XAI_MODEL_POLICY', 'latest'),
('XAI_MODEL_PIN', None),
('SCRAPECREATORS_API_KEY', None),
('APIFY_API_TOKEN', None),
@@ -372,6 +256,11 @@ def get_config() -> Dict[str, Any]:
('BSKY_HANDLE', None),
('BSKY_APP_PASSWORD', None),
('TRUTHSOCIAL_TOKEN', None),
('BRAVE_API_KEY', None),
('EXA_API_KEY', None),
('SERPER_API_KEY', None),
('OPENROUTER_API_KEY', None),
('PARALLEL_API_KEY', None),
('FROM_BROWSER', None),
('SETUP_COMPLETE', None),
('INCLUDE_SOURCES', None),
@@ -380,24 +269,6 @@ def get_config() -> Dict[str, Any]:
for key, default in keys:
config[key] = os.environ.get(key) or merged_env.get(key, default)
# Inject browser cookies for any credentials not already set
browser_creds = extract_browser_credentials(config)
for key, value in browser_creds.items():
if not config.get(key):
config[key] = value
# Track AUTH_TOKEN source for status reporting
if config.get('AUTH_TOKEN'):
if os.environ.get('AUTH_TOKEN') or merged_env.get('AUTH_TOKEN'):
config['_AUTH_TOKEN_SOURCE'] = 'env'
elif browser_creds.get('AUTH_TOKEN'):
browser_name = browser_creds.get('__X_BROWSER', 'unknown')
config['_AUTH_TOKEN_SOURCE'] = f'browser-{browser_name}'
else:
config['_AUTH_TOKEN_SOURCE'] = 'env' # fallback
else:
config['_AUTH_TOKEN_SOURCE'] = None
# Track which config source was used
if project_env_path:
config['_CONFIG_SOURCE'] = f'project:{project_env_path}'
@@ -406,9 +277,87 @@ def get_config() -> Dict[str, Any]:
else:
config['_CONFIG_SOURCE'] = 'env_only'
# Extract browser credentials if configured
browser_creds = extract_browser_credentials(config)
for key, value in browser_creds.items():
if not config.get(key):
config[key] = value
config[f"_{key}_SOURCE"] = "browser"
return config
# ---------------------------------------------------------------------------
# Browser cookie extraction
# ---------------------------------------------------------------------------
COOKIE_DOMAINS: dict[str, dict[str, Any]] = {
"x": {
"domain": ".x.com",
"cookies": ["auth_token", "ct0"],
"mapping": {"auth_token": "AUTH_TOKEN", "ct0": "CT0"},
},
"truthsocial": {
"domain": ".truthsocial.com",
"cookies": ["_session_id"],
"mapping": {"_session_id": "TRUTHSOCIAL_TOKEN"},
},
}
def extract_browser_credentials(config: dict[str, Any]) -> dict[str, str]:
"""Extract auth cookies from local browsers.
Default behavior (FROM_BROWSER unset): tries Firefox and Safari only.
These read local files silently with no system dialogs. Chrome is
skipped because ``security find-generic-password`` triggers a macOS
Keychain prompt that cannot be reliably suppressed.
Set ``FROM_BROWSER=auto`` to also try Chrome (accepts the dialog),
or ``FROM_BROWSER=off`` to disable extraction entirely.
"""
from_browser = (config.get("FROM_BROWSER") or "").strip().lower()
if from_browser == "off":
return {}
try:
from . import cookie_extract
except ImportError:
return {}
# Determine which browsers to try
if from_browser in ("firefox", "chrome", "safari"):
browsers = [from_browser]
elif from_browser == "auto":
browsers = ["firefox", "safari", "chrome"]
else:
# Default: silent browsers only (no Keychain dialog)
browsers = ["firefox", "safari"]
extracted: dict[str, str] = {}
for _service, spec in COOKIE_DOMAINS.items():
if all(config.get(env_key) for env_key in spec["mapping"].values()):
continue
for browser in browsers:
try:
cookies = cookie_extract.extract_cookies(browser, spec["domain"], spec["cookies"])
except Exception:
continue
if cookies:
for cookie_name, env_key in spec["mapping"].items():
if cookie_name in cookies and not config.get(env_key):
extracted[env_key] = cookies[cookie_name]
break # Found cookies for this service, stop trying browsers
return extracted
def get_x_source_with_method(config: dict[str, Any]) -> tuple[str | None, str]:
"""Return (source, method) for X search, where method describes the auth origin."""
if config.get("XAI_API_KEY"):
return "xai", "xai"
if config.get("AUTH_TOKEN") and config.get("CT0"):
method = config.get("_AUTH_TOKEN_SOURCE", "env")
return "bird", method
return None, "none"
def config_exists() -> bool:
"""Check if any configuration source exists."""
if _find_project_env():
@@ -418,235 +367,60 @@ def config_exists() -> bool:
return False
def is_reddit_available(config: Dict[str, Any]) -> bool:
def is_reddit_available(config: dict[str, Any]) -> bool:
"""Check if Reddit search is available.
Reddit can use either ScrapeCreators (preferred) or OpenAI.
v3 uses ScrapeCreators only.
"""
has_sc = bool(config.get('SCRAPECREATORS_API_KEY'))
has_openai = bool(config.get('OPENAI_API_KEY')) and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK
return has_sc or has_openai
return bool(config.get('SCRAPECREATORS_API_KEY'))
def get_reddit_source(config: Dict[str, Any]) -> Optional[str]:
def get_reddit_source(config: dict[str, Any]) -> str | None:
"""Determine which Reddit backend to use.
Priority: ScrapeCreators (cheaper, faster) > OpenAI (legacy)
Returns: 'scrapecreators', 'openai', or None
Returns: 'scrapecreators' or None
"""
if config.get('SCRAPECREATORS_API_KEY'):
return 'scrapecreators'
if config.get('OPENAI_API_KEY') and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK:
return 'openai'
return None
def get_available_sources(config: Dict[str, Any]) -> str:
"""Determine which sources are available.
def get_x_source(config: dict[str, Any]) -> str | None:
"""Determine the best available explicit X/Twitter source.
X is available if ANY auth method works: AUTH_TOKEN/CT0 (env or cookies),
XAI_API_KEY, or Bird installed+authenticated.
Reddit is always available (public JSON fallback).
HN and Polymarket are always available.
YouTube available if yt-dlp installed.
Priority: explicit backend pin, then xAI, then Bird with explicit cookies.
Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
"""
has_reddit = True
has_x = get_x_source(config) is not None
has_web = has_web_search_keys(config)
if has_reddit and has_x:
return 'all' if has_web else 'both'
elif has_reddit:
return 'reddit-web' if has_web else 'reddit'
return 'web' if has_web else 'none'
def has_web_search_keys(config: Dict[str, Any]) -> bool:
"""Check if any web search API keys are configured."""
return bool(config.get('EXA_API_KEY') or config.get('OPENROUTER_API_KEY') or config.get('PARALLEL_API_KEY') or config.get('BRAVE_API_KEY'))
def get_web_search_source(config: Dict[str, Any]) -> Optional[str]:
"""Determine the best available web search backend.
Priority: Exa (free) > Parallel AI > Brave > OpenRouter/Sonar Pro
Returns: 'exa', 'parallel', 'brave', 'openrouter', or None
"""
if config.get('EXA_API_KEY'):
return 'exa'
if config.get('PARALLEL_API_KEY'):
return 'parallel'
if config.get('BRAVE_API_KEY'):
return 'brave'
if config.get('OPENROUTER_API_KEY'):
return 'openrouter'
return None
def get_missing_keys(config: Dict[str, Any]) -> str:
"""Determine which sources are missing (accounting for Bird).
Returns: 'all', 'both', 'reddit', 'x', 'web', or 'none'
"""
has_reddit = True
has_xai = bool(config.get('XAI_API_KEY'))
has_web = has_web_search_keys(config)
# Check if Bird provides X access (import here to avoid circular dependency)
from . import bird_x
has_bird = bird_x.is_bird_installed() and bird_x.is_bird_authenticated()
has_x = has_xai or has_bird
if has_reddit and has_x and has_web:
return 'none'
elif has_reddit and has_x:
return 'web' # Missing web search keys
elif has_reddit and has_web:
return 'x' # Missing X source
elif has_reddit:
return 'x' # Missing X source (and possibly web)
return 'all'
def validate_sources(requested: str, available: str, include_web: bool = False) -> tuple[str, Optional[str]]:
"""Validate requested sources against available keys.
Args:
requested: 'auto', 'reddit', 'x', 'both', or 'web'
available: Result from get_available_sources()
include_web: If True, add WebSearch to available sources
Returns:
Tuple of (effective_sources, error_message)
"""
has_reddit = available in ('reddit', 'both', 'reddit-web', 'all')
has_x = available in ('x', 'both', 'x-web', 'all')
has_web = available in ('web', 'reddit-web', 'x-web', 'all')
if requested == 'auto':
if has_reddit and has_x:
base = 'both'
elif has_reddit:
base = 'reddit'
elif has_x:
base = 'x'
elif has_web:
base = 'web'
else:
return 'none', "No sources are available."
if include_web:
if base == 'both':
return 'all', None
if base == 'reddit':
return 'reddit-web', None
if base == 'x':
return 'x-web', None
return base, None
if requested == 'web':
return 'web', None
if requested == 'both':
if not (has_reddit and has_x):
return 'none', "Requested both sources but X source is missing."
if include_web:
return 'all', None
return 'both', None
if requested == 'reddit':
if not has_reddit:
return 'none', "Requested Reddit but only xAI key is available."
if include_web:
return 'reddit-web', None
return 'reddit', None
if requested == 'x':
if not has_x:
return 'none', "Requested X but no X source is available (need Bird auth or XAI_API_KEY)."
if include_web:
return 'x-web', None
return 'x', None
return requested, None
def get_x_source(config: Dict[str, Any]) -> Optional[str]:
"""Determine the best available X/Twitter source.
Priority chain:
1. AUTH_TOKEN/CT0 from env var or .env file → Bird with method "env"
2. AUTH_TOKEN/CT0 from browser cookie extraction → Bird with method "browser-{browser}"
3. XAI_API_KEY → xAI with method "api"
4. None
Use get_x_source_with_method() to also get the method string.
Browser-cookie probing is intentionally not used here. Automatic Keychain
access causes popups during normal pipeline runs. Bird is only considered
available when AUTH_TOKEN and CT0 are present explicitly.
Args:
config: Configuration dict from get_config()
Returns:
'bird' if Bird is installed and authenticated,
'bird' if Bird is installed and explicit cookies are configured,
'xai' if XAI_API_KEY is configured,
None if no X source available.
"""
source, _method = get_x_source_with_method(config)
return source
def get_x_source_with_method(config: Dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
"""Determine the best available X/Twitter source and auth method.
Priority chain:
1. AUTH_TOKEN/CT0 (env var or .env) → Bird with method "env"
2. AUTH_TOKEN/CT0 (browser cookies) → Bird with method "browser-{browser}"
3. XAI_API_KEY → xAI with method "api"
4. None
Args:
config: Configuration dict from get_config()
Returns:
Tuple of (source, method) where source is 'bird', 'xai', or None
and method is 'env', 'browser-chrome', 'browser-firefox', 'browser-safari', 'api', or None.
"""
# Import here to avoid circular dependency
from . import bird_x
setup_complete = config.get('SETUP_COMPLETE')
preferred = (config.get('LAST30DAYS_X_BACKEND') or '').lower()
has_bird_creds = bool(config.get('AUTH_TOKEN') and config.get('CT0'))
if has_bird_creds:
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0'))
# Check Bird first (free option — uses AUTH_TOKEN/CT0 from any source)
if bird_x.is_bird_installed():
auth_source = config.get('_AUTH_TOKEN_SOURCE')
if preferred == 'xai':
return 'xai' if config.get('XAI_API_KEY') else None
if preferred == 'bird':
return 'bird' if has_bird_creds and bird_x.is_bird_installed() else None
# If SETUP_COMPLETE is not set, only allow explicit env var credentials.
# Do NOT call is_bird_authenticated() for browser-cookie probing —
# that requires user consent via the setup wizard.
if not setup_complete:
# Explicit AUTH_TOKEN from env var / .env file is always allowed
if auth_source == 'env' and config.get('AUTH_TOKEN'):
username = bird_x.is_bird_authenticated()
if username:
return 'bird', 'env'
else:
# SETUP_COMPLETE is set — normal flow, probe cookies if needed
username = bird_x.is_bird_authenticated()
if username:
if auth_source and auth_source.startswith('browser-'):
method = auth_source # e.g. "browser-firefox"
else:
method = 'env'
return 'bird', method
# Fall back to xAI if key exists
if config.get('XAI_API_KEY'):
return 'xai', 'api'
return 'xai'
if has_bird_creds and bird_x.is_bird_installed():
return 'bird'
return None, None
return None
def is_ytdlp_available() -> bool:
@@ -655,6 +429,25 @@ def is_ytdlp_available() -> bool:
return youtube_yt.is_ytdlp_installed()
def is_youtube_comments_available(config: dict[str, Any]) -> bool:
"""Check if YouTube comment enrichment is available.
Requires SCRAPECREATORS_API_KEY AND youtube_comments in INCLUDE_SOURCES.
"""
if not config.get('SCRAPECREATORS_API_KEY'):
return False
include = _parse_include_sources(config)
return 'youtube_comments' in include
def is_youtube_sc_available(config: dict[str, Any]) -> bool:
"""Check if ScrapeCreators YouTube search fallback is available.
Used when yt-dlp is not installed or fails.
"""
return bool(config.get('SCRAPECREATORS_API_KEY'))
def is_hackernews_available() -> bool:
"""Check if Hacker News source is available.
@@ -663,7 +456,7 @@ def is_hackernews_available() -> bool:
return True
def is_bluesky_available(config: Dict[str, Any]) -> bool:
def is_bluesky_available(config: dict[str, Any]) -> bool:
"""Check if Bluesky source is available.
Requires BSKY_HANDLE and BSKY_APP_PASSWORD (app password from bsky.app/settings).
@@ -671,7 +464,7 @@ def is_bluesky_available(config: Dict[str, Any]) -> bool:
return bool(config.get('BSKY_HANDLE') and config.get('BSKY_APP_PASSWORD'))
def is_truthsocial_available(config: Dict[str, Any]) -> bool:
def is_truthsocial_available(config: dict[str, Any]) -> bool:
"""Check if Truth Social source is available.
Requires TRUTHSOCIAL_TOKEN (bearer token from browser dev tools).
@@ -687,7 +480,7 @@ def is_polymarket_available() -> bool:
return True
def is_tiktok_available(config: Dict[str, Any]) -> bool:
def is_tiktok_available(config: dict[str, Any]) -> bool:
"""Check if TikTok source is available (ScrapeCreators or legacy Apify).
Returns True if SCRAPECREATORS_API_KEY or APIFY_API_TOKEN is set.
@@ -695,12 +488,29 @@ def is_tiktok_available(config: Dict[str, Any]) -> bool:
return bool(config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN'))
def get_tiktok_token(config: Dict[str, Any]) -> str:
def get_tiktok_token(config: dict[str, Any]) -> str:
"""Get TikTok API token, preferring ScrapeCreators over legacy Apify."""
return config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN') or ''
def is_instagram_available(config: Dict[str, Any]) -> bool:
def _parse_include_sources(config: dict[str, Any]) -> set[str]:
"""Parse INCLUDE_SOURCES config value into a set of lowercase source names."""
raw = config.get('INCLUDE_SOURCES') or ''
return {s.strip().lower() for s in raw.split(',') if s.strip()}
def is_threads_available(config: dict[str, Any]) -> bool:
"""Check if Threads source is available.
Requires SCRAPECREATORS_API_KEY AND 'threads' in INCLUDE_SOURCES.
Threads is an opt-in source - it is not activated by default.
"""
if not config.get('SCRAPECREATORS_API_KEY'):
return False
return 'threads' in _parse_include_sources(config)
def is_instagram_available(config: dict[str, Any]) -> bool:
"""Check if Instagram source is available (ScrapeCreators).
Returns True if SCRAPECREATORS_API_KEY is set.
@@ -709,12 +519,12 @@ def is_instagram_available(config: Dict[str, Any]) -> bool:
return bool(config.get('SCRAPECREATORS_API_KEY'))
def get_instagram_token(config: Dict[str, Any]) -> str:
def get_instagram_token(config: dict[str, Any]) -> str:
"""Get Instagram API token (same ScrapeCreators key as TikTok)."""
return config.get('SCRAPECREATORS_API_KEY') or ''
def get_xiaohongshu_api_base(config: Dict[str, Any]) -> str:
def get_xiaohongshu_api_base(config: dict[str, Any]) -> str:
"""Get Xiaohongshu HTTP API base URL.
Defaults to host.docker.internal so OpenClaw Docker can reach host service.
@@ -722,7 +532,7 @@ def get_xiaohongshu_api_base(config: Dict[str, Any]) -> str:
return (config.get('XIAOHONGSHU_API_BASE') or "http://host.docker.internal:18060").rstrip("/")
def is_xiaohongshu_available(config: Dict[str, Any]) -> bool:
def is_xiaohongshu_available(config: dict[str, Any]) -> bool:
"""Check whether Xiaohongshu HTTP API is reachable and logged in."""
# Import here to avoid heavy imports at module load.
from . import http
@@ -744,7 +554,14 @@ def is_xiaohongshu_available(config: Dict[str, Any]) -> bool:
if isinstance(login, dict) else False
)
return bool(is_logged_in)
except Exception:
except (OSError, http.HTTPError):
return False
except Exception as exc:
sys.stderr.write(
f"[last30days] WARNING: unexpected error checking Xiaohongshu: "
f"{type(exc).__name__}: {exc}\n"
)
sys.stderr.flush()
return False
@@ -752,56 +569,47 @@ def is_xiaohongshu_available(config: Dict[str, Any]) -> bool:
is_apify_available = is_tiktok_available
def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
def get_x_source_status(config: dict[str, Any]) -> dict[str, Any]:
"""Get detailed X source status for UI decisions.
Returns:
Dict with keys: source, method, bird_installed, bird_authenticated,
Dict with keys: source, bird_installed, bird_authenticated,
bird_username, xai_available, can_install_bird
The ``method`` field indicates HOW the active source is authenticated:
- "env" — AUTH_TOKEN came from an env var or .env file
- "browser-chrome", "browser-firefox", "browser-safari" — from cookie extraction
- "api" — using xAI API key
- None — no X source available
"""
from . import bird_x
setup_complete = config.get('SETUP_COMPLETE')
bird_status = bird_x.get_bird_status()
xai_available = bool(config.get('XAI_API_KEY'))
if not setup_complete:
# Before consent: do NOT call get_bird_status() which probes cookies.
# Only check if Bird is installed (no cookie probing) and use the
# gated get_x_source_with_method() which blocks cookie detection.
bird_installed = bird_x.is_bird_installed()
source, method = get_x_source_with_method(config)
# Bird "authenticated" only if get_x_source_with_method found explicit creds
bird_authenticated = (source == 'bird')
return {
"source": source,
"method": method,
"bird_installed": bird_installed,
"bird_authenticated": bird_authenticated,
"bird_username": None if not bird_authenticated else "env AUTH_TOKEN",
"xai_available": xai_available,
"can_install_bird": True,
}
# SETUP_COMPLETE is set — normal flow
bird_status = bird_x.get_bird_status()
# Use the unified resolution function for source + method
source, method = get_x_source_with_method(config)
# Determine active source
if bird_status["authenticated"]:
source = 'bird'
elif xai_available:
source = 'xai'
else:
source = None
return {
"source": source,
"method": method,
"bird_installed": bird_status["installed"],
"bird_authenticated": bird_status["authenticated"],
"bird_username": bird_status["username"],
"xai_available": xai_available,
"can_install_bird": bird_status["can_install"],
}
# Pinterest
def is_pinterest_available(config: dict[str, Any]) -> bool:
"""Check if Pinterest source is available.
Returns True when SCRAPECREATORS_API_KEY is set AND 'pinterest' is in
INCLUDE_SOURCES (or requested_sources at the pipeline level). Pinterest
is opt-in because not every topic benefits from visual pin results.
"""
return bool(config.get('SCRAPECREATORS_API_KEY'))
def get_pinterest_token(config: dict[str, Any]) -> str:
"""Get Pinterest API token (same ScrapeCreators key as TikTok/Instagram)."""
return config.get('SCRAPECREATORS_API_KEY') or ''
scripts/lib/exa_search.py
-176
View File
@@ -1,176 +0,0 @@
"""Exa AI web search for last30days skill.
Uses the Exa Search API as a free web search backend.
Free tier: 1,000 searches/month, semantic search, no credit card required.
API docs: https://docs.exa.ai/reference/search
"""
import sys
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from . import http
ENDPOINT = "https://api.exa.ai/search"
# Domains to exclude (handled by Reddit/X search)
EXCLUDED_DOMAINS = {
"reddit.com", "www.reddit.com", "old.reddit.com",
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
}
def search_web(
topic: str,
from_date: str,
to_date: str,
api_key: str,
depth: str = "default",
) -> List[Dict[str, Any]]:
"""Search the web via Exa AI Search API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
api_key: Exa API key
depth: 'quick', 'default', or 'deep'
Returns:
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
"""
num_results = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
max_chars = {"quick": 1000, "default": 2000, "deep": 3000}.get(depth, 2000)
payload = {
"query": f"{topic} (from {from_date} to {to_date})",
"type": "auto",
"numResults": num_results,
"contents": {"text": {"maxCharacters": max_chars}},
}
# Add date filtering if dates are provided
if from_date:
payload["startPublishedDate"] = f"{from_date}T00:00:00.000Z"
if to_date:
payload["endPublishedDate"] = f"{to_date}T23:59:59.999Z"
sys.stderr.write(f"[Web] Searching Exa for: {topic}\n")
sys.stderr.flush()
try:
response = http.post(
ENDPOINT,
json_data=payload,
headers={
"x-api-key": api_key,
},
timeout=20,
retries=2,
)
except http.HTTPError as e:
status = e.status_code
if status == 401:
sys.stderr.write("[Web] Exa: invalid API key (401)\n")
sys.stderr.flush()
return []
if status == 429:
sys.stderr.write("[Web] Exa: rate limited (429)\n")
sys.stderr.flush()
return []
sys.stderr.write(f"[Web] Exa: HTTP error {status}: {e}\n")
sys.stderr.flush()
return []
except Exception as e:
sys.stderr.write(f"[Web] Exa: request failed: {e}\n")
sys.stderr.flush()
return []
return _normalize_results(response)
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Convert Exa API response to websearch item schema.
Exa results have: title, url, text, publishedDate, score, author.
"""
items = []
results = response.get("results", [])
if not isinstance(results, list):
return items
for i, result in enumerate(results):
if not isinstance(result, dict):
continue
url = result.get("url", "")
if not url:
continue
# Skip excluded domains
try:
domain = urlparse(url).netloc.lower()
if domain in EXCLUDED_DOMAINS:
continue
if domain.startswith("www."):
domain = domain[4:]
except Exception:
domain = ""
title = str(result.get("title", "")).strip()
# Exa returns page content in "text" field
snippet = str(result.get("text", "")).strip()
if not title and not snippet:
continue
# Parse publishedDate (ISO format from Exa: "2026-03-15T00:00:00.000Z")
date = _parse_exa_date(result.get("publishedDate"))
date_confidence = "med" if date else "low"
# Exa provides a relevance score
relevance = result.get("score", 0.6)
try:
relevance = min(1.0, max(0.0, float(relevance)))
except (TypeError, ValueError):
relevance = 0.6
items.append({
"id": f"W{i+1}",
"title": title[:200],
"url": url,
"source_domain": domain,
"snippet": snippet[:500],
"date": date,
"date_confidence": date_confidence,
"relevance": relevance,
"why_relevant": "",
})
sys.stderr.write(f"[Web] Exa: {len(items)} results\n")
sys.stderr.flush()
return items
def _parse_exa_date(published_date: Optional[str]) -> Optional[str]:
"""Parse Exa's publishedDate to YYYY-MM-DD.
Exa returns ISO format like "2026-03-15T00:00:00.000Z".
"""
if not published_date:
return None
try:
# Extract YYYY-MM-DD from ISO datetime
if "T" in published_date:
return published_date.split("T")[0]
# Already YYYY-MM-DD
if len(published_date) >= 10:
return published_date[:10]
except Exception:
pass
return None
scripts/lib/fusion.py
+202
View File
@@ -0,0 +1,202 @@
"""Weighted reciprocal rank fusion for per-(subquery, source) streams."""
from __future__ import annotations
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
from . import schema
# Standard RRF smoothing constant (Cormack et al. 2009)
RRF_K = 60
def _candidate_sort_key(c: schema.Candidate) -> tuple:
return (-c.rrf_score, -c.local_relevance, -c.freshness, schema.candidate_source_label(c), c.title)
def _normalize_url(url: str) -> str:
"""Normalize URL for dedup: lowercase, strip www/old/m prefixes, remove tracking params."""
parsed = urlparse(url.strip().lower())
netloc = parsed.netloc
for prefix in ("www.", "old.", "m."):
if netloc.startswith(prefix):
netloc = netloc[len(prefix):]
# Strip tracking params
params = parse_qs(parsed.query)
clean_params = {k: v for k, v in params.items() if not k.startswith("utm_")}
query = urlencode(clean_params, doseq=True)
return urlunparse((parsed.scheme, netloc, parsed.path.rstrip("/"), "", query, ""))
def candidate_key(item: schema.SourceItem) -> str:
if item.url:
return _normalize_url(item.url)
return f"{item.source}:{item.item_id}"
_DIVERSITY_RELEVANCE_THRESHOLD = 0.25
# Per-author cap: no single author/handle should dominate the pool.
_MAX_ITEMS_PER_AUTHOR = 3
def _extract_author(candidate: schema.Candidate) -> str | None:
"""Return a normalized author key from a candidate's source items."""
for item in candidate.source_items:
if item.author:
return item.author.strip().lower()
return None
def _apply_per_author_cap(
candidates: list[schema.Candidate],
max_per_author: int = _MAX_ITEMS_PER_AUTHOR,
) -> list[schema.Candidate]:
"""Keep at most *max_per_author* items from any single author.
Candidates are assumed to already be sorted by quality (rrf_score etc.),
so the first N encountered per author are the best ones.
"""
author_counts: dict[str, int] = {}
result: list[schema.Candidate] = []
for c in candidates:
author = _extract_author(c)
if author is None:
result.append(c)
continue
count = author_counts.get(author, 0)
if count < max_per_author:
result.append(c)
author_counts[author] = count + 1
return result
def _diversify_pool(
fused: list[schema.Candidate],
pool_limit: int,
min_per_source: int = 2,
) -> list[schema.Candidate]:
"""Ensure at least *min_per_source* items per qualifying source survive truncation.
Sources only qualify for reserved slots if their best item exceeds
the relevance threshold. Low-relevance sources compete on merit only.
"""
max_relevance: dict[str, float] = {}
for c in fused:
current = max_relevance.get(c.source, 0.0)
if c.local_relevance > current:
max_relevance[c.source] = c.local_relevance
reserved: dict[str, list[schema.Candidate]] = {}
remainder: list[schema.Candidate] = []
for c in fused:
qualifies = max_relevance.get(c.source, 0.0) >= _DIVERSITY_RELEVANCE_THRESHOLD
bucket = reserved.setdefault(c.source, [])
if qualifies and len(bucket) < min_per_source:
bucket.append(c)
else:
remainder.append(c)
pool = [c for per_source in reserved.values() for c in per_source]
seen = {c.candidate_id for c in pool}
for c in remainder:
if len(pool) >= pool_limit:
break
if c.candidate_id not in seen:
pool.append(c)
pool.sort(key=_candidate_sort_key)
return pool[:pool_limit]
def weighted_rrf(
streams: dict[tuple[str, str], list[schema.SourceItem]],
plan: schema.QueryPlan,
*,
pool_limit: int,
) -> list[schema.Candidate]:
"""Fuse ranked lists into a single candidate pool."""
subqueries = {subquery.label: subquery for subquery in plan.subqueries}
candidates: dict[str, schema.Candidate] = {}
for (label, source), items in streams.items():
subquery = subqueries[label]
weight = subquery.weight * plan.source_weights.get(source, 1.0)
for rank, item in enumerate(items, start=1):
key = candidate_key(item)
score = weight / (RRF_K + rank)
item_local_relevance = item.local_relevance if item.local_relevance is not None else float(item.metadata.get("local_relevance", item.relevance_hint))
item_freshness = item.freshness if item.freshness is not None else int(item.metadata.get("freshness", 0))
item_source_quality = item.source_quality if item.source_quality is not None else float(item.metadata.get("source_quality", 0.6))
if key not in candidates:
candidates[key] = schema.Candidate(
candidate_id=key,
item_id=item.item_id,
source=item.source,
title=item.title,
url=item.url,
snippet=item.snippet,
subquery_labels=[label],
native_ranks={f"{label}:{source}": rank},
local_relevance=item_local_relevance,
freshness=item_freshness,
engagement=item.engagement_score if item.engagement_score is not None else item.metadata.get("engagement_score"),
source_quality=item_source_quality,
rrf_score=score,
sources=[item.source],
source_items=[item],
metadata={
"provenance": [
{
"source": source,
"subquery_label": label,
"native_rank": rank,
"item_id": item.item_id,
}
]
},
)
continue
candidate = candidates[key]
candidate.rrf_score += score
previous_primary_score = (candidate.local_relevance * 100.0) + candidate.freshness + (candidate.source_quality * 10.0)
incoming_primary_score = (item_local_relevance * 100.0) + item_freshness + (item_source_quality * 10.0)
candidate.local_relevance = max(
candidate.local_relevance,
item_local_relevance,
)
candidate.freshness = max(candidate.freshness, item_freshness)
item_eng = item.engagement_score if item.engagement_score is not None else item.metadata.get("engagement_score")
if candidate.engagement is None:
candidate.engagement = item_eng
elif item_eng is not None:
candidate.engagement = max(candidate.engagement, item_eng)
candidate.source_quality = max(
candidate.source_quality,
item_source_quality,
)
candidate.native_ranks[f"{label}:{source}"] = rank
if label not in candidate.subquery_labels:
candidate.subquery_labels.append(label)
if item.source not in candidate.sources:
candidate.sources.append(item.source)
if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items):
candidate.source_items.append(item)
candidate.metadata.setdefault("provenance", []).append(
{
"source": source,
"subquery_label": label,
"native_rank": rank,
"item_id": item.item_id,
}
)
if incoming_primary_score > previous_primary_score:
candidate.item_id = item.item_id
candidate.source = item.source
candidate.title = item.title
candidate.snippet = item.snippet
if len(candidate.snippet.split()) < len(item.snippet.split()):
candidate.snippet = item.snippet
fused = sorted(candidates.values(), key=_candidate_sort_key)
fused = _apply_per_author_cap(fused)
return _diversify_pool(fused, pool_limit)
scripts/lib/github.py
+920
View File
@@ -0,0 +1,920 @@
"""GitHub Issues/PRs search via the public GitHub Search API.
Uses api.github.com/search/issues for issue/PR discovery and
per-item comment enrichment. Auth via GITHUB_TOKEN env var or
`gh auth token` subprocess fallback.
"""
import json
import math
import os
import re
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Dict, List, Optional
from . import log
from .query import extract_core_subject
from .relevance import token_overlap_relevance
SEARCH_URL = "https://api.github.com/search/issues"
DEPTH_LIMITS = {
"quick": 15,
"default": 30,
"deep": 60,
}
ENRICH_LIMITS = {
"quick": 3,
"default": 5,
"deep": 8,
}
USER_AGENT = "last30days/3.0 (research tool)"
def _log(msg: str):
log.source_log("GitHub", msg, tty_only=False)
def _resolve_token(token: Optional[str] = None) -> Optional[str]:
"""Resolve GitHub auth token from argument, env, or gh CLI."""
if token:
return token
env_token = os.environ.get("GITHUB_TOKEN")
if env_token:
return env_token
# Fallback: try gh CLI
try:
result = subprocess.run(
["gh", "auth", "token"],
capture_output=True, text=True, timeout=5,
)
if result.returncode == 0 and result.stdout.strip():
return result.stdout.strip()
except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
pass
return None
def _fetch_json(
url: str,
token: Optional[str] = None,
timeout: int = 15,
) -> Optional[Dict[str, Any]]:
"""Fetch JSON from GitHub API. Returns None on failure."""
headers = {
"User-Agent": USER_AGENT,
"Accept": "application/vnd.github+json",
}
if token:
headers["Authorization"] = f"Bearer {token}"
req = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
body = resp.read().decode("utf-8")
return json.loads(body)
except urllib.error.HTTPError as e:
if e.code == 403:
_log(f"403 rate limited or forbidden: {url}")
return None
if e.code == 422:
_log(f"422 unprocessable: {url}")
return None
_log(f"HTTP {e.code}: {e.reason}")
return None
except (urllib.error.URLError, OSError, TimeoutError) as e:
_log(f"Network error: {e}")
return None
except json.JSONDecodeError as e:
_log(f"JSON decode error: {e}")
return None
def _parse_repo_from_url(html_url: str) -> str:
"""Extract 'owner/repo' from a GitHub issue/PR URL."""
parts = html_url.replace("https://github.com/", "").split("/")
if len(parts) >= 2:
return f"{parts[0]}/{parts[1]}"
return ""
def _parse_date(iso_str: Optional[str]) -> Optional[str]:
"""Extract YYYY-MM-DD from ISO 8601 datetime string."""
if not iso_str:
return None
try:
return iso_str[:10]
except (IndexError, TypeError):
return None
def _compute_relevance(
query: str,
title: str,
rank_index: int,
reactions: int,
comments: int,
) -> float:
"""Blend text relevance with engagement signals."""
rank_score = max(0.3, 1.0 - (rank_index * 0.02))
engagement_boost = min(0.2, math.log1p(reactions + comments) / 20)
if query:
content_score = token_overlap_relevance(query, title)
relevance = min(1.0, 0.6 * rank_score + 0.4 * content_score + engagement_boost)
else:
relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
return round(relevance, 2)
def search_github(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
token: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Search GitHub Issues and PRs.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: Optional GitHub token (falls back to env/gh CLI)
Returns:
List of normalized item dicts. Empty list on any failure.
"""
resolved_token = _resolve_token(token)
if not resolved_token:
_log("No GitHub token available (set GITHUB_TOKEN or install gh CLI)")
return []
count = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
core = extract_core_subject(topic)
_log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
# Build search query with date filter
q = f"{core} created:>{from_date}"
params = {
"q": q,
"sort": "reactions",
"order": "desc",
"per_page": str(min(count, 100)),
}
url = f"{SEARCH_URL}?{urllib.parse.urlencode(params)}"
data = _fetch_json(url, token=resolved_token, timeout=30)
if not data:
return []
raw_items = data.get("items", [])
_log(f"Found {len(raw_items)} issues/PRs")
items = []
for i, item in enumerate(raw_items[:count]):
html_url = item.get("html_url", "")
repo = _parse_repo_from_url(html_url)
title = item.get("title", "")
body_text = item.get("body") or ""
reactions_total = item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0
comment_count = item.get("comments", 0)
labels = [
lbl.get("name", "") for lbl in (item.get("labels") or [])
if isinstance(lbl, dict)
]
state = item.get("state", "")
is_pr = "pull_request" in item
author = item.get("user", {}).get("login", "") if isinstance(item.get("user"), dict) else ""
relevance = _compute_relevance(core, title, i, reactions_total, comment_count)
items.append({
"id": f"GH{i + 1}",
"title": title,
"url": html_url,
"date": _parse_date(item.get("created_at")),
"author": author,
"source": "github",
"score": reactions_total,
"container": repo,
"snippet": body_text[:300] if body_text else "",
"relevance": relevance,
"why_relevant": f"GitHub {'PR' if is_pr else 'issue'}: {title[:60]}",
"engagement": {
"reactions": reactions_total,
"comments": comment_count,
},
"metadata": {
"labels": labels,
"state": state,
"comment_count": comment_count,
"reactions": reactions_total,
"is_pr": is_pr,
},
})
# Enrich top items with comments
items = _enrich_top_items(items, depth, resolved_token)
# Date filter
filtered = []
for item in items:
d = item.get("date")
if d is None or (from_date <= d <= to_date):
filtered.append(item)
# Sort by relevance
filtered.sort(key=lambda x: x.get("relevance", 0), reverse=True)
return filtered
def _enrich_top_items(
items: List[Dict[str, Any]],
depth: str,
token: str,
) -> List[Dict[str, Any]]:
"""Fetch comments for top N items by reactions."""
if not items:
return items
limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])
by_reactions = sorted(
range(len(items)),
key=lambda i: items[i].get("score", 0),
reverse=True,
)
to_enrich = by_reactions[:limit]
_log(f"Enriching top {len(to_enrich)} items with comments")
with ThreadPoolExecutor(max_workers=5) as executor:
futures = {
executor.submit(
_fetch_item_comments,
items[idx]["url"],
token,
): idx
for idx in to_enrich
}
for future in as_completed(futures):
idx = futures[future]
try:
comments = future.result(timeout=15)
items[idx]["metadata"]["top_comments"] = comments
except (KeyError, TypeError, OSError) as exc:
_log(f"Comment enrichment failed for {items[idx].get('url', '?')}: {type(exc).__name__}: {exc}")
items[idx]["metadata"]["top_comments"] = []
return items
def _fetch_item_comments(
issue_url: str,
token: str,
max_comments: int = 5,
) -> List[Dict[str, Any]]:
"""Fetch comments for a GitHub issue/PR.
Args:
issue_url: HTML URL like https://github.com/owner/repo/issues/123
token: GitHub auth token
max_comments: Max comments to return
Returns:
List of comment dicts with score, excerpt, author.
"""
path = issue_url.replace("https://github.com/", "")
path = path.replace("/pull/", "/issues/")
api_url = f"https://api.github.com/repos/{path}/comments?per_page={max_comments}&sort=reactions&direction=desc"
data = _fetch_json(api_url, token=token, timeout=15)
if not data or not isinstance(data, list):
return []
comments = []
for c in data[:max_comments]:
body = c.get("body") or ""
excerpt = body[:300] + "..." if len(body) > 300 else body
reactions = c.get("reactions", {})
reaction_count = reactions.get("total_count", 0) if isinstance(reactions, dict) else 0
author = c.get("user", {}).get("login", "") if isinstance(c.get("user"), dict) else ""
comments.append({
"score": reaction_count,
"excerpt": excerpt,
"author": author,
})
return comments
# ---------------------------------------------------------------------------
# Person-mode search: author-scoped queries, star enrichment, release notes
# ---------------------------------------------------------------------------
PERSON_DEPTH_LIMITS = {
"quick": {"pr_pages": 1, "own_repos": 3, "external_repos": 5},
"default": {"pr_pages": 1, "own_repos": 5, "external_repos": 10},
"deep": {"pr_pages": 2, "own_repos": 5, "external_repos": 15},
}
def _fetch_readme_snippet(repo: str, token: str, max_chars: int = 500) -> Optional[str]:
"""Fetch README content for a repo, truncated to first ~max_chars."""
url = f"https://api.github.com/repos/{repo}/readme"
headers = {
"User-Agent": USER_AGENT,
"Accept": "application/vnd.github.raw+json",
}
if token:
headers["Authorization"] = f"Bearer {token}"
req = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
raw = resp.read().decode("utf-8", errors="replace")
except (urllib.error.HTTPError, urllib.error.URLError, OSError, TimeoutError):
return None
if not raw:
return None
# Try to break at a paragraph boundary
if len(raw) <= max_chars:
return raw
cut = raw[:max_chars]
last_double_newline = cut.rfind("\n\n")
if last_double_newline > max_chars // 3:
return cut[:last_double_newline].rstrip()
return cut.rstrip() + "..."
def _fetch_latest_releases(
repo: str, token: str, count: int = 3, max_body: int = 300,
) -> List[Dict[str, str]]:
"""Fetch latest releases for a repo."""
url = f"https://api.github.com/repos/{repo}/releases?per_page={count}"
data = _fetch_json(url, token=token, timeout=10)
if not data or not isinstance(data, list):
return []
releases = []
for r in data[:count]:
tag = r.get("tag_name", "")
date = _parse_date(r.get("published_at"))
body = (r.get("body") or "")[:max_body]
name = r.get("name") or tag
releases.append({"tag": tag, "name": name, "date": date, "body": body})
return releases
def _fetch_top_issues(repo: str, token: str) -> Dict[str, Any]:
"""Fetch top feature request (by reactions) and top complaint (by comments)."""
result: Dict[str, Any] = {}
# Top feature request: issues with enhancement label, sorted by reactions
feat_q = urllib.parse.quote(f"repo:{repo} is:issue is:open label:enhancement")
feat_url = f"{SEARCH_URL}?q={feat_q}&sort=reactions&order=desc&per_page=1"
feat_data = _fetch_json(feat_url, token=token, timeout=10)
if feat_data and feat_data.get("items"):
item = feat_data["items"][0]
result["top_feature_request"] = {
"title": item.get("title", ""),
"reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0,
"comments": item.get("comments", 0),
"url": item.get("html_url", ""),
}
elif feat_data and feat_data.get("total_count", 0) == 0:
# No enhancement label; fall back to top issue by reactions
fallback_q = urllib.parse.quote(f"repo:{repo} is:issue is:open")
fallback_url = f"{SEARCH_URL}?q={fallback_q}&sort=reactions&order=desc&per_page=1"
fallback_data = _fetch_json(fallback_url, token=token, timeout=10)
if fallback_data and fallback_data.get("items"):
item = fallback_data["items"][0]
result["top_feature_request"] = {
"title": item.get("title", ""),
"reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0,
"comments": item.get("comments", 0),
"url": item.get("html_url", ""),
}
# Top complaint: most-discussed open issue (by comments)
bug_q = urllib.parse.quote(f"repo:{repo} is:issue is:open")
bug_url = f"{SEARCH_URL}?q={bug_q}&sort=comments&order=desc&per_page=1"
bug_data = _fetch_json(bug_url, token=token, timeout=10)
if bug_data and bug_data.get("items"):
item = bug_data["items"][0]
result["top_complaint"] = {
"title": item.get("title", ""),
"reactions": item.get("reactions", {}).get("total_count", 0) if isinstance(item.get("reactions"), dict) else 0,
"comments": item.get("comments", 0),
"url": item.get("html_url", ""),
}
return result
def _fetch_repo_info(repo: str, token: str) -> Optional[Dict[str, Any]]:
"""Fetch repo metadata (stars, forks, description, language)."""
url = f"https://api.github.com/repos/{repo}"
data = _fetch_json(url, token=token, timeout=10)
if not data or not isinstance(data, dict):
return None
return {
"stars": data.get("stargazers_count", 0),
"forks": data.get("forks_count", 0),
"description": (data.get("description") or "")[:200],
"language": data.get("language") or "",
"open_issues": data.get("open_issues_count", 0),
}
def _format_stars(n: int) -> str:
"""Format star count as human-readable (e.g., 349K, 2.9K, 42)."""
if n >= 1_000_000:
return f"{n / 1_000_000:.1f}M"
if n >= 1_000:
return f"{n / 1_000:.0f}K" if n >= 10_000 else f"{n / 1_000:.1f}K"
return str(n)
def search_github_person(
username: str,
from_date: str,
to_date: str,
depth: str = "default",
token: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Person-mode GitHub search: author-scoped queries with star enrichment.
Returns SourceItems for:
- 1 velocity summary item
- Per-repo items for top external repos (with stars + release notes)
- Per-repo items for own repos (with stars + README + top issues + releases)
"""
resolved_token = _resolve_token(token)
if not resolved_token:
_log("No GitHub token available for person-mode search")
return []
limits = PERSON_DEPTH_LIMITS.get(depth, PERSON_DEPTH_LIMITS["default"])
_log(f"Person-mode search for @{username} (since {from_date})")
# Phase 1: PR velocity via search API
total_q = urllib.parse.quote(f"author:{username} type:pr created:>{from_date}")
merged_q = urllib.parse.quote(f"author:{username} type:pr is:merged created:>{from_date}")
total_url = f"{SEARCH_URL}?q={total_q}&per_page=1"
merged_url = f"{SEARCH_URL}?q={merged_q}&sort=reactions&order=desc&per_page=100"
total_data = _fetch_json(total_url, token=resolved_token, timeout=20)
merged_data = _fetch_json(merged_url, token=resolved_token, timeout=20)
total_prs = total_data.get("total_count", 0) if total_data else 0
merged_count = merged_data.get("total_count", 0) if merged_data else 0
merged_items = merged_data.get("items", []) if merged_data else []
_log(f"Found {total_prs} total PRs, {merged_count} merged")
if total_prs == 0 and merged_count == 0:
_log("No PRs found, falling back to keyword search")
return []
# Phase 2: Group merged PRs by repo
repo_pr_counts: Dict[str, int] = {}
for item in merged_items:
repo = _parse_repo_from_url(item.get("html_url", ""))
if repo:
repo_pr_counts[repo] = repo_pr_counts.get(repo, 0) + 1
# Sort repos by PR count (most active first)
sorted_repos = sorted(repo_pr_counts.items(), key=lambda x: x[1], reverse=True)
# Phase 3: Fetch own repos
own_repos_url = f"https://api.github.com/users/{username}/repos?sort=stars&per_page={limits['own_repos']}&direction=desc"
own_repos_data = _fetch_json(own_repos_url, token=resolved_token, timeout=15)
own_repo_names = set()
own_repos_info: List[Dict[str, Any]] = []
if own_repos_data and isinstance(own_repos_data, list):
for r in own_repos_data:
full_name = r.get("full_name", "")
if full_name and not r.get("fork"):
own_repo_names.add(full_name)
own_repos_info.append({
"full_name": full_name,
"stars": r.get("stargazers_count", 0),
"forks": r.get("forks_count", 0),
"description": (r.get("description") or "")[:200],
"language": r.get("language") or "",
"open_issues": r.get("open_issues_count", 0),
})
# Separate external repos from own repos
external_repos = [(repo, count) for repo, count in sorted_repos if repo not in own_repo_names]
external_repos = external_repos[:limits["external_repos"]]
# Phase 4: Parallel enrichment (star counts, releases, READMEs, top issues)
items: List[Dict[str, Any]] = []
idx = 0
# Build velocity summary
open_prs = total_prs - merged_count
merge_rate = round(100 * merged_count / total_prs) if total_prs > 0 else 0
num_repos = len(repo_pr_counts)
velocity_text = (
f"GitHub Person Profile: @{username}\n\n"
f"CONTRIBUTION VELOCITY (last {(to_date > from_date) and 30 or 30} days)\n"
f"- {merged_count} PRs merged across {num_repos} repos ({merge_rate}% merge rate)\n"
f"- {total_prs} total PRs submitted, {open_prs} still open\n"
)
idx += 1
items.append({
"id": f"GH{idx}",
"title": f"@{username}: {merged_count} PRs merged across {num_repos} repos ({merge_rate}% merge rate)",
"url": f"https://github.com/{username}",
"date": to_date,
"author": username,
"source": "github",
"score": merged_count,
"container": f"@{username}",
"snippet": velocity_text,
"relevance": 0.95,
"why_relevant": f"GitHub profile: @{username} - {merged_count} PRs merged across {num_repos} repos",
"engagement": {"reactions": merged_count, "comments": total_prs},
"metadata": {
"labels": ["person-profile", "velocity"],
"state": "open",
"comment_count": 0,
"reactions": merged_count,
"is_pr": False,
},
})
# Phase 5: Enrich external repos (parallel: star counts + releases)
_log(f"Enriching {len(external_repos)} external repos + {len(own_repos_info)} own repos")
with ThreadPoolExecutor(max_workers=8) as executor:
# External repo enrichment: stars + releases
ext_futures = {}
for repo, pr_count in external_repos:
ext_futures[executor.submit(_enrich_external_repo, repo, resolved_token)] = (repo, pr_count)
# Own repo enrichment: README + releases + top issues
own_futures = {}
for own_repo in own_repos_info:
own_futures[executor.submit(_enrich_own_repo, own_repo["full_name"], resolved_token)] = own_repo
# Collect external repo results
for future in as_completed(ext_futures):
repo, pr_count = ext_futures[future]
try:
enrichment = future.result(timeout=20)
except Exception as exc:
_log(f"External repo enrichment failed for {repo}: {exc}")
enrichment = {}
repo_info = enrichment.get("info")
releases = enrichment.get("releases", [])
stars = repo_info["stars"] if repo_info else 0
stars_str = _format_stars(stars)
desc = repo_info["description"] if repo_info else ""
snippet_parts = [f"Contributed {pr_count} merged PRs to {repo} ({stars_str} stars)"]
if desc:
snippet_parts.append(f" {desc}")
if releases:
for rel in releases[:2]:
body_preview = f" - {rel['body'][:150]}" if rel.get("body") else ""
snippet_parts.append(f" Latest release: {rel['name']} ({rel['date']}){body_preview}")
idx += 1
items.append({
"id": f"GH{idx}",
"title": f"{repo} ({stars_str} stars) - {pr_count} PRs merged",
"url": f"https://github.com/{repo}",
"date": releases[0]["date"] if releases and releases[0].get("date") else to_date,
"author": username,
"source": "github",
"score": stars,
"container": repo,
"snippet": "\n".join(snippet_parts),
"relevance": min(0.9, 0.6 + math.log1p(stars) / 30 + min(0.15, pr_count / 20)),
"why_relevant": f"GitHub contribution: {pr_count} PRs merged to {repo} ({stars_str} stars)",
"engagement": {"reactions": stars, "comments": pr_count},
"metadata": {
"labels": ["person-profile", "external-repo"],
"state": "open",
"comment_count": pr_count,
"reactions": stars,
"is_pr": False,
},
})
# Collect own repo results
for future in as_completed(own_futures):
own_repo = own_futures[future]
try:
enrichment = future.result(timeout=25)
except Exception as exc:
_log(f"Own repo enrichment failed for {own_repo['full_name']}: {exc}")
enrichment = {}
repo_name = own_repo["full_name"]
stars = own_repo["stars"]
stars_str = _format_stars(stars)
open_issues = own_repo["open_issues"]
desc = own_repo["description"]
readme = enrichment.get("readme")
releases = enrichment.get("releases", [])
top_issues = enrichment.get("top_issues", {})
snippet_parts = [f"Own project: {repo_name} ({stars_str} stars, {open_issues} open issues)"]
if desc:
snippet_parts.append(f" {desc}")
if readme:
snippet_parts.append(f" README: {readme[:300]}")
if releases:
for rel in releases[:2]:
body_preview = f" - {rel['body'][:150]}" if rel.get("body") else ""
snippet_parts.append(f" Latest release: {rel['name']} ({rel['date']}){body_preview}")
feat = top_issues.get("top_feature_request")
if feat:
snippet_parts.append(f" Top feature request: \"{feat['title']}\" ({feat['reactions']} reactions, {feat['comments']} comments)")
complaint = top_issues.get("top_complaint")
if complaint:
snippet_parts.append(f" Top complaint: \"{complaint['title']}\" ({complaint['comments']} comments)")
idx += 1
items.append({
"id": f"GH{idx}",
"title": f"{repo_name} ({stars_str} stars) - own project, {open_issues} open issues",
"url": f"https://github.com/{repo_name}",
"date": releases[0]["date"] if releases and releases[0].get("date") else to_date,
"author": username,
"source": "github",
"score": stars,
"container": repo_name,
"snippet": "\n".join(snippet_parts),
"relevance": min(0.95, 0.7 + math.log1p(stars) / 25),
"why_relevant": f"GitHub own project: {repo_name} ({stars_str} stars)",
"engagement": {"reactions": stars, "comments": open_issues},
"metadata": {
"labels": ["person-profile", "own-repo"],
"state": "open",
"comment_count": open_issues,
"reactions": stars,
"is_pr": False,
},
})
# Sort by relevance
items.sort(key=lambda x: x.get("relevance", 0), reverse=True)
_log(f"Person-mode returned {len(items)} items")
return items
def _enrich_external_repo(repo: str, token: str) -> Dict[str, Any]:
"""Fetch star count + releases for an external repo."""
info = _fetch_repo_info(repo, token)
releases = _fetch_latest_releases(repo, token, count=3)
return {"info": info, "releases": releases}
def _enrich_own_repo(repo: str, token: str) -> Dict[str, Any]:
"""Fetch README + releases + top issues for an own repo."""
readme = _fetch_readme_snippet(repo, token, max_chars=500)
releases = _fetch_latest_releases(repo, token, count=3)
top_issues = _fetch_top_issues(repo, token)
return {"readme": readme, "releases": releases, "top_issues": top_issues}
# ---------------------------------------------------------------------------
# Project-mode search: fetch comprehensive data for specific repos
# ---------------------------------------------------------------------------
def search_github_project(
repos: List[str],
from_date: str,
to_date: str,
depth: str = "default",
token: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Project-mode GitHub search: fetch stars, README, releases, top issues for repos.
Args:
repos: List of 'owner/repo' strings.
from_date: Start date (YYYY-MM-DD).
to_date: End date (YYYY-MM-DD).
depth: 'quick', 'default', or 'deep'.
token: Optional GitHub token.
Returns:
List of SourceItems, one per repo.
"""
resolved_token = _resolve_token(token)
if not resolved_token:
_log("No GitHub token available for project-mode search")
return []
_log(f"Project-mode search for {len(repos)} repos: {', '.join(repos)}")
items: List[Dict[str, Any]] = []
with ThreadPoolExecutor(max_workers=min(8, len(repos))) as executor:
futures = {
executor.submit(_enrich_project_repo, repo, resolved_token): repo
for repo in repos
}
for idx, future in enumerate(as_completed(futures)):
repo = futures[future]
try:
enrichment = future.result(timeout=25)
except Exception as exc:
_log(f"Project enrichment failed for {repo}: {exc}")
continue
info = enrichment.get("info")
if not info:
_log(f"No repo info for {repo}, skipping")
continue
readme = enrichment.get("readme")
releases = enrichment.get("releases", [])
top_issues = enrichment.get("top_issues", {})
stars = info["stars"]
stars_str = _format_stars(stars)
open_issues = info["open_issues"]
desc = info["description"]
lang = info["language"]
snippet_parts = [f"Project: {repo} ({stars_str} stars, {open_issues} open issues, {lang})"]
if desc:
snippet_parts.append(f" {desc}")
if readme:
snippet_parts.append(f" README: {readme[:400]}")
if releases:
for rel in releases[:2]:
body_preview = f" - {rel['body'][:150]}" if rel.get("body") else ""
snippet_parts.append(f" Latest release: {rel['name']} ({rel['date']}){body_preview}")
feat = top_issues.get("top_feature_request")
if feat:
snippet_parts.append(f" Top feature request: \"{feat['title']}\" ({feat['reactions']} reactions, {feat['comments']} comments)")
complaint = top_issues.get("top_complaint")
if complaint:
snippet_parts.append(f" Top complaint: \"{complaint['title']}\" ({complaint['comments']} comments)")
items.append({
"id": f"GH{idx + 1}",
"title": f"{repo} ({stars_str} stars) - {open_issues} open issues",
"url": f"https://github.com/{repo}",
"date": releases[0]["date"] if releases and releases[0].get("date") else to_date,
"author": repo.split("/")[0],
"source": "github",
"score": stars,
"container": repo,
"snippet": "\n".join(snippet_parts),
"relevance": min(0.95, 0.7 + math.log1p(stars) / 25),
"why_relevant": f"GitHub project: {repo} ({stars_str} stars, live)",
"engagement": {"reactions": stars, "comments": open_issues},
"metadata": {
"labels": ["project-mode"],
"state": "open",
"comment_count": open_issues,
"reactions": stars,
"is_pr": False,
"github_stars": {repo: stars},
},
})
items.sort(key=lambda x: x.get("relevance", 0), reverse=True)
_log(f"Project-mode returned {len(items)} items")
return items
def _enrich_project_repo(repo: str, token: str) -> Dict[str, Any]:
"""Fetch all project data for a repo: info + README + releases + top issues."""
info = _fetch_repo_info(repo, token)
readme = _fetch_readme_snippet(repo, token, max_chars=500)
releases = _fetch_latest_releases(repo, token, count=3)
top_issues = _fetch_top_issues(repo, token)
return {"info": info, "readme": readme, "releases": releases, "top_issues": top_issues}
# ---------------------------------------------------------------------------
# Post-rerank star enrichment: annotate candidates with live star counts
# ---------------------------------------------------------------------------
_REPO_URL_PATTERN = re.compile(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)")
_SKIP_PATHS = {"topics", "search", "orgs", "settings", "features", "about", "pricing", "enterprise", "explore", "marketplace", "sponsors"}
def extract_repo_refs(candidates: List[Any]) -> List[str]:
"""Extract unique owner/repo strings from candidate URLs, titles, and snippets."""
seen: set = set()
repos: List[str] = []
for c in candidates:
texts = [
getattr(c, "url", "") or "",
getattr(c, "title", "") or "",
]
# Also check evidence snippets if available
evidence = getattr(c, "evidence", None)
if evidence:
texts.append(str(evidence))
for text in texts:
for match in _REPO_URL_PATTERN.findall(text):
# Normalize: strip trailing .git, lowercase
repo = match.rstrip(".git").lower()
owner = repo.split("/")[0]
if owner in _SKIP_PATHS:
continue
if repo not in seen:
seen.add(repo)
repos.append(match) # preserve original case
return repos
def enrich_candidates_with_stars(
candidates: List[Any],
token: Optional[str] = None,
already_enriched: Optional[set] = None,
max_repos: int = 10,
) -> int:
"""Annotate candidates with live GitHub star counts.
Returns the number of repos enriched.
"""
resolved_token = _resolve_token(token)
if not resolved_token:
return 0
refs = extract_repo_refs(candidates)
if not refs:
return 0
skip = already_enriched or set()
to_fetch = [r for r in refs if r.lower() not in {s.lower() for s in skip}][:max_repos]
if not to_fetch:
return 0
_log(f"Star enrichment: fetching {len(to_fetch)} repos")
# Parallel fetch star counts
star_map: Dict[str, int] = {}
with ThreadPoolExecutor(max_workers=min(8, len(to_fetch))) as executor:
futures = {executor.submit(_fetch_repo_info, repo, resolved_token): repo for repo in to_fetch}
for future in as_completed(futures):
repo = futures[future]
try:
info = future.result(timeout=10)
if info:
star_map[repo.lower()] = info["stars"]
except Exception:
pass
if not star_map:
return 0
# Annotate candidates
enriched_count = 0
for c in candidates:
texts = [getattr(c, "url", "") or "", getattr(c, "title", "") or ""]
evidence = getattr(c, "evidence", None)
if evidence:
texts.append(str(evidence))
combined = " ".join(texts)
for match in _REPO_URL_PATTERN.findall(combined):
repo_lower = match.rstrip(".git").lower()
if repo_lower in star_map:
stars = star_map[repo_lower]
stars_str = _format_stars(stars)
# Add to metadata
if not hasattr(c, "metadata") or c.metadata is None:
continue
if "github_stars" not in c.metadata:
c.metadata["github_stars"] = {}
c.metadata["github_stars"][match] = stars
# Append to evidence if present
if hasattr(c, "evidence") and c.evidence and f"(live:" not in c.evidence:
c.evidence = c.evidence + f" (live: {stars_str} stars)"
enriched_count += 1
break # one annotation per candidate
_log(f"Star enrichment: annotated {enriched_count} candidates")
return enriched_count
scripts/lib/grounding.py
+259
View File
@@ -0,0 +1,259 @@
"""Web search retrieval via Brave Search, Exa, and Serper."""
from __future__ import annotations
import urllib.parse
from datetime import datetime
from urllib.parse import urlparse
from . import dates, http
# ---------------------------------------------------------------------------
# Brave Search API
# ---------------------------------------------------------------------------
def brave_search(
query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
) -> tuple[list[dict], dict]:
url = (
"https://api.search.brave.com/res/v1/web/search?"
+ urllib.parse.urlencode(
{
"q": query,
"count": count,
"freshness": f"{date_range[0]}to{date_range[1]}",
}
)
)
data = http.request("GET", url, headers={"X-Subscription-Token": api_key}, timeout=15)
items = []
for i, r in enumerate((data.get("web", {}).get("results", []))[:count]):
raw_date = r.get("page_age") or ""
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
if not _in_date_range(pub_date, date_range):
continue
items.append({
"id": f"WB{i + 1}",
"title": r.get("title", ""),
"url": r.get("url", ""),
"source_domain": _domain(r.get("url", "")),
"snippet": r.get("description", ""),
"date": pub_date,
"relevance": 0.8,
"why_relevant": "Brave web search",
})
artifact = {"label": "brave", "webSearchQueries": [query], "resultCount": len(items)}
return items, artifact
# ---------------------------------------------------------------------------
# Exa AI Search
# ---------------------------------------------------------------------------
def exa_search(
query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
) -> tuple[list[dict], dict]:
data = http.request(
"POST", "https://api.exa.ai/search",
headers={"x-api-key": api_key},
json_data={
"query": query,
"type": "auto",
"numResults": count,
"startPublishedDate": f"{date_range[0]}T00:00:00.000Z",
"endPublishedDate": f"{date_range[1]}T23:59:59.999Z",
"contents": {"text": {"maxCharacters": 2000}},
},
timeout=15,
)
items = []
for i, r in enumerate((data.get("results", []))[:count]):
if not isinstance(r, dict):
continue
url = r.get("url", "")
if not url:
continue
raw_date = r.get("publishedDate") or ""
pub_date = _normalize_date(raw_date.split("T")[0] if "T" in raw_date else raw_date[:10]) if raw_date else None
if not _in_date_range(pub_date, date_range):
continue
items.append({
"id": f"WE{i + 1}",
"title": r.get("title", ""),
"url": url,
"source_domain": _domain(url),
"snippet": (r.get("text") or "")[:500],
"date": pub_date,
"relevance": 0.8,
"why_relevant": "Exa web search",
})
artifact = {"label": "exa", "webSearchQueries": [query], "resultCount": len(items)}
return items, artifact
# ---------------------------------------------------------------------------
# Serper (Google Search wrapper)
# ---------------------------------------------------------------------------
def serper_search(
query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
) -> tuple[list[dict], dict]:
data = http.request(
"POST", "https://google.serper.dev/search",
headers={"X-API-KEY": api_key},
json_data={
"q": query,
"num": count,
"tbs": f"cdr:1,cd_min:{_serper_date_param(date_range[0])},cd_max:{_serper_date_param(date_range[1])}",
},
timeout=15,
)
items = []
for i, r in enumerate((data.get("organic", []))[:count]):
raw_date = r.get("date") or ""
pub_date = _parse_serper_date(raw_date)
if not _in_date_range(pub_date, date_range):
continue
items.append({
"id": f"WS{i + 1}",
"title": r.get("title", ""),
"url": r.get("link", ""),
"source_domain": _domain(r.get("link", "")),
"snippet": r.get("snippet", ""),
"date": pub_date,
"relevance": 0.8,
"why_relevant": "Serper web search",
})
artifact = {"label": "serper", "webSearchQueries": [query], "resultCount": len(items)}
return items, artifact
# ---------------------------------------------------------------------------
# Parallel AI Search
# ---------------------------------------------------------------------------
def parallel_search(
query: str, date_range: tuple[str, str], api_key: str, count: int = 5,
) -> tuple[list[dict], dict]:
data = http.request(
"POST", "https://api.parallel.ai/v1/search",
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
json_data={"query": query, "max_results": count},
timeout=15,
)
items = []
for i, r in enumerate((data.get("results", []))[:count]):
if not isinstance(r, dict):
continue
url = r.get("url", "")
if not url:
continue
raw_date = r.get("published_date") or ""
pub_date = _normalize_date(raw_date[:10]) if raw_date else None
if not _in_date_range(pub_date, date_range):
continue
items.append({
"id": f"WP{i + 1}",
"title": r.get("title", ""),
"url": url,
"source_domain": _domain(url),
"snippet": r.get("snippet", ""),
"date": pub_date,
"relevance": 0.8,
"why_relevant": "Parallel AI web search",
})
artifact = {"label": "parallel", "webSearchQueries": [query], "resultCount": len(items)}
return items, artifact
def _parse_serper_date(raw: str) -> str | None:
if not raw:
return None
normalized = _normalize_date(raw)
if normalized:
return normalized
for fmt in ("%b %d, %Y", "%B %d, %Y", "%Y-%m-%d"):
try:
return datetime.strptime(raw.strip(), fmt).date().isoformat()
except ValueError:
continue
return None
# ---------------------------------------------------------------------------
# Dispatcher
# ---------------------------------------------------------------------------
def web_search(
query: str,
date_range: tuple[str, str],
config: dict,
backend: str = "auto",
) -> tuple[list[dict], dict]:
"""Run web search with the specified or auto-detected backend."""
if backend == "auto":
if config.get("BRAVE_API_KEY"):
backend = "brave"
elif config.get("EXA_API_KEY"):
backend = "exa"
elif config.get("SERPER_API_KEY"):
backend = "serper"
elif config.get("PARALLEL_API_KEY"):
backend = "parallel"
else:
return [], {}
if backend == "brave":
key = config.get("BRAVE_API_KEY")
if not key:
raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'")
return brave_search(query, date_range, key)
if backend == "exa":
key = config.get("EXA_API_KEY")
if not key:
raise RuntimeError("EXA_API_KEY is required when web_backend='exa'")
return exa_search(query, date_range, key)
if backend == "serper":
key = config.get("SERPER_API_KEY")
if not key:
raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'")
return serper_search(query, date_range, key)
if backend == "parallel":
key = config.get("PARALLEL_API_KEY")
if not key:
raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'")
return parallel_search(query, date_range, key)
if backend != "none":
raise ValueError(f"Unsupported web backend: {backend!r}")
return [], {}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _normalize_date(value: object) -> str | None:
if value is None:
return None
parsed = dates.parse_date(str(value).strip())
if not parsed:
return None
return parsed.date().isoformat()
def _serper_date_param(iso_date: str) -> str:
"""Convert YYYY-MM-DD to MM/DD/YYYY for Serper tbs parameter."""
parts = iso_date.split("-")
return f"{parts[1]}/{parts[2]}/{parts[0]}"
def _in_date_range(pub_date: str | None, date_range: tuple[str, str]) -> bool:
if not pub_date:
return False
return date_range[0] <= pub_date <= date_range[1]
def _domain(url: str) -> str:
return urlparse(url).netloc.strip().lower()
scripts/lib/hackernews.py
+47 -12
View File
@@ -4,6 +4,7 @@ Uses hn.algolia.com/api/v1 for story discovery and comment enrichment.
No API key needed - just HTTP calls via stdlib urllib.
"""
import datetime
import html
import math
import sys
@@ -11,10 +12,15 @@ import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Dict, List, Optional
from . import http
import re
from . import http, log
from .query import extract_core_subject
from .relevance import token_overlap_relevance
# Common HN prefixes that can cause false-positive keyword matches
_HN_PREFIXES = re.compile(r"^(Tell HN|Show HN|Ask HN|Launch HN)\s*:\s*", re.IGNORECASE)
ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date"
ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items"
@@ -33,25 +39,19 @@ ENRICH_LIMITS = {
def _log(msg: str):
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
if sys.stderr.isatty():
sys.stderr.write(f"[HN] {msg}\n")
sys.stderr.flush()
log.source_log("HN", msg)
def _date_to_unix(date_str: str) -> int:
"""Convert YYYY-MM-DD to Unix timestamp (start of day UTC)."""
parts = date_str.split("-")
year, month, day = int(parts[0]), int(parts[1]), int(parts[2])
import calendar
import datetime
dt = datetime.datetime(year, month, day, tzinfo=datetime.timezone.utc)
return int(dt.timestamp())
def _unix_to_date(ts: int) -> str:
"""Convert Unix timestamp to YYYY-MM-DD."""
import datetime
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
return dt.strftime("%Y-%m-%d")
@@ -117,6 +117,30 @@ def search_hackernews(
return response
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
"""Check if the query term appears in the title content, not just an HN prefix or author.
Returns True if the query (or any multi-word token) appears in the title
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
and ignoring the author name. Returns True when query is empty (no filter).
"""
if not query:
return True
stripped = _HN_PREFIXES.sub("", title).strip()
# Also check that the match isn't solely in the author's username
check_text = stripped.lower()
query_lower = query.lower()
# Check each word of the query independently; all must appear somewhere
# in the stripped title (not just the prefix).
query_words = query_lower.split()
for word in query_words:
if word in check_text:
continue
# Word not found in stripped title — reject
return False
return True
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
"""Parse Algolia response into normalized item dicts.
@@ -128,6 +152,16 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
List of item dicts ready for normalization.
"""
hits = response.get("hits", [])
# Post-filter: remove items where query only matched an HN prefix like "Tell HN:"
if query:
before = len(hits)
hits = [
h for h in hits
if _title_matches_query(h.get("title", ""), query, h.get("author", ""))
]
dropped = before - len(hits)
if dropped:
_log(f"Prefix filter removed {dropped}/{before} false-positive hits for '{query}'")
items = []
for i, hit in enumerate(hits):
@@ -154,7 +188,7 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
items.append({
"object_id": object_id,
"id": object_id,
"title": hit.get("title", ""),
"url": article_url,
"hn_url": hn_url,
@@ -162,7 +196,7 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
"date": date_str,
"engagement": {
"points": points,
"num_comments": num_comments,
"comments": num_comments,
},
"relevance": round(relevance, 2),
"why_relevant": f"HN story about {hit.get('title', 'topic')[:60]}",
@@ -248,7 +282,7 @@ def enrich_top_stories(
futures = {
executor.submit(
_fetch_item_comments,
items[idx]["object_id"],
items[idx]["id"],
): idx
for idx in to_enrich
}
@@ -259,7 +293,8 @@ def enrich_top_stories(
result = future.result(timeout=15)
items[idx]["top_comments"] = result["comments"]
items[idx]["comment_insights"] = result["comment_insights"]
except Exception:
except (KeyError, TypeError, OSError) as exc:
_log(f"Comment enrichment failed for story {items[idx].get('id', '?')}: {type(exc).__name__}: {exc}")
items[idx]["top_comments"] = []
items[idx]["comment_insights"] = []
scripts/lib/http.py
+25 -12
View File
@@ -1,26 +1,28 @@
"""HTTP utilities for last30days skill (stdlib only)."""
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request
from typing import Any, Dict, Optional
from typing import Any, Dict, Optional, Union
from urllib.parse import urlencode
from . import log as _log
DEFAULT_TIMEOUT = 30
DEBUG = os.environ.get("LAST30DAYS_DEBUG", "").lower() in ("1", "true", "yes")
def log(msg: str):
"""Log debug message to stderr."""
if DEBUG:
sys.stderr.write(f"[DEBUG] {msg}\n")
sys.stderr.flush()
_log.debug(msg)
MAX_RETRIES = 5
MAX_429_RETRIES = 2
RETRY_DELAY = 2.0
USER_AGENT = "last30days-skill/2.1 (Assistant Skill)"
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
class HTTPError(Exception):
@@ -38,8 +40,9 @@ def request(
json_data: Optional[Dict[str, Any]] = None,
timeout: int = DEFAULT_TIMEOUT,
retries: int = MAX_RETRIES,
max_429_retries: int = MAX_429_RETRIES,
raw: bool = False,
) -> Dict[str, Any]:
) -> Union[Dict[str, Any], str]:
"""Make an HTTP request and return JSON response.
Args:
@@ -49,9 +52,11 @@ def request(
json_data: Optional JSON body (for POST)
timeout: Request timeout in seconds
retries: Number of retries on failure
max_429_retries: Maximum 429 retries before giving up (separate cap)
raw: If True, return raw response text instead of parsed JSON
Returns:
Parsed JSON response (or raw text if raw=True)
Parsed JSON response as dict, or raw text string if raw=True.
Raises:
HTTPError: On request failure
@@ -66,9 +71,11 @@ def request(
req = urllib.request.Request(url, data=data, headers=headers, method=method)
log(f"{method} {url}")
safe_url = re.sub(r'([?&])(key|api_key|token|secret)=[^&]*', r'\1\2=***', url)
log(f"{method} {safe_url}")
last_error = None
rate_limit_count = 0
for attempt in range(retries):
try:
with urllib.request.urlopen(req, timeout=timeout) as response:
@@ -81,7 +88,7 @@ def request(
body = None
try:
body = e.read().decode('utf-8')
except:
except (OSError, UnicodeDecodeError):
pass
log(f"HTTP Error {e.code}: {e.reason}")
if body:
@@ -93,6 +100,12 @@ def request(
if 400 <= e.code < 500 and e.code != 429:
raise last_error
# Cap 429 retries separately to avoid wasting latency
if e.code == 429:
rate_limit_count += 1
if rate_limit_count >= max_429_retries:
raise last_error
if attempt < retries - 1:
if e.code == 429:
# Respect Retry-After header, fall back to exponential backoff
@@ -103,7 +116,7 @@ def request(
except ValueError:
delay = RETRY_DELAY * (2 ** attempt) + 1
else:
delay = RETRY_DELAY * (2 ** attempt) + 1 # 2s, 5s, 9s...
delay = RETRY_DELAY * (2 ** attempt) + 1 # 3s, 5s, 9s...
log(f"Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 2}/{retries}")
else:
delay = RETRY_DELAY * (2 ** attempt)
scripts/lib/instagram.py
+219 -76
View File
@@ -9,7 +9,7 @@ API docs: https://scrapecreators.com/docs
import re
import sys
from datetime import datetime, timezone
from datetime import datetime
from typing import Any, Dict, List, Optional, Set
try:
@@ -17,7 +17,7 @@ try:
except ImportError:
_requests = None
from . import http
from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
@@ -49,11 +49,67 @@ def _extract_core_subject(topic: str) -> str:
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
def _infer_query_intent(topic: str) -> str:
"""Tiny local intent classifier for Instagram query expansion."""
text = topic.lower().strip()
if re.search(r"\b(vs|versus|compare|difference between)\b", text):
return "comparison"
if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install)\b", text):
return "how_to"
if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
return "opinion"
if re.search(r"\b(pricing|feature|features|best .* for)\b", text):
return "product"
return "breaking_news"
def expand_instagram_queries(topic: str, depth: str) -> List[str]:
"""Generate multiple Instagram search queries from a topic.
Mirrors reddit.py's expand_reddit_queries() pattern:
1. Extract core subject (strip noise words)
2. Include original topic if different from core
3. Add intent-specific OR-joined content-type variants
4. Cap by depth: 1 for quick, 2 for default, 3 for deep
Returns 1-3 query strings depending on depth.
"""
core = _extract_core_subject(topic)
queries = [core]
# Include cleaned original topic as variant if different from core
original_clean = topic.strip().rstrip('?!.')
if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
queries.append(original_clean)
qtype = _infer_query_intent(topic)
# Intent-specific Instagram content-type variants
if qtype == "breaking_news":
queries.append(f"{core} reaction OR edit")
elif qtype == "opinion":
queries.append(f"{core} reaction OR edit")
elif qtype == "product":
queries.append(f"{core} review OR haul")
elif qtype == "comparison":
queries.append(f"{core} vs OR compared")
elif qtype == "how_to":
queries.append(f"{core} tutorial OR hack")
else:
queries.append(f"{core} reaction OR edit")
# Deep depth: add viral content variant
if depth == "deep":
queries.append(f"{core} viral OR trending OR reel")
# Cap by depth budget
caps = {"quick": 1, "default": 2, "deep": 3}
cap = caps.get(depth, 2)
return queries[:cap]
def _log(msg: str):
"""Log to stderr (only in interactive terminals; spinner handles non-TTY)."""
if sys.stderr.isatty():
sys.stderr.write(f"[Instagram] {msg}\n")
sys.stderr.flush()
log.source_log("Instagram", msg)
def _sc_headers(token: str) -> Dict[str, str]:
@@ -88,9 +144,8 @@ def _parse_date(item: Dict[str, Any]) -> Optional[str]:
# Fall back to unix timestamp
try:
dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError, OSError):
return dates.timestamp_to_date(int(ts))
except (ValueError, TypeError):
pass
return None
@@ -103,6 +158,122 @@ def _extract_hashtags(caption_text: str) -> List[str]:
return re.findall(r'#(\w+)', caption_text)
def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]:
"""Parse raw Instagram items into normalized dicts."""
items = []
for raw in raw_items:
if not isinstance(raw, dict):
continue
# Extract reel ID and shortcode
reel_pk = str(raw.get("id", raw.get("pk", "")))
shortcode = raw.get("shortcode", raw.get("code", ""))
# Caption text -- can be a string or dict depending on endpoint
caption_obj = raw.get("caption", "")
if isinstance(caption_obj, dict):
text = caption_obj.get("text", "")
elif isinstance(caption_obj, str):
text = caption_obj
else:
text = raw.get("desc", raw.get("text", ""))
# Engagement metrics
play_count = raw.get("video_play_count") or raw.get("video_view_count") or raw.get("play_count") or 0
like_count = raw.get("like_count") or 0
comment_count = raw.get("comment_count") or 0
# Author info -- 'owner' in reels/search, 'user' in user/reels
owner_raw = raw.get("owner") or raw.get("user")
if isinstance(owner_raw, dict):
author_name = owner_raw.get("username", "")
elif isinstance(owner_raw, str):
author_name = owner_raw
else:
author_name = ""
# Duration
duration = raw.get("video_duration")
# Date
date_str = _parse_date(raw)
# Hashtags from caption text
hashtags = _extract_hashtags(text)
# Compute relevance with hashtag boost
relevance = _compute_relevance(core_topic, text, hashtags)
# Build URL -- prefer API-provided url, fallback to shortcode
url = raw.get("url", "")
if not url and shortcode:
url = f"https://www.instagram.com/reel/{shortcode}"
items.append({
"video_id": reel_pk,
"text": text,
"url": url,
"author_name": author_name,
"date": date_str,
"engagement": {
"views": play_count,
"likes": like_count,
"comments": comment_count,
},
"hashtags": hashtags,
"duration": duration,
"relevance": relevance,
"why_relevant": f"Instagram: {text[:60]}" if text else f"Instagram: {core_topic}",
"caption_snippet": "", # populated by fetch_captions
})
return items
def _user_reels(
handle: str,
token: str,
) -> List[Dict[str, Any]]:
"""Fetch an Instagram user's recent reels via ScrapeCreators.
Args:
handle: Instagram username (without @)
token: ScrapeCreators API key
Returns:
List of raw Instagram reel dicts.
"""
_log(f"User reels: @{handle}")
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
if not _requests:
try:
from urllib.parse import urlencode
params = urlencode({"handle": handle})
url = f"{reels_url}?{params}"
headers = _sc_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"User reels error (urllib) for @{handle}: {e}")
return []
else:
try:
resp = _requests.get(
reels_url,
params={"handle": handle},
headers=_sc_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"User reels error for @{handle}: {e}")
return []
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
_log(f" -> {len(raw_items)} reels from @{handle}")
return raw_items
def search_instagram(
topic: str,
from_date: str,
@@ -163,67 +334,7 @@ def search_instagram(
raw_items = raw_items[:config["results_per_page"]]
# Parse items
items = []
for raw in raw_items:
if not isinstance(raw, dict):
continue
# Extract reel ID and shortcode
reel_pk = str(raw.get("id", raw.get("pk", "")))
shortcode = raw.get("shortcode", raw.get("code", ""))
# Caption text — can be a string or dict depending on endpoint
caption_obj = raw.get("caption", "")
if isinstance(caption_obj, dict):
text = caption_obj.get("text", "")
elif isinstance(caption_obj, str):
text = caption_obj
else:
text = raw.get("desc", raw.get("text", ""))
# Engagement metrics
play_count = raw.get("video_play_count") or raw.get("video_view_count") or raw.get("play_count") or 0
like_count = raw.get("like_count") or 0
comment_count = raw.get("comment_count") or 0
# Author info — 'owner' in reels/search, 'user' in user/reels
owner = raw.get("owner") or raw.get("user") or {}
author_name = owner.get("username", "")
# Duration
duration = raw.get("video_duration")
# Date
date_str = _parse_date(raw)
# Hashtags from caption text
hashtags = _extract_hashtags(text)
# Compute relevance with hashtag boost
relevance = _compute_relevance(core_topic, text, hashtags)
# Build URL — prefer API-provided url, fallback to shortcode
url = raw.get("url", "")
if not url and shortcode:
url = f"https://www.instagram.com/reel/{shortcode}"
items.append({
"video_id": reel_pk,
"text": text,
"url": url,
"author_name": author_name,
"date": date_str,
"engagement": {
"views": play_count,
"likes": like_count,
"comments": comment_count,
},
"hashtags": hashtags,
"duration": duration,
"relevance": relevance,
"why_relevant": f"Instagram: {text[:60]}" if text else f"Instagram: {core_topic}",
"caption_snippet": "", # populated by fetch_captions
})
items = _parse_items(raw_items, core_topic)
# Hard date filter
in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
@@ -323,25 +434,57 @@ def search_and_enrich(
to_date: str,
depth: str = "default",
token: str = None,
ig_creators: List[str] | None = None,
) -> Dict[str, Any]:
"""Full Instagram search: find reels, then fetch captions for top results.
Uses expand_instagram_queries() to generate multiple search queries,
runs ScrapeCreators for each, and merges/deduplicates results by video ID.
Args:
topic: Search topic
topic: Search topic (raw topic, not planner's narrowed query)
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
ig_creators: Optional list of Instagram creator handles to fetch reels from
Returns:
Dict with 'items' list. Each item has a 'caption_snippet' field.
"""
# Step 1: Search
search_result = search_instagram(topic, from_date, to_date, depth, token)
items = search_result.get("items", [])
core_topic = _extract_core_subject(topic)
seen_ids: Set[str] = set()
items: List[Dict[str, Any]] = []
last_error = None
# Step 0: Creator reels (high-signal, runs first)
if ig_creators and token:
for creator in ig_creators:
raw_items = _user_reels(creator, token)
parsed = _parse_items(raw_items, core_topic)
for item in parsed:
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Step 1: Multi-query keyword search — run ScrapeCreators for each expanded query
queries = expand_instagram_queries(topic, depth)
for q in queries:
search_result = search_instagram(q, from_date, to_date, depth, token)
if search_result.get("error"):
last_error = search_result["error"]
for item in search_result.get("items", []):
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Sort merged results by views descending
items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True)
if not items:
return search_result
return {"items": [], "error": last_error}
# Step 2: Fetch captions for top N
captions = fetch_captions(items, token, depth)
@@ -353,7 +496,7 @@ def search_and_enrich(
if caption:
item["caption_snippet"] = caption
return {"items": items, "error": search_result.get("error")}
return {"items": items, "error": last_error}
def parse_instagram_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
scripts/lib/log.py
+28
View File
@@ -0,0 +1,28 @@
"""Shared logging utilities for last30days skill."""
import os
import sys
DEBUG = os.environ.get("LAST30DAYS_DEBUG", "").lower() in ("1", "true", "yes")
def debug(msg: str) -> None:
"""Log debug message to stderr (only when LAST30DAYS_DEBUG is set)."""
if DEBUG:
sys.stderr.write(f"[DEBUG] {msg}\n")
sys.stderr.flush()
def source_log(prefix: str, msg: str, *, tty_only: bool = True) -> None:
"""Log a source module message to stderr.
Args:
prefix: Source label (e.g. "Reddit", "Bird").
msg: Message text.
tty_only: If True, only log when stderr is a TTY (avoids cluttering
non-interactive output like Claude Code).
"""
if tty_only and not sys.stderr.isatty():
return
sys.stderr.write(f"[{prefix}] {msg}\n")
sys.stderr.flush()
scripts/lib/models.py
-221
View File
@@ -1,221 +0,0 @@
"""Model auto-selection for last30days skill.
Model selection philosophy: this tool uses LLM APIs exclusively for
search tool invocation + structured JSON extraction. This is not
reasoning-heavy or creative work — mini models handle it equally well
at ~3-5x lower cost. We prefer the newest-generation mini model, falling
back to mainline only when mini isn't available.
xAI non-reasoning variant preferred: same pricing as reasoning, but
faster (skips thinking phase, saves reasoning token output costs).
"""
import re
import sys
from typing import Dict, List, Optional, Tuple
from . import cache, http, env
# OpenAI API
OPENAI_MODELS_URL = "https://api.openai.com/v1/models"
# Ordered by cost-efficiency for web_search + JSON extraction tasks.
# Mini models first: same structured extraction quality at ~3x lower cost.
OPENAI_FALLBACK_MODELS = ["gpt-5-mini", "gpt-4.1-mini", "gpt-4.1", "gpt-4o"]
CODEX_FALLBACK_MODELS = ["gpt-5.1-codex-mini", "gpt-5.2"]
# xAI API - Agent Tools API requires grok-4 family
# Non-reasoning: same price, faster, no unnecessary thinking tokens.
# Both variants support function calling and structured outputs.
XAI_MODELS_URL = "https://api.x.ai/v1/models"
XAI_ALIASES = {
"latest": "grok-4-1-fast-non-reasoning",
"stable": "grok-4-1-fast-non-reasoning",
}
def parse_version(model_id: str) -> Optional[Tuple[int, ...]]:
"""Parse semantic version from model ID.
Examples:
gpt-5 -> (5,)
gpt-5.2 -> (5, 2)
gpt-5.2.1 -> (5, 2, 1)
"""
match = re.search(r'(\d+(?:\.\d+)*)', model_id)
if match:
return tuple(int(x) for x in match.group(1).split('.'))
return None
def is_search_capable_model(model_id: str) -> bool:
"""Check if model supports Responses API web_search with domain filtering.
Includes mini variants (same structured extraction quality, lower cost).
Excludes: nano (no web_search), gpt-4o-mini (no domain filtering),
chat/codex/pro/preview/turbo/search (specialized variants).
Note: gpt-5 with reasoning effort="minimal" does NOT support web_search
(per OpenAI docs). We never set reasoning params — our usage is pure
tool invocation + JSON extraction — so gpt-5 is safe to include here.
"""
model_lower = model_id.lower()
# gpt-4o-mini does NOT support web_search with filters — exclude it
if model_lower.startswith("gpt-4o-mini"):
return False
# Must be gpt-4o, gpt-4.1[-mini], or gpt-5[-mini] series
if not re.match(r'^gpt-(?:4o|4\.1|5)(\.\d+)*(-mini)?$', model_lower):
return False
# Exclude unsupported variants
for exc in ['nano', 'chat', 'codex', 'pro', 'preview', 'turbo', 'search']:
if exc in model_lower:
return False
return True
# Backward compat alias
is_mainline_openai_model = is_search_capable_model
def select_openai_model(
api_key: str,
policy: str = "auto",
pin: Optional[str] = None,
mock_models: Optional[List[Dict]] = None,
) -> str:
"""Select the most cost-efficient OpenAI model for web_search + JSON extraction.
Prefers mini models within the newest generation available, since the task
is structured extraction (not reasoning or creative work).
Args:
api_key: OpenAI API key
policy: 'auto' or 'pinned'
pin: Model to use if policy is 'pinned'
mock_models: Mock model list for testing
Returns:
Selected model ID
"""
if policy == "pinned" and pin:
return pin
# Check cache first
cached = cache.get_cached_model("openai")
if cached:
return cached
# Fetch model list
if mock_models is not None:
models = mock_models
else:
try:
headers = {"Authorization": f"Bearer {api_key}"}
response = http.get(OPENAI_MODELS_URL, headers=headers)
models = response.get("data", [])
except http.HTTPError as e:
sys.stderr.write(f"[Models] Failed to fetch OpenAI models: {e}")
if hasattr(e, 'status_code') and e.status_code in (401, 403):
sys.stderr.write(" — API key may be invalid or lack permissions")
sys.stderr.write(f", using fallback {OPENAI_FALLBACK_MODELS[0]}\n")
return OPENAI_FALLBACK_MODELS[0]
candidates = [m for m in models if is_search_capable_model(m.get("id", ""))]
if not candidates:
return OPENAI_FALLBACK_MODELS[0]
# Sort: newest generation first, prefer mini within same generation
def sort_key(m):
model_id = m.get("id", "")
version = parse_version(model_id) or (0,)
major = version[0] if version else 0
is_mini = 1 if "mini" in model_id.lower() else 0
return (major, is_mini, version)
candidates.sort(key=sort_key, reverse=True)
selected = candidates[0]["id"]
cache.set_cached_model("openai", selected)
return selected
def select_xai_model(
api_key: str,
policy: str = "latest",
pin: Optional[str] = None,
mock_models: Optional[List[Dict]] = None,
) -> str:
"""Select the best xAI model based on policy.
Args:
api_key: xAI API key
policy: 'latest', 'stable', or 'pinned'
pin: Model to use if policy is 'pinned'
mock_models: Mock model list for testing
Returns:
Selected model ID
"""
if policy == "pinned" and pin:
return pin
# Use alias system
if policy in XAI_ALIASES:
alias = XAI_ALIASES[policy]
# Check cache first
cached = cache.get_cached_model("xai")
if cached:
return cached
# Cache the alias
cache.set_cached_model("xai", alias)
return alias
# Default to latest
return XAI_ALIASES["latest"]
def get_models(
config: Dict,
mock_openai_models: Optional[List[Dict]] = None,
mock_xai_models: Optional[List[Dict]] = None,
) -> Dict[str, Optional[str]]:
"""Get selected models for both providers.
Returns:
Dict with 'openai' and 'xai' keys
"""
result = {"openai": None, "xai": None}
if config.get("OPENAI_API_KEY"):
if config.get("OPENAI_AUTH_SOURCE") == env.AUTH_SOURCE_CODEX:
# Codex auth doesn't use the OpenAI models list endpoint
policy = config.get("OPENAI_MODEL_POLICY", "auto")
pin = config.get("OPENAI_MODEL_PIN")
if policy == "pinned" and pin:
result["openai"] = pin
else:
result["openai"] = CODEX_FALLBACK_MODELS[0]
else:
result["openai"] = select_openai_model(
config["OPENAI_API_KEY"],
config.get("OPENAI_MODEL_POLICY", "auto"),
config.get("OPENAI_MODEL_PIN"),
mock_openai_models,
)
if config.get("XAI_API_KEY"):
result["xai"] = select_xai_model(
config["XAI_API_KEY"],
config.get("XAI_MODEL_POLICY", "latest"),
config.get("XAI_MODEL_PIN"),
mock_xai_models,
)
return result
scripts/lib/normalize.py
+390 -437
View File
@@ -1,489 +1,442 @@
"""Normalization of raw API data to canonical schema."""
"""Normalization of source-specific payloads into the v3 generic item model."""
from typing import Any, Dict, List, TypeVar, Union
from __future__ import annotations
from typing import Any
from urllib.parse import urlparse
from . import dates, schema
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.BlueskyItem, schema.PolymarketItem)
def filter_by_date_range(
items: List[T],
items: list[schema.SourceItem],
from_date: str,
to_date: str,
require_date: bool = False,
) -> List[T]:
"""Hard filter: Remove items outside the date range.
This is the safety net - even if the prompt lets old content through,
this filter will exclude it.
Args:
items: List of items to filter
from_date: Start date (YYYY-MM-DD) - exclude items before this
to_date: End date (YYYY-MM-DD) - exclude items after this
require_date: If True, also remove items with no date
Returns:
Filtered list with only items in range (or unknown dates if not required)
"""
result = []
) -> list[schema.SourceItem]:
"""Keep only items within the requested window."""
filtered: list[schema.SourceItem] = []
for item in items:
if item.date is None:
if not item.published_at:
if not require_date:
result.append(item) # Keep unknown dates (with scoring penalty)
filtered.append(item)
continue
# Hard filter: if date is before from_date, exclude
if item.date < from_date:
continue # DROP - too old
# Hard filter: if date is after to_date, exclude (likely parsing error)
if item.date > to_date:
continue # DROP - future date
result.append(item)
return result
if item.published_at < from_date or item.published_at > to_date:
continue
filtered.append(item)
return filtered
def normalize_reddit_items(
items: List[Dict[str, Any]],
def normalize_source_items(
source: str,
items: list[dict[str, Any]],
from_date: str,
to_date: str,
) -> List[schema.RedditItem]:
"""Normalize raw Reddit items to schema.
Args:
items: Raw Reddit items from API
from_date: Start of date range
to_date: End of date range
Returns:
List of RedditItem objects
"""
normalized = []
for item in items:
# Parse engagement
engagement = None
eng_raw = item.get("engagement")
if isinstance(eng_raw, dict):
engagement = schema.Engagement(
score=eng_raw.get("score"),
num_comments=eng_raw.get("num_comments"),
upvote_ratio=eng_raw.get("upvote_ratio"),
)
# Parse comments
top_comments = []
for c in item.get("top_comments", []):
top_comments.append(schema.Comment(
score=c.get("score", 0),
date=c.get("date"),
author=c.get("author", ""),
excerpt=c.get("excerpt", ""),
url=c.get("url", ""),
))
# Determine date confidence
date_str = item.get("date")
date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
normalized.append(schema.RedditItem(
id=item.get("id", ""),
title=item.get("title", ""),
url=item.get("url", ""),
subreddit=item.get("subreddit", ""),
date=date_str,
date_confidence=date_confidence,
engagement=engagement,
top_comments=top_comments,
comment_insights=item.get("comment_insights", []),
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
freshness_mode: str = "balanced_recent",
) -> list[schema.SourceItem]:
"""Normalize raw source items, filter by date range, with evergreen fallback for how_to queries."""
source = source.lower()
normalizers = {
"reddit": _normalize_reddit,
"x": _normalize_x,
"youtube": _normalize_youtube,
"tiktok": lambda s, i, idx, fd, td: _normalize_shortform_video(s, i, idx, fd, td, "TK", "TikTok post"),
"instagram": lambda s, i, idx, fd, td: _normalize_shortform_video(s, i, idx, fd, td, "IG", "Instagram reel"),
"hackernews": _normalize_hackernews,
"bluesky": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "BS", "Bluesky post"),
"truthsocial": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TS", "Truth Social post"),
"threads": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TH", "Threads post"),
"pinterest": _normalize_pinterest,
"polymarket": _normalize_polymarket,
"grounding": _normalize_grounding,
"xiaohongshu": _normalize_grounding,
"github": _normalize_github,
"perplexity": _normalize_grounding,
}
normalizer = normalizers.get(source)
if normalizer is None:
raise ValueError(f"Unsupported source: {source}")
normalized = [normalizer(source, item, index, from_date, to_date) for index, item in enumerate(items)]
require_date = source == "grounding"
filtered = filter_by_date_range(normalized, from_date, to_date, require_date=require_date)
if filtered:
return filtered
if freshness_mode == "evergreen_ok" and source == "youtube":
if require_date:
return [item for item in normalized if item.published_at]
return normalized
return filtered
def normalize_x_items(
items: List[Dict[str, Any]],
def _domain_from_url(url: str) -> str | None:
if not url:
return None
domain = urlparse(url).netloc.strip().lower()
return domain or None
def _date_confidence(item: dict[str, Any], from_date: str, to_date: str, default: str = "low") -> str:
if item.get("date_confidence"):
return str(item["date_confidence"])
date_value = item.get("date")
if not date_value:
return default
return dates.get_date_confidence(str(date_value), from_date, to_date)
def _source_item(
*,
item_id: str,
source: str,
title: str,
body: str,
url: str,
published_at: str | None,
date_confidence: str,
relevance_hint: float,
why_relevant: str,
author: str | None = None,
container: str | None = None,
engagement: dict[str, float | int] | None = None,
snippet: str = "",
metadata: dict[str, Any] | None = None,
) -> schema.SourceItem:
return schema.SourceItem(
item_id=item_id,
source=source,
title=title.strip() or body.strip()[:160] or item_id,
body=body.strip(),
url=url.strip(),
author=(author or "").strip() or None,
container=(container or "").strip() or None,
published_at=published_at,
date_confidence=date_confidence,
engagement=engagement or {},
relevance_hint=max(0.0, min(1.0, float(relevance_hint or 0.0))),
why_relevant=why_relevant.strip(),
snippet=snippet.strip(),
metadata=metadata or {},
)
def _normalize_reddit(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.XItem]:
"""Normalize raw X items to schema.
Args:
items: Raw X items from API
from_date: Start of date range
to_date: End of date range
Returns:
List of XItem objects
"""
normalized = []
for item in items:
# Parse engagement
engagement = None
eng_raw = item.get("engagement")
if isinstance(eng_raw, dict):
engagement = schema.Engagement(
likes=eng_raw.get("likes"),
reposts=eng_raw.get("reposts"),
replies=eng_raw.get("replies"),
quotes=eng_raw.get("quotes"),
)
# Determine date confidence
date_str = item.get("date")
date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
normalized.append(schema.XItem(
id=item.get("id", ""),
text=item.get("text", ""),
url=item.get("url", ""),
author_handle=item.get("author_handle", ""),
date=date_str,
date_confidence=date_confidence,
engagement=engagement,
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
) -> schema.SourceItem:
top_comments = item.get("top_comments") or []
comment_text = " ".join(
str(comment.get("excerpt") or "").strip()
for comment in top_comments[:3]
if isinstance(comment, dict)
)
body = "\n".join(
part
for part in [
str(item.get("title") or "").strip(),
str(item.get("selftext") or "").strip(),
comment_text,
]
if part
)
return _source_item(
item_id=str(item.get("id") or f"R{index + 1}"),
source=source,
title=str(item.get("title") or ""),
body=body,
url=str(item.get("url") or ""),
author=None,
container=str(item.get("subreddit") or ""),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=comment_text or str(item.get("selftext") or "")[:400],
metadata={
"top_comments": top_comments,
"comment_insights": item.get("comment_insights") or [],
},
)
def normalize_youtube_items(
items: List[Dict[str, Any]],
def _normalize_x(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.YouTubeItem]:
"""Normalize raw YouTube items to schema.
Args:
items: Raw YouTube items from yt-dlp
from_date: Start of date range
to_date: End of date range
Returns:
List of YouTubeItem objects
"""
normalized = []
for item in items:
# Parse engagement
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
views=eng_raw.get("views"),
likes=eng_raw.get("likes"),
num_comments=eng_raw.get("comments"),
)
# YouTube dates are reliable (always YYYY-MM-DD from yt-dlp)
date_str = item.get("date")
normalized.append(schema.YouTubeItem(
id=item.get("video_id", ""),
title=item.get("title", ""),
url=item.get("url", ""),
channel_name=item.get("channel_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
transcript_snippet=item.get("transcript_snippet", ""),
transcript_highlights=item.get("transcript_highlights", []),
relevance=item.get("relevance", 0.7),
why_relevant=item.get("why_relevant", ""),
))
return normalized
) -> schema.SourceItem:
text = str(item.get("text") or "").strip()
return _source_item(
item_id=str(item.get("id") or f"X{index + 1}"),
source=source,
title=text[:140] or f"X post {index + 1}",
body=text,
url=str(item.get("url") or ""),
author=str(item.get("author_handle") or "").lstrip("@"),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
)
def normalize_tiktok_items(
items: List[Dict[str, Any]],
def _normalize_youtube(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.TikTokItem]:
"""Normalize raw TikTok items to schema.
Args:
items: Raw TikTok items from Apify
from_date: Start of date range
to_date: End of date range
Returns:
List of TikTokItem objects
"""
normalized = []
for i, item in enumerate(items):
# Parse engagement
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
views=eng_raw.get("views"),
likes=eng_raw.get("likes"),
num_comments=eng_raw.get("comments"),
shares=eng_raw.get("shares"),
)
# TikTok dates are reliable (exact timestamps from Apify)
date_str = item.get("date")
normalized.append(schema.TikTokItem(
id=f"TK{i+1}",
text=item.get("text", ""),
url=item.get("url", ""),
author_name=item.get("author_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
caption_snippet=item.get("caption_snippet", ""),
hashtags=item.get("hashtags", []),
relevance=item.get("relevance", 0.7),
why_relevant=item.get("why_relevant", ""),
))
return normalized
) -> schema.SourceItem:
transcript = str(item.get("transcript_snippet") or "").strip()
description = str(item.get("description") or "").strip()
title = str(item.get("title") or "").strip()
highlights = item.get("transcript_highlights") or []
metadata: dict[str, Any] = {}
if highlights:
metadata["transcript_highlights"] = highlights
return _source_item(
item_id=str(item.get("video_id") or item.get("id") or f"YT{index + 1}"),
source=source,
title=title,
body="\n".join(part for part in [title, description, transcript] if part),
url=str(item.get("url") or ""),
author=str(item.get("channel_name") or ""),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=transcript,
metadata=metadata,
)
def normalize_instagram_items(
items: List[Dict[str, Any]],
def _normalize_shortform_video(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.InstagramItem]:
"""Normalize raw Instagram items to schema.
Args:
items: Raw Instagram items from ScrapeCreators
from_date: Start of date range
to_date: End of date range
Returns:
List of InstagramItem objects
"""
normalized = []
for i, item in enumerate(items):
# Parse engagement
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
views=eng_raw.get("views"),
likes=eng_raw.get("likes"),
num_comments=eng_raw.get("comments"),
)
# Instagram dates are reliable (exact timestamps from ScrapeCreators)
date_str = item.get("date")
normalized.append(schema.InstagramItem(
id=f"IG{i+1}",
text=item.get("text", ""),
url=item.get("url", ""),
author_name=item.get("author_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
caption_snippet=item.get("caption_snippet", ""),
hashtags=item.get("hashtags", []),
relevance=item.get("relevance", 0.7),
why_relevant=item.get("why_relevant", ""),
))
return normalized
id_prefix: str,
default_title: str,
) -> schema.SourceItem:
"""Shared normalizer for TikTok and Instagram (identical structure)."""
caption = str(item.get("caption_snippet") or "").strip()
text = str(item.get("text") or "").strip()
return _source_item(
item_id=str(item.get("id") or f"{id_prefix}{index + 1}"),
source=source,
title=text[:140] or caption[:140] or f"{default_title} {index + 1}",
body="\n".join(part for part in [text, caption] if part),
url=str(item.get("url") or ""),
author=str(item.get("author_name") or ""),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=caption,
metadata={"hashtags": item.get("hashtags") or []},
)
def normalize_hackernews_items(
items: List[Dict[str, Any]],
def _normalize_pinterest(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.HackerNewsItem]:
"""Normalize raw Hacker News items to schema.
) -> schema.SourceItem:
"""Normalizer for Pinterest pins (visual content with descriptions).
Args:
items: Raw HN items from Algolia API
from_date: Start of date range
to_date: End of date range
Returns:
List of HackerNewsItem objects
Saves are the primary engagement signal, analogous to likes/upvotes.
"""
normalized = []
for i, item in enumerate(items):
# Parse engagement
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
score=eng_raw.get("points"),
num_comments=eng_raw.get("num_comments"),
)
# Parse comments (from enrichment)
top_comments = []
for c in item.get("top_comments", []):
top_comments.append(schema.Comment(
score=c.get("points", 0),
date=None,
author=c.get("author", ""),
excerpt=c.get("text", ""),
url="",
))
# HN dates are always high confidence (exact timestamps from Algolia)
date_str = item.get("date")
normalized.append(schema.HackerNewsItem(
id=f"HN{i+1}",
title=item.get("title", ""),
url=item.get("url", ""),
hn_url=item.get("hn_url", ""),
author=item.get("author", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
top_comments=top_comments,
comment_insights=item.get("comment_insights", []),
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
description = str(item.get("description") or "").strip()
return _source_item(
item_id=str(item.get("pin_id") or item.get("id") or f"PI{index + 1}"),
source=source,
title=description[:140] or f"Pinterest pin {index + 1}",
body=description,
url=str(item.get("url") or ""),
author=str(item.get("author") or ""),
container=str(item.get("board") or ""),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="low"),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=description[:400],
)
def normalize_bluesky_items(
items: List[Dict[str, Any]],
def _normalize_hackernews(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.BlueskyItem]:
"""Normalize raw Bluesky items to schema.
Args:
items: Raw Bluesky items from AT Protocol API
from_date: Start of date range
to_date: End of date range
Returns:
List of BlueskyItem objects
"""
normalized = []
for i, item in enumerate(items):
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
likes=eng_raw.get("likes"),
reposts=eng_raw.get("reposts"),
replies=eng_raw.get("replies"),
quotes=eng_raw.get("quotes"),
)
date_str = item.get("date")
normalized.append(schema.BlueskyItem(
id=f"BS{i+1}",
text=item.get("text", ""),
url=item.get("url", ""),
author_handle=item.get("handle", ""),
display_name=item.get("display_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
) -> schema.SourceItem:
top_comments = item.get("top_comments") or []
comment_text = " ".join(
str(comment.get("text") or "").strip()
for comment in top_comments[:3]
if isinstance(comment, dict)
)
title = str(item.get("title") or "").strip()
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
return _source_item(
item_id=str(item.get("id") or f"HN{index + 1}"),
source=source,
title=title or f"HN story {index + 1}",
body=body,
url=str(item.get("url") or item.get("hn_url") or ""),
author=str(item.get("author") or ""),
container="Hacker News",
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=comment_text,
metadata={
"hn_url": item.get("hn_url"),
"top_comments": top_comments,
"comment_insights": item.get("comment_insights") or [],
},
)
def normalize_truthsocial_items(
items: List[Dict[str, Any]],
def _normalize_microblog(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.TruthSocialItem]:
"""Normalize raw Truth Social items to schema.
Args:
items: Raw Truth Social items from Mastodon API
from_date: Start of date range
to_date: End of date range
Returns:
List of TruthSocialItem objects
"""
normalized = []
for i, item in enumerate(items):
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
likes=eng_raw.get("likes"),
reposts=eng_raw.get("reposts"),
replies=eng_raw.get("replies"),
)
date_str = item.get("date")
normalized.append(schema.TruthSocialItem(
id=f"TS{i+1}",
text=item.get("text", ""),
url=item.get("url", ""),
author_handle=item.get("handle", ""),
display_name=item.get("display_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
id_prefix: str,
default_title: str,
) -> schema.SourceItem:
"""Shared normalizer for Bluesky and Truth Social (identical structure)."""
text = str(item.get("text") or "").strip()
return _source_item(
item_id=str(item.get("id") or f"{id_prefix}{index + 1}"),
source=source,
title=text[:140] or f"{default_title} {index + 1}",
body=text,
url=str(item.get("url") or ""),
author=str(item.get("handle") or item.get("author_handle") or "").lstrip("@"),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
metadata={"display_name": item.get("display_name")},
)
def normalize_polymarket_items(
items: List[Dict[str, Any]],
def _normalize_polymarket(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> List[schema.PolymarketItem]:
"""Normalize raw Polymarket items to schema.
Args:
items: Raw Polymarket items from Gamma API
from_date: Start of date range
to_date: End of date range
Returns:
List of PolymarketItem objects
"""
normalized = []
for i, item in enumerate(items):
# Prefer volume1mo (more stable) for engagement scoring, fall back to volume24hr
volume = item.get("volume1mo") or item.get("volume24hr", 0.0)
engagement = schema.Engagement(
volume=volume,
liquidity=item.get("liquidity", 0.0),
)
date_str = item.get("date")
normalized.append(schema.PolymarketItem(
id=f"PM{i+1}",
title=item.get("title", ""),
question=item.get("question", ""),
url=item.get("url", ""),
outcome_prices=item.get("outcome_prices", []),
outcomes_remaining=item.get("outcomes_remaining", 0),
price_movement=item.get("price_movement"),
date=date_str,
date_confidence="high",
engagement=engagement,
end_date=item.get("end_date"),
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
) -> schema.SourceItem:
title = str(item.get("title") or "").strip()
question = str(item.get("question") or "").strip()
engagement = {
"volume": item.get("volume1mo") or item.get("volume24hr") or 0,
"liquidity": item.get("liquidity") or 0,
}
return _source_item(
item_id=str(item.get("id") or f"PM{index + 1}"),
source=source,
title=title or question or f"Polymarket event {index + 1}",
body="\n".join(part for part in [title, question, str(item.get("price_movement") or "")] if part),
url=str(item.get("url") or ""),
author=None,
container="Polymarket",
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
engagement=engagement,
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=str(item.get("price_movement") or ""),
metadata={
"question": question,
"end_date": item.get("end_date"),
"outcome_prices": item.get("outcome_prices") or [],
"outcomes_remaining": item.get("outcomes_remaining"),
},
)
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
"""Convert schema items to dicts for JSON serialization."""
return [item.to_dict() for item in items]
def _normalize_github(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> schema.SourceItem:
title = str(item.get("title") or "").strip()
snippet_text = str(item.get("snippet") or "").strip()
top_comments = item.get("metadata", {}).get("top_comments") or []
comment_text = " ".join(
str(comment.get("excerpt") or "").strip()
for comment in top_comments[:3]
if isinstance(comment, dict)
)
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
metadata = item.get("metadata") or {}
return _source_item(
item_id=str(item.get("id") or f"GH{index + 1}"),
source=source,
title=title or f"GitHub item {index + 1}",
body=body,
url=str(item.get("url") or ""),
author=str(item.get("author") or ""),
container=str(item.get("container") or ""),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date, default="high"),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=comment_text or snippet_text[:400],
metadata={
"top_comments": top_comments,
"labels": metadata.get("labels") or [],
"state": metadata.get("state", ""),
"is_pr": metadata.get("is_pr", False),
},
)
def _normalize_grounding(
source: str,
item: dict[str, Any],
index: int,
from_date: str,
to_date: str,
) -> schema.SourceItem:
title = str(item.get("title") or "").strip()
snippet = str(item.get("snippet") or "").strip()
url = str(item.get("url") or "").strip()
return _source_item(
item_id=str(item.get("id") or f"W{index + 1}"),
source=source,
title=title or _domain_from_url(url) or f"Web result {index + 1}",
body="\n".join(part for part in [title, snippet] if part),
url=url,
author=None,
container=str(item.get("source_domain") or _domain_from_url(url) or ""),
published_at=item.get("date"),
date_confidence=_date_confidence(item, from_date, to_date),
engagement=item.get("engagement") or {},
relevance_hint=item.get("relevance", 0.5),
why_relevant=str(item.get("why_relevant") or ""),
snippet=snippet,
metadata=item.get("metadata") or {},
)
scripts/lib/openai_reddit.py
-631
View File
@@ -1,631 +0,0 @@
"""OpenAI Responses API client for Reddit discovery."""
import json
import re
import sys
from typing import Any, Dict, List, Optional
from . import http, env
# Fallback models when the selected model isn't accessible (e.g., org not verified).
# Ordered by cost-efficiency: mini models handle structured extraction equally well.
# Note: gpt-4o-mini does NOT support web_search with filters — excluded.
MODEL_FALLBACK_ORDER = ["gpt-5-mini", "gpt-4.1-mini", "gpt-4.1", "gpt-4o"]
def _log_error(msg: str):
"""Log error to stderr."""
sys.stderr.write(f"[REDDIT ERROR] {msg}\n")
sys.stderr.flush()
def _log_info(msg: str):
"""Log info to stderr."""
sys.stderr.write(f"[REDDIT] {msg}\n")
sys.stderr.flush()
def _is_model_access_error(error: http.HTTPError) -> bool:
"""Check if error is due to model access/verification issues."""
if error.status_code not in (400, 403):
return False
if not error.body:
return False
body_lower = error.body.lower()
# Check for common access/verification error messages
return any(phrase in body_lower for phrase in [
"verified",
"organization must be",
"does not have access",
"not available",
"not found",
])
OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
CODEX_INSTRUCTIONS = (
"You are a research assistant for a skill that summarizes what people are "
"discussing in the last 30 days. Your goal is to find relevant Reddit threads "
"about the topic and return ONLY the required JSON. Be inclusive (return more "
"rather than fewer), but avoid irrelevant results. Prefer threads with discussion "
"and comments. If you can infer a date, include it; otherwise use null. "
"Do not include developers.reddit.com or business.reddit.com."
)
def _parse_sse_chunk(chunk: str) -> Optional[Dict[str, Any]]:
"""Parse a single SSE chunk into a JSON object."""
lines = chunk.split("\n")
data_lines = []
for line in lines:
if line.startswith("data:"):
data_lines.append(line[5:].strip())
if not data_lines:
return None
data = "\n".join(data_lines).strip()
if not data or data == "[DONE]":
return None
try:
return json.loads(data)
except json.JSONDecodeError:
return None
def _parse_sse_stream_raw(raw: str) -> List[Dict[str, Any]]:
"""Parse SSE stream from raw text and return JSON events."""
events: List[Dict[str, Any]] = []
buffer = ""
for chunk in raw.splitlines(keepends=True):
buffer += chunk
while "\n\n" in buffer:
event_chunk, buffer = buffer.split("\n\n", 1)
event = _parse_sse_chunk(event_chunk)
if event is not None:
events.append(event)
if buffer.strip():
event = _parse_sse_chunk(buffer)
if event is not None:
events.append(event)
return events
def _parse_codex_stream(raw: str) -> Dict[str, Any]:
"""Parse SSE stream from Codex responses into a response-like dict."""
events = _parse_sse_stream_raw(raw)
# Prefer explicit completed response payload if present
for evt in reversed(events):
if isinstance(evt, dict):
if evt.get("type") == "response.completed" and isinstance(evt.get("response"), dict):
return evt["response"]
if isinstance(evt.get("response"), dict):
return evt["response"]
# Fallback: reconstruct output text from deltas
output_text = ""
for evt in events:
if not isinstance(evt, dict):
continue
delta = evt.get("delta")
if isinstance(delta, str):
output_text += delta
continue
text = evt.get("text")
if isinstance(text, str):
output_text += text
if output_text:
return {
"output": [
{
"type": "message",
"content": [{"type": "output_text", "text": output_text}],
}
]
}
return {}
# Depth configurations: (min, max) threads to request
# Request MORE than needed since many get filtered by date
DEPTH_CONFIG = {
"quick": (15, 25),
"default": (30, 50),
"deep": (70, 100),
}
REDDIT_SEARCH_PROMPT = """Find Reddit discussion threads about: {topic}
STEP 1: EXTRACT THE CORE SUBJECT
Get the MAIN NOUN/PRODUCT/TOPIC:
- "best nano banana prompting practices" → "nano banana"
- "killer features of clawdbot" → "clawdbot"
- "top Claude Code skills" → "Claude Code"
DO NOT include "best", "top", "tips", "practices", "features" in your search.
STEP 2: SEARCH BROADLY
Search for the core subject:
1. "[core subject] site:reddit.com"
2. "reddit [core subject]"
3. "[core subject] reddit"
Return as many relevant threads as you find. We filter by date server-side.
STEP 3: INCLUDE ALL MATCHES
- Include ALL threads about the core subject
- Set date to "YYYY-MM-DD" if you can determine it, otherwise null
- We verify dates and filter old content server-side
- DO NOT pre-filter aggressively - include anything relevant
REQUIRED: URLs must contain "/r/" AND "/comments/"
REJECT: developers.reddit.com, business.reddit.com
Find {min_items}-{max_items} threads. Return MORE rather than fewer.
Return JSON:
{{
"items": [
{{
"title": "Thread title",
"url": "https://www.reddit.com/r/sub/comments/xyz/title/",
"subreddit": "subreddit_name",
"date": "YYYY-MM-DD or null",
"why_relevant": "Why relevant",
"relevance": 0.85
}}
]
}}"""
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for retry."""
noise = ['best', 'top', 'how to', 'tips for', 'practices', 'features',
'killer', 'guide', 'tutorial', 'recommendations', 'advice',
'prompting', 'using', 'for', 'with', 'the', 'of', 'in', 'on']
words = topic.lower().split()
result = [w for w in words if w not in noise]
return ' '.join(result[:3]) or topic # Keep max 3 words
def _build_subreddit_query(topic: str) -> str:
"""Build a subreddit-targeted search query for fallback.
When standard search returns few results, try searching for the
subreddit itself: 'r/kanye', 'r/howie', etc.
"""
core = _extract_core_subject(topic)
# Remove dots and special chars for subreddit name guess
sub_name = core.replace('.', '').replace(' ', '').lower()
return f"r/{sub_name} site:reddit.com"
def _build_payload(model: str, instructions_text: str, input_text: str, auth_source: str) -> Dict[str, Any]:
"""Build responses payload for OpenAI or Codex endpoints."""
payload = {
"model": model,
"store": False,
"tools": [
{
"type": "web_search",
"filters": {
"allowed_domains": ["reddit.com"]
}
}
],
"include": ["web_search_call.action.sources"],
"instructions": instructions_text,
"input": input_text,
}
if auth_source == env.AUTH_SOURCE_CODEX:
payload["input"] = [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": input_text}],
}
]
payload["stream"] = True
return payload
def search_reddit(
api_key: str,
model: str,
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
auth_source: str = "api_key",
account_id: Optional[str] = None,
mock_response: Optional[Dict] = None,
_retry: bool = False,
) -> Dict[str, Any]:
"""Search Reddit for relevant threads using OpenAI Responses API.
Args:
api_key: OpenAI API key
model: Model to use
topic: Search topic
from_date: Start date (YYYY-MM-DD) - only include threads after this
to_date: End date (YYYY-MM-DD) - only include threads before this
depth: Research depth - "quick", "default", or "deep"
mock_response: Mock response for testing
Returns:
Raw API response
"""
if mock_response is not None:
return mock_response
min_items, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
if auth_source == env.AUTH_SOURCE_CODEX:
if not account_id:
raise ValueError("Missing chatgpt_account_id for Codex auth")
headers = {
"Authorization": f"Bearer {api_key}",
"chatgpt-account-id": account_id,
"OpenAI-Beta": "responses=experimental",
"originator": "pi",
"Content-Type": "application/json",
}
url = CODEX_RESPONSES_URL
else:
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
url = OPENAI_RESPONSES_URL
# Adjust timeout based on depth (generous for OpenAI web_search which can be slow)
timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
# Build list of models to try: requested model first, then fallbacks
models_to_try = [model] + [m for m in MODEL_FALLBACK_ORDER if m != model]
# Note: allowed_domains accepts base domain, not subdomains
# We rely on prompt to filter out developers.reddit.com, etc.
input_text = REDDIT_SEARCH_PROMPT.format(
topic=topic,
from_date=from_date,
to_date=to_date,
min_items=min_items,
max_items=max_items,
)
if auth_source == env.AUTH_SOURCE_CODEX:
# Codex auth: try model with fallback chain
from . import models as models_mod
codex_models_to_try = [model] + [m for m in models_mod.CODEX_FALLBACK_MODELS if m != model]
instructions_text = CODEX_INSTRUCTIONS + "\n\n" + input_text
last_error = None
for current_model in codex_models_to_try:
try:
payload = _build_payload(current_model, instructions_text, topic, auth_source)
raw = http.post_raw(url, payload, headers=headers, timeout=timeout)
return _parse_codex_stream(raw or "")
except http.HTTPError as e:
last_error = e
if e.status_code == 400:
_log_info(f"Model {current_model} not supported on Codex, trying fallback...")
continue
raise
if last_error:
raise last_error
raise http.HTTPError("No Codex-compatible models available")
# Standard API key auth: try model fallback chain
last_error = None
for current_model in models_to_try:
payload = {
"model": current_model,
"tools": [
{
"type": "web_search",
"filters": {
"allowed_domains": ["reddit.com"]
}
}
],
"include": ["web_search_call.action.sources"],
"input": input_text,
}
try:
return http.post(url, payload, headers=headers, timeout=timeout)
except http.HTTPError as e:
last_error = e
if _is_model_access_error(e):
_log_info(f"Model {current_model} not accessible, trying fallback...")
continue
if e.status_code == 429:
_log_info(f"Rate limited on {current_model}, trying fallback model...")
continue
# Non-access error, don't retry with different model
raise
# All models failed with access errors
if last_error:
_log_error(f"All models failed. Last error: {last_error}")
raise last_error
raise http.HTTPError("No models available")
def _public_relevance(score: int, num_comments: int) -> float:
"""Estimate relevance for public Reddit search results."""
# Lightweight heuristic: blend normalized score + comments.
score_component = min(1.0, max(0.0, score / 500.0))
comments_component = min(1.0, max(0.0, num_comments / 200.0))
return round((score_component * 0.6) + (comments_component * 0.4), 3)
def search_reddit_public(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
) -> List[Dict[str, Any]]:
"""Search Reddit directly via public JSON endpoint (no OpenAI key required).
This is a fallback mode for environments where OpenAI auth is unavailable.
It uses reddit.com/search/.json with recency filter (t=month).
"""
_, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
limit = min(100, max(20, max_items))
core = _extract_core_subject(topic)
queries = [topic]
if core and core.lower() != topic.lower():
queries.append(core)
queries.append(f'"{core}"')
seen_urls = set()
all_items: List[Dict[str, Any]] = []
headers = {
"User-Agent": http.USER_AGENT,
"Accept": "application/json",
}
for query in queries:
try:
url = (
"https://www.reddit.com/search/.json"
f"?q={_url_encode(query)}&sort=new&t=month&limit={limit}&raw_json=1"
)
data = http.get(url, headers=headers, timeout=20, retries=2)
children = data.get("data", {}).get("children", [])
for child in children:
if child.get("kind") != "t3":
continue
post = child.get("data", {})
permalink = str(post.get("permalink", "")).strip()
if not permalink or "/comments/" not in permalink:
continue
full_url = f"https://www.reddit.com{permalink}"
if full_url in seen_urls:
continue
seen_urls.add(full_url)
score = int(post.get("score", 0) or 0)
num_comments = int(post.get("num_comments", 0) or 0)
# Parse date from created_utc
created_utc = post.get("created_utc")
date_value = None
if created_utc:
from . import dates as dates_mod
date_value = dates_mod.timestamp_to_date(created_utc)
all_items.append({
"id": f"R{len(all_items)+1}",
"title": str(post.get("title", "")).strip(),
"url": full_url,
"subreddit": str(post.get("subreddit", "")).strip(),
"date": date_value,
"why_relevant": "Found via Reddit public search",
"relevance": _public_relevance(score, num_comments),
"engagement": {
"score": score,
"num_comments": num_comments,
"upvote_ratio": post.get("upvote_ratio"),
},
})
except http.HTTPError as e:
_log_info(f"Public Reddit search failed for query '{query}': {e}")
# Continue with next query; partial results are still useful.
continue
except Exception as e:
_log_info(f"Public Reddit search error for query '{query}': {e}")
continue
# Sort by date (desc, unknown dates last), then relevance desc
def _sort_key(item: Dict[str, Any]):
date_str = item.get("date") or ""
return (date_str, float(item.get("relevance", 0.0)))
all_items.sort(key=_sort_key, reverse=True)
return all_items[: max_items * 2]
def search_subreddits(
subreddits: List[str],
topic: str,
from_date: str,
to_date: str,
count_per: int = 5,
) -> List[Dict[str, Any]]:
"""Search specific subreddits via Reddit's free JSON endpoint.
No API key needed. Uses reddit.com/r/{sub}/search/.json endpoint.
Used in Phase 2 supplemental search after entity extraction.
Args:
subreddits: List of subreddit names (without r/)
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
count_per: Results to request per subreddit
Returns:
List of raw item dicts (same format as parse_reddit_response output).
"""
all_items = []
core = _extract_core_subject(topic)
for sub in subreddits:
sub = sub.lstrip("r/")
try:
url = f"https://www.reddit.com/r/{sub}/search/.json"
params = f"q={_url_encode(core)}&restrict_sr=on&sort=new&limit={count_per}&raw_json=1"
full_url = f"{url}?{params}"
headers = {
"User-Agent": http.USER_AGENT,
"Accept": "application/json",
}
data = http.get(full_url, headers=headers, timeout=15, retries=1)
# Reddit search returns {"data": {"children": [...]}}
children = data.get("data", {}).get("children", [])
for i, child in enumerate(children):
if child.get("kind") != "t3": # t3 = link/submission
continue
post = child.get("data", {})
permalink = post.get("permalink", "")
if not permalink:
continue
item = {
"id": f"RS{len(all_items)+1}",
"title": str(post.get("title", "")).strip(),
"url": f"https://www.reddit.com{permalink}",
"subreddit": str(post.get("subreddit", sub)).strip(),
"date": None,
"why_relevant": f"Found in r/{sub} supplemental search",
"relevance": 0.65, # Slightly lower default for supplemental
}
# Parse date from created_utc
created_utc = post.get("created_utc")
if created_utc:
from . import dates as dates_mod
item["date"] = dates_mod.timestamp_to_date(created_utc)
all_items.append(item)
except http.HTTPError as e:
_log_info(f"Subreddit search failed for r/{sub}: {e}")
if e.status_code == 429:
_log_info("Reddit rate-limited (429) — skipping remaining subreddits")
break
except Exception as e:
_log_info(f"Subreddit search error for r/{sub}: {e}")
return all_items
def _url_encode(text: str) -> str:
"""Simple URL encoding for query parameters."""
import urllib.parse
return urllib.parse.quote_plus(text)
def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse OpenAI response to extract Reddit items.
Args:
response: Raw API response
Returns:
List of item dicts
"""
items = []
# Check for API errors first
if "error" in response and response["error"]:
error = response["error"]
err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error)
_log_error(f"OpenAI API error: {err_msg}")
if http.DEBUG:
_log_error(f"Full error response: {json.dumps(response, indent=2)[:1000]}")
return items
# Try to find the output text
output_text = ""
if "output" in response:
output = response["output"]
if isinstance(output, str):
output_text = output
elif isinstance(output, list):
for item in output:
if isinstance(item, dict):
if item.get("type") == "message":
content = item.get("content", [])
for c in content:
if isinstance(c, dict) and c.get("type") == "output_text":
output_text = c.get("text", "")
break
elif "text" in item:
output_text = item["text"]
elif isinstance(item, str):
output_text = item
if output_text:
break
# Also check for choices (older format)
if not output_text and "choices" in response:
for choice in response["choices"]:
if "message" in choice:
output_text = choice["message"].get("content", "")
break
if not output_text:
print(f"[REDDIT WARNING] No output text found in OpenAI response. Keys present: {list(response.keys())}", flush=True)
return items
# Extract JSON from the response
json_match = re.search(r'\{[\s\S]*"items"[\s\S]*\}', output_text)
if json_match:
try:
data = json.loads(json_match.group())
items = data.get("items", [])
except json.JSONDecodeError:
pass
# Validate and clean items
clean_items = []
for i, item in enumerate(items):
if not isinstance(item, dict):
continue
url = item.get("url", "")
if not url or "reddit.com" not in url:
continue
clean_item = {
"id": f"R{i+1}",
"title": str(item.get("title", "")).strip(),
"url": url,
"subreddit": str(item.get("subreddit", "")).strip().lstrip("r/"),
"date": item.get("date"),
"why_relevant": str(item.get("why_relevant", "")).strip(),
"relevance": min(1.0, max(0.0, float(item.get("relevance", 0.5)))),
}
# Validate date format
if clean_item["date"]:
if not re.match(r'^\d{4}-\d{2}-\d{2}$', str(clean_item["date"])):
clean_item["date"] = None
clean_items.append(clean_item)
return clean_items
scripts/lib/openrouter_search.py
-216
View File
@@ -1,216 +0,0 @@
"""Perplexity Sonar Pro web search via OpenRouter for last30days skill.
Uses OpenRouter's chat completions API with Perplexity's Sonar Pro model,
which has built-in web search and returns citations with URLs, titles, and dates.
This is the recommended web search backend -- highest quality results.
API docs: https://openrouter.ai/docs/quickstart
Model: perplexity/sonar-pro
"""
import re
import sys
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from . import http
ENDPOINT = "https://openrouter.ai/api/v1/chat/completions"
MODEL = "perplexity/sonar-pro"
# Domains to exclude (handled by Reddit/X search)
EXCLUDED_DOMAINS = {
"reddit.com", "www.reddit.com", "old.reddit.com",
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
}
def search_web(
topic: str,
from_date: str,
to_date: str,
api_key: str,
depth: str = "default",
) -> List[Dict[str, Any]]:
"""Search the web via Perplexity Sonar Pro on OpenRouter.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
api_key: OpenRouter API key
depth: 'quick', 'default', or 'deep'
Returns:
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
Raises:
http.HTTPError: On API errors
"""
max_tokens = {"quick": 1024, "default": 2048, "deep": 4096}.get(depth, 2048)
prompt = (
f"Find recent blog posts, news articles, tutorials, and discussions "
f"about {topic} published between {from_date} and {to_date}. "
f"Exclude results from reddit.com, x.com, and twitter.com. "
f"For each result, provide the title, URL, publication date, "
f"and a brief summary of why it's relevant."
)
payload = {
"model": MODEL,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
}
sys.stderr.write(f"[Web] Searching Sonar Pro via OpenRouter for: {topic}\n")
sys.stderr.flush()
response = http.post(
ENDPOINT,
json_data=payload,
headers={
"Authorization": f"Bearer {api_key}",
"HTTP-Referer": "https://github.com/mvanhorn/last30days-openclaw",
"X-Title": "last30days",
},
timeout=30,
)
return _normalize_results(response)
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Convert Sonar Pro response to websearch item schema.
Sonar Pro returns:
- search_results: [{title, url, date}] -- structured source metadata
- citations: [url, ...] -- flat list of cited URLs
- choices[0].message.content -- the synthesized text with [N] references
We prefer search_results (richer metadata), fall back to citations.
"""
items = []
# Try search_results first (has title, url, date)
search_results = response.get("search_results", [])
if isinstance(search_results, list) and search_results:
items = _parse_search_results(search_results)
# Fall back to citations if no search_results
if not items:
citations = response.get("citations", [])
content = _get_content(response)
if isinstance(citations, list) and citations:
items = _parse_citations(citations, content)
sys.stderr.write(f"[Web] Sonar Pro: {len(items)} results\n")
sys.stderr.flush()
return items
def _parse_search_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Parse the search_results array from Sonar Pro."""
items = []
for i, result in enumerate(results):
if not isinstance(result, dict):
continue
url = result.get("url", "")
if not url:
continue
# Skip excluded domains
try:
domain = urlparse(url).netloc.lower()
if domain in EXCLUDED_DOMAINS:
continue
if domain.startswith("www."):
domain = domain[4:]
except Exception:
domain = ""
title = str(result.get("title", "")).strip()
if not title:
continue
# Sonar Pro provides dates in search_results
date = result.get("date")
date_confidence = "med" if date else "low"
items.append({
"id": f"W{i+1}",
"title": title[:200],
"url": url,
"source_domain": domain,
"snippet": str(result.get("snippet", result.get("description", ""))).strip()[:500],
"date": date,
"date_confidence": date_confidence,
"relevance": 0.7, # Sonar Pro results are generally high quality
"why_relevant": "",
})
return items
def _parse_citations(citations: List[str], content: str) -> List[Dict[str, Any]]:
"""Parse the flat citations array, enriching with content context."""
items = []
for i, url in enumerate(citations):
if not isinstance(url, str) or not url:
continue
# Skip excluded domains
try:
domain = urlparse(url).netloc.lower()
if domain in EXCLUDED_DOMAINS:
continue
if domain.startswith("www."):
domain = domain[4:]
except Exception:
domain = ""
# Try to extract title from content references like [1] Title...
title = _extract_title_for_citation(content, i + 1) or domain
items.append({
"id": f"W{i+1}",
"title": title[:200],
"url": url,
"source_domain": domain,
"snippet": "",
"date": None,
"date_confidence": "low",
"relevance": 0.6,
"why_relevant": "",
})
return items
def _get_content(response: Dict[str, Any]) -> str:
"""Extract the text content from the chat completion response."""
try:
return response["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError):
return ""
def _extract_title_for_citation(content: str, index: int) -> Optional[str]:
"""Try to extract a title near a citation reference [N] in the content."""
if not content:
return None
# Look for patterns like [1] Title or [1](url) Title
pattern = rf'\[{index}\][)\s]*([^\[\n]{{5,80}})'
match = re.search(pattern, content)
if match:
title = match.group(1).strip().rstrip('.')
# Clean up markdown artifacts
title = re.sub(r'[*_`]', '', title)
return title if len(title) > 3 else None
return None
scripts/lib/parallel_search.py
-139
View File
@@ -1,139 +0,0 @@
"""Parallel AI web search for last30days skill.
Uses the Parallel AI Search API to find web content (blogs, docs, news, tutorials).
This is the preferred web search backend -- it returns LLM-optimized results
with extended excerpts ranked by relevance.
API docs: https://docs.parallel.ai/search-api/search-quickstart
"""
import json
import sys
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from . import http
ENDPOINT = "https://api.parallel.ai/v1beta/search"
# Domains to exclude (handled by Reddit/X search)
EXCLUDED_DOMAINS = {
"reddit.com", "www.reddit.com", "old.reddit.com",
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
}
def search_web(
topic: str,
from_date: str,
to_date: str,
api_key: str,
depth: str = "default",
) -> List[Dict[str, Any]]:
"""Search the web via Parallel AI Search API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
api_key: Parallel AI API key
depth: 'quick', 'default', or 'deep'
Returns:
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
Raises:
http.HTTPError: On API errors
"""
max_results = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
payload = {
"objective": (
f"Find recent blog posts, tutorials, news articles, and discussions "
f"about {topic} from {from_date} to {to_date}. "
f"Exclude reddit.com, x.com, and twitter.com."
),
"max_results": max_results,
"max_chars_per_result": 500,
}
sys.stderr.write(f"[Web] Searching Parallel AI for: {topic}\n")
sys.stderr.flush()
response = http.post(
ENDPOINT,
json_data=payload,
headers={
"Authorization": f"Bearer {api_key}",
"parallel-beta": "search-extract-2025-10-10",
},
timeout=30,
)
return _normalize_results(response)
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Convert Parallel AI response to websearch item schema.
Args:
response: Raw API response
Returns:
List of normalized result dicts
"""
items = []
# Handle different response shapes
results = response.get("results", [])
if not isinstance(results, list):
return items
for i, result in enumerate(results):
if not isinstance(result, dict):
continue
url = result.get("url", "")
if not url:
continue
# Skip excluded domains
try:
domain = urlparse(url).netloc.lower()
if domain in EXCLUDED_DOMAINS:
continue
# Clean domain for display
if domain.startswith("www."):
domain = domain[4:]
except Exception:
domain = ""
title = str(result.get("title", "")).strip()
snippet = str(result.get("excerpt", result.get("snippet", result.get("description", "")))).strip()
if not title and not snippet:
continue
# Extract relevance score if provided
relevance = result.get("relevance_score", result.get("relevance", 0.6))
try:
relevance = min(1.0, max(0.0, float(relevance)))
except (TypeError, ValueError):
relevance = 0.6
items.append({
"id": f"W{i+1}",
"title": title[:200],
"url": url,
"source_domain": domain,
"snippet": snippet[:500],
"date": result.get("published_date", result.get("date")),
"date_confidence": "med" if result.get("published_date") or result.get("date") else "low",
"relevance": relevance,
"why_relevant": str(result.get("summary", "")).strip()[:200],
})
sys.stderr.write(f"[Web] Parallel AI: {len(items)} results\n")
sys.stderr.flush()
return items
scripts/lib/perplexity.py
+164
View File
@@ -0,0 +1,164 @@
"""Perplexity Sonar Pro / Deep Research via OpenRouter API.
Queries Perplexity models through OpenRouter for AI-synthesized research
with citation annotations. Returns normalized items with synthesis text
and individual citation entries.
"""
from __future__ import annotations
import sys
from urllib.parse import urlparse
from . import http, log
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
MODEL_SONAR_PRO = "perplexity/sonar-pro"
MODEL_DEEP_RESEARCH = "perplexity/sonar-deep-research"
def _log(msg: str):
log.source_log("Perplexity", msg)
def _domain(url: str) -> str:
return urlparse(url).netloc.strip().lower()
def search(
query: str,
date_range: tuple[str, str],
config: dict,
deep: bool = False,
) -> tuple[list[dict], dict]:
"""Search via Perplexity Sonar Pro or Deep Research through OpenRouter.
Args:
query: Search topic
date_range: (from_date, to_date) as YYYY-MM-DD strings
config: Must contain OPENROUTER_API_KEY
deep: Use Deep Research model (~$0.90/query) instead of Sonar Pro
Returns:
Tuple of (items list, artifact dict).
"""
api_key = config.get("OPENROUTER_API_KEY")
if not api_key:
_log("No OPENROUTER_API_KEY configured, skipping")
return [], {}
from_date, to_date = date_range
model = MODEL_DEEP_RESEARCH if deep else MODEL_SONAR_PRO
timeout = 120 if deep else 30
if deep:
print("[Perplexity] Using Deep Research (~$0.90/query)", file=sys.stderr)
prompt = (
f"What has been happening with {query} between {from_date} and {to_date}? "
"Include specific dates, names, numbers, and sources."
)
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
json_data = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
}
_log(f"Querying {model} for '{query}' ({from_date} to {to_date})")
try:
data = http.post(OPENROUTER_URL, json_data, headers=headers, timeout=timeout)
except http.HTTPError as e:
if e.status_code == 401:
_log("Invalid OpenRouter API key (401)")
elif e.status_code == 429:
_log("Rate limited by OpenRouter (429)")
else:
_log(f"HTTP error: {e}")
return [], {}
except Exception as e:
_log(f"Request failed: {e}")
return [], {}
# Parse response
choices = data.get("choices", [])
if not choices:
_log("No choices in response")
return [], {}
synthesis = choices[0].get("message", {}).get("content", "")
if not synthesis:
_log("Empty synthesis content")
return [], {}
# Extract citations from annotations
annotations = choices[0].get("message", {}).get("annotations", [])
citations = []
for ann in annotations:
url_citation = ann.get("url_citation", {})
url = url_citation.get("url", "")
title = url_citation.get("title", "")
if url:
citations.append({"url": url, "title": title})
# Deduplicate citations by URL
seen_urls = set()
unique_citations = []
for c in citations:
if c["url"] not in seen_urls:
seen_urls.add(c["url"])
unique_citations.append(c)
citations = unique_citations
_log(f"Got synthesis ({len(synthesis)} chars) with {len(citations)} citations")
# Build items list
items = []
# Primary item: the synthesis itself
snippet = synthesis[:2000]
items.append({
"id": "PX1",
"title": f"Perplexity {'Deep Research' if deep else 'Sonar Pro'}: {query}",
"url": "",
"source_domain": "perplexity.ai",
"snippet": snippet,
"date": to_date,
"relevance": 0.9,
"why_relevant": f"AI synthesis of recent activity for '{query}'",
"engagement": {"citations": len(citations)},
"metadata": {"citations": citations},
})
# Individual items for each citation
for i, cit in enumerate(citations):
items.append({
"id": f"PX{i + 2}",
"title": cit["title"] or _domain(cit["url"]),
"url": cit["url"],
"source_domain": _domain(cit["url"]),
"snippet": "",
"date": None,
"relevance": 0.7,
"why_relevant": f"Cited in Perplexity synthesis for '{query}'",
"engagement": {"citations": 1},
"metadata": {"citations": [cit]},
})
artifact = {
"label": "perplexity",
"model": model,
"deep": deep,
"query": query,
"synthesisLength": len(synthesis),
"citationCount": len(citations),
}
return items, artifact
scripts/lib/pinterest.py
+190
View File
@@ -0,0 +1,190 @@
"""Pinterest search via ScrapeCreators API for /last30days.
Uses ScrapeCreators REST API to search Pinterest by keyword, extract
engagement metrics (saves, comments), and return pin descriptions.
Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
API docs: https://scrapecreators.com/docs
"""
import re
import sys
from typing import Any, Dict, List, Optional, Set
try:
import requests as _requests
except ImportError:
_requests = None
from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
# Depth configurations: how many results to fetch
DEPTH_CONFIG = {
"quick": {"results_per_page": 10},
"default": {"results_per_page": 20},
"deep": {"results_per_page": 40},
}
from .relevance import token_overlap_relevance as _compute_relevance
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Pinterest search."""
from .query import extract_core_subject
_PINTEREST_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features',
'recommendations', 'advice',
'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches',
})
return extract_core_subject(topic, noise=_PINTEREST_NOISE)
def _log(msg: str):
log.source_log("Pinterest", msg)
def _sc_headers(token: str) -> Dict[str, str]:
"""Build ScrapeCreators request headers."""
return {
"x-api-key": token,
"Content-Type": "application/json",
}
def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]:
"""Parse raw Pinterest items into normalized dicts.
Pinterest pins are visual content with descriptions. Saves are the
primary engagement signal (analogous to upvotes/likes on other platforms).
"""
items = []
for raw in raw_items:
if not isinstance(raw, dict):
continue
pin_id = str(raw.get("id", raw.get("pin_id", "")))
description = str(raw.get("description") or raw.get("title") or "")
# Engagement metrics - saves are the primary signal
save_count = raw.get("save_count") or raw.get("saves") or raw.get("repin_count") or 0
comment_count = raw.get("comment_count") or raw.get("comments") or 0
# Author info
pinner = raw.get("pinner") or raw.get("creator") or raw.get("user") or {}
if isinstance(pinner, dict):
author_name = pinner.get("username") or pinner.get("full_name") or ""
elif isinstance(pinner, str):
author_name = pinner
else:
author_name = ""
# URL
url = raw.get("link") or raw.get("url") or ""
if not url and pin_id:
url = f"https://www.pinterest.com/pin/{pin_id}/"
# Board info (container for pins)
board = raw.get("board") or {}
board_name = board.get("name", "") if isinstance(board, dict) else ""
# Compute relevance
relevance = _compute_relevance(core_topic, description, [])
items.append({
"pin_id": pin_id,
"description": description,
"url": url,
"author": author_name,
"board": board_name,
"engagement": {
"saves": save_count,
"comments": comment_count,
},
"relevance": relevance,
"why_relevant": f"Pinterest: {description[:60]}" if description else f"Pinterest: {core_topic}",
})
return items
def parse_pinterest_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Pinterest search response to normalized format.
Returns:
List of item dicts ready for normalization.
"""
return response.get("items", [])
def search_pinterest(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
token: str = None,
) -> Dict[str, Any]:
"""Search Pinterest via ScrapeCreators API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
Returns:
Dict with 'items' list and optional 'error'.
"""
if not token:
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
core_topic = _extract_core_subject(topic)
_log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
if not _requests:
_log("requests library not installed, falling back to urllib")
try:
from urllib.parse import urlencode
params = urlencode({"keyword": core_topic})
url = f"{SCRAPECREATORS_BASE}/search?{params}"
headers = _sc_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search",
params={"keyword": core_topic},
headers=_sc_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Extract items from response - try common SC response shapes
raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []
# Limit to configured count
raw_items = raw_items[:config["results_per_page"]]
# Parse items
items = _parse_items(raw_items, core_topic)
# Sort by saves descending (primary engagement signal)
items.sort(key=lambda x: x["engagement"]["saves"], reverse=True)
_log(f"Found {len(items)} Pinterest pins")
return {"items": items}
scripts/lib/pipeline.py
+990
View File
@@ -0,0 +1,990 @@
"""v3.0.0 orchestration pipeline."""
from __future__ import annotations
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from shutil import which
from typing import Any
from . import (
bird_x,
bluesky,
dates,
dedupe,
entity_extract,
env,
github,
grounding,
hackernews,
instagram,
normalize,
perplexity,
pinterest,
planner,
polymarket,
providers,
query,
reddit,
reddit_public,
rerank,
schema,
signals,
snippet,
threads,
tiktok,
truthsocial,
xai_x,
xiaohongshu_api,
youtube_yt,
)
from .cluster import cluster_candidates
from .fusion import weighted_rrf
DEPTH_SETTINGS = {
"quick": {"per_stream_limit": 6, "pool_limit": 15, "rerank_limit": 12},
"default": {"per_stream_limit": 12, "pool_limit": 40, "rerank_limit": 40},
"deep": {"per_stream_limit": 20, "pool_limit": 60, "rerank_limit": 60},
}
SEARCH_ALIAS = {
"hn": "hackernews",
"bsky": "bluesky",
"truth": "truthsocial",
"web": "grounding",
"xhs": "xiaohongshu",
}
MAX_SOURCE_FETCHES: dict[str, int] = {"x": 2}
MOCK_AVAILABLE_SOURCES = [
"reddit",
"x",
"youtube",
"tiktok",
"instagram",
"hackernews",
"bluesky",
"truthsocial",
"polymarket",
"grounding",
"xiaohongshu",
"github",
"perplexity",
]
def normalize_requested_sources(sources: list[str] | None) -> list[str] | None:
if not sources:
return None
normalized = []
for source in sources:
key = SEARCH_ALIAS.get(source.lower(), source.lower())
if key not in normalized:
normalized.append(key)
return normalized
def available_sources(config: dict[str, Any], requested_sources: list[str] | None = None) -> list[str]:
available: list[str] = []
# reddit_public needs no API key - always available
available.append("reddit")
if config.get("SCRAPECREATORS_API_KEY"):
available.extend(["tiktok", "instagram"])
if env.get_x_source(config):
available.append("x")
if which("yt-dlp") or env.is_youtube_sc_available(config):
available.append("youtube")
available.extend(["hackernews", "polymarket"])
if config.get("GITHUB_TOKEN") or which("gh"):
available.append("github")
if env.is_bluesky_available(config):
available.append("bluesky")
if env.is_truthsocial_available(config):
available.append("truthsocial")
if config.get("BRAVE_API_KEY") or config.get("EXA_API_KEY") or config.get("SERPER_API_KEY") or config.get("PARALLEL_API_KEY"):
available.append("grounding")
# Perplexity Sonar: opt-in additive source via INCLUDE_SOURCES=perplexity
include_sources = (config.get("INCLUDE_SOURCES") or "").lower().split(",")
if config.get("OPENROUTER_API_KEY") and "perplexity" in include_sources:
available.append("perplexity")
if requested_sources and "xiaohongshu" in requested_sources and env.is_xiaohongshu_available(config):
available.append("xiaohongshu")
if env.is_threads_available(config):
available.append("threads")
if requested_sources and "pinterest" in requested_sources and env.is_pinterest_available(config):
available.append("pinterest")
return available
def diagnose(config: dict[str, Any], requested_sources: list[str] | None = None) -> dict[str, Any]:
requested_sources = normalize_requested_sources(requested_sources)
google_key = _google_key(config)
x_status = env.get_x_source_status(config)
native_web_backend = None
if config.get("BRAVE_API_KEY"):
native_web_backend = "brave"
elif config.get("EXA_API_KEY"):
native_web_backend = "exa"
elif config.get("SERPER_API_KEY"):
native_web_backend = "serper"
elif config.get("PARALLEL_API_KEY"):
native_web_backend = "parallel"
providers_status = {
"google": bool(google_key),
"openai": bool(config.get("OPENAI_API_KEY")) and config.get("OPENAI_AUTH_STATUS") == env.AUTH_STATUS_OK,
"xai": bool(config.get("XAI_API_KEY")),
"openrouter": bool(config.get("OPENROUTER_API_KEY")),
}
return {
"providers": providers_status,
"local_mode": not any(providers_status.values()),
"reasoning_provider": (config.get("LAST30DAYS_REASONING_PROVIDER") or "auto").lower(),
"x_backend": x_status["source"],
"bird_installed": x_status["bird_installed"],
"bird_authenticated": x_status["bird_authenticated"],
"bird_username": x_status["bird_username"],
"native_web_backend": native_web_backend,
"has_scrapecreators": bool(config.get("SCRAPECREATORS_API_KEY")),
"has_github": bool(config.get("GITHUB_TOKEN") or which("gh")),
"available_sources": available_sources(config, requested_sources),
}
def run(
*,
topic: str,
config: dict[str, Any],
depth: str,
requested_sources: list[str] | None = None,
mock: bool = False,
x_handle: str | None = None,
x_related: list[str] | None = None,
web_backend: str = "auto",
external_plan: dict | None = None,
subreddits: list[str] | None = None,
tiktok_hashtags: list[str] | None = None,
tiktok_creators: list[str] | None = None,
ig_creators: list[str] | None = None,
lookback_days: int = 30,
github_user: str | None = None,
github_repos: list[str] | None = None,
) -> schema.Report:
settings = DEPTH_SETTINGS[depth]
requested_sources = normalize_requested_sources(requested_sources)
from_date, to_date = dates.get_date_range(lookback_days)
if mock:
runtime = providers.mock_runtime(config, depth)
reasoning_provider = None
available = list(requested_sources or MOCK_AVAILABLE_SOURCES)
else:
runtime, reasoning_provider = providers.resolve_runtime(config, depth)
available = available_sources(config, requested_sources)
if requested_sources:
available = [source for source in available if source in requested_sources]
if web_backend == "none":
available = [s for s in available if s != "grounding"]
elif web_backend in ("brave", "exa", "serper") and "grounding" not in available:
available.append("grounding")
if not available:
raise RuntimeError("No sources are available for this run.")
if external_plan:
# External plan provided (e.g., from Claude Code via --plan flag).
# Parse it through the same sanitizer to validate structure.
plan = planner._sanitize_plan(
external_plan, topic, available, requested_sources, depth,
)
print(f"[Planner] Using external plan ({len(plan.subqueries)} subqueries)", file=sys.stderr)
else:
plan = planner.plan_query(
topic=topic,
available_sources=available,
requested_sources=requested_sources,
depth=depth,
provider=None if mock else reasoning_provider,
model=None if mock else runtime.planner_model,
context=config.get("_auto_resolve_context", ""),
)
# Safety net: ensure grounding appears in all subqueries even if the planner
# omits it. This is redundant when the planner includes grounding via
# SOURCE_CAPABILITIES, but kept as a fallback.
if web_backend != "none" and "grounding" in available:
for sq in plan.subqueries:
if "grounding" not in sq.sources:
sq.sources.append("grounding")
bundle = schema.RetrievalBundle(artifacts={"grounding": []})
# Project-mode or person-mode GitHub: run once before the main subquery loop
_github_custom_done = False
_github_enriched_repos: set[str] = set()
# Project mode takes priority over person mode
if github_repos and "github" in available:
try:
project_items = github.search_github_project(
github_repos, from_date, to_date,
depth=depth, token=config.get("GITHUB_TOKEN"),
)
if project_items:
normalized = _normalize_score_dedupe(
"github", project_items, from_date, to_date,
freshness_mode=plan.freshness_mode,
ranking_query=f"What are {', '.join(github_repos)} doing on GitHub?",
)
primary_label = plan.subqueries[0].label if plan.subqueries else "primary"
bundle.add_items(primary_label, "github", normalized)
_github_custom_done = True
_github_enriched_repos = {r.lower() for r in github_repos}
except Exception as exc:
bundle.errors_by_source["github"] = f"Project-mode failed: {exc}"
_github_person_done = False
if github_user and "github" in available and not _github_custom_done:
try:
person_items = github.search_github_person(
github_user, from_date, to_date,
depth=depth, token=config.get("GITHUB_TOKEN"),
)
if person_items:
normalized = _normalize_score_dedupe(
"github", person_items, from_date, to_date,
freshness_mode=plan.freshness_mode,
ranking_query=f"What is @{github_user} doing on GitHub?",
)
# Use the first subquery's label so RRF can look up the weight
primary_label = plan.subqueries[0].label if plan.subqueries else "primary"
bundle.add_items(primary_label, "github", normalized)
_github_person_done = True
except Exception as exc:
bundle.errors_by_source["github"] = f"Person-mode failed: {exc}"
# Thread-safe set prevents redundant fetches after a source returns 429
rate_limited_sources: set[str] = set()
rate_limit_lock = threading.Lock()
futures = {}
# Per-source fetch budget prevents redundant API calls
source_fetch_count: dict[str, int] = {}
stream_count = sum(
1
for subquery in plan.subqueries
for source in subquery.sources
if source in available
)
max_workers = max(4, min(16, stream_count or 1))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
for subquery in plan.subqueries:
for source in subquery.sources:
if source not in available:
continue
# Skip GitHub keyword search if person-mode already ran
if source == "github" and (_github_person_done or _github_custom_done):
continue
# Enforce per-source fetch cap
cap = MAX_SOURCE_FETCHES.get(source)
if cap is not None:
current = source_fetch_count.get(source, 0)
if current >= cap:
continue
source_fetch_count[source] = current + 1
futures[
executor.submit(
_retrieve_stream,
topic=topic,
subquery=subquery,
source=source,
config=config,
depth=depth,
date_range=(from_date, to_date),
runtime=runtime,
mock=mock,
rate_limited_sources=rate_limited_sources,
rate_limit_lock=rate_limit_lock,
web_backend=web_backend,
raw_topic=topic,
subreddits=subreddits,
tiktok_hashtags=tiktok_hashtags,
tiktok_creators=tiktok_creators,
ig_creators=ig_creators,
)
] = (subquery, source)
for future in as_completed(futures):
subquery, source = futures[future]
try:
raw_items, artifact = future.result()
except Exception as exc:
# Share 429 signal so pending futures skip this source
if _is_rate_limit_error(exc):
with rate_limit_lock:
rate_limited_sources.add(source)
bundle.errors_by_source[source] = str(exc)
continue
# Retry once for transient 5xx errors
if _is_transient_error(exc):
time.sleep(3)
try:
raw_items, artifact = _retrieve_stream(
topic=topic, subquery=subquery, source=source,
config=config, depth=depth, date_range=(from_date, to_date),
runtime=runtime, mock=mock,
rate_limited_sources=rate_limited_sources,
rate_limit_lock=rate_limit_lock,
web_backend=web_backend,
raw_topic=topic,
subreddits=subreddits,
tiktok_hashtags=tiktok_hashtags,
tiktok_creators=tiktok_creators,
ig_creators=ig_creators,
)
except Exception as retry_exc:
bundle.errors_by_source[source] = f"{exc} (retried once, still failed: {retry_exc})"
continue
else:
bundle.errors_by_source[source] = str(exc)
continue
normalized = _normalize_score_dedupe(
source, raw_items, from_date, to_date,
freshness_mode=plan.freshness_mode,
ranking_query=subquery.ranking_query,
)
normalized = normalized[: settings["per_stream_limit"]]
bundle.add_items(subquery.label, source, normalized)
if artifact:
bundle.artifacts.setdefault("grounding", []).append(artifact)
# Phase 2: supplemental entity-based searches
_run_supplemental_searches(
topic=topic,
bundle=bundle,
plan=plan,
config=config,
depth=depth,
date_range=(from_date, to_date),
runtime=runtime,
mock=mock,
rate_limited_sources=rate_limited_sources,
rate_limit_lock=rate_limit_lock,
x_handle=x_handle,
x_related=x_related,
)
# Phase 2b: retry thin sources with simplified query
# Note: _github_skip_sources tells the retry to not re-run GitHub keyword search
# when project-mode or person-mode already provided authoritative data.
_github_skip_retry = {"github"} if (_github_person_done or _github_custom_done) else set()
_retry_thin_sources(
topic=topic,
bundle=bundle,
plan=plan,
config=config,
depth=depth,
date_range=(from_date, to_date),
runtime=runtime,
mock=mock,
rate_limited_sources=rate_limited_sources,
rate_limit_lock=rate_limit_lock,
settings=settings,
web_backend=web_backend,
skip_sources=_github_skip_retry,
)
# Clear errors for sources that returned items despite partial failures.
# A source that 429'd on one subquery but succeeded on another is not "errored".
for source in list(bundle.errors_by_source):
if bundle.items_by_source.get(source):
del bundle.errors_by_source[source]
items_by_source = _finalize_items_by_source(bundle.items_by_source)
candidates = weighted_rrf(bundle.items_by_source_and_query, plan, pool_limit=settings["pool_limit"])
ranked_candidates = rerank.rerank_candidates(
topic=topic,
plan=plan,
candidates=candidates,
provider=None if mock else reasoning_provider,
model=None if mock else runtime.rerank_model,
shortlist_size=settings["rerank_limit"],
)
rerank.score_fun(
topic=topic,
candidates=ranked_candidates,
provider=None if mock else reasoning_provider,
model=None if mock else runtime.rerank_model,
)
# Phase 3: post-rerank GitHub star enrichment
if "github" in available and not mock:
github.enrich_candidates_with_stars(
ranked_candidates,
token=config.get("GITHUB_TOKEN"),
already_enriched=_github_enriched_repos,
)
clusters = cluster_candidates(ranked_candidates, plan)
warnings = _warnings(items_by_source, ranked_candidates, bundle.errors_by_source)
return schema.Report(
topic=topic,
range_from=from_date,
range_to=to_date,
generated_at=datetime.now(timezone.utc).isoformat(),
provider_runtime=runtime,
query_plan=plan,
clusters=clusters,
ranked_candidates=ranked_candidates,
items_by_source=items_by_source,
errors_by_source=bundle.errors_by_source,
warnings=warnings,
artifacts=bundle.artifacts,
)
def _normalize_score_dedupe(
source: str,
raw_items: list[dict],
from_date: str,
to_date: str,
freshness_mode: str,
ranking_query: str,
) -> list[schema.SourceItem]:
"""Normalize, annotate, prune, dedupe, and extract snippets for a batch of raw items."""
normalized = normalize.normalize_source_items(
source, raw_items, from_date, to_date,
freshness_mode=freshness_mode,
)
normalized = signals.annotate_stream(normalized, ranking_query, freshness_mode)
normalized = signals.prune_low_relevance(normalized)
normalized = dedupe.dedupe_items(normalized)
for item in normalized:
item.snippet = snippet.extract_best_snippet(item, ranking_query)
return normalized
def _finalize_items_by_source(items_by_source_raw: dict[str, list[schema.SourceItem]]) -> dict[str, list[schema.SourceItem]]:
finalized = {}
for source, items in items_by_source_raw.items():
items = sorted(items, key=lambda item: item.local_rank_score or 0.0, reverse=True)
finalized[source] = dedupe.dedupe_items(items)
return finalized
def _warnings(
items_by_source: dict[str, list[schema.SourceItem]],
candidates: list[schema.Candidate],
errors_by_source: dict[str, str],
) -> list[str]:
warnings: list[str] = []
if not candidates:
warnings.append("No candidates survived retrieval and ranking.")
if len(candidates) < 5:
warnings.append("Evidence is thin for this topic.")
top_sources = {
source
for candidate in candidates[:5]
for source in schema.candidate_sources(candidate)
}
if len(top_sources) <= 1 and len(candidates) >= 3:
warnings.append("Top evidence is highly concentrated in one source.")
if errors_by_source:
warnings.append(f"Some sources failed: {', '.join(sorted(errors_by_source))}")
if not items_by_source:
warnings.append("No source returned usable items.")
return warnings
def _is_rate_limit_error(exc: Exception) -> bool:
"""Detect 429 rate-limit errors by status code or message text."""
if hasattr(exc, "status_code") and getattr(exc, "status_code", None) == 429:
return True
return "429" in str(exc)
def _is_transient_error(exc: Exception) -> bool:
"""Detect 5xx server errors that are worth retrying."""
status = getattr(exc, "status_code", None)
if isinstance(status, int) and 500 <= status < 600:
return True
msg = str(exc)
return any(code in msg for code in ("500", "502", "503", "504"))
def _run_supplemental_searches(
*,
topic: str,
bundle: schema.RetrievalBundle,
plan: schema.QueryPlan,
config: dict[str, Any],
depth: str,
date_range: tuple[str, str],
runtime: schema.ProviderRuntime,
mock: bool,
rate_limited_sources: set[str],
rate_limit_lock: threading.Lock,
x_handle: str | None = None,
x_related: list[str] | None = None,
) -> None:
"""Phase 2: extract entities from Phase 1 results, run targeted supplemental searches."""
if depth == "quick" or mock:
return
from_date, to_date = date_range
# Convert SourceItems to dicts for entity_extract
x_dicts = [
{"author_handle": item.author or "", "text": item.body or ""}
for item in bundle.items_by_source.get("x", [])
]
reddit_dicts = [
{
"subreddit": item.container or "",
"comment_insights": item.metadata.get("comment_insights", []),
"top_comments": [
{"excerpt": c.get("excerpt", c.get("text", ""))}
for c in (item.metadata.get("top_comments") or [])
if isinstance(c, dict)
],
}
for item in bundle.items_by_source.get("reddit", [])
]
if not x_dicts and not reddit_dicts and not x_handle and not x_related:
return
entities = entity_extract.extract_entities(
reddit_dicts, x_dicts,
max_handles=3, max_subreddits=3,
)
handles = entities.get("x_handles", [])
# Add explicit --x-handle if provided
if x_handle:
handle_clean = x_handle.lstrip("@").lower()
if handle_clean not in [h.lower() for h in handles]:
handles.insert(0, handle_clean)
# Collect related handles (searched separately with lower weight)
related_handles = []
if x_related:
primary_lower = x_handle.lstrip("@").lower() if x_handle else ""
for rh in x_related:
rh_clean = rh.lstrip("@").lower().strip()
if rh_clean and rh_clean != primary_lower and rh_clean not in [h.lower() for h in handles]:
related_handles.append(rh_clean)
if not handles and not related_handles:
return
# Check if X is rate-limited
if "x" in rate_limited_sources:
return
backend = runtime.x_search_backend or env.get_x_source(config)
if backend != "bird":
return # Handle search only works with Bird CLI
# Collect existing URLs for deduplication
existing_urls = {
item.url
for items in bundle.items_by_source.values()
for item in items
if item.url
}
ranking_query = plan.subqueries[0].ranking_query if plan.subqueries else topic
primary_label = plan.subqueries[0].label if plan.subqueries else "primary"
# Search primary handles (full weight)
if handles:
try:
raw_items = bird_x.search_handles(
handles, topic, from_date, count_per=3,
)
except Exception as exc:
print(f"[Pipeline] Phase 2 handle search failed: {exc}", file=sys.stderr)
if not bundle.items_by_source.get("x"):
bundle.errors_by_source["x"] = f"Phase 2 handle search: {exc}"
raw_items = []
if raw_items:
normalized = _normalize_score_dedupe(
"x", raw_items, from_date, to_date,
freshness_mode=plan.freshness_mode,
ranking_query=ranking_query,
)
# Deduplicate against Phase 1 URLs
normalized = [item for item in normalized if item.url not in existing_urls]
if normalized:
bundle.add_items(primary_label, "x", normalized)
# Update existing URLs for related-handle dedup
for item in normalized:
if item.url:
existing_urls.add(item.url)
# Search related handles with lower weight (0.3)
if related_handles:
try:
raw_items = bird_x.search_handles(
related_handles, topic, from_date, count_per=3,
)
except Exception as exc:
print(f"[Pipeline] Phase 2 related handle search failed: {exc}", file=sys.stderr)
raw_items = []
if raw_items:
normalized = _normalize_score_dedupe(
"x", raw_items, from_date, to_date,
freshness_mode=plan.freshness_mode,
ranking_query=ranking_query,
)
# Deduplicate against all existing URLs (Phase 1 + primary handles)
normalized = [item for item in normalized if item.url not in existing_urls]
if normalized:
# Use a separate subquery label with lower weight so RRF
# scores related-handle results below primary results.
bundle.add_items("supplemental-related", "x", normalized)
# Register the supplemental-related label in the plan for fusion
if not any(sq.label == "supplemental-related" for sq in plan.subqueries):
plan.subqueries.append(
schema.SubQuery(
label="supplemental-related",
search_query=", ".join(related_handles),
ranking_query=ranking_query,
sources=["x"],
weight=0.3,
)
)
def _retry_thin_sources(
*,
topic: str,
bundle: schema.RetrievalBundle,
plan: schema.QueryPlan,
config: dict[str, Any],
depth: str,
date_range: tuple[str, str],
runtime: schema.ProviderRuntime,
mock: bool,
rate_limited_sources: set[str],
rate_limit_lock: threading.Lock,
settings: dict[str, Any],
web_backend: str = "auto",
skip_sources: set[str] | None = None,
) -> None:
"""Retry sources with thin results using simplified core subject query."""
if depth == "quick":
return
planned_sources: list[str] = []
for subquery in plan.subqueries:
for source in subquery.sources:
if source not in planned_sources:
planned_sources.append(source)
_skip = skip_sources or set()
thin_sources = [
source
for source in planned_sources
if len(bundle.items_by_source.get(source, [])) < 3
and source not in bundle.errors_by_source
and source not in _skip
]
if not thin_sources:
return
core = query.extract_core_subject(topic, max_words=3)
if not core:
return
# Note: we intentionally do NOT skip when core == topic. For short topics
# like "Kanye West", the 3-word core IS the topic — but the planner may
# have sent a different (worse) query to the source. Retrying with the
# raw core subject is still valuable.
from_date, to_date = date_range
# Create a retry subquery with the simplified core subject
retry_subquery = schema.SubQuery(
label="retry",
search_query=core,
ranking_query=f"What recent evidence from the last 30 days matters for {core}?",
sources=thin_sources,
weight=0.3,
)
for source in thin_sources:
if source in rate_limited_sources:
continue
try:
raw_items, _artifact = _retrieve_stream(
topic=topic,
subquery=retry_subquery,
source=source,
config=config,
depth=depth,
date_range=date_range,
runtime=runtime,
mock=mock,
rate_limited_sources=rate_limited_sources,
rate_limit_lock=rate_limit_lock,
web_backend=web_backend,
raw_topic=topic,
)
normalized = _normalize_score_dedupe(
source,
raw_items,
from_date,
to_date,
freshness_mode=plan.freshness_mode,
ranking_query=retry_subquery.ranking_query,
)
normalized = normalized[:settings["per_stream_limit"]]
existing_urls = {item.url for item in bundle.items_by_source.get(source, []) if item.url}
new_items = [item for item in normalized if item.url not in existing_urls]
if new_items:
bundle.items_by_source.setdefault(source, []).extend(new_items)
primary_label = plan.subqueries[0].label if plan.subqueries else "primary"
existing = bundle.items_by_source_and_query.get((primary_label, source), [])
bundle.items_by_source_and_query[(primary_label, source)] = existing + new_items
except Exception as exc:
print(f"[Pipeline] Retry failed for {source}: {type(exc).__name__}: {exc}", file=sys.stderr)
def _retrieve_stream(
*,
topic: str,
subquery: schema.SubQuery,
source: str,
config: dict[str, Any],
depth: str,
date_range: tuple[str, str],
runtime: schema.ProviderRuntime,
mock: bool,
rate_limited_sources: set[str] | None = None,
rate_limit_lock: threading.Lock | None = None,
web_backend: str = "auto",
raw_topic: str = "",
subreddits: list[str] | None = None,
tiktok_hashtags: list[str] | None = None,
tiktok_creators: list[str] | None = None,
ig_creators: list[str] | None = None,
) -> tuple[list[dict], dict]:
# Early exit if source was rate-limited by a sibling future
if rate_limited_sources is not None and source in rate_limited_sources:
return [], {}
from_date, to_date = date_range
if mock:
return _mock_stream_results(source, subquery)
if source == "grounding":
return grounding.web_search(
subquery.search_query, date_range, config, backend=web_backend)
if source == "reddit":
# Use raw_topic so expand_reddit_queries() generates diverse variants
# from the original user topic, not the planner's narrowed search_query.
reddit_query = raw_topic or subquery.search_query
# Public Reddit first (free, gets comments); SC as backup
try:
public_results = reddit_public.search_reddit_public(
reddit_query, from_date, to_date, depth=depth,
subreddits=subreddits,
)
if public_results:
return public_results, {}
except Exception as exc:
sys.stderr.write(
f"[Reddit] Public search failed ({type(exc).__name__}: {exc})"
)
if not config.get("SCRAPECREATORS_API_KEY"):
sys.stderr.write("\n")
return [], {}
sys.stderr.write(", using ScrapeCreators backup\n")
# Fallback to ScrapeCreators if public returned empty or raised
if config.get("SCRAPECREATORS_API_KEY"):
try:
result = reddit.search_and_enrich(
reddit_query,
from_date,
to_date,
depth=depth,
token=config.get("SCRAPECREATORS_API_KEY"),
subreddits=subreddits,
)
return reddit.parse_reddit_response(result), {}
except Exception as exc:
sys.stderr.write(
f"[Reddit] ScrapeCreators backup also failed "
f"({type(exc).__name__}: {exc})\n"
)
return [], {}
if source == "x":
backend = runtime.x_search_backend or env.get_x_source(config)
if backend == "bird":
result = bird_x.search_x(subquery.search_query, from_date, to_date, depth=depth)
return bird_x.parse_bird_response(result, query=subquery.search_query), {}
if backend == "xai":
model = config.get("LAST30DAYS_X_MODEL") or config.get("XAI_MODEL_PIN") or providers.XAI_DEFAULT
result = xai_x.search_x(
config["XAI_API_KEY"],
model,
subquery.search_query,
from_date,
to_date,
depth=depth,
)
return xai_x.parse_x_response(result), {}
raise RuntimeError("No X backend is available.")
if source == "youtube":
# Use raw_topic so expand_youtube_queries() generates diverse variants
# from the original user topic, not the planner's narrowed search_query.
yt_query = raw_topic or subquery.search_query
result = None
# Try yt-dlp first, fall back to SC YouTube if it fails or isn't installed
if which("yt-dlp"):
try:
result = youtube_yt.search_and_transcribe(yt_query, from_date, to_date, depth=depth)
except Exception:
result = None
if (result is None or not result.get("items")) and env.is_youtube_sc_available(config):
sc_token = config.get("SCRAPECREATORS_API_KEY", "")
result = youtube_yt.search_youtube_sc(yt_query, from_date, to_date, depth=depth, token=sc_token)
if result is None:
result = {"items": []}
# Enrich top videos with comments when SC key is available
items = youtube_yt.parse_youtube_response(result)
if items and env.is_youtube_comments_available(config):
sc_token = config.get("SCRAPECREATORS_API_KEY", "")
youtube_yt.enrich_with_comments(items, token=sc_token)
return items, {}
if source == "tiktok":
# Use raw_topic so expand_tiktok_queries() generates diverse variants
# from the original user topic, not the planner's narrowed search_query.
tiktok_query = raw_topic or subquery.search_query
result = tiktok.search_and_enrich(
tiktok_query,
from_date,
to_date,
depth=depth,
token=env.get_tiktok_token(config),
hashtags=tiktok_hashtags,
creators=tiktok_creators,
)
return tiktok.parse_tiktok_response(result), {}
if source == "instagram":
# Use raw_topic so expand_instagram_queries() generates diverse variants
# from the original user topic, not the planner's narrowed search_query.
ig_query = raw_topic or subquery.search_query
result = instagram.search_and_enrich(
ig_query,
from_date,
to_date,
depth=depth,
token=env.get_instagram_token(config),
ig_creators=ig_creators,
)
return instagram.parse_instagram_response(result), {}
if source == "hackernews":
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
return hackernews.parse_hackernews_response(result, query=subquery.search_query), {}
if source == "bluesky":
result = bluesky.search_bluesky(subquery.search_query, from_date, to_date, depth=depth, config=config)
return bluesky.parse_bluesky_response(result), {}
if source == "threads":
result = threads.search_threads(
subquery.search_query, from_date, to_date,
depth=depth,
token=config.get("SCRAPECREATORS_API_KEY"),
)
return threads.parse_threads_response(result), {}
if source == "truthsocial":
result = truthsocial.search_truthsocial(subquery.search_query, from_date, to_date, depth=depth, config=config)
return truthsocial.parse_truthsocial_response(result), {}
if source == "polymarket":
result = polymarket.search_polymarket(subquery.search_query, from_date, to_date, depth=depth)
return polymarket.parse_polymarket_response(result, topic=subquery.search_query), {}
if source == "github":
result = github.search_github(subquery.search_query, from_date, to_date, depth=depth, token=config.get("GITHUB_TOKEN"))
return result, {}
if source == "pinterest":
result = pinterest.search_pinterest(
subquery.search_query, from_date, to_date,
depth=depth,
token=env.get_pinterest_token(config),
)
return pinterest.parse_pinterest_response(result), {}
if source == "xiaohongshu":
return xiaohongshu_api.search_feeds(
subquery.search_query,
from_date,
to_date,
env.get_xiaohongshu_api_base(config),
depth=depth,
), {}
if source == "perplexity":
return perplexity.search(subquery.search_query, date_range, config, deep=config.get("_deep_research", False))
raise RuntimeError(f"Unsupported source: {source}")
def _google_key(config: dict[str, Any]) -> str | None:
return config.get("GOOGLE_API_KEY") or config.get("GEMINI_API_KEY") or config.get("GOOGLE_GENAI_API_KEY")
def _mock_stream_results(source: str, subquery: schema.SubQuery) -> tuple[list[dict], dict]:
payloads = {
"reddit": [
{
"id": "R1",
"title": f"{subquery.search_query} discussion thread",
"url": "https://reddit.com/r/example/comments/1",
"subreddit": "example",
"date": dates.get_date_range(5)[0],
"engagement": {"score": 120, "num_comments": 48, "upvote_ratio": 0.91},
"selftext": f"Community discussion about {subquery.search_query}.",
"top_comments": [{"excerpt": "Strong firsthand feedback from users."}],
"relevance": 0.82,
"why_relevant": "Mock Reddit result",
}
],
"x": [
{
"id": "X1",
"text": f"People on X are discussing {subquery.search_query} right now.",
"url": "https://x.com/example/status/1",
"author_handle": "example",
"date": dates.get_date_range(2)[0],
"engagement": {"likes": 200, "reposts": 35, "replies": 18, "quotes": 4},
"relevance": 0.79,
"why_relevant": "Mock X result",
}
],
"grounding": [
{
"id": "WB1",
"title": f"{subquery.search_query} article",
"url": "https://example.com/article",
"source_domain": "example.com",
"snippet": f"Recent web reporting about {subquery.search_query}.",
"date": dates.get_date_range(7)[0],
"relevance": 0.88,
"why_relevant": "Brave web search",
}
],
}
if source == "grounding":
return payloads.get(source, []), {
"label": subquery.label,
"mock": True,
"webSearchQueries": [subquery.search_query],
"resultCount": 1,
}
return payloads.get(source, []), {}
scripts/lib/planner.py
+576
View File
@@ -0,0 +1,576 @@
"""LLM-first query planning with deterministic guards for risky queries."""
from __future__ import annotations
import json
import re
from . import http, providers, query, schema
ALLOWED_INTENTS = {
"factual",
"product",
"concept",
"opinion",
"how_to",
"comparison",
"breaking_news",
"prediction",
}
ALLOWED_CLUSTER_MODES = {"none", "story", "workflow", "market", "debate"}
QUICK_SOURCE_PRIORITY = {
"factual": ["hackernews", "reddit", "x", "youtube"],
"product": ["youtube", "reddit", "x", "tiktok"],
"concept": ["hackernews", "reddit", "x", "youtube"],
"opinion": ["reddit", "x", "youtube", "hackernews"],
"how_to": ["youtube", "reddit", "x", "hackernews"],
"comparison": ["reddit", "x", "hackernews", "youtube"],
"breaking_news": ["x", "reddit", "hackernews", "youtube", "polymarket"],
"prediction": ["polymarket", "x", "hackernews", "reddit", "youtube"],
}
SOURCE_PRIORITY = {
"factual": ["hackernews", "reddit", "x", "youtube"],
"product": ["youtube", "reddit", "x", "tiktok", "hackernews"],
"concept": ["hackernews", "reddit", "x", "youtube"],
"opinion": ["reddit", "x", "youtube", "hackernews"],
"how_to": ["youtube", "reddit", "x", "hackernews"],
"comparison": ["reddit", "x", "hackernews", "youtube"],
"breaking_news": ["x", "reddit", "hackernews", "youtube", "polymarket"],
"prediction": ["polymarket", "x", "hackernews", "reddit", "youtube"],
}
SOURCE_LIMITS = {
"quick": {
"factual": 2,
"product": 2,
"concept": 2,
"opinion": 2,
"how_to": 2,
"comparison": 2,
"breaking_news": 2,
"prediction": 2,
},
# "default" intentionally absent: all available sources are searched
# at default depth. Fusion and reranking handle quality. quick mode
# uses tight budgets above for latency.
}
INTENT_SOURCE_EXCLUSIONS: dict[str, set[str]] = {
"concept": {"polymarket"},
"how_to": {"polymarket"},
}
SOURCE_CAPABILITIES = {
"reddit": {"discussion", "social"},
"x": {"discussion", "social"},
"youtube": {"video", "video_longform", "discussion"},
"tiktok": {"video", "video_shortform", "social"},
"instagram": {"video", "video_shortform", "social"},
"hackernews": {"discussion", "link"},
"bluesky": {"discussion", "social"},
"truthsocial": {"discussion", "social"},
"polymarket": {"market"},
"xiaohongshu": {"video", "video_shortform", "social"},
"github": {"discussion", "link"},
"grounding": {"web", "reference", "link"},
"perplexity": {"web", "reference", "analysis"},
}
DEFAULT_INTENT_CAPABILITIES = {
"comparison": {"discussion", "video", "web", "reference", "social", "link", "market"},
"how_to": {"discussion", "video", "web", "reference", "link"},
}
def plan_query(
*,
topic: str,
available_sources: list[str],
requested_sources: list[str] | None,
depth: str,
provider: providers.ReasoningClient | None,
model: str | None,
context: str = "",
) -> schema.QueryPlan:
"""Create a query plan. Comparison queries with extractable entities use a
deterministic plan; other intents prefer the configured reasoning provider."""
if _should_force_deterministic_plan(topic):
return _fallback_plan(
topic,
available_sources,
requested_sources,
depth,
note="deterministic-comparison-plan",
)
prompt = _build_prompt(topic, available_sources, requested_sources, depth)
if context:
prompt += f"\n\nCurrent context (from web search): {context}"
if provider and model:
try:
raw = provider.generate_json(model, prompt)
plan = _sanitize_plan(raw, topic, available_sources, requested_sources, depth)
if plan.subqueries:
return plan
except (ValueError, KeyError, json.JSONDecodeError, OSError, http.HTTPError) as exc:
import sys
print(f"[Planner] LLM planning failed, using deterministic fallback: {type(exc).__name__}: {exc}", file=sys.stderr)
return _fallback_plan(
topic, available_sources, requested_sources, depth,
note=f"fallback-plan (LLM error: {type(exc).__name__})",
)
return _fallback_plan(topic, available_sources, requested_sources, depth)
def _build_prompt(
topic: str,
available_sources: list[str],
requested_sources: list[str] | None,
depth: str,
) -> str:
requested = ", ".join(requested_sources or ["auto"])
available = ", ".join(available_sources)
return f"""
You are the query planner for a live last-30-days research pipeline.
Topic: {topic}
Depth: {depth}
Available sources: {available}
Requested sources: {requested}
Return JSON only with this shape:
{{
"intent": "factual|product|concept|opinion|how_to|comparison|breaking_news|prediction",
"freshness_mode": "strict_recent|balanced_recent|evergreen_ok",
"cluster_mode": "none|story|workflow|market|debate",
"source_weights": {{"source_name": 0.0}},
"subqueries": [
{{
"label": "short label",
"search_query": "keyword style query for search APIs",
"ranking_query": "natural language rewrite for reranking",
"sources": ["reddit", "x", "grounding"],
"weight": 1.0
}}
],
"notes": ["optional short notes"]
}}
Rules:
- emit 1 to 4 subqueries
- every subquery must include both search_query and ranking_query
- sources must be drawn from Available sources only
- use cluster_mode=none for factual or many how-to queries
- use strict_recent for breaking news and most predictions
- use debate for comparison/opinion, market for prediction, workflow for how_to, story for breaking_news
- search_query should be concise and keyword-heavy
- ranking_query should read like a natural-language question
- preserve exact proper nouns and entity strings from the topic
- NEVER include temporal phrases in search_query: no 'last 30 days', 'recent', month names, year numbers
- NEVER include meta-research phrases: no 'news', 'updates', 'public appearances', 'latest developments'
- search_query should match how content is TITLED on platforms
- GitHub (Issues/PRs) is best for engineering, developer tools, and open source topics: 'kanye west bully' not 'kanye west album news March 2026'
""".strip()
def _sanitize_plan(
raw: dict,
topic: str,
available_sources: list[str],
requested_sources: list[str] | None,
depth: str,
) -> schema.QueryPlan:
intent_hint = str(raw.get("intent") or _infer_intent(topic)).strip()
if intent_hint not in ALLOWED_INTENTS:
intent_hint = _infer_intent(topic)
requested = set(requested_sources or [])
available = set(available_sources)
eligible_sources = [
source for source in available_sources
if (not requested or source in requested)
]
source_weights = {
source: float(weight)
for source, weight in (raw.get("source_weights") or {}).items()
if source in available
}
if requested:
source_weights = {source: weight for source, weight in source_weights.items() if source in requested}
if not source_weights:
source_weights = _default_source_weights(_infer_intent(topic), eligible_sources)
# Ensure all eligible sources are available for subqueries. The LLM may
# assign high weights to its preferred sources, but omitted sources still
# participate with base weight so retrieval can overfetch and let fusion
# decide quality.
for source in eligible_sources:
source_weights.setdefault(source, 1.0)
if intent_hint in DEFAULT_INTENT_CAPABILITIES and depth != "quick":
for source in _default_sources_for_intent(intent_hint, eligible_sources):
source_weights.setdefault(source, 1.0)
source_weights = _normalize_weights(source_weights)
subqueries: list[schema.SubQuery] = []
for index, subquery in enumerate((raw.get("subqueries") or [])[:_max_subqueries(intent_hint)], start=1):
if not isinstance(subquery, dict):
continue
sources = [source for source in subquery.get("sources") or [] if source in source_weights]
if requested:
sources = [source for source in sources if source in requested]
if not sources:
sources = list(source_weights)
search_query = str(subquery.get("search_query") or "").strip()
ranking_query = str(subquery.get("ranking_query") or "").strip()
if not search_query or not ranking_query:
continue
subqueries.append(
schema.SubQuery(
label=str(subquery.get("label") or f"q{index}").strip() or f"q{index}",
search_query=search_query,
ranking_query=ranking_query,
sources=sources,
weight=max(0.05, float(subquery.get("weight") or 1.0)),
)
)
if depth == "quick" and subqueries:
subqueries = subqueries[:1]
if not subqueries:
return _fallback_plan(topic, available_sources, requested_sources, depth)
intent = intent_hint
freshness_mode = str(raw.get("freshness_mode") or _default_freshness(intent)).strip()
if intent == "how_to":
freshness_mode = "evergreen_ok"
cluster_mode = str(raw.get("cluster_mode") or _default_cluster_mode(intent)).strip()
if cluster_mode not in ALLOWED_CLUSTER_MODES:
cluster_mode = _default_cluster_mode(intent)
return schema.QueryPlan(
intent=intent,
freshness_mode=freshness_mode,
cluster_mode=cluster_mode,
raw_topic=topic,
subqueries=_normalize_subquery_weights(_trim_subqueries_for_depth(subqueries, intent, depth, eligible_sources)),
source_weights=source_weights,
notes=[str(note).strip() for note in raw.get("notes") or [] if str(note).strip()],
)
def _normalize_subquery_weights(subqueries: list[schema.SubQuery]) -> list[schema.SubQuery]:
total = sum(subquery.weight for subquery in subqueries) or 1.0
return [
schema.SubQuery(
label=subquery.label,
search_query=subquery.search_query,
ranking_query=subquery.ranking_query,
sources=subquery.sources,
weight=subquery.weight / total,
)
for subquery in subqueries
]
def _normalize_weights(weights: dict[str, float]) -> dict[str, float]:
total = sum(max(weight, 0.0) for weight in weights.values()) or 1.0
return {
source: max(weight, 0.0) / total
for source, weight in weights.items()
}
def _trim_subqueries_for_depth(
subqueries: list[schema.SubQuery],
intent: str,
depth: str,
available_sources: list[str],
) -> list[schema.SubQuery]:
# At non-quick depth, expand sources: use capability routing for intents
# that define it, or all available sources otherwise. The LLM planner may
# assign narrow source lists; we override to let fusion decide quality.
if depth != "quick":
expanded_sources = _default_sources_for_intent(intent, available_sources)
return [
schema.SubQuery(
label=subquery.label,
search_query=subquery.search_query,
ranking_query=subquery.ranking_query,
sources=expanded_sources,
weight=subquery.weight,
)
for subquery in subqueries
]
limits = SOURCE_LIMITS.get(depth)
if not limits:
return subqueries
priority_table = QUICK_SOURCE_PRIORITY if depth == "quick" else SOURCE_PRIORITY
priority = priority_table.get(intent, priority_table["breaking_news"])
limit = limits.get(intent, 3)
ranked_sources = [source for source in priority if source in available_sources]
if not ranked_sources:
ranked_sources = list(available_sources)
trimmed = []
for subquery in subqueries:
if depth in {"quick", "default"}:
preferred_sources = ranked_sources[:limit]
else:
preferred_sources = [source for source in ranked_sources if source in subquery.sources][:limit]
if len(preferred_sources) < limit:
for source in ranked_sources:
if source in preferred_sources:
continue
preferred_sources.append(source)
if len(preferred_sources) >= limit:
break
trimmed.append(
schema.SubQuery(
label=subquery.label,
search_query=subquery.search_query,
ranking_query=subquery.ranking_query,
sources=preferred_sources,
weight=subquery.weight,
)
)
return trimmed
def _fallback_plan(
topic: str,
available_sources: list[str],
requested_sources: list[str] | None,
depth: str,
note: str = "fallback-plan",
) -> schema.QueryPlan:
intent = _infer_intent(topic)
allowed_sources = requested_sources or available_sources
source_weights = _default_source_weights(intent, allowed_sources)
core = query.extract_core_subject(topic, max_words=6, strip_suffixes=True)
base_search = _keyword_query(topic, core)
base_ranking = _ranking_query(topic, core)
subqueries = [schema.SubQuery(
label="primary",
search_query=base_search,
ranking_query=base_ranking,
sources=list(source_weights),
weight=1.0,
)]
if depth != "quick" and intent == "comparison":
entities = _comparison_entities(topic)
if entities:
for index, entity in enumerate(entities, start=1):
subqueries.append(
schema.SubQuery(
label=f"entity-{index}",
search_query=entity,
ranking_query=f"What recent evidence from the last 30 days is most relevant to {entity} in the comparison '{topic}'?",
sources=list(source_weights),
weight=0.65,
)
)
elif depth != "quick" and intent == "prediction":
subqueries.append(
schema.SubQuery(
label="odds",
search_query=f"{base_search} odds forecast",
ranking_query=f"What are the current odds, forecasts, or market signals about {topic}?",
sources=[source for source in source_weights if source in {"polymarket", "grounding", "x", "reddit"}] or list(source_weights),
weight=0.7,
)
)
elif depth != "quick" and intent == "breaking_news":
subqueries.append(
schema.SubQuery(
label="reaction",
search_query=f"{base_search} reaction update",
ranking_query=f"What new reactions or follow-up reporting from the last 30 days matter for {topic}?",
sources=[source for source in source_weights if source in {"x", "reddit", "grounding", "hackernews"}] or list(source_weights),
weight=0.7,
)
)
return schema.QueryPlan(
intent=intent,
freshness_mode=_default_freshness(intent),
cluster_mode=_default_cluster_mode(intent),
raw_topic=topic,
subqueries=_normalize_subquery_weights(
_trim_subqueries_for_depth(subqueries[:_max_subqueries(intent)], intent, depth, list(source_weights))
),
source_weights=_normalize_weights(source_weights),
notes=[note],
)
def _infer_intent(topic: str) -> str:
text = topic.lower().strip()
if re.search(r"\b(vs|versus|compare|compared to|difference between)\b", text):
return "comparison"
# Slash-separated proper nouns: "React/Vue/Svelte" (not URLs, not acronyms like CI/CD or I/O)
if not re.search(r"https?://", topic) and re.search(r"\b[A-Z][a-z]{2,}(?:/[A-Z][a-z]{2,})+\b", topic):
return "comparison"
if re.search(r"\b(odds|predict|prediction|forecast|chance|probability|will .* win)\b", text):
return "prediction"
if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install)\b", text):
return "how_to"
if re.search(r"\b(what is|what are|who is|who acquired|when did|parameter count|release date)\b", text):
return "factual"
if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
return "opinion"
if re.search(r"\b(latest|news|announced|just shipped|launched|released|update)\b", text):
return "breaking_news"
if re.search(r"\b(pricing|feature|features|best .* for|top .* for)\b", text):
return "product"
if re.search(r"\b(explain|concept|protocol|architecture|what does)\b", text):
return "concept"
if re.search(r"\b(tournament|championship|playoffs|march madness|world cup|olympics|super bowl|final four|ceremony|awards|keynote)\b", text):
return "breaking_news"
return "breaking_news"
def _default_freshness(intent: str) -> str:
if intent in {"breaking_news", "prediction"}:
return "strict_recent"
if intent in {"concept", "how_to"}:
return "evergreen_ok"
return "balanced_recent"
def _default_cluster_mode(intent: str) -> str:
return {
"breaking_news": "story",
"comparison": "debate",
"opinion": "debate",
"prediction": "market",
"how_to": "workflow",
"factual": "none",
"product": "none",
"concept": "none",
}.get(intent, "none")
def _default_source_weights(intent: str, sources: list[str]) -> dict[str, float]:
base = {source: 1.0 for source in sources}
if intent == "prediction":
for source, bonus in {"polymarket": 2.5, "x": 1.3}.items():
if source in base:
base[source] += bonus
elif intent == "breaking_news":
for source, bonus in {"x": 1.5, "reddit": 1.3, "hackernews": 0.8}.items():
if source in base:
base[source] += bonus
elif intent == "how_to":
for source, bonus in {"youtube": 2.0, "hackernews": 0.8}.items():
if source in base:
base[source] += bonus
elif intent == "factual":
for source, bonus in {"reddit": 0.8, "x": 0.5}.items():
if source in base:
base[source] += bonus
return base
def _keyword_query(topic: str, core: str) -> str:
compounds = query.extract_compound_terms(topic)
quoted = " ".join(f"\"{term}\"" for term in compounds[:2])
keywords = [quoted.strip(), core.strip() or topic.strip()]
return " ".join(part for part in keywords if part).strip()
def _ranking_query(topic: str, core: str) -> str:
if topic.strip().endswith("?"):
return topic.strip()
if core and core.lower() != topic.lower():
return f"What recent evidence from the last 30 days is most relevant to {topic}, especially about {core}?"
return f"What recent evidence from the last 30 days is most relevant to {topic}?"
_TRAILING_CONTEXT = re.compile(
r"\s+\b(?:for|in|on|at|to|with|about|from|by|during|since|after|before|using|via)\b.*$",
re.I,
)
def _comparison_entities(topic: str) -> list[str]:
# "difference between X and Y" -> "X vs Y" (replace "and" only in this context)
normalized = re.sub(
r"\bdifference between\s+(.+?)\s+and\s+",
r"\1 vs ",
topic,
flags=re.I,
)
normalized = re.sub(r"\b(compared to)\b", " vs ", normalized, flags=re.I)
parts = [
part.strip(" \t\r\n?.,:;!()[]{}\"'")
for part in re.split(r"\bvs\.?\b|\bversus\b|/", normalized, flags=re.I)
if part.strip(" \t\r\n?.,:;!()[]{}\"'")
]
# Strip trailing context from parts ("Svelte for frontend in 2026" -> "Svelte")
if len(parts) >= 2:
parts = [_TRAILING_CONTEXT.sub("", part).strip() or part for part in parts]
deduped = []
for part in parts:
if part and part not in deduped:
deduped.append(part)
return deduped[:_max_subqueries("comparison")]
return []
def _should_force_deterministic_plan(topic: str) -> bool:
return _infer_intent(topic) == "comparison" and len(_comparison_entities(topic)) >= 2
def _max_subqueries(intent: str) -> int:
if intent == "comparison":
return 4
if intent in {"factual", "concept"}:
return 2
return 3
def _default_sources_for_intent(intent: str, available_sources: list[str]) -> list[str]:
if intent == "how_to":
sources = _how_to_sources(available_sources)
else:
target_capabilities = DEFAULT_INTENT_CAPABILITIES.get(intent)
if not target_capabilities:
sources = list(available_sources)
else:
matched = [
source
for source in available_sources
if SOURCE_CAPABILITIES.get(source, set()) & target_capabilities
]
sources = matched or list(available_sources)
excluded = INTENT_SOURCE_EXCLUSIONS.get(intent, set())
if excluded:
filtered = [s for s in sources if s not in excluded]
return filtered or sources
return sources
def _how_to_sources(available_sources: list[str]) -> list[str]:
"""Pick one source per role: web/reference, video (prefer longform), discussion."""
selected: set[str] = set()
has_video = False
# Order matters: web first, then longform video, generic video, discussion.
role_capabilities = [
{"web", "reference"},
{"video_longform"},
{"video"},
{"discussion"},
]
for role in role_capabilities:
is_video_role = role & {"video", "video_longform"}
if is_video_role and has_video:
continue
for source in available_sources:
if source in selected:
continue
if SOURCE_CAPABILITIES.get(source, set()) & role:
selected.add(source)
if is_video_role:
has_video = True
break
# After core role-based selection, include remaining sources with any
# how_to-relevant capability (video, discussion, web, reference, link).
how_to_caps = DEFAULT_INTENT_CAPABILITIES.get("how_to", set())
for source in available_sources:
if source not in selected and SOURCE_CAPABILITIES.get(source, set()) & how_to_caps:
selected.add(source)
if not selected:
return list(available_sources)
return [source for source in available_sources if source in selected]
scripts/lib/polymarket.py
+114 -8
View File
@@ -12,8 +12,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Dict, List, Optional
from urllib.parse import quote_plus, urlencode
from . import http
from .query_type import detect_query_type
from . import http, log
from .relevance import LOW_SIGNAL_QUERY_TOKENS, token_overlap_relevance
GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search"
@@ -34,10 +33,7 @@ RESULT_CAP = {
def _log(msg: str):
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
if sys.stderr.isatty():
sys.stderr.write(f"[PM] {msg}\n")
sys.stderr.flush()
log.source_log("PM", msg)
def _extract_core_subject(topic: str) -> str:
@@ -75,7 +71,7 @@ def _expand_queries(topic: str) -> List[str]:
words = core.split()
if len(words) >= 2:
for word in words:
if len(word) > 1 and word.lower() not in LOW_SIGNAL_QUERY_TOKENS:
if len(word) > 1 and word.lower() not in LOW_SIGNAL_QUERY_TOKENS and word.lower() not in _NOISE_WORDS:
queries.append(word)
# Add the full topic if different from core
@@ -95,6 +91,79 @@ def _expand_queries(topic: str) -> List[str]:
_GENERIC_TAGS = frozenset({"sports", "politics", "crypto", "science", "culture", "pop culture"})
# Words that are too generic to serve as the sole topic-match signal.
# If ALL core words from the topic are in this set, we skip filtering (can't meaningfully filter).
# But if some words are informative and some are generic, we require at least one informative word.
_NOISE_WORDS = frozenset({
# Articles, prepositions, conjunctions
"the", "a", "an", "in", "on", "at", "of", "for", "and", "or", "to", "is", "are",
"was", "were", "will", "be", "by", "with", "from", "as", "it", "its", "not", "no",
"but", "if", "so", "do", "has", "had", "have", "this", "that", "what", "who",
# Directional / geographic terms that cause false matches
"west", "east", "north", "south", "central", "southern", "northern", "eastern", "western",
# Common sports / category terms
"champion", "championship", "league", "division", "conference", "cup", "series",
"team", "game", "match", "season", "win", "winner", "finals",
# Common geographic / place nouns that cause false matches
# "club" -> Athletic Club, Racing Club; "island" -> Epstein's Island, Rhode Island
"club", "island", "city", "park", "hill", "lake", "bay", "beach", "valley",
"river", "mountain", "county", "state", "village", "town", "point", "creek",
"springs", "heights", "ridge", "bridge", "harbor", "port", "station", "center",
"square", "field", "forest", "garden", "tower", "school", "church", "camp",
"ranch", "crossing", "shore", "rock", "summit", "falls", "grove", "haven",
# Generic tech terms that match too broadly on Polymarket
# "cli" -> any CLI tool market; "mcp" -> protocol markets; "ai" -> every AI market
"cli", "mcp", "protocol", "tool", "app", "code", "model", "ai", "api",
"software", "plugin", "skill", "agent", "bot", "search", "research",
# Generic prediction market terms
"market", "odds", "prediction", "forecast", "chance", "probability",
})
def _passes_topic_filter(topic: str, event_title: str) -> bool:
"""Check if event title contains enough informative words from the topic.
Prevents noise like "Meek Mill" matching "Mill.com food recycler" by requiring
proportional word overlap. For topics with 3+ informative words, at least 2 must
match. For shorter topics, 1 match suffices (existing behavior).
Returns True if the event should be kept, False if it should be filtered out.
"""
core = _extract_core_subject(topic).lower()
core_words = [w for w in re.sub(r"[^\w\s]", " ", core).split() if len(w) > 1]
if not core_words:
return True # No words to check against
# Split into informative vs generic
informative = [w for w in core_words if w not in _NOISE_WORDS]
# If ALL words are generic, we can't meaningfully filter — keep everything
if not informative:
return True
# Normalize the title for matching
title_lower = " ".join(re.sub(r"[^\w\s]", " ", event_title.lower()).split())
title_words = set(title_lower.split())
# Count how many informative words appear in the title
match_count = 0
for word in informative:
# Check as whole word in the title word set
if word in title_words:
match_count += 1
continue
# Also check as substring for compound words (e.g., "kanye" in "kanyewest")
if len(word) >= 4 and word in title_lower:
match_count += 1
# For topics with 3+ informative words, require at least 2 matches.
# This prevents single-word false positives like "mill" in "Meek Mill"
# when the topic is "Mill.com food recycler" (3 informative words).
min_matches = 2 if len(informative) >= 3 else 1
return match_count >= min_matches
def _extract_domain_queries(topic: str, events: List[Dict]) -> List[str]:
"""Extract domain-indicator search terms from first-pass event tags.
@@ -130,6 +199,14 @@ def _extract_domain_queries(topic: str, events: List[Dict]) -> List[str]:
return domain_queries
def _infer_query_intent(topic: str) -> str:
"""Tiny local fallback for Polymarket search tuning only."""
text = topic.lower().strip()
if re.search(r"\b(predict|prediction|odds|forecast|chance|probability|will .* win)\b", text):
return "prediction"
return "breaking_news"
def _search_single_query(query: str, page: int = 1) -> Dict[str, Any]:
"""Run a single search query against Gamma API."""
params = {
@@ -328,7 +405,7 @@ def _compute_text_similarity(topic: str, title: str, outcomes: List[str] = None)
if core in title_lower:
return 1.0
query_type = detect_query_type(topic)
query_type = _infer_query_intent(topic)
title_score = token_overlap_relevance(core, title)
best_score = title_score
@@ -392,6 +469,7 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
events = response.get("events", [])
items = []
filtered_count = 0
for i, event in enumerate(events):
event_id = event.get("id", "")
title = event.get("title", "")
@@ -403,6 +481,12 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
if not event.get("active", True):
continue
# Filter: skip events that don't match the topic's core subject
# This prevents "NFC West" from matching a "Kanye West" search
if topic and not _passes_topic_filter(topic, title):
filtered_count += 1
continue
# Get markets for this event
markets = event.get("markets", [])
if not markets:
@@ -574,7 +658,29 @@ def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List
"why_relevant": f"Prediction market: {title[:60]}",
})
if filtered_count:
_log(f"Filtered {filtered_count} noise events (topic: '{topic}')")
# Sort by relevance (quality-signal ranked) and apply cap
items.sort(key=lambda x: x["relevance"], reverse=True)
# Drop ALL results if nothing is genuinely on-topic.
# If the best item's relevance is below the threshold, the Gamma API
# returned only tangential matches (e.g., "Anthropic best AI model"
# for a "CLI vs MCP" query). Better to show 0 than noise.
_MIN_RELEVANCE = 0.15
if items and items[0]["relevance"] < _MIN_RELEVANCE:
_log(f"All {len(items)} Polymarket results below relevance threshold "
f"({items[0]['relevance']:.2f} < {_MIN_RELEVANCE}), dropping all")
return []
# Per-item floor: drop individual noise items even if the best item passed
_ITEM_MIN_RELEVANCE = 0.10
before_count = len(items)
items = [i for i in items if i["relevance"] >= _ITEM_MIN_RELEVANCE]
dropped = before_count - len(items)
if dropped:
_log(f"Dropped {dropped} Polymarket items below per-item relevance floor ({_ITEM_MIN_RELEVANCE})")
cap = response.get("_cap", len(items))
return items[:cap]
scripts/lib/providers.py
+471
View File
@@ -0,0 +1,471 @@
"""Static provider catalog and runtime client implementations."""
from __future__ import annotations
import json
import re
import sys
from typing import Any
from . import env, http, schema
GEMINI_FLASH_LITE = "gemini-3.1-flash-lite-preview"
GEMINI_PRO = "gemini-3.1-pro-preview"
OPENAI_DEFAULT = "gpt-5.4-nano"
XAI_DEFAULT = "grok-4-1-fast"
GEMINI_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses"
XAI_RESPONSES_URL = "https://api.x.ai/v1/responses"
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
OPENROUTER_DEFAULT = "google/gemini-flash-2.0"
class ReasoningClient:
"""Shared interface for planner and rerank providers."""
name: str
def generate_text(
self,
model: str,
prompt: str,
*,
tools: list[dict[str, Any]] | None = None,
response_mime_type: str | None = None,
) -> str:
raise NotImplementedError
def generate_json(
self,
model: str,
prompt: str,
*,
tools: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
text = self.generate_text(model, prompt, tools=tools, response_mime_type="application/json")
return extract_json(text)
class GeminiClient(ReasoningClient):
name = "gemini"
def __init__(self, api_key: str):
self.api_key = api_key
def _generate_content(
self,
model: str,
prompt: str,
*,
tools: list[dict[str, Any]] | None = None,
response_mime_type: str | None = None,
) -> dict[str, Any]:
body: dict[str, Any] = {
"contents": [{"parts": [{"text": prompt}]}],
"generationConfig": {"temperature": 0},
}
if response_mime_type:
body["generationConfig"]["responseMimeType"] = response_mime_type
if tools:
body["tools"] = tools
return http.post(
GEMINI_URL.format(model=model, api_key=self.api_key),
body,
headers={"Content-Type": "application/json"},
timeout=90,
)
def generate_text(
self,
model: str,
prompt: str,
*,
tools: list[dict[str, Any]] | None = None,
response_mime_type: str | None = None,
) -> str:
payload = self._generate_content(
model,
prompt,
tools=tools,
response_mime_type=response_mime_type,
)
return extract_gemini_text(payload)
def ground_search(self, model: str, prompt: str) -> dict[str, Any]:
return self._generate_content(model, prompt, tools=[{"google_search": {}}])
def url_context_json(self, model: str, prompt: str) -> dict[str, Any]:
return self.generate_json(model, prompt, tools=[{"url_context": {}}])
class OpenAIClient(ReasoningClient):
name = "openai"
def __init__(self, token: str, auth_source: str, account_id: str | None):
self.token = token
self.auth_source = auth_source
self.account_id = account_id
def generate_text(
self,
model: str,
prompt: str,
*,
tools: list[dict[str, Any]] | None = None,
response_mime_type: str | None = None,
) -> str:
del tools, response_mime_type
if self.auth_source == env.AUTH_SOURCE_CODEX:
payload = {
"model": model,
"stream": True,
"store": False,
"input": [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": prompt}],
}
],
}
headers = {
"Authorization": f"Bearer {self.token}",
"chatgpt-account-id": self.account_id or "",
"OpenAI-Beta": "responses=experimental",
"originator": "pi",
"Content-Type": "application/json",
}
raw = http.post_raw(CODEX_RESPONSES_URL, payload, headers=headers, timeout=90)
return extract_openai_text(_parse_codex_stream(raw))
payload = {
"model": model,
"store": False,
"input": prompt,
"temperature": 0,
}
response = http.post(
OPENAI_RESPONSES_URL,
payload,
headers={
"Authorization": f"Bearer {self.token}",
"Content-Type": "application/json",
},
timeout=90,
)
return extract_openai_text(response)
class XAIClient(ReasoningClient):
name = "xai"
def __init__(self, api_key: str):
self.api_key = api_key
def generate_text(
self,
model: str,
prompt: str,
*,
tools: list[dict[str, Any]] | None = None,
response_mime_type: str | None = None,
) -> str:
del tools, response_mime_type
payload = {
"model": model,
"input": [{"role": "user", "content": prompt}],
}
response = http.post(
XAI_RESPONSES_URL,
payload,
headers={
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
},
timeout=90,
)
return extract_openai_text(response)
class OpenRouterClient(ReasoningClient):
name = "openrouter"
def __init__(self, api_key: str):
self.api_key = api_key
def generate_text(
self,
model: str,
prompt: str,
*,
tools: list[dict[str, Any]] | None = None,
response_mime_type: str | None = None,
) -> str:
del tools, response_mime_type
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0,
}
response = http.post(
OPENROUTER_URL,
payload,
headers={
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
},
timeout=90,
)
return extract_openai_text(response)
_MODEL_DEFAULTS: dict[str, tuple[str, str]] = {
"gemini": (GEMINI_FLASH_LITE, GEMINI_FLASH_LITE),
"openai": (OPENAI_DEFAULT, OPENAI_DEFAULT),
"xai": (XAI_DEFAULT, XAI_DEFAULT),
"openrouter": (OPENROUTER_DEFAULT, OPENROUTER_DEFAULT),
}
def _resolve_model_pins(config: dict[str, Any], depth: str, provider_name: str) -> tuple[str, str, str]:
"""Resolve planner, rerank, and grounding model pins for a provider."""
default_planner, default_rerank = _MODEL_DEFAULTS.get(provider_name, (GEMINI_FLASH_LITE, GEMINI_FLASH_LITE))
if depth == "deep" and provider_name == "gemini":
default_rerank = GEMINI_PRO
planner_model = config.get("LAST30DAYS_PLANNER_MODEL") or default_planner
rerank_model = config.get("LAST30DAYS_RERANK_MODEL") or default_rerank
if provider_name == "gemini":
_require_gemini_31_preview(planner_model, role="planner")
_require_gemini_31_preview(rerank_model, role="rerank")
return planner_model, rerank_model
def mock_runtime(config: dict[str, Any], depth: str) -> schema.ProviderRuntime:
"""Resolve model pins for mock mode without requiring live credentials."""
provider_name = (config.get("LAST30DAYS_REASONING_PROVIDER") or "gemini").lower()
if provider_name == "auto":
provider_name = "gemini"
if provider_name not in _MODEL_DEFAULTS:
raise RuntimeError(f"Unsupported reasoning provider: {provider_name}")
planner_model, rerank_model = _resolve_model_pins(config, depth, provider_name)
return schema.ProviderRuntime(
reasoning_provider=provider_name,
planner_model=planner_model,
rerank_model=rerank_model,
x_search_backend=_resolve_x_backend(config),
)
def resolve_runtime(config: dict[str, Any], depth: str) -> tuple[schema.ProviderRuntime, ReasoningClient | None]:
"""Resolve the reasoning provider and pinned models."""
provider_name = (config.get("LAST30DAYS_REASONING_PROVIDER") or "auto").lower()
google_key = config.get("GOOGLE_API_KEY") or config.get("GEMINI_API_KEY") or config.get("GOOGLE_GENAI_API_KEY")
openai_token = config.get("OPENAI_API_KEY")
xai_key = config.get("XAI_API_KEY")
if provider_name == "auto":
if google_key:
provider_name = "gemini"
elif openai_token and config.get("OPENAI_AUTH_STATUS") == env.AUTH_STATUS_OK:
provider_name = "openai"
elif xai_key:
provider_name = "xai"
elif config.get("OPENROUTER_API_KEY"):
provider_name = "openrouter"
else:
return schema.ProviderRuntime(
reasoning_provider="local",
planner_model="deterministic",
rerank_model="local-score",
x_search_backend=_resolve_x_backend(config),
), None
planner_model, rerank_model = _resolve_model_pins(config, depth, provider_name)
if provider_name == "gemini":
if not google_key:
raise RuntimeError("Gemini selected but no Google API key is configured.")
runtime = schema.ProviderRuntime(
reasoning_provider="gemini",
planner_model=planner_model,
rerank_model=rerank_model,
x_search_backend=_resolve_x_backend(config),
)
return runtime, GeminiClient(google_key)
if provider_name == "openai":
if not openai_token or config.get("OPENAI_AUTH_STATUS") != env.AUTH_STATUS_OK:
raise RuntimeError("OpenAI selected but no valid OpenAI auth is configured.")
runtime = schema.ProviderRuntime(
reasoning_provider="openai",
planner_model=planner_model,
rerank_model=rerank_model,
x_search_backend=_resolve_x_backend(config),
)
return runtime, OpenAIClient(
openai_token,
config.get("OPENAI_AUTH_SOURCE") or env.AUTH_SOURCE_API_KEY,
config.get("OPENAI_CHATGPT_ACCOUNT_ID"),
)
if provider_name == "xai":
if not xai_key:
raise RuntimeError("xAI selected but XAI_API_KEY is not configured.")
runtime = schema.ProviderRuntime(
reasoning_provider="xai",
planner_model=planner_model,
rerank_model=rerank_model,
x_search_backend=_resolve_x_backend(config),
)
return runtime, XAIClient(xai_key)
if provider_name == "openrouter":
openrouter_key = config.get("OPENROUTER_API_KEY")
if not openrouter_key:
raise RuntimeError("OpenRouter selected but OPENROUTER_API_KEY is not configured.")
runtime = schema.ProviderRuntime(
reasoning_provider="openrouter",
planner_model=planner_model,
rerank_model=rerank_model,
x_search_backend=_resolve_x_backend(config),
)
return runtime, OpenRouterClient(openrouter_key)
raise RuntimeError(f"Unsupported reasoning provider: {provider_name}")
def _resolve_x_backend(config: dict[str, Any]) -> str | None:
preferred = (config.get("LAST30DAYS_X_BACKEND") or "").lower()
if preferred in {"xai", "bird"}:
return preferred
return env.get_x_source(config)
def _require_gemini_31_preview(model: str, *, role: str) -> None:
if model.startswith("gemini-3.1-") and model.endswith("-preview"):
return
raise RuntimeError(
f"{role} must use a Gemini 3.1 preview model. Got: {model}"
)
def extract_json(text: str) -> dict[str, Any]:
"""Extract the first JSON object from a model response."""
text = text.strip()
if not text:
raise ValueError("Expected JSON response, got empty text")
try:
return json.loads(text)
except json.JSONDecodeError:
match = re.search(r"\{[\s\S]*\}", text)
if not match:
raise
return json.loads(match.group(0))
def extract_gemini_text(payload: dict[str, Any]) -> str:
for candidate in payload.get("candidates", []):
content = candidate.get("content") or {}
for part in content.get("parts", []):
text = part.get("text")
if text:
return text
if payload:
print(f"[Providers] extract_gemini_text: no text in payload keys: {list(payload.keys())}", file=sys.stderr)
return ""
def extract_openai_text(payload: dict[str, Any]) -> str:
if isinstance(payload.get("output_text"), str):
return payload["output_text"]
output = payload.get("output") or payload.get("choices") or []
for item in output:
if isinstance(item, str):
return item
if isinstance(item, dict):
if isinstance(item.get("text"), str):
return item["text"]
content = item.get("content") or []
if isinstance(content, list):
for part in content:
if isinstance(part, dict) and isinstance(part.get("text"), str):
return part["text"]
if isinstance(part, dict) and part.get("type") == "output_text" and isinstance(part.get("text"), str):
return part["text"]
message = item.get("message") or {}
if isinstance(message, dict) and isinstance(message.get("content"), str):
return message["content"]
if payload:
print(f"[Providers] extract_openai_text: no text in payload keys: {list(payload.keys())}", file=sys.stderr)
return ""
def _parse_sse_chunk(chunk: str) -> dict[str, Any] | None:
data_lines = [
line[5:].strip()
for line in chunk.split("\n")
if line.startswith("data:")
]
if not data_lines:
return None
data = "\n".join(data_lines).strip()
if not data or data == "[DONE]":
return None
try:
return json.loads(data)
except json.JSONDecodeError:
print(f"[Providers] _parse_sse_chunk: invalid JSON: {data[:100]}", file=sys.stderr)
return None
def _parse_codex_stream(raw: str) -> dict[str, Any]:
events: list[dict[str, Any]] = []
buffer = ""
for chunk in raw.splitlines(keepends=True):
buffer += chunk
while "\n\n" in buffer:
event_chunk, buffer = buffer.split("\n\n", 1)
event = _parse_sse_chunk(event_chunk)
if event is not None:
events.append(event)
if buffer.strip():
event = _parse_sse_chunk(buffer)
if event is not None:
events.append(event)
for event in reversed(events):
if event.get("type") == "response.completed" and isinstance(event.get("response"), dict):
return event["response"]
if isinstance(event.get("response"), dict):
return event["response"]
output_text = ""
for event in events:
delta = event.get("delta")
if isinstance(delta, str):
output_text += delta
text = event.get("text")
if isinstance(text, str):
output_text += text
if output_text:
return {
"output": [
{
"type": "message",
"content": [{"type": "output_text", "text": output_text}],
}
]
}
if raw.strip():
print(f"[Providers] _parse_codex_stream: received {len(raw)} bytes but could not extract text", file=sys.stderr)
return {}
scripts/lib/quality_nudge.py
+49 -60
View File
@@ -4,11 +4,11 @@ Computes a quality score based on 5 core sources and builds
a nudge message describing what the user missed and how to fix it.
"""
from typing import Any, Dict, List, Optional
from typing import List
# The 5 core sources
CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit_comments"]
CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit"]
# Labels for display
SOURCE_LABELS = {
@@ -16,7 +16,7 @@ SOURCE_LABELS = {
"polymarket": "Polymarket",
"x": "X/Twitter",
"youtube": "YouTube",
"reddit_comments": "Reddit with comments",
"reddit": "Reddit",
}
@@ -45,16 +45,6 @@ def _is_youtube_active(config: dict, research_results: dict) -> bool:
return True
def _is_reddit_comments_active(config: dict, research_results: dict) -> bool:
"""Check if Reddit with comments is active (ScrapeCreators)."""
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
if not has_sc:
return False
if research_results.get("reddit_error"):
return False
return True
def compute_quality_score(config: dict, research_results: dict) -> dict:
"""Compute research quality score based on 5 core sources.
@@ -67,8 +57,8 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
{
"score_pct": 40-100,
"core_active": ["hn", "polymarket", ...],
"core_missing": ["x", "youtube", "reddit_comments"],
"core_errored": ["reddit_comments"], # configured but errored
"core_missing": ["x", "youtube"],
"core_errored": [], # configured but errored
"nudge_text": "..." or None if 100%
}
"""
@@ -76,9 +66,10 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
core_missing: List[str] = []
core_errored: List[str] = []
# HN and Polymarket are always active
# HN, Polymarket, and Reddit are always active
core_active.append("hn")
core_active.append("polymarket")
core_active.append("reddit")
# X
has_x_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
@@ -90,30 +81,25 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
core_errored.append("x")
# YouTube
try:
from . import youtube_yt
has_ytdlp = youtube_yt.is_ytdlp_installed()
except Exception:
has_ytdlp = False
if _is_youtube_active(config, research_results):
yt_active = _is_youtube_active(config, research_results)
if yt_active:
core_active.append("youtube")
else:
core_missing.append("youtube")
# Check if configured but errored (yt-dlp installed but failed this run)
try:
from . import youtube_yt
has_ytdlp = youtube_yt.is_ytdlp_installed()
except Exception:
has_ytdlp = False
if has_ytdlp and research_results.get("youtube_error"):
core_errored.append("youtube")
# Reddit with comments (ScrapeCreators)
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
if _is_reddit_comments_active(config, research_results):
core_active.append("reddit_comments")
else:
core_missing.append("reddit_comments")
if has_sc and research_results.get("reddit_error"):
core_errored.append("reddit_comments")
score_pct = int(len(core_active) / 5 * 100)
nudge_text = _build_nudge_text(core_missing, core_errored) if core_missing else None
has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
active_sources = research_results.get("active_sources") or []
nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None
return {
"score_pct": score_pct,
@@ -124,10 +110,11 @@ def compute_quality_score(config: dict, research_results: dict) -> dict:
}
def _build_nudge_text(core_missing: List[str], core_errored: List[str]) -> str:
def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str:
"""Build human-readable nudge text describing what was missed.
Prioritizes free suggestions before paid ones.
Prioritizes free suggestions. Optionally mentions bonus sources
(TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured.
"""
lines: List[str] = []
@@ -145,43 +132,44 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str]) -> str:
lines.append(f"Missing: {', '.join(missed_parts)}.")
lines.append("")
# Free suggestions first
# Free suggestions
free_suggestions: List[str] = []
paid_suggestions: List[str] = []
if "x" in core_missing:
if "x" in core_errored:
free_suggestions.append(
"X errored — try refreshing your browser cookies "
"(log into x.com, then re-run)."
"X/Twitter errored - log into x.com in your browser, then re-run."
)
else:
free_suggestions.append(
"X/Twitter: scan browser cookies automatically — "
"just log into x.com in any browser and re-run."
"X/Twitter: real-time posts with likes and reposts - the fastest "
"signal for breaking topics. Two options: log into x.com in your "
"browser and re-run (cookies detected automatically), or add "
"XAI_API_KEY to your .env (no browser access, get key at api.x.ai)."
)
if "youtube" in core_missing:
if "youtube" in core_errored:
free_suggestions.append(
"YouTube errored — check that yt-dlp is up to date: "
"brew upgrade yt-dlp"
"YouTube errored - update yt-dlp: brew upgrade yt-dlp"
)
else:
free_suggestions.append(
"YouTube: install yt-dlp — brew install yt-dlp"
"YouTube: video transcripts with key moments - often the deepest "
"explanations on any topic. Install yt-dlp: brew install yt-dlp (free)"
)
if "reddit_comments" in core_missing:
if "reddit_comments" in core_errored:
paid_suggestions.append(
"Reddit comments errored — check your ScrapeCreators API key "
"at scrapecreators.com."
)
else:
paid_suggestions.append(
"Reddit with comments: add SCRAPECREATORS_API_KEY — "
"100 free API calls, no credit card — scrapecreators.com"
# Mention bonus opt-in sources when SC key is present
if has_sc:
bonus_hints = []
if "threads" not in (active_sources or []):
bonus_hints.append("Threads")
if "pinterest" not in (active_sources or []):
bonus_hints.append("Pinterest")
if bonus_hints:
free_suggestions.append(
f"Your SC key also powers {', '.join(bonus_hints)} and YouTube comments. "
"Add them to INCLUDE_SOURCES in your .env to enable."
)
if free_suggestions:
@@ -190,12 +178,13 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str]) -> str:
lines.append(f" - {s}")
lines.append("")
if paid_suggestions:
lines.append("Paid options:")
for s in paid_suggestions:
lines.append(f" - {s}")
lines.append("")
lines.append("last30days has no affiliation with any API provider.")
# Bonus sources mention (non-blocking)
if not has_sc:
lines.append(
"Bonus: TikTok and Instagram are available with a free "
"ScrapeCreators key at scrapecreators.com (no affiliation)."
)
else:
lines.append("last30days has no affiliation with any API provider.")
return "\n".join(lines)
scripts/lib/query_type.py
-111
View File
@@ -1,111 +0,0 @@
"""Query type detection for source selection and scoring adjustments."""
import re
from typing import Literal
QueryType = Literal["product", "concept", "opinion", "how_to", "comparison", "breaking_news", "prediction"]
# Pattern-based classification (no LLM, no external deps)
_PRODUCT_PATTERNS = re.compile(
r"\b(price|pricing|cost|buy|purchase|deal|discount|subscription|plan|tier|free tier|alternative|prompt|prompts|prompting|template|templates)\b", re.I
)
_CONCEPT_PATTERNS = re.compile(
r"\b(what is|what are|explain|definition|how does|how do|overview|introduction|guide to|primer)\b", re.I
)
_OPINION_PATTERNS = re.compile(
r"\b(worth it|thoughts on|opinion|review|experience with|recommend|should i|pros and cons|good or bad)\b", re.I
)
_HOWTO_PATTERNS = re.compile(
r"\b(how to|tutorial|step by step|setup|install|configure|deploy|migrate|implement|build a|create a|prompting|prompts?|best practices|tips|examples|animation|animations|video workflow|render pipeline)\b",
re.I,
)
_COMPARISON_PATTERNS = re.compile(
r"\b(vs\.?|versus|compared to|comparison|better than|difference between|switch from)\b", re.I
)
_BREAKING_PATTERNS = re.compile(
r"\b(latest|breaking|just announced|launched|released|new|update|news|happened|today|this week)\b", re.I
)
_PREDICTION_PATTERNS = re.compile(
r"\b(predict|forecast|odds|chance|probability|election|outcome|bet on|market for)\b", re.I
)
def detect_query_type(topic: str) -> QueryType:
"""Classify a query into a type using pattern matching.
Returns the first match in priority order:
comparison > how_to > product > opinion > prediction > concept > breaking_news.
"""
# Most specific first
if _COMPARISON_PATTERNS.search(topic):
return "comparison"
if _HOWTO_PATTERNS.search(topic):
return "how_to"
if _PRODUCT_PATTERNS.search(topic):
return "product"
if _OPINION_PATTERNS.search(topic):
return "opinion"
if _PREDICTION_PATTERNS.search(topic):
return "prediction"
if _CONCEPT_PATTERNS.search(topic):
return "concept"
if _BREAKING_PATTERNS.search(topic):
return "breaking_news"
# Default: treat as breaking news (most common use case for "last 30 days")
return "breaking_news"
# Source tiering by query type.
# Tier 1: always run. Tier 2: run if available. Tier 3: opt-in only.
# Sources not listed are implicitly tier 3 (opt-in).
SOURCE_TIERS = {
"product": {"tier1": {"reddit", "x", "youtube"}, "tier2": {"web", "tiktok"}},
"concept": {"tier1": {"reddit", "hn", "web"}, "tier2": {"youtube", "x"}},
"opinion": {"tier1": {"reddit", "x"}, "tier2": {"youtube", "bluesky"}},
"how_to": {"tier1": {"youtube", "reddit", "hn"}, "tier2": {"web", "x"}},
"comparison": {"tier1": {"reddit", "hn", "youtube"}, "tier2": {"x", "web"}},
"breaking_news": {"tier1": {"x", "reddit", "web"}, "tier2": {"hn", "bluesky", "youtube"}},
"prediction": {"tier1": {"polymarket", "x", "reddit"}, "tier2": {"web", "hn", "youtube"}},
}
# WebSearch penalty adjustment by query type.
# Points subtracted from websearch score (0-100 scale). 0 = no penalty, 15 = full penalty.
# Concept/how_to queries benefit from authoritative web sources.
WEBSEARCH_PENALTY_BY_TYPE = {
"product": 15, # default: social discussion > blog posts
"concept": 0, # web docs are the best source
"opinion": 15, # social discussion > blog posts
"how_to": 5, # tutorials on web are valuable
"comparison": 10, # mix of social and web
"breaking_news": 10, # news sites are valuable
"prediction": 15, # social/market data > web articles
}
# Tiebreaker priority overrides by query type.
# Maps source type name to priority (lower = higher priority).
TIEBREAKER_BY_TYPE = {
"product": {"reddit": 0, "x": 1, "youtube": 2, "tiktok": 3, "instagram": 4, "hn": 5, "web": 6, "polymarket": 7},
"concept": {"hn": 0, "reddit": 1, "web": 2, "youtube": 3, "x": 4, "tiktok": 5, "instagram": 6, "polymarket": 7},
"opinion": {"reddit": 0, "x": 1, "bluesky": 2, "youtube": 3, "hn": 4, "tiktok": 5, "web": 6, "polymarket": 7},
"how_to": {"youtube": 0, "reddit": 1, "hn": 2, "web": 3, "x": 4, "tiktok": 5, "instagram": 6, "polymarket": 7},
"comparison": {"reddit": 0, "hn": 1, "youtube": 2, "x": 3, "web": 4, "tiktok": 5, "instagram": 6, "polymarket": 7},
"breaking_news": {"x": 0, "reddit": 1, "web": 2, "hn": 3, "bluesky": 4, "tiktok": 5, "youtube": 6, "polymarket": 7},
"prediction": {"polymarket": 0, "x": 1, "reddit": 2, "web": 3, "hn": 4, "bluesky": 5, "youtube": 6, "tiktok": 7},
}
def is_source_enabled(source: str, query_type: QueryType, explicitly_requested: bool = False) -> bool:
"""Check if a source should run for a given query type.
Tier 1 and Tier 2 sources are enabled. Tier 3 (unlisted) sources only run
if explicitly requested via --search flag. Truth Social is always opt-in.
"""
if source == "truthsocial":
return explicitly_requested
if explicitly_requested:
return True
tiers = SOURCE_TIERS.get(query_type, SOURCE_TIERS["breaking_news"])
return source in tiers["tier1"] or source in tiers["tier2"]
scripts/lib/reddit.py
+254 -90
View File
@@ -1,19 +1,17 @@
"""Reddit search via ScrapeCreators API for /last30days.
"""Reddit search via ScrapeCreators API for the v3 pipeline.
Uses ScrapeCreators REST API to search Reddit globally, discover relevant
subreddits, run targeted subreddit searches, and fetch comment trees.
Replaces openai_reddit.py as the primary Reddit search backend.
Falls back to openai_reddit.py if SCRAPECREATORS_API_KEY is missing but
OPENAI_API_KEY is present.
Requires SCRAPECREATORS_API_KEY in config (same key as TikTok + Instagram).
API docs: https://scrapecreators.com/docs
"""
import re
import sys
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed, wait as futures_wait
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Set
@@ -22,7 +20,15 @@ try:
except ImportError:
_requests = None
from . import http
def _first_of(*values, default=None):
"""Return first value that is not None."""
for v in values:
if v is not None:
return v
return default
from . import http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/reddit"
@@ -49,7 +55,6 @@ DEPTH_CONFIG = {
}
from .query import extract_core_subject as _query_extract
from .query_type import detect_query_type
from .relevance import token_overlap_relevance
# Reddit-specific noise words (preserves original smaller set)
@@ -68,9 +73,7 @@ NOISE_WORDS = frozenset({
def _log(msg: str):
"""Log to stderr."""
sys.stderr.write(f"[Reddit] {msg}\n")
sys.stderr.flush()
log.source_log("Reddit", msg, tty_only=False)
def _sc_headers(token: str) -> Dict[str, str]:
@@ -108,9 +111,18 @@ def expand_reddit_queries(topic: str, depth: str) -> List[str]:
if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
queries.append(original_clean)
# Opinion/review variants help mostly for product and opinion queries.
# They contaminate broader searches like predictions or breaking news.
qtype = detect_query_type(topic)
qtype = _infer_query_intent(topic)
# Product queries: always include review-oriented variant to bias toward
# review communities instead of keyword-matching unrelated subreddits.
if qtype == "product":
queries.append(f"{core} review OR recommendation OR best")
# Comparison queries: include head-to-head discussion variant.
if qtype == "comparison":
queries.append(f"{core} worth it OR vs OR compared")
# Opinion/review variants for default/deep depth.
if depth in ("default", "deep") and qtype in ("product", "opinion"):
queries.append(f"{core} worth it OR thoughts OR review")
@@ -121,6 +133,22 @@ def expand_reddit_queries(topic: str, depth: str) -> List[str]:
return queries
def _infer_query_intent(topic: str) -> str:
"""Tiny local fallback for Reddit query expansion only."""
text = topic.lower().strip()
if re.search(r"\b(vs|versus|compare|difference between)\b", text):
return "comparison"
if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install|configuration|configure|troubleshoot|troubleshooting|error|errors|fix|debug)\b", text):
return "how_to"
if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
return "opinion"
if re.search(r"\b(pricing|feature|features|best .* for)\b", text):
return "product"
if re.search(r"\b(predict|prediction|odds|forecast|chance)\b", text):
return "prediction"
return "breaking_news"
# Known utility/meta subreddits that match queries but aren't discussion subs.
# These get a 0.3x penalty (not banned) in subreddit discovery scoring.
UTILITY_SUBS = frozenset({
@@ -153,7 +181,7 @@ def discover_subreddits(
scores = Counter()
for post in results:
sub = post.get("subreddit", "")
sub = _extract_subreddit_name(post.get("subreddit", ""))
if not sub:
continue
@@ -170,7 +198,7 @@ def discover_subreddits(
base *= 0.3
# Bonus: post engagement (high-engagement posts = better sub)
ups = post.get("ups") or post.get("score", 0)
ups = _first_of(post.get("ups"), post.get("score"), post.get("votes"), default=0)
if ups and ups > 100:
base += 0.5
@@ -179,19 +207,84 @@ def discover_subreddits(
return [sub for sub, _ in scores.most_common(max_subs)]
def _parse_date(created_utc) -> Optional[str]:
"""Convert Unix timestamp to YYYY-MM-DD."""
if not created_utc:
def _parse_date(value) -> Optional[str]:
"""Convert Unix timestamp or ISO-8601 string to YYYY-MM-DD.
Global search returns ``created_at`` as an ISO string
(e.g. "2018-05-03T01:09:17.620000+0000"); subreddit search returns
``created_utc`` as a Unix timestamp. Handle both.
"""
if not value:
return None
# ISO-8601 string (contains 'T' or '-')
if isinstance(value, str) and ("T" in value or "-" in value):
try:
# Strip trailing offset variations (+0000, Z) for fromisoformat
clean = value.replace("Z", "+00:00")
if clean.endswith("+0000"):
clean = clean[:-5] + "+00:00"
dt = datetime.fromisoformat(clean)
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError):
pass
# Unix timestamp (int or float or numeric string)
try:
dt = datetime.fromtimestamp(float(created_utc), tz=timezone.utc)
dt = datetime.fromtimestamp(float(value), tz=timezone.utc)
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError, OSError):
return None
def _extract_subreddit_name(value: Any) -> str:
"""Extract subreddit name from string or API object dict."""
if isinstance(value, dict):
return str(value.get("name") or value.get("display_name") or "").strip()
return str(value).strip()
def _extract_score(post: Dict[str, Any]) -> int:
"""Extract post score from either API schema.
Global search uses ``votes``; subreddit search uses ``ups``/``score``.
"""
return _first_of(post.get("ups"), post.get("score"), post.get("votes"), default=0)
def _extract_date(post: Dict[str, Any]) -> Optional[str]:
"""Extract date from either API schema.
Global search uses ``created_at`` (ISO); subreddit search uses ``created_utc`` (Unix).
"""
return _parse_date(
post.get("created_utc") or post.get("created_at") or post.get("created_at_iso")
)
def _normalize_reddit_id(raw_id: str) -> str:
"""Strip Reddit fullname prefix (t3_) for consistent dedup."""
s = str(raw_id or "")
return s[3:] if s.startswith("t3_") else s
def _total_engagement(item: Dict[str, Any]) -> int:
"""Combined engagement score: upvotes + comment count.
Used for selecting which threads to enrich with comments.
Threads with lots of comments are high-value even if upvote score is low.
"""
eng = item.get("engagement", {})
score = eng.get("score", 0) or 0
num_comments = eng.get("num_comments", 0) or 0
return score + num_comments
def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global", query: str = "") -> Dict[str, Any]:
"""Normalize a ScrapeCreators Reddit post to our internal format."""
"""Normalize a ScrapeCreators Reddit post to our internal format.
Handles both the global-search schema (``votes``, ``created_at``,
``subreddit`` as dict) and the subreddit-search schema (``ups``/``score``,
``created_utc``, ``subreddit`` as string).
"""
permalink = post.get("permalink", "")
url = f"https://www.reddit.com{permalink}" if permalink else post.get("url", "")
@@ -208,13 +301,13 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global"
return {
"id": f"R{idx}",
"reddit_id": post.get("id", ""),
"reddit_id": _normalize_reddit_id(post.get("id", "")),
"title": title,
"url": url,
"subreddit": str(post.get("subreddit", "")).strip(),
"date": _parse_date(post.get("created_utc")),
"subreddit": _extract_subreddit_name(post.get("subreddit", "")),
"date": _extract_date(post),
"engagement": {
"score": post.get("ups") or post.get("score", 0),
"score": _extract_score(post),
"num_comments": post.get("num_comments", 0),
"upvote_ratio": post.get("upvote_ratio"),
},
@@ -268,6 +361,11 @@ def _global_search(
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
return data.get("posts", data.get("data", []))
except http.HTTPError as e:
if e.status_code and e.status_code in (401, 403):
raise
_log(f"Global search error (urllib): {e}")
return []
except Exception as e:
_log(f"Global search error (urllib): {e}")
return []
@@ -282,6 +380,11 @@ def _global_search(
resp.raise_for_status()
data = resp.json()
return data.get("posts", data.get("data", []))
except _requests.exceptions.HTTPError as e:
if e.response is not None and e.response.status_code in (401, 403):
raise http.HTTPError(f"Auth error: {e}", e.response.status_code)
_log(f"Global search error: {e}")
return []
except Exception as e:
_log(f"Global search error: {e}")
return []
@@ -409,10 +512,11 @@ def search_reddit(
to_date: str,
depth: str = "default",
token: str = None,
subreddits: List[str] | None = None,
) -> Dict[str, Any]:
"""Full Reddit search: multi-query global discovery + subreddit drill-down.
This is the main entry point. Replaces openai_reddit.search_reddit().
This is the main v3 Reddit entry point.
Args:
topic: Search topic
@@ -420,6 +524,7 @@ def search_reddit(
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
subreddits: Optional list of subreddit names to search first (pre-resolved)
Returns:
Dict with 'items' list and optional 'error'.
@@ -429,40 +534,72 @@ def search_reddit(
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
timeframe = config["timeframe"]
intent = _infer_query_intent(topic)
# === Phase 1: Query Expansion ===
queries = expand_reddit_queries(topic, depth)
_log(f"Expanded '{topic}' into {len(queries)} queries: {queries}")
# === Phase 2: Global Discovery ===
core = _extract_core_subject(topic)
# === Phase 1.5: Pre-resolved subreddit search (high-signal) ===
all_raw_posts = []
all_items: List[Dict[str, Any]] = []
if subreddits:
_log(f"Searching pre-resolved subreddits: {subreddits}")
with ThreadPoolExecutor(max_workers=min(5, len(subreddits))) as executor:
futures = {}
for sub in subreddits:
futures[executor.submit(_subreddit_search, sub, core, token, "relevance", timeframe)] = sub
for future in as_completed(futures):
sub = futures[future]
sub_posts = future.result()
_log(f" -> {len(sub_posts)} results from pre-resolved r/{sub}")
for j, post in enumerate(sub_posts):
item = _normalize_post(post, len(all_items) + j + 1, f"r/{sub}", query=core)
all_items.append(item)
# === Phase 2: Global Discovery ===
max_global = config["global_searches"]
for i, query in enumerate(queries[:max_global]):
sort = "relevance" if i == 0 else "top"
_log(f"Global search {i+1}/{max_global}: '{query}' (sort={sort})")
posts = _global_search(query, token, sort=sort, timeframe=timeframe)
_log(f" -> {len(posts)} results")
all_raw_posts.extend(posts)
with ThreadPoolExecutor(max_workers=max_global or 1) as executor:
futures = {}
for i, query in enumerate(queries[:max_global]):
# Product/comparison queries: sort=top surfaces high-engagement posts
# from relevant communities instead of keyword-matched noise.
sort = "top" if intent in ("product", "comparison") else ("relevance" if i == 0 else "top")
_log(f"Global search {i+1}/{max_global}: '{query}' (sort={sort})")
futures[executor.submit(_global_search, query, token, sort, timeframe)] = query
for future in as_completed(futures):
query = futures[future]
posts = future.result()
_log(f" -> {len(posts)} results for '{query}'")
all_raw_posts.extend(posts)
# Normalize all posts (with query for relevance scoring)
core = _extract_core_subject(topic)
all_items = []
for i, post in enumerate(all_raw_posts):
item = _normalize_post(post, i + 1, "global", query=core)
all_items.append(item)
# === Phase 3: Subreddit Discovery + Targeted Search ===
discovered_subs = discover_subreddits(all_raw_posts, topic=topic, max_subs=config["subreddit_searches"])
subreddit_budget = 0 if intent == "how_to" else config["subreddit_searches"]
discovered_subs = discover_subreddits(all_raw_posts, topic=topic, max_subs=subreddit_budget)
_log(f"Discovered subreddits: {discovered_subs}")
for sub in discovered_subs[:config["subreddit_searches"]]:
_log(f"Subreddit search: r/{sub} for '{core}'")
sub_posts = _subreddit_search(sub, core, token, sort="relevance", timeframe=timeframe)
_log(f" -> {len(sub_posts)} results from r/{sub}")
for j, post in enumerate(sub_posts):
item = _normalize_post(post, len(all_items) + j + 1, f"r/{sub}", query=core)
all_items.append(item)
subreddit_limit = subreddit_budget
if subreddit_limit > 0:
with ThreadPoolExecutor(max_workers=subreddit_limit) as executor:
futures = {}
for sub in discovered_subs[:subreddit_limit]:
_log(f"Subreddit search: r/{sub} for '{core}'")
futures[executor.submit(_subreddit_search, sub, core, token, "relevance", timeframe)] = sub
for future in as_completed(futures):
sub = futures[future]
sub_posts = future.result()
_log(f" -> {len(sub_posts)} results from r/{sub}")
for j, post in enumerate(sub_posts):
item = _normalize_post(post, len(all_items) + j + 1, f"r/{sub}", query=core)
all_items.append(item)
# === Phase 4: Deduplicate ===
all_items = _dedupe_posts(all_items)
@@ -486,9 +623,9 @@ def search_reddit(
else:
_log(f"No posts within date range, keeping all {len(all_items)}")
# === Phase 6: Sort by engagement ===
# === Phase 6: Sort by engagement (upvotes + comment count) ===
all_items.sort(
key=lambda x: (x.get("engagement", {}).get("score", 0) or 0),
key=lambda x: _total_engagement(x),
reverse=True,
)
@@ -504,6 +641,7 @@ def enrich_with_comments(
items: List[Dict[str, Any]],
token: str,
depth: str = "default",
budget_seconds: int = 60,
) -> List[Dict[str, Any]]:
"""Enrich top items with comment data from ScrapeCreators.
@@ -511,6 +649,8 @@ def enrich_with_comments(
items: Reddit items from search_reddit()
token: ScrapeCreators API key
depth: Depth for comment limit
budget_seconds: Maximum total time for enrichment. If exceeded,
returns items with whatever enrichment completed. Never discards items.
Returns:
Items with top_comments and comment_insights added.
@@ -518,62 +658,84 @@ def enrich_with_comments(
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
max_comments = config["comment_enrichments"]
if not items or not token:
if not items or not token or max_comments <= 0:
return items
top_items = items[:max_comments]
_log(f"Enriching comments for {len(top_items)} posts")
# Select the top threads by total engagement (upvotes + comment count),
# not by list position. This ensures high-comment threads like [FRESH ALBUM]
# always get enriched even if their upvote score is low.
ranked = sorted(items, key=_total_engagement, reverse=True)
top_items = ranked[:max_comments]
_log(f"Enriching comments for {len(top_items)} posts (by total engagement)")
for item in top_items:
url = item.get("url", "")
if not url:
continue
start = time.monotonic()
raw_comments = fetch_post_comments(url, token)
if not raw_comments:
continue
with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:
futures = {
executor.submit(fetch_post_comments, item.get("url", ""), token): item
for item in top_items
if item.get("url")
}
# Parse comments into our format
top_comments = []
insights = []
# Wait with budget instead of unbounded as_completed
remaining = max(0, budget_seconds - (time.monotonic() - start))
done, not_done = futures_wait(futures, timeout=remaining)
for ci, c in enumerate(raw_comments[:10]): # Take top 10 comments
body = c.get("body", "")
if not body or body in ("[deleted]", "[removed]"):
enriched_count = 0
for future in done:
item = futures[future]
try:
raw_comments = future.result(timeout=0)
except Exception:
continue
if not raw_comments:
continue
score = c.get("ups") or c.get("score", 0)
author = c.get("author", "[deleted]")
permalink = c.get("permalink", "")
comment_url = f"https://reddit.com{permalink}" if permalink else ""
top_comments = []
insights = []
# Top comment gets more room (400 chars) — funny/clever comments need it
max_excerpt = 400 if ci == 0 else 300
top_comments.append({
"score": score,
"date": _parse_date(c.get("created_utc")),
"author": author,
"excerpt": body[:max_excerpt],
"url": comment_url,
})
for ci, c in enumerate(raw_comments[:10]):
body = c.get("body", "")
if not body or body in ("[deleted]", "[removed]"):
continue
# Extract insights from substantive comments
if len(body) >= 30 and author not in ("[deleted]", "[removed]", "AutoModerator"):
insight = body[:150]
if len(body) > 150:
for i, char in enumerate(insight):
if char in '.!?' and i > 50:
insight = insight[:i+1]
break
else:
insight = insight.rstrip() + "..."
insights.append(insight)
score = c.get("ups") or c.get("score", 0)
author = c.get("author", "[deleted]")
permalink = c.get("permalink", "")
comment_url = f"https://reddit.com{permalink}" if permalink else ""
# Sort comments by score
top_comments.sort(key=lambda c: c.get("score", 0), reverse=True)
max_excerpt = 400 if ci == 0 else 300
top_comments.append({
"score": score,
"date": _parse_date(c.get("created_utc")),
"author": author,
"excerpt": body[:max_excerpt],
"url": comment_url,
})
item["top_comments"] = top_comments[:10]
item["comment_insights"] = insights[:10]
if len(body) >= 30 and author not in ("[deleted]", "[removed]", "AutoModerator"):
insight = body[:150]
if len(body) > 150:
for i, char in enumerate(insight):
if char in '.!?' and i > 50:
insight = insight[:i+1]
break
else:
insight = insight.rstrip() + "..."
insights.append(insight)
top_comments.sort(key=lambda c: c.get("score", 0), reverse=True)
item["top_comments"] = top_comments[:10]
item["comment_insights"] = insights[:10]
enriched_count += 1
if not_done:
_log(f"Enrichment budget hit ({budget_seconds}s): {enriched_count}/{len(futures)} posts enriched, {len(not_done)} skipped")
for future in not_done:
future.cancel()
else:
elapsed = time.monotonic() - start
_log(f"Enriched {enriched_count}/{len(futures)} posts in {elapsed:.1f}s")
return items
@@ -584,6 +746,7 @@ def search_and_enrich(
to_date: str,
depth: str = "default",
token: str = None,
subreddits: List[str] | None = None,
) -> Dict[str, Any]:
"""Full Reddit pipeline: search + comment enrichment.
@@ -595,11 +758,12 @@ def search_and_enrich(
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
subreddits: Optional list of subreddit names to search first (pre-resolved)
Returns:
Dict with 'items' list. Items include top_comments and comment_insights.
"""
result = search_reddit(topic, from_date, to_date, depth, token)
result = search_reddit(topic, from_date, to_date, depth, token, subreddits=subreddits)
items = result.get("items", [])
if items and token:
@@ -612,6 +776,6 @@ def search_and_enrich(
def parse_reddit_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse ScrapeCreators response to item list.
Compatibility shim matching openai_reddit.parse_reddit_response() signature.
Parse raw Reddit search output into the generic item shape.
"""
return response.get("items", [])
scripts/lib/reddit_enrich.py
+3 -6
View File
@@ -21,13 +21,10 @@ def extract_reddit_path(url: str) -> Optional[str]:
Returns:
Path component or None
"""
try:
parsed = urlparse(url)
if "reddit.com" not in parsed.netloc:
return None
return parsed.path
except:
parsed = urlparse(url)
if "reddit.com" not in parsed.netloc:
return None
return parsed.path
class RedditRateLimitError(Exception):
scripts/lib/reddit_public.py
+121 -3
View File
@@ -17,6 +17,7 @@ import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
from typing import Any, Dict, List, Optional
@@ -29,6 +30,13 @@ DEPTH_LIMITS = {
"deep": 50,
}
# How many top posts to enrich with comments, by depth
ENRICH_LIMITS = {
"quick": 3,
"default": 5,
"deep": 8,
}
MAX_RETRIES = 3
BASE_BACKOFF = 2.0 # seconds
@@ -156,6 +164,7 @@ def _parse_posts(data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
},
"relevance": _compute_relevance(score, num_comments),
"why_relevant": "Reddit public search",
"metadata": {},
})
return posts
@@ -217,27 +226,133 @@ def search(
return unique[:limit]
def _enrich_post(item: Dict[str, Any], timeout: int = 10) -> Dict[str, Any]:
"""Enrich a single post with top comments. Never raises."""
try:
from . import reddit_enrich
thread_data = reddit_enrich.fetch_thread_data(item["url"], timeout=timeout)
if not thread_data:
return item
parsed = reddit_enrich.parse_thread_data(thread_data)
comments = parsed.get("comments", [])
top = reddit_enrich.get_top_comments(comments)
item["top_comments"] = [
{
"score": c.get("score", 0),
"excerpt": (c.get("body") or "")[:200],
"author": c.get("author", ""),
}
for c in top[:10]
]
except Exception:
# Never discard — keep post with empty metadata
pass
return item
def _enrich_posts(posts: List[Dict[str, Any]], depth: str = "default") -> List[Dict[str, Any]]:
"""Enrich top N posts with comment data using threads. Total budget 45s."""
limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])
to_enrich = posts[:limit]
rest = posts[limit:]
if not to_enrich:
return posts
enriched = []
try:
with ThreadPoolExecutor(max_workers=min(limit, 4)) as executor:
futures = {
executor.submit(_enrich_post, post, 10): i
for i, post in enumerate(to_enrich)
}
# Collect results with 45s total budget
import concurrent.futures
done, not_done = concurrent.futures.wait(futures, timeout=45)
# Build result list preserving order
result_map: Dict[int, Dict[str, Any]] = {}
for future in done:
idx = futures[future]
try:
result_map[idx] = future.result(timeout=0)
except Exception:
result_map[idx] = to_enrich[idx]
# Any not-done futures: keep original post
for future in not_done:
idx = futures[future]
result_map[idx] = to_enrich[idx]
future.cancel()
enriched = [result_map[i] for i in range(len(to_enrich))]
except Exception:
enriched = to_enrich
return enriched + rest
def _search_subreddit(sub: str, topic: str, depth: str, timeout: int = 15) -> List[Dict[str, Any]]:
"""Search a single subreddit. Never raises."""
try:
return search(topic, depth=depth, subreddit=sub, timeout=timeout)
except Exception as e:
_log(f"Subreddit search failed for r/{sub}: {e}")
return []
def search_reddit_public(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
subreddits: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
"""High-level Reddit public search matching the openai_reddit interface.
Runs global search, deduplicates, filters by date range, and sorts
by engagement. Compatible as a drop-in replacement in the fallback chain.
When subreddits are provided (from agent planning), searches each targeted
sub first, then does global search, and deduplicates across both. This
mirrors the SC search_and_enrich() flow where pre-resolved subreddits get
priority.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
subreddits: Optional list of subreddit names (without r/) for targeted search
Returns:
List of normalized item dicts matching ScrapeCreators output format.
"""
results = search(topic, depth=depth)
all_posts: List[Dict[str, Any]] = []
# Phase 1: Search targeted subreddits in parallel (if provided)
if subreddits:
_log(f"Searching {len(subreddits)} targeted subreddits: {subreddits}")
workers = min(4, len(subreddits))
with ThreadPoolExecutor(max_workers=workers) as executor:
futures = {
executor.submit(_search_subreddit, sub, topic, depth): sub
for sub in subreddits
}
for future in futures:
sub = futures[future]
try:
sub_posts = future.result(timeout=30)
_log(f" -> {len(sub_posts)} results from r/{sub}")
all_posts.extend(sub_posts)
except (Exception, FuturesTimeoutError) as e:
_log(f" -> r/{sub} failed: {e}")
# Phase 2: Global search
global_posts = search(topic, depth=depth)
all_posts.extend(global_posts)
# Deduplicate by URL (targeted results keep priority since they come first)
seen_urls: set = set()
results: List[Dict[str, Any]] = []
for post in all_posts:
if post["url"] not in seen_urls:
seen_urls.add(post["url"])
results.append(post)
# Date filter: keep posts in range or with unknown dates
filtered = []
@@ -252,6 +367,9 @@ def search_reddit_public(
reverse=True,
)
# Enrich top posts with comments
filtered = _enrich_posts(filtered, depth=depth)
# Re-index IDs
for i, item in enumerate(filtered):
item["id"] = f"R{i + 1}"
scripts/lib/relevance.py
+1 -1
View File
@@ -84,7 +84,7 @@ def token_overlap_relevance(
- a small precision term to penalize extra noise
- an exact phrase bonus
Generic tokens alone are capped below the post-retrieval 0.3 threshold.
Generic tokens alone are capped below typical relevance filter thresholds.
Args:
query: Search query
scripts/lib/render.py
+616 -992
View File
File diff suppressed because it is too large Load Diff
scripts/lib/rerank.py
+296
View File
@@ -0,0 +1,296 @@
"""Reranking with LLM-scored relevance and demotion of low-confidence candidates."""
from __future__ import annotations
import json
from . import http, providers, schema
INTENT_SCORING_HINTS: dict[str, str] = {
"comparison": (
"Prefer items that directly compare, contrast, or benchmark the entities"
" mentioned in the topic. Head-to-head comparisons score higher than items"
" covering only one entity."
),
"how_to": (
"Prefer tutorials, step-by-step guides, and practical demonstrations."
" Video walkthroughs and code examples score higher than theoretical discussion."
),
"prediction": (
"Prefer items with quantitative forecasts, odds, market data, or expert"
" predictions. Vague speculation scores lower."
),
"factual": (
"Prefer items with specific facts, dates, numbers, and primary sources."
" News reports with direct quotes score higher than commentary."
),
"opinion": (
"Prefer items with substantive opinions backed by reasoning or evidence."
" Hot takes without substance score lower."
),
"breaking_news": (
"Prefer the latest updates, eyewitness reports, and official statements."
" Recency matters more than depth."
),
"concept": (
"Prefer clear explanations with examples or analogies. Accessible content"
" scores higher than dense academic papers unless the topic is highly technical."
),
"product": (
"Prefer hands-on reviews, benchmarks, and user experience reports."
" Marketing copy and listicles score lower."
),
}
def rerank_candidates(
*,
topic: str,
plan: schema.QueryPlan,
candidates: list[schema.Candidate],
provider: providers.ReasoningClient | None,
model: str | None,
shortlist_size: int,
) -> list[schema.Candidate]:
"""Rerank the fused shortlist, demoting candidates the reranker scored as irrelevant."""
shortlisted = candidates[:shortlist_size]
if provider and model and shortlisted:
try:
response = provider.generate_json(model, _build_prompt(topic, plan, shortlisted))
_apply_llm_scores(shortlisted, response)
except (ValueError, KeyError, json.JSONDecodeError, OSError, http.HTTPError) as exc:
import sys
print(f"[Rerank] LLM reranking failed, using local fallback: {type(exc).__name__}: {exc}", file=sys.stderr)
_apply_fallback_scores(shortlisted)
else:
_apply_fallback_scores(shortlisted)
if len(candidates) > shortlist_size:
tail = candidates[shortlist_size:]
_apply_fallback_scores(tail)
return sorted(
candidates,
key=lambda candidate: (
-candidate.final_score,
-(candidate.engagement or -1),
min(candidate.native_ranks.values(), default=999),
candidate.title,
),
)
def _intent_hint_block(plan: schema.QueryPlan) -> str:
hint = INTENT_SCORING_HINTS.get(plan.intent, "")
if hint:
return f"\nIntent-specific guidance ({plan.intent}):\n- {hint}\n"
return ""
def _build_prompt(topic: str, plan: schema.QueryPlan, candidates: list[schema.Candidate]) -> str:
ranking_queries = "\n".join(
f"- {subquery.label}: {subquery.ranking_query}"
for subquery in plan.subqueries
)
candidate_block = "\n".join(
"\n".join(
[
f"- candidate_id: {candidate.candidate_id}",
f" sources: {schema.candidate_source_label(candidate)}",
f" title: {candidate.title[:220]}",
f" snippet: {candidate.snippet[:420]}",
f" date: {schema.candidate_best_published_at(candidate) or 'unknown'}",
f" matched_subqueries: {', '.join(candidate.subquery_labels)}",
]
)
for candidate in candidates
)
return f"""
Judge search-result relevance for a last-30-days research pipeline.
Topic: {topic}
Intent: {plan.intent}
Ranking queries:
{ranking_queries}
Return JSON only:
{{
"scores": [
{{
"candidate_id": "id",
"relevance": 0-100,
"reason": "short reason"
}}
]
}}
Scoring guidance:
- 90 to 100: one of the strongest pieces of evidence
- 70 to 89: clearly relevant and useful
- 40 to 69: somewhat relevant but weaker
- 0 to 39: weak, redundant, or off-target
{_intent_hint_block(plan)}
Candidates:
{candidate_block}
""".strip()
def _apply_llm_scores(candidates: list[schema.Candidate], payload: dict) -> None:
scores = {}
for row in payload.get("scores") or []:
if not isinstance(row, dict):
continue
candidate_id = str(row.get("candidate_id") or "").strip()
if not candidate_id:
continue
scores[candidate_id] = (
max(0.0, min(100.0, float(row.get("relevance") or 0.0))),
str(row.get("reason") or "").strip() or None,
)
for candidate in candidates:
rerank_score, reason = scores.get(candidate.candidate_id, _fallback_tuple(candidate))
candidate.rerank_score = rerank_score
candidate.explanation = reason
candidate.final_score = _final_score(candidate)
def _apply_fallback_scores(candidates: list[schema.Candidate]) -> None:
for candidate in candidates:
rerank_score, reason = _fallback_tuple(candidate)
candidate.rerank_score = rerank_score
candidate.explanation = reason
candidate.final_score = _final_score(candidate)
def _fallback_tuple(candidate: schema.Candidate) -> tuple[float, str]:
score = (
(candidate.local_relevance * 100.0 * 0.7)
+ (candidate.freshness * 0.2)
+ (candidate.source_quality * 100.0 * 0.1)
)
return max(0.0, min(100.0, score)), "fallback-local-score"
def _final_score(candidate: schema.Candidate) -> float:
normalized_rrf = _normalized_rrf(candidate.rrf_score)
rerank_score = candidate.rerank_score or 0.0
# Engagement bonus: high-engagement items (viral TikToks, popular YouTube videos)
# get a boost so they aren't buried by lower-engagement but text-relevant items.
# Engagement is log1p-normalized (0-100 range via signals.py), so a 2.5M-view
# TikTok scores ~15 and a 1500-view one scores ~7. The 0.05 weight gives a
# meaningful but not dominant boost.
engagement_val = candidate.engagement if candidate.engagement is not None else 0.0
base = (
0.60 * rerank_score
+ 0.20 * normalized_rrf
+ 0.10 * candidate.freshness
+ 0.05 * (candidate.source_quality * 100.0)
+ 0.05 * min(engagement_val * 6.0, 100.0)
)
if candidate.rerank_score is not None and candidate.rerank_score < 20.0:
base *= 0.3
return base
def score_fun(
*,
topic: str,
candidates: list[schema.Candidate],
provider: providers.ReasoningClient | None,
model: str | None,
max_candidates: int = 60,
) -> None:
"""Score candidates for humor, cleverness, and virality (the fun judge)."""
pool = candidates[:max_candidates]
if provider and model and pool:
try:
response = provider.generate_json(model, _build_fun_prompt(topic, pool))
_apply_fun_scores(pool, response)
except (ValueError, KeyError, json.JSONDecodeError, OSError, http.HTTPError) as exc:
import sys
print(f"[FunJudge] LLM scoring failed: {type(exc).__name__}: {exc}", file=sys.stderr)
_apply_fun_fallback(pool)
else:
_apply_fun_fallback(pool)
def _build_fun_prompt(topic: str, candidates: list[schema.Candidate]) -> str:
candidate_block = "\n".join(
"\n".join([
f"- candidate_id: {c.candidate_id}",
f" source: {schema.candidate_source_label(c)}",
f" title: {c.title[:220]}",
f" snippet: {c.snippet[:420]}",
f" comments: {_extract_comment_text(c)[:300]}",
])
for c in candidates
)
return (
"Score each item for humor, cleverness, wit, and shareability.\n"
"You are the fun judge. A press conference is 0. A one-liner that makes you laugh is 95.\n\n"
f"Topic: {topic}\n\n"
"Return JSON only:\n"
'{\n \"scores\": [{\"candidate_id\": \"id\", \"fun\": 0-100, \"reason\": \"short reason\"}]\n}\n\n'
"Scoring: 90-100=genuinely hilarious, 70-89=witty/clever, "
"40-69=has personality, 20-39=straight news, 0-19=dry/official.\n"
"Prefer SHORT PUNCHY content. A 15-word tweet > a 500-word analysis.\n\n"
f"Candidates:\n{candidate_block}"
)
def _extract_comment_text(candidate: schema.Candidate) -> str:
parts = []
for item in candidate.source_items:
for comment in item.metadata.get("top_comments", [])[:3]:
body = comment.get("body", "") if isinstance(comment, dict) else str(comment)
if body:
parts.append(body[:150])
for insight in item.metadata.get("comment_insights", [])[:2]:
if insight:
parts.append(str(insight)[:150])
return " | ".join(parts) if parts else ""
def _apply_fun_scores(candidates: list[schema.Candidate], payload: dict) -> None:
scores = {}
for row in payload.get("scores") or []:
if not isinstance(row, dict):
continue
cid = str(row.get("candidate_id") or "").strip()
if not cid:
continue
scores[cid] = (
max(0.0, min(100.0, float(row.get("fun") or 0.0))),
str(row.get("reason") or "").strip() or None,
)
for c in candidates:
if c.candidate_id in scores:
c.fun_score, c.fun_explanation = scores[c.candidate_id]
else:
_apply_single_fun_fallback(c)
def _apply_fun_fallback(candidates: list[schema.Candidate]) -> None:
for c in candidates:
_apply_single_fun_fallback(c)
def _apply_single_fun_fallback(candidate: schema.Candidate) -> None:
text = candidate.title + " " + (candidate.snippet or "") + " " + _extract_comment_text(candidate)
text_len = len(text.strip())
eng = candidate.engagement if candidate.engagement is not None else 0.0
shortness = max(0, (200 - text_len) / 200) * 30
eng_bonus = min(eng * 2.0, 40)
markers = ["lol", "lmao", "dead", "hilarious", "funny", "bruh", "ratio", "nah", "bro", "ain't no way", "i'm crying", "rent free"]
marker_bonus = 10 if any(m in text.lower() for m in markers) else 0
candidate.fun_score = max(0.0, min(100.0, shortness + eng_bonus + marker_bonus))
candidate.fun_explanation = "heuristic-fallback"
def _normalized_rrf(rrf_score: float) -> float:
# Empirical ceiling for normalized RRF scores at the pool sizes we use.
# Max single-stream RRF at rank 1 is 1/(K+1) ~ 0.016; multi-stream
# accumulation reaches ~0.08.
return max(0.0, min(100.0, (rrf_score / 0.08) * 100.0))
scripts/lib/resolve.py
+196
View File
@@ -0,0 +1,196 @@
"""Auto-resolve subreddits, X handles, and current events context for a topic.
Uses web search (Brave/Exa/Serper) to discover relevant communities and context
before the planner runs. This is the engine-side equivalent of SKILL.md Steps
0.55/0.75 which use Claude Code's WebSearch tool.
"""
from __future__ import annotations
import re
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from . import dates, grounding
def _log(msg: str) -> None:
print(f"[Resolve] {msg}", file=sys.stderr)
def _has_backend(config: dict) -> bool:
"""Check if any web search backend is available."""
return bool(
config.get("BRAVE_API_KEY")
or config.get("EXA_API_KEY")
or config.get("SERPER_API_KEY")
or config.get("PARALLEL_API_KEY")
or config.get("OPENROUTER_API_KEY")
)
def _extract_subreddits(items: list[dict]) -> list[str]:
"""Parse subreddit names from search result titles and snippets."""
pattern = re.compile(r"r/([A-Za-z0-9_]{2,21})")
seen: set[str] = set()
results: list[str] = []
for item in items:
text = f"{item.get('title', '')} {item.get('snippet', '')} {item.get('url', '')}"
for match in pattern.findall(text):
lower = match.lower()
if lower not in seen:
seen.add(lower)
results.append(match)
return results
def _extract_x_handle(items: list[dict]) -> str:
"""Extract the most likely X/Twitter handle from search results."""
pattern = re.compile(r"@([A-Za-z0-9_]{1,15})")
url_pattern = re.compile(r"(?:twitter\.com|x\.com)/([A-Za-z0-9_]{1,15})(?:/|$|\?)")
counts: dict[str, int] = {}
for item in items:
text = f"{item.get('title', '')} {item.get('snippet', '')}"
url = item.get("url", "")
for match in pattern.findall(text):
lower = match.lower()
counts[lower] = counts.get(lower, 0) + 1
for match in url_pattern.findall(url):
lower = match.lower()
# URL matches are stronger signals
counts[lower] = counts.get(lower, 0) + 3
# Filter out generic handles
skip = {"twitter", "x", "search", "hashtag", "intent", "share", "i", "home", "explore", "settings"}
counts = {k: v for k, v in counts.items() if k not in skip}
if not counts:
return ""
return max(counts, key=counts.get)
def _extract_github_user(items: list[dict]) -> str:
"""Extract GitHub username from search results."""
url_pattern = re.compile(r"github\.com/([A-Za-z0-9_-]{1,39})(?:/|$|\?)")
counts: dict[str, int] = {}
for item in items:
url = item.get("url", "")
text = f"{item.get('title', '')} {item.get('snippet', '')}"
for match in url_pattern.findall(url):
lower = match.lower()
counts[lower] = counts.get(lower, 0) + 3
for match in url_pattern.findall(text):
lower = match.lower()
counts[lower] = counts.get(lower, 0) + 1
# Filter out org/repo-like names and generic pages
skip = {"topics", "explore", "settings", "orgs", "search", "features", "about", "pricing", "enterprise"}
counts = {k: v for k, v in counts.items() if k not in skip}
if not counts:
return ""
return max(counts, key=counts.get)
def _extract_github_repos(items: list[dict]) -> list[str]:
"""Extract owner/repo strings from search results."""
repo_pattern = re.compile(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)")
skip_owners = {"topics", "explore", "settings", "orgs", "search", "features", "about", "pricing", "enterprise"}
seen: set[str] = set()
repos: list[str] = []
for item in items:
url = item.get("url", "")
text = f"{item.get('title', '')} {item.get('snippet', '')}"
for source in [url, text]:
for match in repo_pattern.findall(source):
owner = match.split("/")[0].lower()
if owner in skip_owners:
continue
lower = match.lower()
if lower not in seen:
seen.add(lower)
repos.append(match)
return repos[:5] # cap at 5 repos
def _build_context_summary(items: list[dict]) -> str:
"""Build a 1-2 sentence current events summary from news search results."""
snippets: list[str] = []
for item in items[:3]:
snippet = item.get("snippet", "").strip()
if snippet:
snippets.append(snippet)
if not snippets:
return ""
# Take the first two meaningful snippets and truncate to keep it concise
combined = " ".join(snippets[:2])
if len(combined) > 300:
combined = combined[:297] + "..."
return combined
def auto_resolve(topic: str, config: dict) -> dict:
"""Discover subreddits, X handles, and current events context for a topic.
Args:
topic: The research topic.
config: Dict with API keys (BRAVE_API_KEY, EXA_API_KEY, SERPER_API_KEY).
Returns:
Dict with keys: subreddits, x_handle, context, searches_run.
Returns empty result if no web search backend is available.
"""
empty = {"subreddits": [], "x_handle": "", "context": "", "searches_run": 0}
if not _has_backend(config):
_log("No web search backend available, skipping resolve")
return empty
from_date, to_date = dates.get_date_range(30)
date_range = (from_date, to_date)
now = datetime.now(timezone.utc)
current_month = now.strftime("%B")
current_year = now.strftime("%Y")
queries = {
"subreddit": f"{topic} subreddit reddit",
"news": f"{topic} news {current_month} {current_year}",
"x_handle": f"{topic} X twitter handle",
"github": f"{topic} github profile site:github.com",
}
results: dict[str, list[dict]] = {}
searches_run = 0
def _search(label: str, query: str) -> tuple[str, list[dict]]:
items, _artifact = grounding.web_search(query, date_range, config)
return label, items
with ThreadPoolExecutor(max_workers=3) as executor:
futures = {
executor.submit(_search, label, q): label
for label, q in queries.items()
}
for future in as_completed(futures):
label = futures[future]
try:
_label, items = future.result()
results[label] = items
searches_run += 1
except Exception as exc:
_log(f"Search failed for {label}: {exc}")
results[label] = []
subreddits = _extract_subreddits(results.get("subreddit", []))
x_handle = _extract_x_handle(results.get("x_handle", []))
github_user = _extract_github_user(results.get("github", []))
github_repos = _extract_github_repos(results.get("github", []))
context = _build_context_summary(results.get("news", []))
_log(f"Resolved {len(subreddits)} subreddits, x_handle={x_handle!r}, github_user={github_user!r}, github_repos={github_repos!r}, context_len={len(context)}")
return {
"subreddits": subreddits,
"x_handle": x_handle,
"github_user": github_user,
"github_repos": github_repos,
"context": context,
"searches_run": searches_run,
}
scripts/lib/schema.py
+288 -811
View File
File diff suppressed because it is too large Load Diff
scripts/lib/score.py
-775
View File
@@ -1,775 +0,0 @@
"""Popularity-aware scoring for last30days skill."""
import math
from typing import List, Optional, Union
from . import dates, schema
from .query_type import QueryType, WEBSEARCH_PENALTY_BY_TYPE, TIEBREAKER_BY_TYPE
# Score weights for Reddit/X (has engagement)
WEIGHT_RELEVANCE = 0.45
WEIGHT_RECENCY = 0.25
WEIGHT_ENGAGEMENT = 0.30
# Polymarket needs stronger semantic weighting because volume/liquidity already
# show up as engagement and lightly influence parse-time relevance.
PM_WEIGHT_RELEVANCE = 0.60
PM_WEIGHT_RECENCY = 0.20
PM_WEIGHT_ENGAGEMENT = 0.20
# WebSearch weights (no engagement data available)
WEBSEARCH_WEIGHT_RELEVANCE = 0.55
WEBSEARCH_WEIGHT_RECENCY = 0.45
# Default web search penalty (fallback when query_type is not provided).
# Per-type penalties in query_type.WEBSEARCH_PENALTY_BY_TYPE.
WEBSEARCH_SOURCE_PENALTY = 15
# WebSearch date confidence adjustments
WEBSEARCH_VERIFIED_BONUS = 10 # Bonus for URL-verified recent date (high confidence)
WEBSEARCH_NO_DATE_PENALTY = 20 # Heavy penalty for no date signals (low confidence)
# Default engagement score for unknown
DEFAULT_ENGAGEMENT = 35
UNKNOWN_ENGAGEMENT_PENALTY = 3
def log1p_safe(x: Optional[int]) -> float:
"""Safe log1p that handles None and negative values."""
if x is None or x < 0:
return 0.0
return math.log1p(x)
def compute_reddit_engagement_raw(
engagement: Optional[schema.Engagement],
top_comment_score: Optional[int] = None,
) -> Optional[float]:
"""Compute raw engagement score for Reddit item.
Formula: 0.50*log1p(score) + 0.35*log1p(num_comments) + 0.05*(upvote_ratio*10) + 0.10*log1p(top_comment_score)
The 10% comment quality weight rewards posts where the community engaged deeply
— a highly upvoted top comment means the thread sparked real discussion.
"""
if engagement is None:
return None
if engagement.score is None and engagement.num_comments is None:
return None
score = log1p_safe(engagement.score)
comments = log1p_safe(engagement.num_comments)
ratio = (engagement.upvote_ratio or 0.5) * 10
top_cmt = log1p_safe(top_comment_score)
return 0.50 * score + 0.35 * comments + 0.05 * ratio + 0.10 * top_cmt
def compute_x_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for X item.
Formula: 0.55*log1p(likes) + 0.25*log1p(reposts) + 0.15*log1p(replies) + 0.05*log1p(quotes)
"""
if engagement is None:
return None
if engagement.likes is None and engagement.reposts is None:
return None
likes = log1p_safe(engagement.likes)
reposts = log1p_safe(engagement.reposts)
replies = log1p_safe(engagement.replies)
quotes = log1p_safe(engagement.quotes)
return 0.55 * likes + 0.25 * reposts + 0.15 * replies + 0.05 * quotes
def normalize_to_100(values: List[float], default: float = 50) -> List[float]:
"""Normalize a list of values to 0-100 scale.
Args:
values: Raw values (None values are preserved)
default: Default value for None entries
Returns:
Normalized values
"""
# Filter out None
valid = [v for v in values if v is not None]
if not valid:
return [default if v is None else 50 for v in values]
min_val = min(valid)
max_val = max(valid)
range_val = max_val - min_val
if range_val == 0:
return [50 if v is None else 50 for v in values]
result = []
for v in values:
if v is None:
result.append(None)
else:
normalized = ((v - min_val) / range_val) * 100
result.append(normalized)
return result
def score_reddit_items(items: List[schema.RedditItem]) -> List[schema.RedditItem]:
"""Compute scores for Reddit items.
Args:
items: List of Reddit items
Returns:
Items with updated scores
"""
if not items:
return items
# Compute raw engagement scores (with top comment quality signal)
eng_raw = []
for item in items:
top_cmt_score = None
if item.top_comments:
top_cmt_score = item.top_comments[0].score
eng_raw.append(compute_reddit_engagement_raw(item.engagement, top_cmt_score))
# Normalize engagement to 0-100
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
# Relevance subscore (model-provided, convert to 0-100)
rel_score = int(item.relevance * 100)
# Recency subscore
rec_score = dates.recency_score(item.date)
# Engagement subscore
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
# Store subscores
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
# Compute overall score
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
# Apply penalty for unknown engagement
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
# Apply penalty for low date confidence
if item.date_confidence == "low":
overall -= 5
elif item.date_confidence == "med":
overall -= 2
item.score = max(0, min(100, int(overall)))
return items
def score_x_items(items: List[schema.XItem]) -> List[schema.XItem]:
"""Compute scores for X items.
Args:
items: List of X items
Returns:
Items with updated scores
"""
if not items:
return items
# Compute raw engagement scores
eng_raw = [compute_x_engagement_raw(item.engagement) for item in items]
# Normalize engagement to 0-100
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
# Relevance subscore (model-provided, convert to 0-100)
rel_score = int(item.relevance * 100)
# Recency subscore
rec_score = dates.recency_score(item.date)
# Engagement subscore
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
# Store subscores
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
# Compute overall score
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
# Apply penalty for unknown engagement
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
# Apply penalty for low date confidence
if item.date_confidence == "low":
overall -= 5
elif item.date_confidence == "med":
overall -= 2
item.score = max(0, min(100, int(overall)))
return items
def compute_youtube_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for YouTube item.
Formula: 0.50*log1p(views) + 0.35*log1p(likes) + 0.15*log1p(comments)
Views dominate on YouTube — they're the primary discovery signal.
"""
if engagement is None:
return None
if engagement.views is None and engagement.likes is None:
return None
views = log1p_safe(engagement.views)
likes = log1p_safe(engagement.likes)
comments = log1p_safe(engagement.num_comments)
return 0.50 * views + 0.35 * likes + 0.15 * comments
def score_youtube_items(items: List[schema.YouTubeItem]) -> List[schema.YouTubeItem]:
"""Compute scores for YouTube items.
Uses same weight structure as Reddit/X (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_youtube_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def compute_tiktok_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for TikTok item.
Formula: 0.50*log1p(views) + 0.30*log1p(likes) + 0.20*log1p(comments)
Views dominate on TikTok — they're the primary discovery signal.
"""
if engagement is None:
return None
if engagement.views is None and engagement.likes is None:
return None
views = log1p_safe(engagement.views)
likes = log1p_safe(engagement.likes)
comments = log1p_safe(engagement.num_comments)
return 0.50 * views + 0.30 * likes + 0.20 * comments
def score_tiktok_items(items: List[schema.TikTokItem]) -> List[schema.TikTokItem]:
"""Compute scores for TikTok items.
Uses same weight structure as YouTube (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_tiktok_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def compute_instagram_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Instagram item.
Formula: 0.50*log1p(views) + 0.30*log1p(likes) + 0.20*log1p(comments)
Views dominate on Instagram Reels — they're the primary discovery signal.
"""
if engagement is None:
return None
if engagement.views is None and engagement.likes is None:
return None
views = log1p_safe(engagement.views)
likes = log1p_safe(engagement.likes)
comments = log1p_safe(engagement.num_comments)
return 0.50 * views + 0.30 * likes + 0.20 * comments
def score_instagram_items(items: List[schema.InstagramItem]) -> List[schema.InstagramItem]:
"""Compute scores for Instagram items.
Uses same weight structure as TikTok (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_instagram_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def compute_hackernews_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Hacker News item.
Formula: 0.55*log1p(points) + 0.45*log1p(num_comments)
Points are the primary signal on HN; comments indicate depth of discussion.
"""
if engagement is None:
return None
if engagement.score is None and engagement.num_comments is None:
return None
points = log1p_safe(engagement.score)
comments = log1p_safe(engagement.num_comments)
return 0.55 * points + 0.45 * comments
def score_hackernews_items(items: List[schema.HackerNewsItem]) -> List[schema.HackerNewsItem]:
"""Compute scores for Hacker News items.
Uses same weight structure as Reddit/X (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_hackernews_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def compute_bluesky_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Bluesky item.
Formula: 0.40*log1p(likes) + 0.30*log1p(reposts) + 0.20*log1p(replies) + 0.10*log1p(quotes)
Likes are primary signal; reposts indicate reach; replies indicate discussion depth.
"""
if engagement is None:
return None
if engagement.likes is None and engagement.reposts is None:
return None
likes = log1p_safe(engagement.likes)
reposts = log1p_safe(engagement.reposts)
replies = log1p_safe(engagement.replies)
quotes = log1p_safe(engagement.quotes)
return 0.40 * likes + 0.30 * reposts + 0.20 * replies + 0.10 * quotes
def score_bluesky_items(items: List[schema.BlueskyItem]) -> List[schema.BlueskyItem]:
"""Compute scores for Bluesky items.
Uses same weight structure as Reddit/X (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_bluesky_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def compute_truthsocial_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Truth Social item.
Formula: 0.45*log1p(likes) + 0.30*log1p(reposts) + 0.25*log1p(replies)
Likes are primary signal; reposts indicate reach; replies indicate discussion.
"""
if engagement is None:
return None
if engagement.likes is None and engagement.reposts is None:
return None
likes = log1p_safe(engagement.likes)
reposts = log1p_safe(engagement.reposts)
replies = log1p_safe(engagement.replies)
return 0.45 * likes + 0.30 * reposts + 0.25 * replies
def score_truthsocial_items(items: List[schema.TruthSocialItem]) -> List[schema.TruthSocialItem]:
"""Compute scores for Truth Social items."""
if not items:
return items
eng_raw = [compute_truthsocial_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Polymarket item.
Formula: 0.60*log1p(volume) + 0.40*log1p(liquidity)
Volume is the primary signal (money flowing); liquidity indicates market depth.
"""
if engagement is None:
return None
if engagement.volume is None and engagement.liquidity is None:
return None
volume = math.log1p(engagement.volume or 0)
liquidity = math.log1p(engagement.liquidity or 0)
return 0.60 * volume + 0.40 * liquidity
def score_polymarket_items(items: List[schema.PolymarketItem]) -> List[schema.PolymarketItem]:
"""Compute scores for Polymarket items.
Uses same weight structure as Reddit/X (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_polymarket_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
PM_WEIGHT_RELEVANCE * rel_score +
PM_WEIGHT_RECENCY * rec_score +
PM_WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def score_websearch_items(items: List[schema.WebSearchItem], query_type: QueryType = None) -> List[schema.WebSearchItem]:
"""Compute scores for WebSearch items WITHOUT engagement metrics.
Uses reweighted formula: 55% relevance + 45% recency - penalty.
Penalty varies by query type: concept queries get 0 penalty (web docs
are authoritative), while product/opinion queries get full 15pt penalty
(social discussion is more valuable).
Args:
items: List of WebSearch items
query_type: Query classification for penalty adjustment
Returns:
Items with updated scores
"""
if not items:
return items
for item in items:
# Relevance subscore (model-provided, convert to 0-100)
rel_score = int(item.relevance * 100)
# Recency subscore
rec_score = dates.recency_score(item.date)
# Store subscores (engagement is 0 for WebSearch - no data)
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=0, # Explicitly zero - no engagement data available
)
# Compute overall score using WebSearch weights
overall = (
WEBSEARCH_WEIGHT_RELEVANCE * rel_score +
WEBSEARCH_WEIGHT_RECENCY * rec_score
)
# Apply source penalty (varies by query type)
penalty = WEBSEARCH_PENALTY_BY_TYPE.get(query_type, WEBSEARCH_SOURCE_PENALTY) if query_type else WEBSEARCH_SOURCE_PENALTY
overall -= penalty
# Apply date confidence adjustments
# High confidence (URL-verified): reward with bonus
# Med confidence (snippet-extracted): neutral
# Low confidence (no date signals): heavy penalty
if item.date_confidence == "high":
overall += WEBSEARCH_VERIFIED_BONUS # Reward verified recent dates
elif item.date_confidence == "low":
overall -= WEBSEARCH_NO_DATE_PENALTY # Heavy penalty for unknown
item.score = max(0, min(100, int(overall)))
return items
_ITEM_SOURCE_MAP = {
schema.RedditItem: "reddit",
schema.XItem: "x",
schema.YouTubeItem: "youtube",
schema.TikTokItem: "tiktok",
schema.InstagramItem: "instagram",
schema.HackerNewsItem: "hn",
schema.BlueskyItem: "bluesky",
schema.TruthSocialItem: "truthsocial",
schema.PolymarketItem: "polymarket",
}
_DEFAULT_TIEBREAKER = {"reddit": 0, "x": 1, "youtube": 2, "tiktok": 3, "instagram": 4, "hn": 5, "bluesky": 6, "truthsocial": 7, "polymarket": 8, "web": 9}
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.BlueskyItem, schema.TruthSocialItem, schema.PolymarketItem]], query_type: QueryType = None) -> List:
"""Sort items by score (descending), then date, then source tiebreaker.
Tiebreaker (tertiary sort key, after score and date): source priority
varies by query type. YouTube ranks first for how_to, X ranks first
for breaking_news, Polymarket ranks first for prediction.
Args:
items: List of items to sort
query_type: Query classification for tiebreaker adjustment
Returns:
Sorted items
"""
tiebreaker = TIEBREAKER_BY_TYPE.get(query_type, _DEFAULT_TIEBREAKER) if query_type else _DEFAULT_TIEBREAKER
def sort_key(item):
# Primary: score descending (negate for descending)
score = -item.score
# Secondary: date descending (recent first)
date = item.date or "0000-00-00"
date_key = -int(date.replace("-", ""))
# Tertiary: query-type-aware source priority
source_name = _ITEM_SOURCE_MAP.get(type(item), "web")
source_priority = tiebreaker.get(source_name, 99)
# Quaternary: title/text for stability
text = getattr(item, "title", "") or getattr(item, "text", "")
return (score, date_key, source_priority, text)
return sorted(items, key=sort_key)
def relevance_filter(items, source_name: str, threshold: float = 0.3):
"""Filter items below relevance threshold with minimum-result guarantee.
Items with no relevance attribute are treated as 0.0 (fail the filter).
If all items are below threshold, keeps the top 3 by relevance.
Lists with 3 or fewer items are returned unchanged.
"""
import sys
if len(items) <= 3:
return items
passed = [i for i in items if getattr(i, 'relevance', 0.0) >= threshold]
if not passed:
print(f"[{source_name} WARNING] All results below relevance {threshold}, keeping top 3", file=sys.stderr)
by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.0), reverse=True)
return by_rel[:3]
return passed
scripts/lib/scrapecreators_x.py
-182
View File
@@ -1,182 +0,0 @@
"""X/Twitter search via ScrapeCreators API for /last30days.
Uses ScrapeCreators REST API to search Twitter/X by keyword.
Same API key as Reddit, TikTok, and Instagram - one key covers all social sources.
Requires SCRAPECREATORS_API_KEY in config.
API docs: https://scrapecreators.com/docs
"""
import sys
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
try:
import requests as _requests
except ImportError:
_requests = None
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/twitter"
DEPTH_CONFIG = {
"quick": {"results_per_page": 10},
"default": {"results_per_page": 20},
"deep": {"results_per_page": 40},
}
from .relevance import token_overlap_relevance as _compute_relevance
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Twitter search."""
from .query import extract_core_subject
_SC_X_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice',
})
return extract_core_subject(topic, noise=_SC_X_NOISE)
def _log(msg: str):
if sys.stderr.isatty():
sys.stderr.write(f"[X/SC] {msg}\n")
sys.stderr.flush()
def _sc_headers(token: str) -> Dict[str, str]:
return {
"x-api-key": token,
"Content-Type": "application/json",
}
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
"""Parse date from ScrapeCreators Twitter item to YYYY-MM-DD."""
# Try created_at string (e.g. "Wed Oct 10 20:19:24 +0000 2018")
created_at = item.get("created_at")
if created_at and isinstance(created_at, str):
try:
dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError):
pass
# Try unix timestamp
ts = item.get("timestamp") or item.get("created_at_timestamp")
if ts:
try:
dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError, OSError):
pass
# Try ISO format
for key in ("created_at", "date"):
val = item.get(key)
if val and isinstance(val, str):
try:
dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError):
pass
return None
def search_x(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
token: str = None,
) -> Dict[str, Any]:
"""Search X/Twitter via ScrapeCreators API.
Returns:
Dict with 'items' list (in normalize_x_items format) and optional 'error'.
"""
if not token:
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
if not _requests:
return {"items": [], "error": "requests library not installed"}
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
core_topic = _extract_core_subject(topic)
_log(f"Searching X for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search/tweets",
params={"query": core_topic, "sort_by": "relevance"},
headers=_sc_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
raw_items = data.get("tweets") or data.get("data") or data.get("results") or []
raw_items = raw_items[:config["results_per_page"]]
items = []
for i, raw in enumerate(raw_items):
tweet_id = str(raw.get("id") or raw.get("tweet_id") or raw.get("id_str") or f"sc-x-{i}")
text = raw.get("full_text") or raw.get("text") or ""
user = raw.get("user") or raw.get("author") or {}
author_handle = user.get("screen_name") or user.get("username") or ""
# Engagement metrics
likes = raw.get("favorite_count") or raw.get("likes") or 0
retweets = raw.get("retweet_count") or raw.get("retweets") or 0
replies = raw.get("reply_count") or raw.get("replies") or 0
quotes = raw.get("quote_count") or raw.get("quotes") or 0
date_str = _parse_date(raw)
relevance = _compute_relevance(core_topic, text)
url = ""
if author_handle and tweet_id and not tweet_id.startswith("sc-x-"):
url = f"https://x.com/{author_handle}/status/{tweet_id}"
items.append({
"id": tweet_id,
"text": text,
"url": url,
"author_handle": author_handle,
"date": date_str,
"engagement": {
"likes": likes,
"reposts": retweets,
"replies": replies,
"quotes": quotes,
},
"relevance": relevance,
"why_relevant": f"X: @{author_handle}: {text[:60]}" if text else f"X: {core_topic}",
})
# Date filter
in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
out_of_range = len(items) - len(in_range)
if in_range:
items = in_range
if out_of_range:
_log(f"Filtered {out_of_range} tweets outside date range")
else:
_log(f"No tweets within date range, keeping all {len(items)}")
# Sort by engagement (likes + retweets)
items.sort(key=lambda x: (x["engagement"]["likes"] + x["engagement"]["reposts"]), reverse=True)
_log(f"Found {len(items)} tweets")
return {"items": items}
def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse search response to normalized format."""
return response.get("items", [])
scripts/lib/setup_wizard.py
+350 -1
View File
@@ -5,11 +5,15 @@ and writes configuration. The actual wizard UI is SKILL.md-driven (the LLM
presents it), but this module provides the detection and setup actions.
"""
import json
import logging
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any, Dict, Optional
from typing import Any, Dict, Optional, Tuple
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
logger = logging.getLogger(__name__)
@@ -184,3 +188,348 @@ def get_setup_status_text(results: Dict[str, Any]) -> str:
lines.append("Configuration saved. Future runs will auto-detect your browsers.")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# OpenClaw server-side setup (no browser, JSON output)
# ---------------------------------------------------------------------------
_OPENCLAW_KEY_NAMES = [
"SCRAPECREATORS_API_KEY",
"XAI_API_KEY",
"BRAVE_API_KEY",
"EXA_API_KEY",
"SERPER_API_KEY",
"OPENAI_API_KEY",
"AUTH_TOKEN",
]
def run_openclaw_setup(config: Dict[str, Any]) -> Dict[str, Any]:
"""Server-side setup probe: no cookies, just tool + key availability.
Returns a dict suitable for JSON output to stdout so that SKILL.md
can present appropriate options to the user.
"""
yt_dlp = shutil.which("yt-dlp") is not None
node = shutil.which("node") is not None
python3 = shutil.which("python3") is not None
keys: Dict[str, bool] = {}
for key_name in _OPENCLAW_KEY_NAMES:
short = key_name.lower().replace("_api_key", "").replace("_key", "").replace("_token", "")
# Normalize: AUTH_TOKEN -> auth, SCRAPECREATORS_API_KEY -> scrapecreators
keys[short] = bool(config.get(key_name))
# Determine x_method
if config.get("XAI_API_KEY"):
x_method: Optional[str] = "xai"
elif config.get("AUTH_TOKEN") and config.get("CT0"):
x_method = "cookies"
else:
x_method = None
return {
"yt_dlp": yt_dlp,
"node": node,
"python3": python3,
"keys": keys,
"x_method": x_method,
}
# ---------------------------------------------------------------------------
# PAT auth flow (GitHub token via ScrapeCreators)
# ---------------------------------------------------------------------------
_PAT_BASE = "https://api.scrapecreators.com/v1/github/pat"
def auth_with_pat(github_token: str) -> Optional[Dict[str, Any]]:
"""Authenticate with ScrapeCreators using a GitHub PAT.
POSTs the token to the PAT auth endpoint. ScrapeCreators verifies it
against GitHub's API, creates/finds the account, and returns an API key.
Returns:
Dict with api_key, github_username, etc. on success, None on failure.
"""
try:
req = Request(f"{_PAT_BASE}/auth", data=b"", method="POST")
req.add_header("Authorization", f"Bearer {github_token}")
with urlopen(req, timeout=15) as resp:
data = json.loads(resp.read())
except HTTPError as exc:
if exc.code == 422:
logger.warning("PAT auth: insufficient scope — user needs user:email")
else:
logger.warning("PAT auth failed: %s", exc)
return None
except (URLError, OSError) as exc:
logger.warning("PAT auth request failed: %s", exc)
return None
if not data.get("api_key"):
logger.warning("PAT auth returned no api_key: %s", data)
return None
return data
# ---------------------------------------------------------------------------
# Device auth flow (GitHub OAuth via ScrapeCreators)
# ---------------------------------------------------------------------------
_DEVICE_BASE = "https://api.scrapecreators.com/v1/github/device"
def run_device_auth() -> Optional[Tuple[str, str, str, int]]:
"""Start the device authorization flow.
POSTs to the ScrapeCreators device/code endpoint.
Returns:
(device_code, user_code, verification_uri, interval) on success,
None on failure.
"""
try:
body = json.dumps({}).encode()
req = Request(f"{_DEVICE_BASE}/code", data=body, method="POST")
req.add_header("Content-Type", "application/json")
with urlopen(req, timeout=15) as resp:
data = json.loads(resp.read())
except (HTTPError, URLError, OSError) as exc:
logger.warning("Device auth code request failed: %s", exc)
return None
device_code = data.get("device_code")
user_code = data.get("user_code")
verification_uri = data.get("verification_uri")
interval = data.get("interval", 5)
if not device_code or not user_code:
logger.warning("Device auth returned incomplete response: %s", data)
return None
return (device_code, user_code, verification_uri or "", interval)
def poll_device_auth(
device_code: str,
interval: int,
timeout: int = 300,
user_code: str = "",
clipboard_ok: bool = False,
) -> Optional[str]:
"""Poll for an access token after the user authorizes the device.
Args:
device_code: The device_code from run_device_auth().
interval: Polling interval in seconds.
timeout: Maximum time to poll in seconds.
user_code: The user code to remind about during polling.
clipboard_ok: Whether the code was copied to clipboard.
Returns:
access_token on success, None on timeout or failure.
"""
import sys
deadline = time.time() + timeout
last_reminder = time.time()
reminder_count = 0
max_reminders = 4
reminder_interval = 30 # seconds between reminders
while time.time() < deadline:
time.sleep(interval)
# Periodic reminder of the code while waiting
if (
user_code
and reminder_count < max_reminders
and time.time() - last_reminder >= reminder_interval
):
clipboard_hint = " (on your clipboard)" if clipboard_ok else ""
print(
f" Still waiting... Your code: {user_code}{clipboard_hint}",
file=sys.stderr,
flush=True,
)
last_reminder = time.time()
reminder_count += 1
try:
body = json.dumps({"device_code": device_code}).encode()
req = Request(f"{_DEVICE_BASE}/token", data=body, method="POST")
req.add_header("Content-Type", "application/json")
with urlopen(req, timeout=15) as resp:
data = json.loads(resp.read())
except HTTPError as exc:
if exc.code in (400, 403, 428):
continue
logger.warning("Device auth poll error: %s", exc)
return None
except (URLError, OSError):
continue
if data.get("access_token"):
return data["access_token"]
error = data.get("error")
if error == "slow_down":
interval = min(interval + 2, 30)
continue
if error == "authorization_pending":
continue
if error in ("expired_token", "access_denied"):
logger.warning("Device auth failed: %s", error)
return None
return None
def fetch_api_key(access_token: str) -> Optional[str]:
"""Fetch the ScrapeCreators API key using the GitHub access token.
GETs the device/profile endpoint with Bearer auth.
Returns:
api_key string on success, None on failure.
"""
try:
req = Request(f"{_DEVICE_BASE}/profile")
req.add_header("Authorization", f"Bearer {access_token}")
with urlopen(req, timeout=15) as resp:
data = json.loads(resp.read())
except (HTTPError, URLError, OSError) as exc:
logger.warning("Failed to fetch API key: %s", exc)
return None
return data.get("api_key")
def run_full_device_auth(timeout: int = 300) -> Dict[str, Any]:
"""Run the complete GitHub device auth flow and return JSON-serializable result.
Chains: start device flow -> open browser -> poll -> fetch API key.
Designed to be called from the CLI and have its stdout parsed by the LLM.
Returns:
Dict with status and relevant fields:
- {"status": "success", "api_key": "sc_...", "user_code": "ABCD-1234"}
- {"status": "error", "message": "..."}
- {"status": "timeout", "user_code": "ABCD-1234"}
- {"status": "denied"}
"""
import webbrowser
# Step 1: Start device flow
result = run_device_auth()
if result is None:
return {"status": "error", "message": "Failed to start device auth flow"}
device_code, user_code, verification_uri, interval = result
import sys
# Step 2: Copy code to clipboard BEFORE opening browser
clipboard_ok = False
if sys.platform == "darwin":
try:
subprocess.run(
["pbcopy"], input=user_code.encode(), check=True, timeout=5,
)
clipboard_ok = True
except Exception:
pass # pbcopy unavailable or failed, fall through
# Step 3: Show code prominently, then open browser
clipboard_hint = " (copied to clipboard)" if clipboard_ok else ""
code_line = f" Your code: {user_code}{clipboard_hint}"
action_line = " Paste it on the GitHub page that just opened"
width = max(len(code_line), len(action_line)) + 2
border = "-" * width
print(f"\n+{border}+", file=sys.stderr)
print(f"|{code_line.ljust(width)}|", file=sys.stderr)
print(f"|{action_line.ljust(width)}|", file=sys.stderr)
print(f"+{border}+", file=sys.stderr)
if verification_uri:
try:
webbrowser.open(verification_uri)
except Exception:
print(f"Open: {verification_uri}", file=sys.stderr)
print("Waiting for authorization...", file=sys.stderr, flush=True)
# Step 4: Poll for token (with periodic code reminders)
access_token = poll_device_auth(
device_code, interval, timeout=timeout,
user_code=user_code, clipboard_ok=clipboard_ok,
)
if access_token is None:
return {"status": "timeout", "user_code": user_code, "clipboard_ok": clipboard_ok}
# Step 4: Fetch API key
api_key = fetch_api_key(access_token)
if api_key is None:
return {
"status": "error",
"message": "Authorized but failed to fetch API key",
"clipboard_ok": clipboard_ok,
}
return {"status": "success", "method": "device", "api_key": api_key, "user_code": user_code, "clipboard_ok": clipboard_ok}
# ---------------------------------------------------------------------------
# Unified GitHub auth: PAT first, device flow fallback
# ---------------------------------------------------------------------------
def run_github_auth(timeout: int = 300) -> Dict[str, Any]:
"""Try PAT auth via gh CLI, fall back to device flow.
1. Check for `gh` CLI
2. If found, run `gh auth token` to get a PAT
3. POST PAT to ScrapeCreators — if it works, done
4. If PAT fails for any reason, fall through to device flow
Returns JSON-serializable dict with status, method, and api_key.
"""
import sys
# Step 1: Try PAT via gh CLI
gh_path = shutil.which("gh")
if gh_path:
try:
result = subprocess.run(
["gh", "auth", "token"],
capture_output=True, text=True, timeout=10,
)
if result.returncode == 0 and result.stdout.strip():
token = result.stdout.strip()
print("Found gh CLI — trying PAT auth...", file=sys.stderr)
pat_result = auth_with_pat(token)
if pat_result and pat_result.get("api_key"):
return {
"status": "success",
"method": "pat",
"api_key": pat_result["api_key"],
"github_username": pat_result.get("github_username", ""),
}
# PAT failed — might be insufficient scope
print(
"PAT auth didn't work (scope or endpoint issue). "
"Falling back to GitHub device flow...",
file=sys.stderr,
)
except Exception as exc:
logger.debug("gh auth token failed: %s", exc)
# Step 2: Fall back to device flow
if not gh_path:
print("gh CLI not found — using GitHub device flow...", file=sys.stderr)
return run_full_device_auth(timeout=timeout)
scripts/lib/signals.py
+221
View File
@@ -0,0 +1,221 @@
"""Reusable local scoring signals for v3 pipeline stages."""
from __future__ import annotations
import math
from . import dates, relevance, schema
# Editorial signal-to-noise scores. Grounding (Google Search) is 1.0 baseline;
# social platforms discounted for noise.
SOURCE_QUALITY = {
"xiaohongshu": 0.7,
"hackernews": 0.8,
"youtube": 0.85,
"reddit": 0.6,
"x": 0.68,
"bluesky": 0.66,
"truthsocial": 0.6,
"polymarket": 0.5,
"instagram": 0.58,
"tiktok": 0.58,
}
def source_quality(source: str) -> float:
return SOURCE_QUALITY.get(source, 0.6)
def local_relevance(item: schema.SourceItem, ranking_query: str) -> float:
text = "\n".join(
part
for part in [item.title, item.body, item.snippet]
if part
)
hashtags = item.metadata.get("hashtags") if isinstance(item.metadata, dict) else None
score = relevance.token_overlap_relevance(ranking_query, text, hashtags=hashtags)
# High-engagement YouTube floor: official videos with millions of views
# often have titles that don't keyword-match the query (e.g., "YE - FATHER
# (feat. TRAVIS SCOTT)" doesn't match "kanye west"). The engagement signals
# say "this is important" even when text overlap is weak.
if item.source == "youtube" and item.engagement.get("views", 0) > 100_000:
score = max(score, 0.3)
# Project-mode GitHub floor: items fetched via --github-repo are explicitly
# requested by the user and relevant by construction. Without this floor,
# repos with low token diversity (e.g., "openclaw/openclaw" -> 1 unique token)
# get pruned despite being the primary search target.
labels = item.metadata.get("labels", []) if isinstance(item.metadata, dict) else []
if "project-mode" in labels:
score = max(score, 0.8)
return score
def freshness(item: schema.SourceItem, freshness_mode: str = "balanced_recent") -> int:
score = dates.recency_score(item.published_at)
if freshness_mode == "strict_recent":
return int(score)
if freshness_mode == "evergreen_ok":
return int((score * 0.6) + 40)
return int((score * 0.8) + 10)
def log1p_safe(value: float | int | None) -> float:
if value is None:
return 0.0
try:
numeric = float(value)
except (TypeError, ValueError):
return 0.0
if numeric <= 0:
return 0.0
return math.log1p(numeric)
def _top_comment_score(item: schema.SourceItem) -> float:
comments = item.metadata.get("top_comments") or []
if not comments or not isinstance(comments[0], dict):
return 0.0
return log1p_safe(comments[0].get("score"))
# Per-source engagement weights: list of (field_name, weight) tuples.
# Reddit uses a custom function because upvote_ratio and top_comment_score
# are not simple log1p fields.
ENGAGEMENT_WEIGHTS: dict[str, list[tuple[str, float]]] = {
"x": [("likes", 0.55), ("reposts", 0.25), ("replies", 0.15), ("quotes", 0.05)],
"youtube": [("views", 0.50), ("likes", 0.35), ("comments", 0.15)],
"tiktok": [("views", 0.50), ("likes", 0.30), ("comments", 0.20)],
"instagram": [("views", 0.50), ("likes", 0.30), ("comments", 0.20)],
"hackernews": [("points", 0.55), ("comments", 0.45)],
"bluesky": [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)],
"truthsocial": [("likes", 0.45), ("reposts", 0.30), ("replies", 0.25)],
"polymarket": [("volume", 0.60), ("liquidity", 0.40)],
}
def _weighted_engagement(item: schema.SourceItem, weights: list[tuple[str, float]]) -> float | None:
values = [(log1p_safe(item.engagement.get(field)), weight) for field, weight in weights]
if not any(v for v, _ in values):
return None
return sum(v * w for v, w in values)
def _reddit_engagement(item: schema.SourceItem) -> float | None:
score = log1p_safe(item.engagement.get("score"))
comments = log1p_safe(item.engagement.get("num_comments"))
ratio = float(item.engagement.get("upvote_ratio") or 0.0)
top_comment = _top_comment_score(item)
if not any([score, comments, ratio, top_comment]):
return None
return (0.50 * score) + (0.35 * comments) + (0.05 * (ratio * 10.0)) + (0.10 * top_comment)
def _generic_engagement(item: schema.SourceItem) -> float | None:
if not item.engagement:
return None
values = [logged for v in item.engagement.values() if (logged := log1p_safe(v)) > 0]
if not values:
return None
return sum(values) / len(values)
def engagement_raw(item: schema.SourceItem) -> float | None:
if item.source == "reddit":
return _reddit_engagement(item)
weights = ENGAGEMENT_WEIGHTS.get(item.source)
if weights:
return _weighted_engagement(item, weights)
return _generic_engagement(item)
def normalize(values: list[float | None]) -> list[int | None]:
valid = [value for value in values if value is not None]
if not valid:
return [None for _ in values]
low = min(valid)
high = max(valid)
if math.isclose(low, high):
return [50 if value is not None else None for value in values]
return [
None
if value is None
else int(((value - low) / (high - low)) * 100)
for value in values
]
def annotate_stream(
items: list[schema.SourceItem],
ranking_query: str,
freshness_mode: str,
) -> list[schema.SourceItem]:
"""Attach local scoring metadata and return items sorted by local_rank_score."""
engagement_scores = normalize([engagement_raw(item) for item in items])
for item, eng_score in zip(items, engagement_scores, strict=True):
item.local_relevance = local_relevance(item, ranking_query)
item.freshness = freshness(item, freshness_mode)
item.engagement_score = eng_score
item.source_quality = source_quality(item.source)
item.local_rank_score = (
0.65 * item.local_relevance
+ 0.25 * (item.freshness / 100.0)
+ 0.10 * ((eng_score or 0) / 100.0)
)
return sorted(items, key=lambda item: item.local_rank_score or 0, reverse=True)
_SOCIAL_SOURCES = {"reddit", "x", "tiktok", "instagram", "bluesky", "truthsocial"}
# Minimum view count for short-video platforms. Items below this floor
# are typically spam reposts or low-effort clips that add no unique signal.
_VIDEO_ENGAGEMENT_FLOOR_SOURCES = {"tiktok", "instagram"}
_VIDEO_ENGAGEMENT_FLOOR_VIEWS = 1000
def _passes_engagement_floor(item: schema.SourceItem, sole_source: bool) -> bool:
"""Check whether a TikTok/Instagram item meets the minimum view floor.
Items from sources not in _VIDEO_ENGAGEMENT_FLOOR_SOURCES always pass.
If the item's source is the *only* source represented in the batch
(sole_source=True), all items pass so we never return an empty result
for a whole source.
"""
if item.source not in _VIDEO_ENGAGEMENT_FLOOR_SOURCES:
return True
if sole_source:
return True
views = item.engagement.get("views", 0) if item.engagement else 0
return views >= _VIDEO_ENGAGEMENT_FLOOR_VIEWS
def prune_low_relevance(
items: list[schema.SourceItem],
minimum: float = 0.15,
) -> list[schema.SourceItem]:
"""Drop weak lexical matches when stronger evidence exists.
Social-source items with zero engagement get a stricter threshold
because zero engagement on a social platform is a strong noise signal.
TikTok and Instagram items with fewer than 1000 views are pruned
(unless they are the only source represented in the batch).
"""
sources_present = {item.source for item in items}
def passes(item: schema.SourceItem) -> bool:
rel = item.local_relevance if item.local_relevance is not None else 0.0
if rel < minimum:
return False
if item.source in _SOCIAL_SOURCES and (item.engagement_score is None or item.engagement_score == 0):
if rel < minimum * 1.5:
return False
sole_source = sources_present == {item.source}
if not _passes_engagement_floor(item, sole_source):
return False
return True
filtered = [item for item in items if passes(item)]
return filtered or items
scripts/lib/snippet.py
+50
View File
@@ -0,0 +1,50 @@
"""Best-window extraction for rerankable evidence snippets."""
from __future__ import annotations
from . import relevance, schema
def _truncate_words(text: str, max_words: int) -> str:
words = text.split()
if len(words) <= max_words:
return text.strip()
return " ".join(words[:max_words]).strip() + "..."
def _windows(words: list[str], size: int, overlap: int) -> list[str]:
if not words:
return []
if len(words) <= size:
return [" ".join(words)]
step = max(1, size - overlap)
return [
" ".join(words[start:start + size])
for start in range(0, len(words), step)
]
def extract_best_snippet(
item: schema.SourceItem,
ranking_query: str,
max_words: int = 120,
) -> str:
"""Prefer existing snippets, else extract the best matching evidence window."""
preferred = item.snippet.strip()
if preferred:
return _truncate_words(preferred, max_words)
body = item.body.strip()
if not body:
return _truncate_words(item.title, max_words)
words = body.split()
candidates = _windows(words, size=min(max_words, 110), overlap=30)
if not candidates:
return _truncate_words(body, max_words)
best = max(
candidates,
key=lambda candidate: relevance.token_overlap_relevance(ranking_query, candidate),
)
return _truncate_words(best, max_words)
scripts/lib/threads.py
+245
View File
@@ -0,0 +1,245 @@
"""Threads keyword search via ScrapeCreators API for /last30days.
Uses ScrapeCreators REST API to search Threads by keyword, extracting
engagement metrics (likes, replies) from short text posts.
Requires SCRAPECREATORS_API_KEY in config. Opt-in source via INCLUDE_SOURCES.
API docs: https://scrapecreators.com/docs
"""
import math
import re
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from . import http, log
from .relevance import token_overlap_relevance as _compute_relevance
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/threads"
# Depth configurations: how many results to fetch
DEPTH_CONFIG = {
"quick": {"results": 10},
"default": {"results": 20},
"deep": {"results": 40},
}
def _log(msg: str):
log.source_log("Threads", msg)
def _sc_headers(token: str) -> Dict[str, str]:
"""Build ScrapeCreators request headers."""
return {
"x-api-key": token,
"Content-Type": "application/json",
}
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Threads search."""
from .query import extract_core_subject
_THREADS_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice',
})
return extract_core_subject(topic, noise=_THREADS_NOISE)
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
"""Parse date from Threads item to YYYY-MM-DD.
Tries common timestamp fields: taken_at (unix), created_at (ISO),
and falls back to any date-like string field.
"""
# Unix timestamp (taken_at is common in Meta APIs)
for key in ("taken_at", "create_time"):
ts = item.get(key)
if ts:
try:
from . import dates
return dates.timestamp_to_date(int(ts))
except (ValueError, TypeError):
pass
# ISO 8601 string
for key in ("created_at", "published_at", "date"):
val = item.get(key)
if val and isinstance(val, str):
try:
dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError):
pass
return None
def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]:
"""Parse raw Threads items into normalized dicts."""
items = []
for i, raw in enumerate(raw_items):
post_id = str(
raw.get("id")
or raw.get("pk")
or raw.get("code")
or f"TH{i + 1}"
)
text = raw.get("text") or raw.get("caption") or raw.get("content") or ""
if isinstance(text, dict):
text = text.get("text", "")
# Author extraction
user = raw.get("user") or raw.get("author") or {}
if isinstance(user, dict):
handle = user.get("username") or user.get("handle") or ""
display_name = user.get("full_name") or user.get("displayName") or handle
elif isinstance(user, str):
handle = user
display_name = user
else:
handle = ""
display_name = ""
# Engagement metrics
likes = raw.get("like_count") or raw.get("likes") or 0
replies = raw.get("reply_count") or raw.get("replies") or 0
reposts = raw.get("repost_count") or raw.get("reposts") or 0
quotes = raw.get("quote_count") or raw.get("quotes") or 0
date_str = _parse_date(raw)
# Build URL
code = raw.get("code") or raw.get("shortcode") or ""
url = raw.get("url") or raw.get("share_url") or ""
if not url and code:
url = f"https://www.threads.net/post/{code}"
elif not url and handle and post_id:
url = f"https://www.threads.net/@{handle}/post/{post_id}"
# Relevance: position-based + engagement boost (similar to bluesky)
rank_score = max(0.3, 1.0 - (i * 0.02))
engagement_boost = min(0.2, math.log1p(likes + reposts) / 40)
text_relevance = _compute_relevance(core_topic, text)
relevance = min(1.0, text_relevance * 0.5 + rank_score * 0.3 + engagement_boost + 0.1)
items.append({
"id": post_id,
"handle": handle,
"display_name": display_name,
"text": text,
"url": url,
"date": date_str,
"engagement": {
"likes": likes,
"replies": replies,
"reposts": reposts,
"quotes": quotes,
},
"relevance": round(relevance, 2),
"why_relevant": f"Threads: @{handle}: {text[:60]}" if text else f"Threads: {handle}",
})
return items
def search_threads(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
token: str = None,
) -> Dict[str, Any]:
"""Search Threads via ScrapeCreators API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
Returns:
Dict with 'items' list and optional 'error'.
"""
if not token:
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
core_topic = _extract_core_subject(topic)
_log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
try:
import requests as _requests
except ImportError:
_requests = None
if not _requests:
_log("requests library not installed, falling back to urllib")
try:
from urllib.parse import urlencode
params = urlencode({"keyword": core_topic})
url = f"{SCRAPECREATORS_BASE}/search?{params}"
headers = _sc_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search",
params={"keyword": core_topic},
headers=_sc_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Extract items from response (try common SC response shapes)
raw_items = (
data.get("items")
or data.get("data")
or data.get("threads")
or data.get("posts")
or data.get("search_results")
or []
)
# Limit to configured count
raw_items = raw_items[:config["results"]]
# Parse items
items = _parse_items(raw_items, core_topic)
# Date filter
in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
out_of_range = len(items) - len(in_range)
if in_range:
items = in_range
if out_of_range:
_log(f"Filtered {out_of_range} posts outside date range")
else:
_log(f"No posts within date range, keeping all {len(items)}")
# Sort by engagement (likes) descending
items.sort(key=lambda x: x["engagement"]["likes"], reverse=True)
_log(f"Found {len(items)} Threads posts")
return {"items": items}
def parse_threads_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Threads search response to normalized format.
Returns:
List of item dicts ready for normalization.
"""
return response.get("items", [])
scripts/lib/tiktok.py
+265 -65
View File
@@ -9,7 +9,6 @@ API docs: https://scrapecreators.com/docs
import re
import sys
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Set
try:
@@ -17,7 +16,7 @@ try:
except ImportError:
_requests = None
from . import http
from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
@@ -49,11 +48,65 @@ def _extract_core_subject(topic: str) -> str:
return extract_core_subject(topic, noise=_TIKTOK_NOISE)
def _infer_query_intent(topic: str) -> str:
"""Tiny local intent classifier for TikTok query expansion."""
text = topic.lower().strip()
if re.search(r"\b(vs|versus|compare|difference between)\b", text):
return "comparison"
if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install)\b", text):
return "how_to"
if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
return "opinion"
if re.search(r"\b(pricing|feature|features|best .* for)\b", text):
return "product"
return "breaking_news"
def expand_tiktok_queries(topic: str, depth: str) -> List[str]:
"""Generate multiple TikTok search queries from a topic.
Mirrors reddit.py's expand_reddit_queries() pattern:
1. Extract core subject (strip noise words)
2. Include original topic if different from core
3. Add intent-specific OR-joined content-type variants
4. Cap by depth: 1 for quick, 2 for default, 3 for deep
Returns 1-3 query strings depending on depth.
"""
core = _extract_core_subject(topic)
queries = [core]
# Include cleaned original topic as variant if different from core
original_clean = topic.strip().rstrip('?!.')
if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
queries.append(original_clean)
qtype = _infer_query_intent(topic)
# Intent-specific TikTok content-type variants
if qtype in ("breaking_news", "opinion"):
queries.append(f"{core} edit OR reaction OR trend")
elif qtype == "product":
queries.append(f"{core} review OR haul OR unboxing")
elif qtype == "comparison":
queries.append(f"{core} vs OR compared OR which is better")
elif qtype == "how_to":
queries.append(f"{core} tutorial OR hack OR tip")
else:
queries.append(f"{core} edit OR reaction OR trend")
# Deep depth: add viral content variant
if depth == "deep":
queries.append(f"{core} viral OR fyp OR trending")
# Cap by depth budget
caps = {"quick": 1, "default": 2, "deep": 3}
cap = caps.get(depth, 2)
return queries[:cap]
def _log(msg: str):
"""Log to stderr (only in interactive terminals; spinner handles non-TTY)."""
if sys.stderr.isatty():
sys.stderr.write(f"[TikTok] {msg}\n")
sys.stderr.flush()
log.source_log("TikTok", msg)
def _sc_headers(token: str) -> Dict[str, str]:
@@ -65,18 +118,13 @@ def _sc_headers(token: str) -> Dict[str, str]:
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
"""Parse date from ScrapeCreators TikTok item to YYYY-MM-DD.
Handles create_time (unix timestamp).
"""
"""Parse date from ScrapeCreators TikTok item to YYYY-MM-DD."""
ts = item.get("create_time")
if ts:
try:
dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError, OSError):
return dates.timestamp_to_date(int(ts))
except (ValueError, TypeError):
pass
return None
@@ -100,6 +148,157 @@ def _clean_webvtt(text: str) -> str:
return ' '.join(cleaned)
def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]:
"""Parse raw TikTok items into normalized dicts."""
items = []
for raw in raw_items:
video_id = str(raw.get("aweme_id", ""))
text = raw.get("desc", "")
stats = raw.get("statistics") if isinstance(raw.get("statistics"), dict) else {}
play_count = stats.get("play_count") if stats.get("play_count") is not None else 0
digg_count = stats.get("digg_count") if stats.get("digg_count") is not None else 0
comment_count = stats.get("comment_count") if stats.get("comment_count") is not None else 0
share_count = stats.get("share_count") if stats.get("share_count") is not None else 0
author_raw = raw.get("author")
if isinstance(author_raw, dict):
author_name = author_raw.get("unique_id", "")
elif isinstance(author_raw, str):
author_name = author_raw
else:
author_name = ""
share_url = raw.get("share_url", "")
text_extra = raw.get("text_extra") or []
hashtag_names = [t.get("hashtag_name", "") for t in text_extra
if isinstance(t, dict) and t.get("hashtag_name")]
video_raw = raw.get("video")
duration = video_raw.get("duration") if isinstance(video_raw, dict) else None
date_str = _parse_date(raw)
# Compute relevance with hashtag boost
relevance = _compute_relevance(core_topic, text, hashtag_names)
# Build URL: prefer share_url, fallback to constructed URL
url = share_url.split("?")[0] if share_url else ""
if not url and author_name and video_id:
url = f"https://www.tiktok.com/@{author_name}/video/{video_id}"
items.append({
"video_id": video_id,
"text": text,
"url": url,
"author_name": author_name,
"date": date_str,
"engagement": {
"views": play_count,
"likes": digg_count,
"comments": comment_count,
"shares": share_count,
},
"hashtags": hashtag_names,
"duration": duration,
"relevance": relevance,
"why_relevant": f"TikTok: {text[:60]}" if text else f"TikTok: {core_topic}",
"caption_snippet": "", # populated by fetch_captions
})
return items
def _hashtag_search(
hashtag: str,
token: str,
) -> List[Dict[str, Any]]:
"""Search TikTok by hashtag via ScrapeCreators.
Args:
hashtag: Hashtag name (without #)
token: ScrapeCreators API key
Returns:
List of raw TikTok item dicts (aweme_info format).
"""
_log(f"Hashtag search: #{hashtag}")
if not _requests:
try:
from urllib.parse import urlencode
params = urlencode({"hashtag": hashtag})
url = f"{SCRAPECREATORS_BASE}/search/hashtag?{params}"
headers = _sc_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"Hashtag search error (urllib) for #{hashtag}: {e}")
return []
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search/hashtag",
params={"hashtag": hashtag},
headers=_sc_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"Hashtag search error for #{hashtag}: {e}")
return []
raw_items = data.get("aweme_list") or data.get("data") or []
_log(f" -> {len(raw_items)} results for #{hashtag}")
return raw_items
def _profile_videos(
handle: str,
token: str,
count: int = 10,
) -> List[Dict[str, Any]]:
"""Fetch a TikTok creator's recent videos via ScrapeCreators.
Args:
handle: TikTok username (without @)
token: ScrapeCreators API key
count: Max videos to return
Returns:
List of raw TikTok item dicts (aweme_info format).
"""
_log(f"Profile videos: @{handle}")
profile_url = "https://api.scrapecreators.com/v3/tiktok/profile/videos"
if not _requests:
try:
from urllib.parse import urlencode
params = urlencode({"handle": handle, "sort_by": "latest"})
url = f"{profile_url}?{params}"
headers = _sc_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"Profile videos error (urllib) for @{handle}: {e}")
return []
else:
try:
resp = _requests.get(
profile_url,
params={"handle": handle, "sort_by": "latest"},
headers=_sc_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"Profile videos error for @{handle}: {e}")
return []
raw_items = data.get("aweme_list") or data.get("data") or []
_log(f" -> {len(raw_items)} videos from @{handle}")
return raw_items[:count]
def search_tiktok(
topic: str,
from_date: str,
@@ -165,51 +364,7 @@ def search_tiktok(
raw_items = raw_items[:config["results_per_page"]]
# Parse items
items = []
for raw in raw_items:
video_id = str(raw.get("aweme_id", ""))
text = raw.get("desc", "")
stats = raw.get("statistics") or {}
play_count = stats.get("play_count") or 0
digg_count = stats.get("digg_count") or 0
comment_count = stats.get("comment_count") or 0
share_count = stats.get("share_count") or 0
author = raw.get("author") or {}
author_name = author.get("unique_id", "")
share_url = raw.get("share_url", "")
text_extra = raw.get("text_extra") or []
hashtag_names = [t.get("hashtag_name", "") for t in text_extra
if isinstance(t, dict) and t.get("hashtag_name")]
duration = (raw.get("video") or {}).get("duration")
date_str = _parse_date(raw)
# Compute relevance with hashtag boost
relevance = _compute_relevance(core_topic, text, hashtag_names)
# Build URL: prefer share_url, fallback to constructed URL
url = share_url.split("?")[0] if share_url else ""
if not url and author_name and video_id:
url = f"https://www.tiktok.com/@{author_name}/video/{video_id}"
items.append({
"video_id": video_id,
"text": text,
"url": url,
"author_name": author_name,
"date": date_str,
"engagement": {
"views": play_count,
"likes": digg_count,
"comments": comment_count,
"shares": share_count,
},
"hashtags": hashtag_names,
"duration": duration,
"relevance": relevance,
"why_relevant": f"TikTok: {text[:60]}" if text else f"TikTok: {core_topic}",
"caption_snippet": "", # populated by fetch_captions
})
items = _parse_items(raw_items, core_topic)
# Hard date filter
in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
@@ -307,25 +462,70 @@ def search_and_enrich(
to_date: str,
depth: str = "default",
token: str = None,
hashtags: List[str] | None = None,
creators: List[str] | None = None,
) -> Dict[str, Any]:
"""Full TikTok search: find videos, then fetch captions for top results.
Uses expand_tiktok_queries() to generate multiple search queries,
runs ScrapeCreators for each, and merges/deduplicates results by video ID.
Args:
topic: Search topic
topic: Search topic (raw topic, not planner's narrowed query)
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
hashtags: Optional list of TikTok hashtags to search (without #)
creators: Optional list of TikTok creator handles to fetch videos from
Returns:
Dict with 'items' list. Each item has a 'caption_snippet' field.
"""
# Step 1: Search
search_result = search_tiktok(topic, from_date, to_date, depth, token)
items = search_result.get("items", [])
core_topic = _extract_core_subject(topic)
seen_ids: Set[str] = set()
items: List[Dict[str, Any]] = []
last_error = None
# Step 0a: Hashtag search (high-signal, runs first)
if hashtags and token:
for hashtag in hashtags:
raw_items = _hashtag_search(hashtag, token)
parsed = _parse_items(raw_items, core_topic)
for item in parsed:
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Step 0b: Creator profile videos (high-signal)
if creators and token:
for creator in creators:
raw_items = _profile_videos(creator, token)
parsed = _parse_items(raw_items, core_topic)
for item in parsed:
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Step 1: Multi-query keyword search — run ScrapeCreators for each expanded query
queries = expand_tiktok_queries(topic, depth)
for q in queries:
search_result = search_tiktok(q, from_date, to_date, depth, token)
if search_result.get("error"):
last_error = search_result["error"]
for item in search_result.get("items", []):
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Sort merged results by views descending
items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True)
if not items:
return search_result
return {"items": [], "error": last_error}
# Step 2: Fetch captions for top N
captions = fetch_captions(items, token, depth)
@@ -337,7 +537,7 @@ def search_and_enrich(
if caption:
item["caption_snippet"] = caption
return {"items": items, "error": search_result.get("error")}
return {"items": items, "error": last_error}
def parse_tiktok_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
scripts/lib/truthsocial.py
+6 -21
View File
@@ -9,7 +9,7 @@ import re
import sys
from typing import Any, Dict, List, Optional
from . import http
from . import http, log
TRUTHSOCIAL_SEARCH_URL = "https://truthsocial.com/api/v2/search"
@@ -21,10 +21,7 @@ DEPTH_CONFIG = {
def _log(msg: str):
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
if sys.stderr.isatty():
sys.stderr.write(f"[TruthSocial] {msg}\n")
sys.stderr.flush()
log.source_log("TruthSocial", msg)
def _strip_html(html: str) -> str:
@@ -36,26 +33,14 @@ def _strip_html(html: str) -> str:
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Truth Social search."""
text = topic.lower().strip()
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
noise = {
from .query import extract_core_subject
_TS_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice',
}
words = text.split()
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
})
return extract_core_subject(topic, noise=_TS_NOISE)
def _parse_date(status: Dict[str, Any]) -> Optional[str]:
scripts/lib/ui.py
+200 -210
View File
@@ -112,13 +112,68 @@ WEB_ONLY_MESSAGES = [
"Discovering tutorials...",
]
SOURCE_COMPLETION_ORDER = [
"reddit",
"x",
"youtube",
"tiktok",
"instagram",
"hackernews",
"bluesky",
"truthsocial",
"polymarket",
"grounding",
"xiaohongshu",
]
SOURCE_COMPLETION_META = {
"reddit": ("Reddit", "thread", "threads", Colors.YELLOW),
"x": ("X", "post", "posts", Colors.CYAN),
"youtube": ("YouTube", "video", "videos", Colors.RED),
"tiktok": ("TikTok", "video", "videos", Colors.PURPLE),
"instagram": ("Instagram", "reel", "reels", Colors.PURPLE),
"hackernews": ("HN", "story", "stories", Colors.YELLOW),
"bluesky": ("Bluesky", "post", "posts", Colors.BLUE),
"truthsocial": ("Truth Social", "post", "posts", Colors.CYAN),
"polymarket": ("Polymarket", "market", "markets", Colors.GREEN),
"grounding": ("Web", "result", "results", Colors.GREEN),
"xiaohongshu": ("Xiaohongshu", "post", "posts", Colors.RED),
}
def _completion_sources(source_counts: dict[str, int], display_sources: list[str] | None) -> list[str]:
requested = list(dict.fromkeys(display_sources or []))
if not requested:
requested = [source for source, count in source_counts.items() if count]
if not requested and source_counts:
requested = list(source_counts)
candidate_set = set(requested) | set(source_counts)
ordered = [source for source in SOURCE_COMPLETION_ORDER if source in candidate_set]
for source in requested + list(source_counts):
if source in candidate_set and source not in ordered:
ordered.append(source)
return ordered
def _format_completion_part(source: str, count: int, tty: bool) -> str:
label, singular, plural, color = SOURCE_COMPLETION_META.get(
source,
(source.replace("_", " ").title(), "result", "results", Colors.RESET),
)
unit = singular if count == 1 else plural
if tty:
return f"{color}{label}:{Colors.RESET} {count} {unit}"
return f"{label}: {count} {unit}"
def _build_nux_message(diag: dict = None) -> str:
"""Build conversational NUX message with dynamic source status."""
available = set((diag or {}).get("available_sources", []))
if diag:
reddit = "✓" if diag.get("openai") else "✗"
x = "✓" if diag.get("x_source") else "✗"
youtube = "✓" if diag.get("youtube") else "✗"
web = "✓" if diag.get("web_search_backend") else "✗"
reddit = "✓" if "reddit" in available else "✗"
x = "✓" if "x" in available else "✗"
youtube = "✓" if "youtube" in available else "✗"
web = "✓" if "grounding" in available else "✗"
status_line = f"Reddit {reddit}, X {x}, YouTube {youtube}, Web {web}"
else:
status_line = "YouTube ✓, Web ✓, Reddit ✗, X ✗"
@@ -128,12 +183,11 @@ I just researched that for you. Here's what I've got right now:
{status_line}
You can unlock more sources with API keys or by signing in to Codex — just ask me how and I'll walk you through it. More sources means better research, but it works fine as-is.
More sources means better research, but it works fine as-is. You can unlock more for free - log into x.com in your browser for X, and run `brew install yt-dlp` for YouTube transcripts. That gives you Reddit (with comments), X, YouTube, HN, and Polymarket - all free.
Some examples of what you can do:
- "last30 what are people saying about Figma"
- "last30 watch my biggest competitor every week"
- "last30 watch Peter Steinberger every 30 days"
- "last30 watch AI video tools monthly"
- "last30 what have you found about AI video?"
@@ -142,8 +196,9 @@ Just start with "last30" and talk to me like normal.
# Shorter promo for single missing key
PROMO_SINGLE_KEY = {
"reddit": "\n💡 You can unlock Reddit with an OpenAI API key or by running `codex login` — just ask me how.\n",
"x": "\n💡 You can unlock X with AUTH_TOKEN/CT0 or XAI_API_KEY - just ask me how.\n",
"reddit": "\n💡 Unlock TikTok and Instagram with SCRAPECREATORS_API_KEY - 10,000 free calls, no CC - scrapecreators.com\n",
"x": "\n💡 Unlock X: log into x.com in Firefox or Safari, then re-run. Or add AUTH_TOKEN/CT0 or XAI_API_KEY.\n",
"web": "\n💡 You can unlock native grounded web search with BRAVE_API_KEY or SERPER_API_KEY.\n",
}
# Bird auth help (for local users with vendored Bird CLI)
@@ -328,36 +383,46 @@ class ProgressDisplay:
if self.spinner:
self.spinner.stop()
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0, pm_count: int = 0, tiktok_count: int = 0, ig_count: int = 0):
def show_complete(
self,
reddit_count: int = 0,
x_count: int = 0,
youtube_count: int = 0,
hn_count: int = 0,
pm_count: int = 0,
tiktok_count: int = 0,
ig_count: int = 0,
*,
source_counts: dict[str, int] | None = None,
display_sources: list[str] | None = None,
):
elapsed = time.time() - self.start_time
if source_counts is None:
source_counts = {
"reddit": reddit_count,
"x": x_count,
"youtube": youtube_count,
"tiktok": tiktok_count,
"instagram": ig_count,
"hackernews": hn_count,
"polymarket": pm_count,
}
if display_sources is None:
display_sources = [source for source, count in source_counts.items() if count]
if not display_sources:
display_sources = ["reddit", "x"]
ordered_sources = _completion_sources(source_counts, display_sources)
parts = [
_format_completion_part(source, source_counts.get(source, 0), tty=IS_TTY)
for source in ordered_sources
]
if IS_TTY:
sys.stderr.write(f"\n{Colors.GREEN}{Colors.BOLD}✓ Research complete{Colors.RESET} ")
sys.stderr.write(f"{Colors.DIM}({elapsed:.1f}s){Colors.RESET}\n")
sys.stderr.write(f" {Colors.YELLOW}Reddit:{Colors.RESET} {reddit_count} threads ")
sys.stderr.write(f"{Colors.CYAN}X:{Colors.RESET} {x_count} posts")
if youtube_count:
sys.stderr.write(f" {Colors.RED}YouTube:{Colors.RESET} {youtube_count} videos")
if tiktok_count:
sys.stderr.write(f" {Colors.PURPLE}TikTok:{Colors.RESET} {tiktok_count} videos")
if ig_count:
sys.stderr.write(f" {Colors.PURPLE}Instagram:{Colors.RESET} {ig_count} reels")
if hn_count:
sys.stderr.write(f" {Colors.YELLOW}HN:{Colors.RESET} {hn_count} stories")
if pm_count:
sys.stderr.write(f" {Colors.GREEN}Polymarket:{Colors.RESET} {pm_count} markets")
sys.stderr.write(" " + " ".join(parts))
sys.stderr.write("\n\n")
else:
parts = [f"Reddit: {reddit_count} threads", f"X: {x_count} posts"]
if youtube_count:
parts.append(f"YouTube: {youtube_count} videos")
if tiktok_count:
parts.append(f"TikTok: {tiktok_count} videos")
if ig_count:
parts.append(f"Instagram: {ig_count} reels")
if hn_count:
parts.append(f"HN: {hn_count} stories")
if pm_count:
parts.append(f"Polymarket: {pm_count} markets")
sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - {', '.join(parts)}\n")
sys.stderr.flush()
@@ -417,190 +482,115 @@ class ProgressDisplay:
sys.stderr.flush()
def _build_status_banner(diag: dict) -> list[str]:
"""Build the status banner lines (plain text, no ANSI).
Returns a list of strings, each being a line of the banner box.
Args:
diag: Dict with keys:
setup_complete, reddit_source, x_source, x_method,
youtube, tiktok, instagram, hackernews, polymarket,
bluesky, truthsocial, xiaohongshu, scrapecreators,
web_search_backend
"""
setup_complete = diag.get("setup_complete", False)
has_sc = diag.get("scrapecreators", False)
# --- Build active sources list: (label, method_label) ---
active: list[str] = []
# Reddit — always available; what matters to users is comments or not
reddit_src = diag.get("reddit_source")
if reddit_src == "scrapecreators":
active.append("Reddit (with comments)")
else:
active.append("Reddit (threads only)")
# X/Twitter
x_source = diag.get("x_source")
x_method = diag.get("x_method")
if x_source:
if x_method and x_method.startswith("browser-"):
browser = x_method.split("-", 1)[1].capitalize()
active.append(f"X ({browser})")
elif x_method == "env":
active.append("X (env)")
elif x_method == "api":
active.append("X (xAI)")
else:
active.append("X")
# YouTube
if diag.get("youtube"):
active.append("YouTube")
# HN — always available
if diag.get("hackernews"):
active.append("HN")
# Polymarket — always available
if diag.get("polymarket"):
active.append("Polymarket")
# TikTok (requires SC or Apify)
if diag.get("tiktok"):
active.append("TikTok")
# Instagram (requires SC)
if diag.get("instagram"):
active.append("Instagram")
# Bluesky
if diag.get("bluesky"):
active.append("Bluesky")
# Truth Social
if diag.get("truthsocial"):
active.append("Truth Social")
# Xiaohongshu
if diag.get("xiaohongshu"):
active.append("Xiaohongshu")
# --- Format active sources into wrapped lines ---
BOX_INNER = 53 # characters inside the box (between │ and │)
PREFIX = " " # 2-space indent inside box
def _wrap_sources(sources: list[str]) -> list[str]:
"""Wrap source labels into lines that fit the box width."""
result_lines: list[str] = []
current = PREFIX
for i, s in enumerate(sources):
token = f"✅ {s}"
sep = " " if current != PREFIX else ""
if len(current) + len(sep) + len(token) > BOX_INNER:
result_lines.append(current)
current = PREFIX + token
else:
current += sep + token
if current.strip():
result_lines.append(current)
return result_lines
source_lines = _wrap_sources(active)
# --- Title ---
if not setup_complete:
title = "/last30days v3.0 — First Run"
else:
title = "/last30days v3.0 — Source Status"
# --- Build upgrade suggestions ---
suggestions: list[str] = []
if not setup_complete:
suggestions.append("Run /last30days setup to unlock more sources")
else:
# Recommend ScrapeCreators if missing
if not has_sc:
suggestions.append("⭐ Add SCRAPECREATORS_API_KEY for Reddit comments")
suggestions.append(" + TikTok + Instagram")
suggestions.append(" 100 free calls, no CC — scrapecreators.com (no affiliation)")
# --- Assemble box lines ---
# Collect all content lines, then determine box width dynamically.
content: list[str] = []
content.append(f" {title}")
content.append("") # blank line
for sl in source_lines:
content.append(sl)
if suggestions:
content.append("") # blank line
for sg in suggestions:
content.append(f" {sg}")
content.append("") # blank line
content.append(" Config: ~/.config/last30days/.env")
# Width = widest content line + 1 for right margin
width = max(len(line) for line in content) + 1
if width < 53:
width = 53
lines: list[str] = []
lines.append("\u250c" + "\u2500" * width + "\u2510")
for c in content:
lines.append("\u2502" + c.ljust(width) + "\u2502")
lines.append("\u2514" + "\u2500" * width + "\u2518")
return lines
def _colorize_banner(lines: list[str]) -> list[str]:
"""Apply ANSI colors to plain-text banner lines for TTY output."""
colored: list[str] = []
for line in lines:
if line.startswith("\u250c") or line.startswith("\u2514"):
colored.append(f"{Colors.DIM}{line}{Colors.RESET}")
elif line.startswith("\u2502"):
inner = line[1:-1] # strip box chars on both sides
inner_width = len(inner)
# Colorize check marks green, star yellow
inner = inner.replace("\u2705", f"{Colors.GREEN}\u2705{Colors.RESET}")
inner = inner.replace("\u2b50", f"{Colors.YELLOW}\u2b50{Colors.RESET}")
# Bold the title line
if "/last30days v3.0" in inner:
stripped = inner.strip()
inner = f" {Colors.BOLD}{stripped}{Colors.RESET}"
# Re-pad to original width (ANSI codes are zero-width)
visible_len = 1 + len(stripped)
inner = inner + " " * max(0, inner_width - visible_len)
colored.append(f"{Colors.DIM}\u2502{Colors.RESET}{inner}{Colors.DIM}\u2502{Colors.RESET}")
else:
colored.append(line)
return colored
def show_diagnostic_banner(diag: dict):
"""Show pre-flight source status banner.
Free-first design: leads with what's working (✅), not what's broken.
Shows upgrade suggestions only when relevant.
"""Show pre-flight source status banner when sources are missing.
Args:
diag: Dict with keys:
setup_complete, reddit_source, x_source, x_method,
youtube, tiktok, instagram, hackernews, polymarket,
bluesky, truthsocial, xiaohongshu, scrapecreators,
web_search_backend
diag: Dict from pipeline.diagnose() with available_sources, x_backend,
bird status, provider availability, and native web backend info.
"""
lines = _build_status_banner(diag)
available_sources = set(diag.get("available_sources") or [])
has_reddit = "reddit" in available_sources
has_scrapecreators = diag.get("has_scrapecreators", False)
has_x = "x" in available_sources
has_youtube = "youtube" in available_sources
has_web = "grounding" in available_sources
has_xiaohongshu = "xiaohongshu" in available_sources
x_backend = diag.get("x_backend")
native_web_backend = diag.get("native_web_backend")
# If everything is available, no banner needed
if has_reddit and has_x and has_youtube and has_web:
return
lines = []
if IS_TTY:
lines = _colorize_banner(lines)
lines.append(f"{Colors.DIM}┌─────────────────────────────────────────────────────┐{Colors.RESET}")
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.BOLD}/last30days v3.0.0 - Source Status{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
# Reddit
if has_reddit and has_scrapecreators:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Reddit{Colors.RESET} — full threads with comments {Colors.DIM}│{Colors.RESET}")
elif has_reddit:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Reddit{Colors.RESET} — public threads (titles + scores) {Colors.DIM}│{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ Reddit{Colors.RESET} — unavailable {Colors.DIM}│{Colors.RESET}")
# X/Twitter
if has_x:
username = diag.get("bird_username", "")
label = f"Bird ({username})" if x_backend == "bird" and username else str(x_backend or "xai").upper()
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ X/Twitter{Colors.RESET} — {label} {Colors.DIM}│{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ X/Twitter{Colors.RESET} — No X auth or fallback key {Colors.DIM}│{Colors.RESET}")
if diag.get("bird_installed"):
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY {Colors.DIM}│{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Needs Node.js 22+ (Bird is bundled) {Colors.DIM}│{Colors.RESET}")
# YouTube
if has_youtube:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ YouTube{Colors.RESET} — yt-dlp found {Colors.DIM}│{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ YouTube{Colors.RESET} — yt-dlp not installed {Colors.DIM}│{Colors.RESET}")
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Fix: brew install yt-dlp (free) {Colors.DIM}│{Colors.RESET}")
# Xiaohongshu (only show when configured)
if has_xiaohongshu:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Xiaohongshu{Colors.RESET} — API connected + logged in {Colors.DIM}│{Colors.RESET}")
# Web
if has_web:
backend = native_web_backend or "native"
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Web{Colors.RESET} — {backend} API {Colors.DIM}│{Colors.RESET}")
else:
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.YELLOW}⚡ Web{Colors.RESET} — Add BRAVE_API_KEY or SERPER_API_KEY {Colors.DIM}│{Colors.RESET}")
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
lines.append(f"{Colors.DIM}│{Colors.RESET} Config: {Colors.BOLD}~/.config/last30days/.env{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
lines.append(f"{Colors.DIM}└─────────────────────────────────────────────────────┘{Colors.RESET}")
else:
# Plain text for non-TTY (Claude Code / Codex)
lines.append("┌─────────────────────────────────────────────────────┐")
lines.append("│ /last30days v3.0.0 - Source Status │")
lines.append("│ │")
if has_reddit and has_scrapecreators:
lines.append("│ ✅ Reddit — full threads with comments │")
elif has_reddit:
lines.append("│ ✅ Reddit — public threads (titles + scores) │")
else:
lines.append("│ ❌ Reddit — unavailable │")
if has_x:
lines.append("│ ✅ X/Twitter — available │")
else:
lines.append("│ ❌ X/Twitter — No X auth or fallback key │")
if diag.get("bird_installed"):
lines.append("│ └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY │")
else:
lines.append("│ └─ Needs Node.js 22+ (Bird is bundled) │")
if has_youtube:
lines.append("│ ✅ YouTube — yt-dlp found │")
else:
lines.append("│ ❌ YouTube — yt-dlp not installed │")
lines.append("│ └─ Fix: brew install yt-dlp (free) │")
if has_xiaohongshu:
lines.append("│ ✅ Xiaohongshu — API connected + logged in │")
if has_web:
backend = native_web_backend or "native"
lines.append(f"│ ✅ Web — {backend} API available{' ' * max(0, 13 - len(backend))}│")
else:
lines.append("│ ⚡ Web — Add BRAVE_API_KEY or SERPER_API_KEY │")
lines.append("│ │")
lines.append("│ Config: ~/.config/last30days/.env │")
lines.append("└─────────────────────────────────────────────────────┘")
sys.stderr.write("\n".join(lines) + "\n\n")
sys.stderr.flush()
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/LICENSE
Generated Vendored
-22
View File
@@ -1,22 +0,0 @@
MIT License
Copyright (c) 2025 Peter Steinberger
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/README.md
Generated Vendored
-29
View File
@@ -1,29 +0,0 @@
# @steipete/sweet-cookie
Inline-first browser cookie extraction for local tooling (no native addons).
Supports:
- Inline payloads (JSON / base64 / file) — most reliable path.
- Local browser reads (best effort): Chrome, Edge, Firefox, Safari (macOS).
Install:
```bash
npm i @steipete/sweet-cookie
```
Usage:
```ts
import { getCookies, toCookieHeader } from '@steipete/sweet-cookie';
const { cookies, warnings } = await getCookies({
url: 'https://example.com/',
names: ['session', 'csrf'],
browsers: ['chrome', 'edge', 'firefox', 'safari'],
});
for (const w of warnings) console.warn(w);
const cookieHeader = toCookieHeader(cookies, { dedupeByName: true });
```
Docs + extension exporter: see the repo root README.
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/index.d.ts
Generated Vendored
-3
View File
@@ -1,3 +0,0 @@
export { getCookies, toCookieHeader } from './public.js';
export type { BrowserName, Cookie, CookieHeaderOptions, CookieMode, CookieSameSite, GetCookiesOptions, GetCookiesResult, } from './types.js';
//# sourceMappingURL=index.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/index.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACzD,YAAY,EACX,WAAW,EACX,MAAM,EACN,mBAAmB,EACnB,UAAU,EACV,cAAc,EACd,iBAAiB,EACjB,gBAAgB,GAChB,MAAM,YAAY,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/index.js
Generated Vendored
-2
View File
@@ -1,2 +0,0 @@
export { getCookies, toCookieHeader } from './public.js';
//# sourceMappingURL=index.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/index.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chrome.d.ts
Generated Vendored
-8
View File
@@ -1,8 +0,0 @@
import type { GetCookiesResult } from '../types.js';
export declare function getCookiesFromChrome(options: {
profile?: string;
timeoutMs?: number;
includeExpired?: boolean;
debug?: boolean;
}, origins: string[], allowlistNames: Set<string> | null): Promise<GetCookiesResult>;
//# sourceMappingURL=chrome.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chrome.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chrome.d.ts","sourceRoot":"","sources":["../../src/providers/chrome.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAU,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAK5D,wBAAsB,oBAAoB,CACzC,OAAO,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,EAC5F,OAAO,EAAE,MAAM,EAAE,EACjB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAChC,OAAO,CAAC,gBAAgB,CAAC,CA0B3B"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chrome.js
Generated Vendored
-27
View File
@@ -1,27 +0,0 @@
import { getCookiesFromChromeSqliteLinux } from './chromeSqliteLinux.js';
import { getCookiesFromChromeSqliteMac } from './chromeSqliteMac.js';
import { getCookiesFromChromeSqliteWindows } from './chromeSqliteWindows.js';
export async function getCookiesFromChrome(options, origins, allowlistNames) {
const warnings = [];
// Platform dispatch only. All real logic lives in the per-OS providers.
if (process.platform === 'darwin') {
const r = await getCookiesFromChromeSqliteMac(options, origins, allowlistNames);
warnings.push(...r.warnings);
const cookies = r.cookies;
return { cookies, warnings };
}
if (process.platform === 'linux') {
const r = await getCookiesFromChromeSqliteLinux(options, origins, allowlistNames);
warnings.push(...r.warnings);
const cookies = r.cookies;
return { cookies, warnings };
}
if (process.platform === 'win32') {
const r = await getCookiesFromChromeSqliteWindows(options, origins, allowlistNames);
warnings.push(...r.warnings);
const cookies = r.cookies;
return { cookies, warnings };
}
return { cookies: [], warnings };
}
//# sourceMappingURL=chrome.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chrome.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chrome.js","sourceRoot":"","sources":["../../src/providers/chrome.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,iCAAiC,EAAE,MAAM,0BAA0B,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,OAA4F,EAC5F,OAAiB,EACjB,cAAkC;IAElC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,wEAAwE;IACxE,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,MAAM,6BAA6B,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;QAChF,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAa,CAAC,CAAC,OAAO,CAAC;QACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,MAAM,+BAA+B,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;QAClF,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAa,CAAC,CAAC,OAAO,CAAC;QACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,MAAM,iCAAiC,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;QACpF,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAa,CAAC,CAAC,OAAO,CAAC;QACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;AAClC,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/crypto.d.ts
Generated Vendored
-11
View File
@@ -1,11 +0,0 @@
export declare function deriveAes128CbcKeyFromPassword(password: string, options: {
iterations: number;
}): Buffer;
export declare function decryptChromiumAes128CbcCookieValue(encryptedValue: Uint8Array, keyCandidates: readonly Buffer[], options: {
stripHashPrefix: boolean;
treatUnknownPrefixAsPlaintext?: boolean;
}): string | null;
export declare function decryptChromiumAes256GcmCookieValue(encryptedValue: Uint8Array, key: Buffer, options: {
stripHashPrefix: boolean;
}): string | null;
//# sourceMappingURL=crypto.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/crypto.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../../../src/providers/chromeSqlite/crypto.ts"],"names":[],"mappings":"AAIA,wBAAgB,8BAA8B,CAC7C,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,GAC7B,MAAM,CAIR;AAED,wBAAgB,mCAAmC,CAClD,cAAc,EAAE,UAAU,EAC1B,aAAa,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,EAAE;IAAE,eAAe,EAAE,OAAO,CAAC;IAAC,6BAA6B,CAAC,EAAE,OAAO,CAAA;CAAE,GAC5E,MAAM,GAAG,IAAI,CA2Bf;AAED,wBAAgB,mCAAmC,CAClD,cAAc,EAAE,UAAU,EAC1B,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IAAE,eAAe,EAAE,OAAO,CAAA;CAAE,GACnC,MAAM,GAAG,IAAI,CAyBf"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/crypto.js
Generated Vendored
-100
View File
@@ -1,100 +0,0 @@
import { createDecipheriv, pbkdf2Sync } from 'node:crypto';
const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
export function deriveAes128CbcKeyFromPassword(password, options) {
// Chromium derives the AES-128-CBC key from "Chrome Safe Storage" using PBKDF2.
// The salt/length/digest are fixed by Chromium ("saltysalt", 16 bytes, sha1).
return pbkdf2Sync(password, 'saltysalt', options.iterations, 16, 'sha1');
}
export function decryptChromiumAes128CbcCookieValue(encryptedValue, keyCandidates, options) {
const buf = Buffer.from(encryptedValue);
if (buf.length < 3)
return null;
// Chromium prefixes encrypted cookies with `v10`, `v11`, ... (three bytes).
const prefix = buf.subarray(0, 3).toString('utf8');
const hasVersionPrefix = /^v\d\d$/.test(prefix);
if (!hasVersionPrefix) {
// Some platforms (notably macOS) can store plaintext values in `encrypted_value`.
// Callers decide whether unknown prefixes should be treated as plaintext.
if (options.treatUnknownPrefixAsPlaintext === false)
return null;
return decodeCookieValueBytes(buf, false);
}
const ciphertext = buf.subarray(3);
if (!ciphertext.length)
return '';
for (const key of keyCandidates) {
// Try multiple candidates because Linux may fall back to empty passwords depending on keyring state.
const decrypted = tryDecryptAes128Cbc(ciphertext, key);
if (!decrypted)
continue;
const decoded = decodeCookieValueBytes(decrypted, options.stripHashPrefix);
if (decoded !== null)
return decoded;
}
return null;
}
export function decryptChromiumAes256GcmCookieValue(encryptedValue, key, options) {
const buf = Buffer.from(encryptedValue);
if (buf.length < 3)
return null;
const prefix = buf.subarray(0, 3).toString('utf8');
if (!/^v\d\d$/.test(prefix))
return null;
// AES-256-GCM layout:
// - 12-byte nonce
// - ciphertext
// - 16-byte authentication tag
const payload = buf.subarray(3);
if (payload.length < 12 + 16)
return null;
const nonce = payload.subarray(0, 12);
const authenticationTag = payload.subarray(payload.length - 16);
const ciphertext = payload.subarray(12, payload.length - 16);
try {
const decipher = createDecipheriv('aes-256-gcm', key, nonce);
decipher.setAuthTag(authenticationTag);
const plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
return decodeCookieValueBytes(plaintext, options.stripHashPrefix);
}
catch {
return null;
}
}
function tryDecryptAes128Cbc(ciphertext, key) {
try {
// Chromium's legacy AES-128-CBC uses an IV of 16 spaces.
const iv = Buffer.alloc(16, 0x20);
const decipher = createDecipheriv('aes-128-cbc', key, iv);
decipher.setAutoPadding(false);
const plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
return removePkcs7Padding(plaintext);
}
catch {
return null;
}
}
function removePkcs7Padding(value) {
if (!value.length)
return value;
const padding = value[value.length - 1];
if (!padding || padding > 16)
return value;
return value.subarray(0, value.length - padding);
}
function decodeCookieValueBytes(value, stripHashPrefix) {
// Chromium >= 24 prepends a 32-byte hash to cookie values.
const bytes = stripHashPrefix && value.length >= 32 ? value.subarray(32) : value;
try {
return stripLeadingControlChars(UTF8_DECODER.decode(bytes));
}
catch {
return null;
}
}
function stripLeadingControlChars(value) {
let i = 0;
while (i < value.length && value.charCodeAt(i) < 0x20)
i += 1;
return value.slice(i);
}
//# sourceMappingURL=crypto.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/crypto.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../../src/providers/chromeSqlite/crypto.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE3D,MAAM,YAAY,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE/D,MAAM,UAAU,8BAA8B,CAC7C,QAAgB,EAChB,OAA+B;IAE/B,gFAAgF;IAChF,8EAA8E;IAC9E,OAAO,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,mCAAmC,CAClD,cAA0B,EAC1B,aAAgC,EAChC,OAA8E;IAE9E,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAEhC,4EAA4E;IAC5E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvB,kFAAkF;QAClF,0EAA0E;QAC1E,IAAI,OAAO,CAAC,6BAA6B,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;QACjE,OAAO,sBAAsB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAElC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,qGAAqG;QACrG,MAAM,SAAS,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS;YAAE,SAAS;QACzB,MAAM,OAAO,GAAG,sBAAsB,CAAC,SAAS,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;QAC3E,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,OAAO,CAAC;IACtC,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,UAAU,mCAAmC,CAClD,cAA0B,EAC1B,GAAW,EACX,OAAqC;IAErC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,sBAAsB;IACtB,kBAAkB;IAClB,eAAe;IACf,+BAA+B;IAC/B,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IAE1C,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;IAE7D,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,gBAAgB,CAAC,aAAa,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7D,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACjF,OAAO,sBAAsB,CAAC,SAAS,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAkB,EAAE,GAAW;IAC3D,IAAI,CAAC;QACJ,yDAAyD;QACzD,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,aAAa,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC1D,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACjF,OAAO,kBAAkB,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACxC,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO,IAAI,OAAO,GAAG,EAAE;QAAE,OAAO,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa,EAAE,eAAwB;IACtE,2DAA2D;IAC3D,MAAM,KAAK,GAAG,eAAe,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjF,IAAI,CAAC;QACJ,OAAO,wBAAwB,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAa;IAC9C,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI;QAAE,CAAC,IAAI,CAAC,CAAC;IAC9D,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/linuxKeyring.d.ts
Generated Vendored
-25
View File
@@ -1,25 +0,0 @@
export type LinuxKeyringBackend = 'gnome' | 'kwallet' | 'basic';
/**
* Read the "Safe Storage" password from a Linux keyring.
*
* Chromium browsers typically store their cookie encryption password under:
* - service: "<Browser> Safe Storage"
* - account: "<Browser>"
*
* We keep this logic in JS (no native deps) and return an empty password on failure
* (Chromium may still have v10 cookies, and callers can use inline/export escape hatches).
*/
export declare function getLinuxChromiumSafeStoragePassword(options: {
backend?: LinuxKeyringBackend;
app: 'chrome' | 'edge';
}): Promise<{
password: string;
warnings: string[];
}>;
export declare function getLinuxChromeSafeStoragePassword(options?: {
backend?: LinuxKeyringBackend;
}): Promise<{
password: string;
warnings: string[];
}>;
//# sourceMappingURL=linuxKeyring.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/linuxKeyring.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"linuxKeyring.d.ts","sourceRoot":"","sources":["../../../src/providers/chromeSqlite/linuxKeyring.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO,CAAC;AAEhE;;;;;;;;;GASG;AACH,wBAAsB,mCAAmC,CAAC,OAAO,EAAE;IAClE,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,GAAG,EAAE,QAAQ,GAAG,MAAM,CAAC;CACvB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA8DpD;AAED,wBAAsB,iCAAiC,CACtD,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,mBAAmB,CAAA;CAAO,GAC7C,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAInD"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/linuxKeyring.js
Generated Vendored
-104
View File
@@ -1,104 +0,0 @@
import { execCapture } from '../../util/exec.js';
/**
* Read the "Safe Storage" password from a Linux keyring.
*
* Chromium browsers typically store their cookie encryption password under:
* - service: "<Browser> Safe Storage"
* - account: "<Browser>"
*
* We keep this logic in JS (no native deps) and return an empty password on failure
* (Chromium may still have v10 cookies, and callers can use inline/export escape hatches).
*/
export async function getLinuxChromiumSafeStoragePassword(options) {
const warnings = [];
// Escape hatch: if callers already know the password (or want deterministic CI behavior),
// they can bypass keyring probing entirely.
const overrideKey = options.app === 'edge'
? 'SWEET_COOKIE_EDGE_SAFE_STORAGE_PASSWORD'
: 'SWEET_COOKIE_CHROME_SAFE_STORAGE_PASSWORD';
const override = readEnv(overrideKey);
if (override !== undefined)
return { password: override, warnings };
const backend = options.backend ?? parseLinuxKeyringBackend() ?? chooseLinuxKeyringBackend();
// `basic` means "don't try keyrings" (Chrome will fall back to older/less-secure schemes on some setups).
if (backend === 'basic')
return { password: '', warnings };
const service = options.app === 'edge' ? 'Microsoft Edge Safe Storage' : 'Chrome Safe Storage';
const account = options.app === 'edge' ? 'Microsoft Edge' : 'Chrome';
const folder = `${account} Keys`;
if (backend === 'gnome') {
// GNOME keyring: `secret-tool` is the simplest way to read libsecret entries.
const res = await execCapture('secret-tool', ['lookup', 'service', service, 'account', account], { timeoutMs: 3_000 });
if (res.code === 0)
return { password: res.stdout.trim(), warnings };
warnings.push('Failed to read Linux keyring via secret-tool; v11 cookies may be unavailable.');
return { password: '', warnings };
}
// KDE keyring: query KWallet via `kwallet-query`, but the wallet name differs across KDE versions.
const kdeVersion = (readEnv('KDE_SESSION_VERSION') ?? '').trim();
const serviceName = kdeVersion === '6'
? 'org.kde.kwalletd6'
: kdeVersion === '5'
? 'org.kde.kwalletd5'
: 'org.kde.kwalletd';
const walletPath = kdeVersion === '6'
? '/modules/kwalletd6'
: kdeVersion === '5'
? '/modules/kwalletd5'
: '/modules/kwalletd';
const wallet = await getKWalletNetworkWallet(serviceName, walletPath);
const passwordRes = await execCapture('kwallet-query', ['--read-password', service, '--folder', folder, wallet], { timeoutMs: 3_000 });
if (passwordRes.code !== 0) {
warnings.push('Failed to read Linux keyring via kwallet-query; v11 cookies may be unavailable.');
return { password: '', warnings };
}
if (passwordRes.stdout.toLowerCase().startsWith('failed to read'))
return { password: '', warnings };
return { password: passwordRes.stdout.trim(), warnings };
}
export async function getLinuxChromeSafeStoragePassword(options = {}) {
const args = { app: 'chrome' };
if (options.backend !== undefined)
args.backend = options.backend;
return await getLinuxChromiumSafeStoragePassword(args);
}
function parseLinuxKeyringBackend() {
const raw = readEnv('SWEET_COOKIE_LINUX_KEYRING');
if (!raw)
return undefined;
const normalized = raw.toLowerCase();
if (normalized === 'gnome')
return 'gnome';
if (normalized === 'kwallet')
return 'kwallet';
if (normalized === 'basic')
return 'basic';
return undefined;
}
function chooseLinuxKeyringBackend() {
const xdg = readEnv('XDG_CURRENT_DESKTOP') ?? '';
const isKde = xdg.split(':').some((p) => p.trim().toLowerCase() === 'kde') || !!readEnv('KDE_FULL_SESSION');
return isKde ? 'kwallet' : 'gnome';
}
async function getKWalletNetworkWallet(serviceName, walletPath) {
const res = await execCapture('dbus-send', [
'--session',
'--print-reply=literal',
`--dest=${serviceName}`,
walletPath,
'org.kde.KWallet.networkWallet',
], { timeoutMs: 3_000 });
const fallback = 'kdewallet';
if (res.code !== 0)
return fallback;
const raw = res.stdout.trim();
if (!raw)
return fallback;
return raw.replaceAll('"', '').trim() || fallback;
}
function readEnv(key) {
const value = process.env[key];
const trimmed = typeof value === 'string' ? value.trim() : '';
return trimmed.length ? trimmed : undefined;
}
//# sourceMappingURL=linuxKeyring.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/linuxKeyring.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"linuxKeyring.js","sourceRoot":"","sources":["../../../src/providers/chromeSqlite/linuxKeyring.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAIjD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,OAGzD;IACA,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,0FAA0F;IAC1F,4CAA4C;IAC5C,MAAM,WAAW,GAChB,OAAO,CAAC,GAAG,KAAK,MAAM;QACrB,CAAC,CAAC,yCAAyC;QAC3C,CAAC,CAAC,2CAA2C,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;IAEpE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,wBAAwB,EAAE,IAAI,yBAAyB,EAAE,CAAC;IAC7F,0GAA0G;IAC1G,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IAE3D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,qBAAqB,CAAC;IAC/F,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC;IACrE,MAAM,MAAM,GAAG,GAAG,OAAO,OAAO,CAAC;IAEjC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACzB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,WAAW,CAC5B,aAAa,EACb,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,EAClD,EAAE,SAAS,EAAE,KAAK,EAAE,CACpB,CAAC;QACF,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC;QACrE,QAAQ,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAC/F,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IACnC,CAAC;IAED,mGAAmG;IACnG,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACjE,MAAM,WAAW,GAChB,UAAU,KAAK,GAAG;QACjB,CAAC,CAAC,mBAAmB;QACrB,CAAC,CAAC,UAAU,KAAK,GAAG;YACnB,CAAC,CAAC,mBAAmB;YACrB,CAAC,CAAC,kBAAkB,CAAC;IACxB,MAAM,UAAU,GACf,UAAU,KAAK,GAAG;QACjB,CAAC,CAAC,oBAAoB;QACtB,CAAC,CAAC,UAAU,KAAK,GAAG;YACnB,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,mBAAmB,CAAC;IAEzB,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IACtE,MAAM,WAAW,GAAG,MAAM,WAAW,CACpC,eAAe,EACf,CAAC,iBAAiB,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,EACxD,EAAE,SAAS,EAAE,KAAK,EAAE,CACpB,CAAC;IACF,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CACZ,iFAAiF,CACjF,CAAC;QACF,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IACnC,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC;QAChE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IACnC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACtD,UAA6C,EAAE;IAE/C,MAAM,IAAI,GAAqD,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC;IACjF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAClE,OAAO,MAAM,mCAAmC,CAAC,IAAI,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,wBAAwB;IAChC,MAAM,GAAG,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAClD,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IACrC,IAAI,UAAU,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC;IAC3C,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC/C,IAAI,UAAU,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC;IAC3C,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,yBAAyB;IACjC,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,KAAK,GACV,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC/F,OAAO,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,WAAmB,EAAE,UAAkB;IAC7E,MAAM,GAAG,GAAG,MAAM,WAAW,CAC5B,WAAW,EACX;QACC,WAAW;QACX,uBAAuB;QACvB,UAAU,WAAW,EAAE;QACvB,UAAU;QACV,+BAA+B;KAC/B,EACD,EAAE,SAAS,EAAE,KAAK,EAAE,CACpB,CAAC;IACF,MAAM,QAAQ,GAAG,WAAW,CAAC;IAC7B,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IACpC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC1B,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,QAAQ,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,GAAW;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7C,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/shared.d.ts
Generated Vendored
-10
View File
@@ -1,10 +0,0 @@
import type { GetCookiesResult } from '../../types.js';
export declare function getCookiesFromChromeSqliteDb(options: {
dbPath: string;
profile?: string;
includeExpired?: boolean;
debug?: boolean;
}, origins: string[], allowlistNames: Set<string> | null, decrypt: (encryptedValue: Uint8Array, options: {
stripHashPrefix: boolean;
}) => string | null): Promise<GetCookiesResult>;
//# sourceMappingURL=shared.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/shared.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/providers/chromeSqlite/shared.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAA0B,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAkB/E,wBAAsB,4BAA4B,CACjD,OAAO,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,EACxF,OAAO,EAAE,MAAM,EAAE,EACjB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAClC,OAAO,EAAE,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE;IAAE,eAAe,EAAE,OAAO,CAAA;CAAE,KAAK,MAAM,GAAG,IAAI,GAC3F,OAAO,CAAC,gBAAgB,CAAC,CAsD3B"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/shared.js
Generated Vendored
-293
View File
@@ -1,293 +0,0 @@
import { copyFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { normalizeExpiration } from '../../util/expire.js';
import { hostMatchesCookieDomain } from '../../util/hostMatch.js';
import { importNodeSqlite, supportsReadBigInts } from '../../util/nodeSqlite.js';
import { isBunRuntime } from '../../util/runtime.js';
export async function getCookiesFromChromeSqliteDb(options, origins, allowlistNames, decrypt) {
const warnings = [];
// Chrome can keep its cookie DB locked and/or rely on WAL sidecars.
// Copying to a temp dir gives us a stable snapshot that both node:sqlite and bun:sqlite can open.
const tempDir = mkdtempSync(path.join(tmpdir(), 'sweet-cookie-chrome-'));
const tempDbPath = path.join(tempDir, 'Cookies');
try {
copyFileSync(options.dbPath, tempDbPath);
// If WAL is enabled, the latest writes might live in `Cookies-wal`/`Cookies-shm`.
// Copy them too when present so our snapshot reflects the current browser state.
copySidecar(options.dbPath, `${tempDbPath}-wal`, '-wal');
copySidecar(options.dbPath, `${tempDbPath}-shm`, '-shm');
}
catch (error) {
rmSync(tempDir, { recursive: true, force: true });
warnings.push(`Failed to copy Chrome cookie DB: ${error instanceof Error ? error.message : String(error)}`);
return { cookies: [], warnings };
}
try {
const hosts = origins.map((o) => new URL(o).hostname);
const where = buildHostWhereClause(hosts, 'host_key');
const metaVersion = await readChromiumMetaVersion(tempDbPath);
// Chromium >= 24 stores a 32-byte hash prefix in decrypted cookie values.
// We detect this via the `meta` table version and strip it when present.
const stripHashPrefix = metaVersion >= 24;
const rowsResult = await readChromeRows(tempDbPath, where);
if (!rowsResult.ok) {
warnings.push(rowsResult.error);
return { cookies: [], warnings };
}
const collectOptions = {};
if (options.profile)
collectOptions.profile = options.profile;
if (options.includeExpired !== undefined)
collectOptions.includeExpired = options.includeExpired;
const cookies = collectChromeCookiesFromRows(rowsResult.rows, collectOptions, hosts, allowlistNames, (encryptedValue) => decrypt(encryptedValue, { stripHashPrefix }), warnings);
return { cookies: dedupeCookies(cookies), warnings };
}
finally {
rmSync(tempDir, { recursive: true, force: true });
}
}
function collectChromeCookiesFromRows(rows, options, hosts, allowlistNames, decrypt, warnings) {
const cookies = [];
const now = Math.floor(Date.now() / 1000);
let warnedEncryptedType = false;
for (const row of rows) {
const name = typeof row.name === 'string' ? row.name : null;
if (!name)
continue;
if (allowlistNames && allowlistNames.size > 0 && !allowlistNames.has(name))
continue;
const hostKey = typeof row.host_key === 'string' ? row.host_key : null;
if (!hostKey)
continue;
if (!hostMatchesAny(hosts, hostKey))
continue;
const rowPath = typeof row.path === 'string' ? row.path : '';
const valueString = typeof row.value === 'string' ? row.value : null;
let value = valueString;
if (value === null || value.length === 0) {
// Many modern Chromium cookies keep `value` empty and only store `encrypted_value`.
// We decrypt on demand and drop rows we can't interpret.
const encryptedBytes = getEncryptedBytes(row);
if (!encryptedBytes) {
if (!warnedEncryptedType && row.encrypted_value !== undefined) {
warnings.push('Chrome cookie encrypted_value is in an unsupported type.');
warnedEncryptedType = true;
}
continue;
}
value = decrypt(encryptedBytes);
}
if (value === null)
continue;
const expiresRaw = typeof row.expires_utc === 'number' || typeof row.expires_utc === 'bigint'
? row.expires_utc
: tryParseInt(row.expires_utc);
const expires = normalizeExpiration(expiresRaw ?? undefined);
if (!options.includeExpired) {
if (expires && expires < now)
continue;
}
const secure = row.is_secure === 1 ||
row.is_secure === 1n ||
row.is_secure === '1' ||
row.is_secure === true;
const httpOnly = row.is_httponly === 1 ||
row.is_httponly === 1n ||
row.is_httponly === '1' ||
row.is_httponly === true;
const sameSite = normalizeChromiumSameSite(row.samesite);
const source = { browser: 'chrome' };
if (options.profile)
source.profile = options.profile;
const cookie = {
name,
value,
domain: hostKey.startsWith('.') ? hostKey.slice(1) : hostKey,
path: rowPath || '/',
secure,
httpOnly,
source,
};
if (expires !== undefined)
cookie.expires = expires;
if (sameSite !== undefined)
cookie.sameSite = sameSite;
cookies.push(cookie);
}
return cookies;
}
function tryParseInt(value) {
if (typeof value === 'bigint') {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
if (typeof value !== 'string')
return null;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) ? parsed : null;
}
function normalizeChromiumSameSite(value) {
if (typeof value === 'bigint') {
const parsed = Number(value);
return Number.isFinite(parsed) ? normalizeChromiumSameSite(parsed) : undefined;
}
if (typeof value === 'number') {
if (value === 2)
return 'Strict';
if (value === 1)
return 'Lax';
if (value === 0)
return 'None';
return undefined;
}
if (typeof value === 'string') {
const parsed = Number.parseInt(value, 10);
if (Number.isFinite(parsed))
return normalizeChromiumSameSite(parsed);
const normalized = value.toLowerCase();
if (normalized === 'strict')
return 'Strict';
if (normalized === 'lax')
return 'Lax';
if (normalized === 'none' || normalized === 'no_restriction')
return 'None';
}
return undefined;
}
function getEncryptedBytes(row) {
const raw = row.encrypted_value;
if (raw instanceof Uint8Array)
return raw;
return null;
}
async function readChromiumMetaVersion(dbPath) {
const sql = `SELECT value FROM meta WHERE key = 'version'`;
const result = isBunRuntime()
? await queryNodeOrBun({ kind: 'bun', dbPath, sql })
: await queryNodeOrBun({ kind: 'node', dbPath, sql });
if (!result.ok)
return 0;
const first = result.rows[0];
const value = first?.value;
if (typeof value === 'number')
return Math.floor(value);
if (typeof value === 'bigint') {
const parsed = Number(value);
return Number.isFinite(parsed) ? Math.floor(parsed) : 0;
}
if (typeof value === 'string') {
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) ? parsed : 0;
}
return 0;
}
async function readChromeRows(dbPath, where) {
const sqliteKind = isBunRuntime() ? 'bun' : 'node';
const sqliteLabel = sqliteKind === 'bun' ? 'bun:sqlite' : 'node:sqlite';
const sql = `SELECT name, value, host_key, path, expires_utc, samesite, encrypted_value, ` +
`is_secure AS is_secure, is_httponly AS is_httponly ` +
`FROM cookies WHERE (${where}) ORDER BY expires_utc DESC;`;
const result = await queryNodeOrBun({ kind: sqliteKind, dbPath, sql });
if (result.ok)
return { ok: true, rows: result.rows };
// Intentionally strict: only support modern Chromium cookie DB schemas.
// If this fails, assume the local Chrome/Chromium is too old or uses a non-standard schema.
return {
ok: false,
error: `${sqliteLabel} failed reading Chrome cookies (requires modern Chromium, e.g. Chrome >= 100): ${result.error}`,
};
}
async function queryNodeOrBun(options) {
try {
if (options.kind === 'node') {
// Node's `node:sqlite` is synchronous and returns plain JS values. Keep it boxed in a
// small scope so callers don't need to care about runtime differences.
const { DatabaseSync } = await importNodeSqlite();
const dbOptions = { readOnly: true };
if (supportsReadBigInts()) {
dbOptions.readBigInts = true;
}
const db = new DatabaseSync(options.dbPath, dbOptions);
try {
const rows = db.prepare(options.sql).all();
return { ok: true, rows };
}
finally {
db.close();
}
}
// Bun's sqlite API has a different surface (`Database` + `.query().all()`).
const { Database } = await import('bun:sqlite');
const db = new Database(options.dbPath, { readonly: true });
try {
const rows = db.query(options.sql).all();
return { ok: true, rows };
}
finally {
db.close();
}
}
catch (error) {
return { ok: false, error: error instanceof Error ? error.message : String(error) };
}
}
function copySidecar(sourceDbPath, target, suffix) {
const sidecar = `${sourceDbPath}${suffix}`;
if (!existsSync(sidecar))
return;
try {
copyFileSync(sidecar, target);
}
catch {
// ignore
}
}
function buildHostWhereClause(hosts, column) {
const clauses = [];
for (const host of hosts) {
// Chrome cookies often live on parent domains (e.g. .google.com for gemini.google.com).
// Include parent domains so the SQL filter doesn't drop valid session cookies.
for (const candidate of expandHostCandidates(host)) {
const escaped = sqlLiteral(candidate);
const escapedDot = sqlLiteral(`.${candidate}`);
const escapedLike = sqlLiteral(`%.${candidate}`);
clauses.push(`${column} = ${escaped}`);
clauses.push(`${column} = ${escapedDot}`);
clauses.push(`${column} LIKE ${escapedLike}`);
}
}
return clauses.length ? clauses.join(' OR ') : '1=0';
}
function sqlLiteral(value) {
const escaped = value.replaceAll("'", "''");
return `'${escaped}'`;
}
function expandHostCandidates(host) {
const parts = host.split('.').filter(Boolean);
if (parts.length <= 1)
return [host];
const candidates = new Set();
candidates.add(host);
// Include parent domains down to two labels (avoid TLD-only fragments).
for (let i = 1; i <= parts.length - 2; i += 1) {
const candidate = parts.slice(i).join('.');
if (candidate)
candidates.add(candidate);
}
return Array.from(candidates);
}
function hostMatchesAny(hosts, cookieHost) {
const cookieDomain = cookieHost.startsWith('.') ? cookieHost.slice(1) : cookieHost;
return hosts.some((host) => hostMatchesCookieDomain(host, cookieDomain));
}
function dedupeCookies(cookies) {
const merged = new Map();
for (const cookie of cookies) {
const key = `${cookie.name}|${cookie.domain ?? ''}|${cookie.path ?? ''}`;
if (!merged.has(key))
merged.set(key, cookie);
}
return Array.from(merged.values());
}
//# sourceMappingURL=shared.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/shared.js.map
Generated Vendored
-1
View File
File diff suppressed because one or more lines are too long
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/windowsDpapi.d.ts
Generated Vendored
-10
View File
@@ -1,10 +0,0 @@
export declare function dpapiUnprotect(data: Buffer, options?: {
timeoutMs?: number;
}): Promise<{
ok: true;
value: Buffer;
} | {
ok: false;
error: string;
}>;
//# sourceMappingURL=windowsDpapi.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/windowsDpapi.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"windowsDpapi.d.ts","sourceRoot":"","sources":["../../../src/providers/chromeSqlite/windowsDpapi.ts"],"names":[],"mappings":"AAEA,wBAAsB,cAAc,CACnC,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA+BrE"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/windowsDpapi.js
Generated Vendored
-26
View File
@@ -1,26 +0,0 @@
import { execCapture } from '../../util/exec.js';
export async function dpapiUnprotect(data, options = {}) {
const timeoutMs = options.timeoutMs ?? 5_000;
// There is no cross-platform JS API for Windows DPAPI, and we explicitly avoid native addons.
// PowerShell can call ProtectedData.Unprotect for the current user, which matches Chrome's behavior.
const inputB64 = data.toString('base64');
const prelude = 'try { Add-Type -AssemblyName System.Security.Cryptography.ProtectedData -ErrorAction Stop } catch { try { Add-Type -AssemblyName System.Security -ErrorAction Stop } catch {} };';
const script = prelude +
`$in=[Convert]::FromBase64String('${inputB64}');` +
`$out=[System.Security.Cryptography.ProtectedData]::Unprotect($in,$null,[System.Security.Cryptography.DataProtectionScope]::CurrentUser);` +
`[Convert]::ToBase64String($out)`;
const res = await execCapture('powershell', ['-NoProfile', '-NonInteractive', '-Command', script], {
timeoutMs,
});
if (res.code !== 0) {
return { ok: false, error: res.stderr.trim() || `powershell exit ${res.code}` };
}
try {
const out = Buffer.from(res.stdout.trim(), 'base64');
return { ok: true, value: out };
}
catch (error) {
return { ok: false, error: error instanceof Error ? error.message : String(error) };
}
}
//# sourceMappingURL=windowsDpapi.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqlite/windowsDpapi.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"windowsDpapi.js","sourceRoot":"","sources":["../../../src/providers/chromeSqlite/windowsDpapi.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,IAAY,EACZ,UAAkC,EAAE;IAEpC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAE7C,8FAA8F;IAC9F,qGAAqG;IACrG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,OAAO,GACZ,kLAAkL,CAAC;IACpL,MAAM,MAAM,GACX,OAAO;QACP,oCAAoC,QAAQ,KAAK;QACjD,0IAA0I;QAC1I,iCAAiC,CAAC;IAEnC,MAAM,GAAG,GAAG,MAAM,WAAW,CAC5B,YAAY,EACZ,CAAC,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,CAAC,EACrD;QACC,SAAS;KACT,CACD,CAAC;IACF,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,mBAAmB,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACrF,CAAC;AACF,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteLinux.d.ts
Generated Vendored
-7
View File
@@ -1,7 +0,0 @@
import type { GetCookiesResult } from '../types.js';
export declare function getCookiesFromChromeSqliteLinux(options: {
profile?: string;
includeExpired?: boolean;
debug?: boolean;
}, origins: string[], allowlistNames: Set<string> | null): Promise<GetCookiesResult>;
//# sourceMappingURL=chromeSqliteLinux.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteLinux.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chromeSqliteLinux.d.ts","sourceRoot":"","sources":["../../src/providers/chromeSqliteLinux.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AASpD,wBAAsB,+BAA+B,CACpD,OAAO,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,EACxE,OAAO,EAAE,MAAM,EAAE,EACjB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAkD3B"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteLinux.js
Generated Vendored
-51
View File
@@ -1,51 +0,0 @@
import { decryptChromiumAes128CbcCookieValue, deriveAes128CbcKeyFromPassword, } from './chromeSqlite/crypto.js';
import { getLinuxChromeSafeStoragePassword } from './chromeSqlite/linuxKeyring.js';
import { getCookiesFromChromeSqliteDb } from './chromeSqlite/shared.js';
import { resolveChromiumCookiesDbLinux } from './chromium/linuxPaths.js';
export async function getCookiesFromChromeSqliteLinux(options, origins, allowlistNames) {
const args = {
configDirName: 'google-chrome',
};
if (options.profile !== undefined)
args.profile = options.profile;
const dbPath = resolveChromiumCookiesDbLinux(args);
if (!dbPath) {
return { cookies: [], warnings: ['Chrome cookies database not found.'] };
}
const { password, warnings: keyringWarnings } = await getLinuxChromeSafeStoragePassword();
// Linux uses multiple schemes depending on distro/keyring availability.
// - v10 often uses the hard-coded "peanuts" password
// - v11 uses "Chrome Safe Storage" from the keyring (may be empty/unavailable)
const v10Key = deriveAes128CbcKeyFromPassword('peanuts', { iterations: 1 });
const emptyKey = deriveAes128CbcKeyFromPassword('', { iterations: 1 });
const v11Key = deriveAes128CbcKeyFromPassword(password, { iterations: 1 });
const decrypt = (encryptedValue, opts) => {
const prefix = Buffer.from(encryptedValue).subarray(0, 3).toString('utf8');
if (prefix === 'v10') {
return decryptChromiumAes128CbcCookieValue(encryptedValue, [v10Key, emptyKey], {
stripHashPrefix: opts.stripHashPrefix,
treatUnknownPrefixAsPlaintext: false,
});
}
if (prefix === 'v11') {
return decryptChromiumAes128CbcCookieValue(encryptedValue, [v11Key, emptyKey], {
stripHashPrefix: opts.stripHashPrefix,
treatUnknownPrefixAsPlaintext: false,
});
}
return null;
};
const dbOptions = {
dbPath,
};
if (options.profile)
dbOptions.profile = options.profile;
if (options.includeExpired !== undefined)
dbOptions.includeExpired = options.includeExpired;
if (options.debug !== undefined)
dbOptions.debug = options.debug;
const result = await getCookiesFromChromeSqliteDb(dbOptions, origins, allowlistNames, decrypt);
result.warnings.unshift(...keyringWarnings);
return result;
}
//# sourceMappingURL=chromeSqliteLinux.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteLinux.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chromeSqliteLinux.js","sourceRoot":"","sources":["../../src/providers/chromeSqliteLinux.ts"],"names":[],"mappings":"AACA,OAAO,EACN,mCAAmC,EACnC,8BAA8B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,iCAAiC,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,4BAA4B,EAAE,MAAM,0BAA0B,CAAC;AACxE,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAC;AAEzE,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACpD,OAAwE,EACxE,OAAiB,EACjB,cAAkC;IAElC,MAAM,IAAI,GAAwD;QACjE,aAAa,EAAE,eAAe;KAC9B,CAAC;IACF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAClE,MAAM,MAAM,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,oCAAoC,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,iCAAiC,EAAE,CAAC;IAE1F,wEAAwE;IACxE,qDAAqD;IACrD,+EAA+E;IAC/E,MAAM,MAAM,GAAG,8BAA8B,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5E,MAAM,QAAQ,GAAG,8BAA8B,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,8BAA8B,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IAE3E,MAAM,OAAO,GAAG,CACf,cAA0B,EAC1B,IAAkC,EAClB,EAAE;QAClB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC3E,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACtB,OAAO,mCAAmC,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;gBAC9E,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,6BAA6B,EAAE,KAAK;aACpC,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACtB,OAAO,mCAAmC,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;gBAC9E,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,6BAA6B,EAAE,KAAK;aACpC,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC;IAEF,MAAM,SAAS,GACd;QACC,MAAM;KACN,CAAC;IACH,IAAI,OAAO,CAAC,OAAO;QAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACzD,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS;QAAE,SAAS,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IAC5F,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;QAAE,SAAS,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;IAC/F,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,eAAe,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC;AACf,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteMac.d.ts
Generated Vendored
-7
View File
@@ -1,7 +0,0 @@
import type { GetCookiesResult } from '../types.js';
export declare function getCookiesFromChromeSqliteMac(options: {
profile?: string;
includeExpired?: boolean;
debug?: boolean;
}, origins: string[], allowlistNames: Set<string> | null): Promise<GetCookiesResult>;
//# sourceMappingURL=chromeSqliteMac.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteMac.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chromeSqliteMac.d.ts","sourceRoot":"","sources":["../../src/providers/chromeSqliteMac.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AASpD,wBAAsB,6BAA6B,CAClD,OAAO,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,EACxE,OAAO,EAAE,MAAM,EAAE,EACjB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAChC,OAAO,CAAC,gBAAgB,CAAC,CA6C3B"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteMac.js
Generated Vendored
-60
View File
@@ -1,60 +0,0 @@
import { homedir } from 'node:os';
import path from 'node:path';
import { decryptChromiumAes128CbcCookieValue, deriveAes128CbcKeyFromPassword, } from './chromeSqlite/crypto.js';
import { getCookiesFromChromeSqliteDb } from './chromeSqlite/shared.js';
import { readKeychainGenericPasswordFirst } from './chromium/macosKeychain.js';
import { resolveCookiesDbFromProfileOrRoots } from './chromium/paths.js';
export async function getCookiesFromChromeSqliteMac(options, origins, allowlistNames) {
const dbPath = resolveChromeCookiesDb(options.profile);
if (!dbPath) {
return { cookies: [], warnings: ['Chrome cookies database not found.'] };
}
const warnings = [];
// On macOS, Chrome stores its "Safe Storage" secret in Keychain.
// `security find-generic-password` is stable and avoids any native Node keychain modules.
const passwordResult = await readKeychainGenericPasswordFirst({
account: 'Chrome',
services: ['Chrome Safe Storage'],
timeoutMs: 3_000,
label: 'Chrome Safe Storage',
});
if (!passwordResult.ok) {
warnings.push(passwordResult.error);
return { cookies: [], warnings };
}
const chromePassword = passwordResult.password.trim();
if (!chromePassword) {
warnings.push('macOS Keychain returned an empty Chrome Safe Storage password.');
return { cookies: [], warnings };
}
// Chromium uses PBKDF2(password, "saltysalt", 1003, 16, sha1) for AES-128-CBC cookie values on macOS.
const key = deriveAes128CbcKeyFromPassword(chromePassword, { iterations: 1003 });
const decrypt = (encryptedValue, opts) => decryptChromiumAes128CbcCookieValue(encryptedValue, [key], {
stripHashPrefix: opts.stripHashPrefix,
treatUnknownPrefixAsPlaintext: true,
});
const dbOptions = {
dbPath,
};
if (options.profile)
dbOptions.profile = options.profile;
if (options.includeExpired !== undefined)
dbOptions.includeExpired = options.includeExpired;
if (options.debug !== undefined)
dbOptions.debug = options.debug;
const result = await getCookiesFromChromeSqliteDb(dbOptions, origins, allowlistNames, decrypt);
result.warnings.unshift(...warnings);
return result;
}
function resolveChromeCookiesDb(profile) {
const home = homedir();
/* c8 ignore next */
const roots = process.platform === 'darwin'
? [path.join(home, 'Library', 'Application Support', 'Google', 'Chrome')]
: [];
const args = { roots };
if (profile !== undefined)
args.profile = profile;
return resolveCookiesDbFromProfileOrRoots(args);
}
//# sourceMappingURL=chromeSqliteMac.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteMac.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chromeSqliteMac.js","sourceRoot":"","sources":["../../src/providers/chromeSqliteMac.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EACN,mCAAmC,EACnC,8BAA8B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,4BAA4B,EAAE,MAAM,0BAA0B,CAAC;AACxE,OAAO,EAAE,gCAAgC,EAAE,MAAM,6BAA6B,CAAC;AAC/E,OAAO,EAAE,kCAAkC,EAAE,MAAM,qBAAqB,CAAC;AAEzE,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAClD,OAAwE,EACxE,OAAiB,EACjB,cAAkC;IAElC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,oCAAoC,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,iEAAiE;IACjE,0FAA0F;IAC1F,MAAM,cAAc,GAAG,MAAM,gCAAgC,CAAC;QAC7D,OAAO,EAAE,QAAQ;QACjB,QAAQ,EAAE,CAAC,qBAAqB,CAAC;QACjC,SAAS,EAAE,KAAK;QAChB,KAAK,EAAE,qBAAqB;KAC5B,CAAC,CAAC;IACH,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IAClC,CAAC;IAED,MAAM,cAAc,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtD,IAAI,CAAC,cAAc,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;QAChF,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IAClC,CAAC;IAED,sGAAsG;IACtG,MAAM,GAAG,GAAG,8BAA8B,CAAC,cAAc,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,CAAC,cAA0B,EAAE,IAAkC,EAAiB,EAAE,CACjG,mCAAmC,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,EAAE;QAC1D,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,6BAA6B,EAAE,IAAI;KACnC,CAAC,CAAC;IAEJ,MAAM,SAAS,GACd;QACC,MAAM;KACN,CAAC;IACH,IAAI,OAAO,CAAC,OAAO;QAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACzD,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS;QAAE,SAAS,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IAC5F,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;QAAE,SAAS,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;IAC/F,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAgB;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;IACvB,oBAAoB;IACpB,MAAM,KAAK,GACV,OAAO,CAAC,QAAQ,KAAK,QAAQ;QAC5B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,qBAAqB,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzE,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,IAAI,GAA6D,EAAE,KAAK,EAAE,CAAC;IACjF,IAAI,OAAO,KAAK,SAAS;QAAE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAClD,OAAO,kCAAkC,CAAC,IAAI,CAAC,CAAC;AACjD,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteWindows.d.ts
Generated Vendored
-7
View File
@@ -1,7 +0,0 @@
import type { GetCookiesResult } from '../types.js';
export declare function getCookiesFromChromeSqliteWindows(options: {
profile?: string;
includeExpired?: boolean;
debug?: boolean;
}, origins: string[], allowlistNames: Set<string> | null): Promise<GetCookiesResult>;
//# sourceMappingURL=chromeSqliteWindows.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteWindows.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chromeSqliteWindows.d.ts","sourceRoot":"","sources":["../../src/providers/chromeSqliteWindows.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAMpD,wBAAsB,iCAAiC,CACtD,OAAO,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,EACxE,OAAO,EAAE,MAAM,EAAE,EACjB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAmC3B"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteWindows.js
Generated Vendored
-38
View File
@@ -1,38 +0,0 @@
import path from 'node:path';
import { decryptChromiumAes256GcmCookieValue } from './chromeSqlite/crypto.js';
import { getCookiesFromChromeSqliteDb } from './chromeSqlite/shared.js';
import { getWindowsChromiumMasterKey } from './chromium/windowsMasterKey.js';
import { resolveChromiumPathsWindows } from './chromium/windowsPaths.js';
export async function getCookiesFromChromeSqliteWindows(options, origins, allowlistNames) {
const resolveArgs = {
localAppDataVendorPath: path.join('Google', 'Chrome', 'User Data'),
};
if (options.profile !== undefined)
resolveArgs.profile = options.profile;
const { dbPath, userDataDir } = resolveChromiumPathsWindows(resolveArgs);
if (!dbPath || !userDataDir) {
return { cookies: [], warnings: ['Chrome cookies database not found.'] };
}
// On Windows, Chrome stores an AES key in `Local State` encrypted with DPAPI (CurrentUser).
// That master key is then used for AES-256-GCM cookie values (`v10`/`v11`/`v20` prefixes).
const masterKey = await getWindowsChromiumMasterKey(userDataDir, 'Chrome');
if (!masterKey.ok) {
return { cookies: [], warnings: [masterKey.error] };
}
const decrypt = (encryptedValue, opts) => {
return decryptChromiumAes256GcmCookieValue(encryptedValue, masterKey.value, {
stripHashPrefix: opts.stripHashPrefix,
});
};
const dbOptions = {
dbPath,
};
if (options.profile)
dbOptions.profile = options.profile;
if (options.includeExpired !== undefined)
dbOptions.includeExpired = options.includeExpired;
if (options.debug !== undefined)
dbOptions.debug = options.debug;
return await getCookiesFromChromeSqliteDb(dbOptions, origins, allowlistNames, decrypt);
}
//# sourceMappingURL=chromeSqliteWindows.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromeSqliteWindows.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"chromeSqliteWindows.js","sourceRoot":"","sources":["../../src/providers/chromeSqliteWindows.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,mCAAmC,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,4BAA4B,EAAE,MAAM,0BAA0B,CAAC;AACxE,OAAO,EAAE,2BAA2B,EAAE,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AAEzE,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACtD,OAAwE,EACxE,OAAiB,EACjB,cAAkC;IAElC,MAAM,WAAW,GAAsD;QACtE,sBAAsB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC;KAClE,CAAC;IACF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,WAAW,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACzE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,2BAA2B,CAAC,WAAW,CAAC,CAAC;IACzE,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,oCAAoC,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED,4FAA4F;IAC5F,2FAA2F;IAC3F,MAAM,SAAS,GAAG,MAAM,2BAA2B,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC3E,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;IACrD,CAAC;IAED,MAAM,OAAO,GAAG,CACf,cAA0B,EAC1B,IAAkC,EAClB,EAAE;QAClB,OAAO,mCAAmC,CAAC,cAAc,EAAE,SAAS,CAAC,KAAK,EAAE;YAC3E,eAAe,EAAE,IAAI,CAAC,eAAe;SACrC,CAAC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,SAAS,GACd;QACC,MAAM;KACN,CAAC;IACH,IAAI,OAAO,CAAC,OAAO;QAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACzD,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS;QAAE,SAAS,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IAC5F,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;QAAE,SAAS,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAEjE,OAAO,MAAM,4BAA4B,CAAC,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;AACxF,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/linuxPaths.d.ts
Generated Vendored
-5
View File
@@ -1,5 +0,0 @@
export declare function resolveChromiumCookiesDbLinux(options: {
configDirName: string;
profile?: string;
}): string | null;
//# sourceMappingURL=linuxPaths.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/linuxPaths.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"linuxPaths.d.ts","sourceRoot":"","sources":["../../../src/providers/chromium/linuxPaths.ts"],"names":[],"mappings":"AAMA,wBAAgB,6BAA6B,CAAC,OAAO,EAAE;IACtD,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,MAAM,GAAG,IAAI,CA0BhB"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/linuxPaths.js
Generated Vendored
-33
View File
@@ -1,33 +0,0 @@
import { existsSync } from 'node:fs';
import { homedir } from 'node:os';
import path from 'node:path';
import { expandPath, looksLikePath } from './paths.js';
export function resolveChromiumCookiesDbLinux(options) {
const home = homedir();
// biome-ignore lint/complexity/useLiteralKeys: process.env is an index signature under strict TS.
const configHome = process.env['XDG_CONFIG_HOME']?.trim() || path.join(home, '.config');
const root = path.join(configHome, options.configDirName);
if (options.profile && looksLikePath(options.profile)) {
const candidate = expandPath(options.profile);
if (candidate.endsWith('Cookies') && existsSync(candidate))
return candidate;
const direct = path.join(candidate, 'Cookies');
if (existsSync(direct))
return direct;
const network = path.join(candidate, 'Network', 'Cookies');
if (existsSync(network))
return network;
return null;
}
const profileDir = options.profile && options.profile.trim().length > 0 ? options.profile.trim() : 'Default';
const candidates = [
path.join(root, profileDir, 'Cookies'),
path.join(root, profileDir, 'Network', 'Cookies'),
];
for (const candidate of candidates) {
if (existsSync(candidate))
return candidate;
}
return null;
}
//# sourceMappingURL=linuxPaths.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/linuxPaths.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"linuxPaths.js","sourceRoot":"","sources":["../../../src/providers/chromium/linuxPaths.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,UAAU,6BAA6B,CAAC,OAG7C;IACA,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;IACvB,kGAAkG;IAClG,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACxF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAE1D,IAAI,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;QAC7E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAI,UAAU,CAAC,OAAO,CAAC;YAAE,OAAO,OAAO,CAAC;QACxC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,UAAU,GACf,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3F,MAAM,UAAU,GAAG;QAClB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC;KACjD,CAAC;IACF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/macosKeychain.d.ts
Generated Vendored
-24
View File
@@ -1,24 +0,0 @@
export declare function readKeychainGenericPassword(options: {
account: string;
service: string;
timeoutMs: number;
}): Promise<{
ok: true;
password: string;
} | {
ok: false;
error: string;
}>;
export declare function readKeychainGenericPasswordFirst(options: {
account: string;
services: string[];
timeoutMs: number;
label: string;
}): Promise<{
ok: true;
password: string;
} | {
ok: false;
error: string;
}>;
//# sourceMappingURL=macosKeychain.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/macosKeychain.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"macosKeychain.d.ts","sourceRoot":"","sources":["../../../src/providers/chromium/macosKeychain.ts"],"names":[],"mappings":"AAEA,wBAAsB,2BAA2B,CAAC,OAAO,EAAE;IAC1D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAczE;AAED,wBAAsB,gCAAgC,CAAC,OAAO,EAAE;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACd,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgBzE"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/macosKeychain.js
Generated Vendored
-30
View File
@@ -1,30 +0,0 @@
import { execCapture } from '../../util/exec.js';
export async function readKeychainGenericPassword(options) {
const res = await execCapture('security', ['find-generic-password', '-w', '-a', options.account, '-s', options.service], { timeoutMs: options.timeoutMs });
if (res.code === 0) {
const password = res.stdout.trim();
return { ok: true, password };
}
return {
ok: false,
error: `${res.stderr.trim() || `exit ${res.code}`}`,
};
}
export async function readKeychainGenericPasswordFirst(options) {
let lastError = null;
for (const service of options.services) {
const r = await readKeychainGenericPassword({
account: options.account,
service,
timeoutMs: options.timeoutMs,
});
if (r.ok)
return r;
lastError = r.error;
}
return {
ok: false,
error: `Failed to read macOS Keychain (${options.label}): ${lastError ?? 'permission denied / keychain locked / entry missing.'}`,
};
}
//# sourceMappingURL=macosKeychain.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/macosKeychain.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"macosKeychain.js","sourceRoot":"","sources":["../../../src/providers/chromium/macosKeychain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,OAIjD;IACA,MAAM,GAAG,GAAG,MAAM,WAAW,CAC5B,UAAU,EACV,CAAC,uBAAuB,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,EAC7E,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAChC,CAAC;IACF,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IACD,OAAO;QACN,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,QAAQ,GAAG,CAAC,IAAI,EAAE,EAAE;KACnD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gCAAgC,CAAC,OAKtD;IACA,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,MAAM,2BAA2B,CAAC;YAC3C,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO;YACP,SAAS,EAAE,OAAO,CAAC,SAAS;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,CAAC,EAAE;YAAE,OAAO,CAAC,CAAC;QACnB,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC;IACrB,CAAC;IAED,OAAO;QACN,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,kCAAkC,OAAO,CAAC,KAAK,MAAM,SAAS,IAAI,sDAAsD,EAAE;KACjI,CAAC;AACH,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/paths.d.ts
Generated Vendored
-11
View File
@@ -1,11 +0,0 @@
export declare function looksLikePath(value: string): boolean;
export declare function expandPath(input: string): string;
export declare function safeStat(candidate: string): {
isFile: () => boolean;
isDirectory: () => boolean;
} | null;
export declare function resolveCookiesDbFromProfileOrRoots(options: {
profile?: string;
roots: string[];
}): string | null;
//# sourceMappingURL=paths.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/paths.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"paths.d.ts","sourceRoot":"","sources":["../../../src/providers/chromium/paths.ts"],"names":[],"mappings":"AAIA,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAEpD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGhD;AAED,wBAAgB,QAAQ,CACvB,SAAS,EAAE,MAAM,GACf;IAAE,MAAM,EAAE,MAAM,OAAO,CAAC;IAAC,WAAW,EAAE,MAAM,OAAO,CAAA;CAAE,GAAG,IAAI,CAM9D;AAED,wBAAgB,kCAAkC,CAAC,OAAO,EAAE;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CAChB,GAAG,MAAM,GAAG,IAAI,CAuBhB"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/paths.js
Generated Vendored
-43
View File
@@ -1,43 +0,0 @@
import { existsSync, statSync } from 'node:fs';
import { homedir } from 'node:os';
import path from 'node:path';
export function looksLikePath(value) {
return value.includes('/') || value.includes('\\');
}
export function expandPath(input) {
if (input.startsWith('~/'))
return path.join(homedir(), input.slice(2));
return path.isAbsolute(input) ? input : path.resolve(process.cwd(), input);
}
export function safeStat(candidate) {
try {
return statSync(candidate);
}
catch {
return null;
}
}
export function resolveCookiesDbFromProfileOrRoots(options) {
const candidates = [];
if (options.profile && looksLikePath(options.profile)) {
const expanded = expandPath(options.profile);
const stat = safeStat(expanded);
if (stat?.isFile())
return expanded;
candidates.push(path.join(expanded, 'Cookies'));
candidates.push(path.join(expanded, 'Network', 'Cookies'));
}
else {
const profileDir = options.profile && options.profile.trim().length > 0 ? options.profile.trim() : 'Default';
for (const root of options.roots) {
candidates.push(path.join(root, profileDir, 'Cookies'));
candidates.push(path.join(root, profileDir, 'Network', 'Cookies'));
}
}
for (const candidate of candidates) {
if (existsSync(candidate))
return candidate;
}
return null;
}
//# sourceMappingURL=paths.js.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/paths.js.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"paths.js","sourceRoot":"","sources":["../../../src/providers/chromium/paths.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,UAAU,aAAa,CAAC,KAAa;IAC1C,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAa;IACvC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,QAAQ,CACvB,SAAiB;IAEjB,IAAI,CAAC;QACJ,OAAO,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,MAAM,UAAU,kCAAkC,CAAC,OAGlD;IACA,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,IAAI,EAAE,MAAM,EAAE;YAAE,OAAO,QAAQ,CAAC;QACpC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QAChD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;IAC5D,CAAC;SAAM,CAAC;QACP,MAAM,UAAU,GACf,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3F,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;YACxD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;QACpE,CAAC;IACF,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IAC7C,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/windowsMasterKey.d.ts
Generated Vendored
-8
View File
@@ -1,8 +0,0 @@
export declare function getWindowsChromiumMasterKey(userDataDir: string, label: string): Promise<{
ok: true;
value: Buffer;
} | {
ok: false;
error: string;
}>;
//# sourceMappingURL=windowsMasterKey.d.ts.map
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/windowsMasterKey.d.ts.map
Generated Vendored
-1
View File
@@ -1 +0,0 @@
{"version":3,"file":"windowsMasterKey.d.ts","sourceRoot":"","sources":["../../../src/providers/chromium/windowsMasterKey.ts"],"names":[],"mappings":"AAKA,wBAAsB,2BAA2B,CAChD,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,GACX,OAAO,CACP;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC3B;IACA,EAAE,EAAE,KAAK,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;CACb,CACH,CAsCA"}
scripts/lib/vendor/bird-search/node_modules/@steipete/sweet-cookie/dist/providers/chromium/windowsMasterKey.js
Generated Vendored
-41
View File
@@ -1,41 +0,0 @@
import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { dpapiUnprotect } from '../chromeSqlite/windowsDpapi.js';
export async function getWindowsChromiumMasterKey(userDataDir, label) {
const localStatePath = path.join(userDataDir, 'Local State');
if (!existsSync(localStatePath)) {
return { ok: false, error: `${label} Local State file not found.` };
}
let encryptedKeyB64 = null;
try {
const raw = readFileSync(localStatePath, 'utf8');
const parsed = JSON.parse(raw);
encryptedKeyB64 =
typeof parsed.os_crypt?.encrypted_key === 'string' ? parsed.os_crypt.encrypted_key : null;
}
catch (error) {
return {
ok: false,
error: `Failed to parse ${label} Local State: ${error instanceof Error ? error.message : String(error)}`,
};
}
if (!encryptedKeyB64)
return { ok: false, error: `${label} Local State missing os_crypt.encrypted_key.` };
let encryptedKey;
try {
encryptedKey = Buffer.from(encryptedKeyB64, 'base64');
}
catch {
return { ok: false, error: `${label} Local State contains an invalid encrypted_key.` };
}
const prefix = Buffer.from('DPAPI', 'utf8');
if (!encryptedKey.subarray(0, prefix.length).equals(prefix)) {
return { ok: false, error: `${label} encrypted_key does not start with DPAPI prefix.` };
}
const unprotected = await dpapiUnprotect(encryptedKey.subarray(prefix.length));
if (!unprotected.ok) {
return { ok: false, error: `DPAPI decrypt failed: ${unprotected.error}` };
}
return { ok: true, value: unprotected.value };
}
//# sourceMappingURL=windowsMasterKey.js.map

Some files were not shown because too many files have changed in this diff Show More