sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add WebSearch as third source with zero-config fallback

Browse Source 
Add Claude's built-in WebSearch tool as a third research source for
/last30days. This enables the skill to work out of the box with zero
API keys while preserving Reddit/X as the primary sources.

Key changes:
- Add WebSearchItem schema for web results (no engagement metrics)
- Add score_websearch_items() with 55/45 relevance/recency weighting
- Apply -15pt source penalty so WebSearch ranks below Reddit/X
- Add --include-web CLI flag to opt-in to WebSearch
- Return 'web' mode when no API keys configured (zero-config)
- Update render.py with [WEB] source label formatting

When WebSearch is enabled, the script outputs instructions for Claude
to use its built-in WebSearch tool, then synthesize results together.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-01-24 10:23:32 -08:00
parent 6d5e4467c2
commit f98a0b9f35
7 changed files with 852 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lib/env.py
+40 -11
View File
@@ -57,7 +57,7 @@ def config_exists() -> bool:
def get_available_sources(config: Dict[str, Any]) -> str:
"""Determine which sources are available based on API keys.
Returns: 'both', 'reddit', 'x', or 'none'
Returns: 'both', 'reddit', 'x', or 'web' (fallback when no keys)
"""
has_openai = bool(config.get('OPENAI_API_KEY'))
has_xai = bool(config.get('XAI_API_KEY'))
@@ -69,34 +69,63 @@ def get_available_sources(config: Dict[str, Any]) -> str:
elif has_xai:
return 'x'
else:
return 'none'
return 'web' # Fallback: WebSearch only (no API keys needed)
def validate_sources(requested: str, available: str) -> tuple[str, Optional[str]]:
def validate_sources(requested: str, available: str, include_web: bool = False) -> tuple[str, Optional[str]]:
"""Validate requested sources against available keys.
Args:
requested: 'auto', 'reddit', 'x', or 'both'
requested: 'auto', 'reddit', 'x', 'both', or 'web'
available: Result from get_available_sources()
include_web: If True, add WebSearch to available sources
Returns:
Tuple of (effective_sources, error_message)
"""
if available == 'none':
return 'none', "No API keys configured. Please add at least one key to ~/.config/last30days/.env"
# WebSearch-only mode (no API keys)
if available == 'web':
if requested == 'auto':
return 'web', None
elif requested == 'web':
return 'web', None
else:
return 'web', f"No API keys configured. Using WebSearch fallback. Add keys to ~/.config/last30days/.env for Reddit/X."
if requested == 'auto':
# Add web to sources if include_web is set
if include_web:
if available == 'both':
return 'all', None # reddit + x + web
elif available == 'reddit':
return 'reddit-web', None
elif available == 'x':
return 'x-web', None
return available, None
if requested == 'web':
return 'web', None
if requested == 'both':
if available != 'both':
if available not in ('both',):
missing = 'xAI' if available == 'reddit' else 'OpenAI'
return 'none', f"Requested both sources but {missing} key is missing. Use --sources=auto to use available keys."
if include_web:
return 'all', None
return 'both', None
if requested == 'reddit' and available == 'x':
return 'none', "Requested Reddit but only xAI key is available."
if requested == 'reddit':
if available == 'x':
return 'none', "Requested Reddit but only xAI key is available."
if include_web:
return 'reddit-web', None
return 'reddit', None
if requested == 'x' and available == 'reddit':
return 'none', "Requested X but only OpenAI key is available."
if requested == 'x':
if available == 'reddit':
return 'none', "Requested X but only OpenAI key is available."
if include_web:
return 'x-web', None
return 'x', None
return requested, None
scripts/lib/render.py
+39 -1
View File
@@ -100,7 +100,7 @@ def render_compact(report: schema.Report, limit: int = 15) -> str:
lines.append("")
lines.append(f"**ERROR:** {report.x_error}")
lines.append("")
elif report.mode in ("both", "x-only") and not report.x:
elif report.mode in ("both", "x-only", "all", "x-web") and not report.x:
lines.append("### X Posts")
lines.append("")
lines.append("*No relevant X posts found for this topic.*")
@@ -129,6 +129,26 @@ def render_compact(report: schema.Report, limit: int = 15) -> str:
lines.append(f" *{item.why_relevant}*")
lines.append("")
# Web items (if any - populated by Claude)
if report.web_error:
lines.append("### Web Results")
lines.append("")
lines.append(f"**ERROR:** {report.web_error}")
lines.append("")
elif report.web:
lines.append("### Web Results")
lines.append("")
for item in report.web[:limit]:
date_str = f" ({item.date})" if item.date else " (date unknown)"
conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else ""
lines.append(f"**{item.id}** [WEB] (score:{item.score}) {item.source_domain}{date_str}{conf_str}")
lines.append(f" {item.title}")
lines.append(f" {item.url}")
lines.append(f" {item.snippet[:150]}...")
lines.append(f" *{item.why_relevant}*")
lines.append("")
return "\n".join(lines)
@@ -156,6 +176,8 @@ def render_context_snippet(report: schema.Report) -> str:
all_items.append((item.score, "Reddit", item.title, item.url))
for item in report.x[:5]:
all_items.append((item.score, "X", item.text[:50] + "...", item.url))
for item in report.web[:5]:
all_items.append((item.score, "Web", item.title[:50] + "...", item.url))
all_items.sort(key=lambda x: -x[0])
for score, source, text, url in all_items[:7]:
@@ -243,6 +265,22 @@ def render_full_report(report: schema.Report) -> str:
lines.append(f"> {item.text}")
lines.append("")
# Web section
if report.web:
lines.append("## Web Results")
lines.append("")
for item in report.web:
lines.append(f"### {item.id}: {item.title}")
lines.append("")
lines.append(f"- **Source:** {item.source_domain}")
lines.append(f"- **URL:** {item.url}")
lines.append(f"- **Date:** {item.date or 'Unknown'} (confidence: {item.date_confidence})")
lines.append(f"- **Score:** {item.score}/100")
lines.append(f"- **Relevance:** {item.why_relevant}")
lines.append("")
lines.append(f"> {item.snippet}")
lines.append("")
# Placeholders for Claude synthesis
lines.append("## Best Practices")
lines.append("")
scripts/lib/schema.py
+57 -1
View File
@@ -138,6 +138,37 @@ class XItem:
}
@dataclass
class WebSearchItem:
"""Normalized web search item (no engagement metrics)."""
id: str
title: str
url: str
source_domain: str # e.g., "medium.com", "github.com"
snippet: str
date: Optional[str] = None
date_confidence: str = "low"
relevance: float = 0.5
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
def to_dict(self) -> Dict[str, Any]:
return {
'id': self.id,
'title': self.title,
'url': self.url,
'source_domain': self.source_domain,
'snippet': self.snippet,
'date': self.date,
'date_confidence': self.date_confidence,
'relevance': self.relevance,
'why_relevant': self.why_relevant,
'subs': self.subs.to_dict(),
'score': self.score,
}
@dataclass
class Report:
"""Full research report."""
@@ -145,17 +176,19 @@ class Report:
range_from: str
range_to: str
generated_at: str
mode: str # 'reddit-only', 'x-only', 'both'
mode: str # 'reddit-only', 'x-only', 'both', 'web-only', etc.
openai_model_used: Optional[str] = None
xai_model_used: Optional[str] = None
reddit: List[RedditItem] = field(default_factory=list)
x: List[XItem] = field(default_factory=list)
web: List[WebSearchItem] = field(default_factory=list)
best_practices: List[str] = field(default_factory=list)
prompt_pack: List[str] = field(default_factory=list)
context_snippet_md: str = ""
# Status tracking
reddit_error: Optional[str] = None
x_error: Optional[str] = None
web_error: Optional[str] = None
# Cache info
from_cache: bool = False
cache_age_hours: Optional[float] = None
@@ -173,6 +206,7 @@ class Report:
'xai_model_used': self.xai_model_used,
'reddit': [r.to_dict() for r in self.reddit],
'x': [x.to_dict() for x in self.x],
'web': [w.to_dict() for w in self.web],
'best_practices': self.best_practices,
'prompt_pack': self.prompt_pack,
'context_snippet_md': self.context_snippet_md,
@@ -181,6 +215,8 @@ class Report:
d['reddit_error'] = self.reddit_error
if self.x_error:
d['x_error'] = self.x_error
if self.web_error:
d['web_error'] = self.web_error
if self.from_cache:
d['from_cache'] = self.from_cache
if self.cache_age_hours is not None:
@@ -240,6 +276,24 @@ class Report:
score=x.get('score', 0),
))
# Reconstruct Web items
web_items = []
for w in data.get('web', []):
subs = SubScores(**w.get('subs', {})) if w.get('subs') else SubScores()
web_items.append(WebSearchItem(
id=w['id'],
title=w['title'],
url=w['url'],
source_domain=w.get('source_domain', ''),
snippet=w.get('snippet', ''),
date=w.get('date'),
date_confidence=w.get('date_confidence', 'low'),
relevance=w.get('relevance', 0.5),
why_relevant=w.get('why_relevant', ''),
subs=subs,
score=w.get('score', 0),
))
return cls(
topic=data['topic'],
range_from=range_from,
@@ -250,11 +304,13 @@ class Report:
xai_model_used=data.get('xai_model_used'),
reddit=reddit_items,
x=x_items,
web=web_items,
best_practices=data.get('best_practices', []),
prompt_pack=data.get('prompt_pack', []),
context_snippet_md=data.get('context_snippet_md', ''),
reddit_error=data.get('reddit_error'),
x_error=data.get('x_error'),
web_error=data.get('web_error'),
from_cache=data.get('from_cache', False),
cache_age_hours=data.get('cache_age_hours'),
)
scripts/lib/score.py
+63 -4
View File
@@ -5,11 +5,16 @@ from typing import List, Optional, Union
from . import dates, schema
# Score weights
# Score weights for Reddit/X (has engagement)
WEIGHT_RELEVANCE = 0.45
WEIGHT_RECENCY = 0.25
WEIGHT_ENGAGEMENT = 0.30
# WebSearch weights (no engagement, reweighted to 100%)
WEBSEARCH_WEIGHT_RELEVANCE = 0.55
WEBSEARCH_WEIGHT_RECENCY = 0.45
WEBSEARCH_SOURCE_PENALTY = 15 # Points deducted for lacking engagement
# Default engagement score for unknown
DEFAULT_ENGAGEMENT = 35
UNKNOWN_ENGAGEMENT_PENALTY = 10
@@ -212,7 +217,56 @@ def score_x_items(items: List[schema.XItem]) -> List[schema.XItem]:
return items
def sort_items(items: List[Union[schema.RedditItem, schema.XItem]]) -> List:
def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
"""Compute scores for WebSearch items WITHOUT engagement metrics.
Uses reweighted formula: 55% relevance + 45% recency - 15pt source penalty.
This ensures WebSearch items rank below comparable Reddit/X items.
Args:
items: List of WebSearch items
Returns:
Items with updated scores
"""
if not items:
return items
for item in items:
# Relevance subscore (model-provided, convert to 0-100)
rel_score = int(item.relevance * 100)
# Recency subscore
rec_score = dates.recency_score(item.date)
# Store subscores (engagement is 0 for WebSearch - no data)
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=0, # Explicitly zero - no engagement data available
)
# Compute overall score using WebSearch weights
overall = (
WEBSEARCH_WEIGHT_RELEVANCE * rel_score +
WEBSEARCH_WEIGHT_RECENCY * rec_score
)
# Apply source penalty (WebSearch < Reddit/X for same relevance/recency)
overall -= WEBSEARCH_SOURCE_PENALTY
# Apply penalty for low date confidence
if item.date_confidence == "low":
overall -= 10
elif item.date_confidence == "med":
overall -= 5
item.score = max(0, min(100, int(overall)))
return items
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem]]) -> List:
"""Sort items by score (descending), then date, then source priority.
Args:
@@ -229,8 +283,13 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem]]) -> List:
date = item.date or "0000-00-00"
date_key = -int(date.replace("-", ""))
# Tertiary: source priority (Reddit before X)
source_priority = 0 if isinstance(item, schema.RedditItem) else 1
# Tertiary: source priority (Reddit > X > WebSearch)
if isinstance(item, schema.RedditItem):
source_priority = 0
elif isinstance(item, schema.XItem):
source_priority = 1
else: # WebSearchItem
source_priority = 2
# Quaternary: title/text for stability
text = getattr(item, "title", "") or getattr(item, "text", "")
scripts/lib/websearch.py
+195
View File
@@ -0,0 +1,195 @@
"""WebSearch module for last30days skill.
NOTE: WebSearch uses Claude's built-in WebSearch tool, which runs INSIDE Claude Code.
Unlike Reddit/X which use external APIs, WebSearch results are obtained by Claude
directly and passed to this module for normalization and scoring.
The typical flow is:
1. Claude invokes WebSearch tool with the topic
2. Claude passes results to parse_websearch_results()
3. Results are normalized into WebSearchItem objects
"""
import re
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from . import schema
# Domains to exclude (Reddit and X are handled separately)
EXCLUDED_DOMAINS = {
"reddit.com",
"www.reddit.com",
"old.reddit.com",
"twitter.com",
"www.twitter.com",
"x.com",
"www.x.com",
"mobile.twitter.com",
}
def extract_domain(url: str) -> str:
"""Extract the domain from a URL.
Args:
url: Full URL
Returns:
Domain string (e.g., "medium.com")
"""
try:
parsed = urlparse(url)
domain = parsed.netloc.lower()
# Remove www. prefix for cleaner display
if domain.startswith("www."):
domain = domain[4:]
return domain
except Exception:
return ""
def is_excluded_domain(url: str) -> bool:
"""Check if URL is from an excluded domain (Reddit/X).
Args:
url: URL to check
Returns:
True if URL should be excluded
"""
try:
parsed = urlparse(url)
domain = parsed.netloc.lower()
return domain in EXCLUDED_DOMAINS
except Exception:
return False
def parse_websearch_results(
results: List[Dict[str, Any]],
topic: str,
) -> List[Dict[str, Any]]:
"""Parse WebSearch results into normalized format.
This function expects results from Claude's WebSearch tool.
Each result should have: title, url, snippet, and optionally date/relevance.
Args:
results: List of WebSearch result dicts
topic: Original search topic (for context)
Returns:
List of normalized item dicts ready for WebSearchItem creation
"""
items = []
for i, result in enumerate(results):
if not isinstance(result, dict):
continue
url = result.get("url", "")
if not url:
continue
# Skip Reddit/X URLs (handled separately)
if is_excluded_domain(url):
continue
title = str(result.get("title", "")).strip()
snippet = str(result.get("snippet", result.get("description", ""))).strip()
if not title and not snippet:
continue
# Parse date if provided
date = result.get("date")
date_confidence = "low"
if date:
# Validate date format
if re.match(r'^\d{4}-\d{2}-\d{2}$', str(date)):
date_confidence = "med" # WebSearch dates are often approximate
else:
date = None
# Get relevance if provided, default to 0.5
relevance = result.get("relevance", 0.5)
try:
relevance = min(1.0, max(0.0, float(relevance)))
except (TypeError, ValueError):
relevance = 0.5
item = {
"id": f"W{i+1}",
"title": title[:200], # Truncate long titles
"url": url,
"source_domain": extract_domain(url),
"snippet": snippet[:500], # Truncate long snippets
"date": date,
"date_confidence": date_confidence,
"relevance": relevance,
"why_relevant": str(result.get("why_relevant", "")).strip(),
}
items.append(item)
return items
def normalize_websearch_items(
items: List[Dict[str, Any]],
from_date: str,
to_date: str,
) -> List[schema.WebSearchItem]:
"""Convert parsed dicts to WebSearchItem objects.
Args:
items: List of parsed item dicts
from_date: Start of date range (YYYY-MM-DD)
to_date: End of date range (YYYY-MM-DD)
Returns:
List of WebSearchItem objects
"""
result = []
for item in items:
web_item = schema.WebSearchItem(
id=item["id"],
title=item["title"],
url=item["url"],
source_domain=item["source_domain"],
snippet=item["snippet"],
date=item.get("date"),
date_confidence=item.get("date_confidence", "low"),
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
)
result.append(web_item)
return result
def dedupe_websearch(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
"""Remove duplicate WebSearch items.
Deduplication is based on URL.
Args:
items: List of WebSearchItem objects
Returns:
Deduplicated list
"""
seen_urls = set()
result = []
for item in items:
# Normalize URL for comparison
url_key = item.url.lower().rstrip("/")
if url_key not in seen_urls:
seen_urls.add(url_key)
result.append(item)
return result