feat(open): Port web search backends, persistence layer, and env merge from openclaw
This commit is contained in:
Matt Van Horn
@@ -0,0 +1,213 @@
|
||||
"""Brave Search web search for last30days skill.
|
||||
|
||||
Uses the Brave Search API as a fallback web search backend.
|
||||
Simple, cheap (free tier: 2,000 queries/month), widely available.
|
||||
|
||||
API docs: https://api-dashboard.search.brave.com/app/documentation/web-search/get-started
|
||||
"""
|
||||
|
||||
import html
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from . import http
|
||||
|
||||
ENDPOINT = "https://api.search.brave.com/res/v1/web/search"
|
||||
|
||||
# Freshness codes: pd=24h, pw=7d, pm=31d
|
||||
FRESHNESS_MAP = {1: "pd", 7: "pw", 31: "pm"}
|
||||
|
||||
# Domains to exclude (handled by Reddit/X search)
|
||||
EXCLUDED_DOMAINS = {
|
||||
"reddit.com", "www.reddit.com", "old.reddit.com",
|
||||
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
|
||||
}
|
||||
|
||||
|
||||
def search_web(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
api_key: str,
|
||||
depth: str = "default",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web via Brave Search API.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
api_key: Brave Search API key
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
|
||||
|
||||
Raises:
|
||||
http.HTTPError: On API errors
|
||||
"""
|
||||
count = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
|
||||
|
||||
# Calculate days for freshness filter
|
||||
days = _days_between(from_date, to_date)
|
||||
freshness = _brave_freshness(days)
|
||||
|
||||
params = {
|
||||
"q": topic,
|
||||
"result_filter": "web,news",
|
||||
"count": count,
|
||||
"safesearch": "strict",
|
||||
"text_decorations": 0,
|
||||
"spellcheck": 0,
|
||||
}
|
||||
if freshness:
|
||||
params["freshness"] = freshness
|
||||
|
||||
url = f"{ENDPOINT}?{urlencode(params)}"
|
||||
|
||||
sys.stderr.write(f"[Web] Searching Brave for: {topic}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
response = http.request(
|
||||
"GET",
|
||||
url,
|
||||
headers={"X-Subscription-Token": api_key},
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
return _normalize_results(response, from_date, to_date)
|
||||
|
||||
|
||||
def _days_between(from_date: str, to_date: str) -> int:
|
||||
"""Calculate days between two YYYY-MM-DD dates."""
|
||||
try:
|
||||
d1 = datetime.strptime(from_date, "%Y-%m-%d")
|
||||
d2 = datetime.strptime(to_date, "%Y-%m-%d")
|
||||
return max(1, (d2 - d1).days)
|
||||
except (ValueError, TypeError):
|
||||
return 30
|
||||
|
||||
|
||||
def _brave_freshness(days: Optional[int]) -> Optional[str]:
|
||||
"""Convert days to Brave freshness parameter.
|
||||
|
||||
Uses canned codes for <=31d, explicit date range for longer periods.
|
||||
"""
|
||||
if days is None:
|
||||
return None
|
||||
code = next((v for d, v in sorted(FRESHNESS_MAP.items()) if days <= d), None)
|
||||
if code:
|
||||
return code
|
||||
start = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
end = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
return f"{start}to{end}"
|
||||
|
||||
|
||||
def _normalize_results(
|
||||
response: Dict[str, Any],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convert Brave Search response to websearch item schema.
|
||||
|
||||
Merges news + web results, cleans HTML entities, filters excluded domains.
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Merge news results (tend to be more recent) with web results
|
||||
raw_results = (
|
||||
response.get("news", {}).get("results", []) +
|
||||
response.get("web", {}).get("results", [])
|
||||
)
|
||||
|
||||
for i, result in enumerate(raw_results):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
|
||||
url = result.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
title = _clean_html(str(result.get("title", "")).strip())
|
||||
snippet = _clean_html(str(result.get("description", "")).strip())
|
||||
|
||||
if not title and not snippet:
|
||||
continue
|
||||
|
||||
# Parse date from Brave's 'age' field or 'page_age'
|
||||
date = _parse_brave_date(result.get("age"), result.get("page_age"))
|
||||
date_confidence = "med" if date else "low"
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": snippet[:500],
|
||||
"date": date,
|
||||
"date_confidence": date_confidence,
|
||||
"relevance": 0.6, # Brave doesn't provide relevance scores
|
||||
"why_relevant": "",
|
||||
})
|
||||
|
||||
sys.stderr.write(f"[Web] Brave: {len(items)} results\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _clean_html(text: str) -> str:
|
||||
"""Remove HTML tags and decode entities."""
|
||||
text = re.sub(r"<[^>]*>", "", text)
|
||||
text = html.unescape(text)
|
||||
return text
|
||||
|
||||
|
||||
def _parse_brave_date(age: Optional[str], page_age: Optional[str]) -> Optional[str]:
|
||||
"""Parse Brave's age/page_age fields to YYYY-MM-DD.
|
||||
|
||||
Brave returns dates like "3 hours ago", "2 days ago", "January 24, 2026".
|
||||
"""
|
||||
text = age or page_age
|
||||
if not text:
|
||||
return None
|
||||
|
||||
text_lower = text.lower().strip()
|
||||
now = datetime.now()
|
||||
|
||||
# "X hours ago" -> today
|
||||
if re.search(r'\d+\s*hours?\s*ago', text_lower):
|
||||
return now.strftime("%Y-%m-%d")
|
||||
|
||||
# "X days ago"
|
||||
match = re.search(r'(\d+)\s*days?\s*ago', text_lower)
|
||||
if match:
|
||||
days = int(match.group(1))
|
||||
if days <= 60:
|
||||
return (now - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
|
||||
# "X weeks ago"
|
||||
match = re.search(r'(\d+)\s*weeks?\s*ago', text_lower)
|
||||
if match:
|
||||
weeks = int(match.group(1))
|
||||
return (now - timedelta(weeks=weeks)).strftime("%Y-%m-%d")
|
||||
|
||||
# ISO format: 2026-01-24T...
|
||||
match = re.search(r'(\d{4}-\d{2}-\d{2})', text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
+65
-21
@@ -1,5 +1,6 @@
|
||||
"""Environment and API key management for last30days skill."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
@@ -48,15 +49,22 @@ def get_config() -> Dict[str, Any]:
|
||||
# Load from config file first (if configured)
|
||||
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {}
|
||||
|
||||
# Environment variables override file
|
||||
config = {
|
||||
'OPENAI_API_KEY': os.environ.get('OPENAI_API_KEY') or file_env.get('OPENAI_API_KEY'),
|
||||
'XAI_API_KEY': os.environ.get('XAI_API_KEY') or file_env.get('XAI_API_KEY'),
|
||||
'OPENAI_MODEL_POLICY': os.environ.get('OPENAI_MODEL_POLICY') or file_env.get('OPENAI_MODEL_POLICY', 'auto'),
|
||||
'OPENAI_MODEL_PIN': os.environ.get('OPENAI_MODEL_PIN') or file_env.get('OPENAI_MODEL_PIN'),
|
||||
'XAI_MODEL_POLICY': os.environ.get('XAI_MODEL_POLICY') or file_env.get('XAI_MODEL_POLICY', 'latest'),
|
||||
'XAI_MODEL_PIN': os.environ.get('XAI_MODEL_PIN') or file_env.get('XAI_MODEL_PIN'),
|
||||
}
|
||||
# Build config: process.env > .env file
|
||||
keys = [
|
||||
('OPENAI_API_KEY', None),
|
||||
('XAI_API_KEY', None),
|
||||
('OPENROUTER_API_KEY', None),
|
||||
('PARALLEL_API_KEY', None),
|
||||
('BRAVE_API_KEY', None),
|
||||
('OPENAI_MODEL_POLICY', 'auto'),
|
||||
('OPENAI_MODEL_PIN', None),
|
||||
('XAI_MODEL_POLICY', 'latest'),
|
||||
('XAI_MODEL_PIN', None),
|
||||
]
|
||||
|
||||
config = {}
|
||||
for key, default in keys:
|
||||
config[key] = os.environ.get(key) or file_env.get(key, default)
|
||||
|
||||
return config
|
||||
|
||||
@@ -69,28 +77,53 @@ def config_exists() -> bool:
|
||||
def get_available_sources(config: Dict[str, Any]) -> str:
|
||||
"""Determine which sources are available based on API keys.
|
||||
|
||||
Returns: 'both', 'reddit', 'x', or 'web' (fallback when no keys)
|
||||
Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
|
||||
"""
|
||||
has_openai = bool(config.get('OPENAI_API_KEY'))
|
||||
has_xai = bool(config.get('XAI_API_KEY'))
|
||||
has_web = has_web_search_keys(config)
|
||||
|
||||
if has_openai and has_xai:
|
||||
return 'both'
|
||||
return 'all' if has_web else 'both'
|
||||
elif has_openai:
|
||||
return 'reddit'
|
||||
return 'reddit-web' if has_web else 'reddit'
|
||||
elif has_xai:
|
||||
return 'x'
|
||||
return 'x-web' if has_web else 'x'
|
||||
elif has_web:
|
||||
return 'web'
|
||||
else:
|
||||
return 'web' # Fallback: WebSearch only (no API keys needed)
|
||||
return 'web' # Fallback: assistant WebSearch (no API keys needed)
|
||||
|
||||
|
||||
def has_web_search_keys(config: Dict[str, Any]) -> bool:
|
||||
"""Check if any web search API keys are configured."""
|
||||
return bool(config.get('OPENROUTER_API_KEY') or config.get('PARALLEL_API_KEY') or config.get('BRAVE_API_KEY'))
|
||||
|
||||
|
||||
def get_web_search_source(config: Dict[str, Any]) -> Optional[str]:
|
||||
"""Determine the best available web search backend.
|
||||
|
||||
Priority: Parallel AI > Brave > OpenRouter/Sonar Pro
|
||||
|
||||
Returns: 'parallel', 'brave', 'openrouter', or None
|
||||
"""
|
||||
if config.get('PARALLEL_API_KEY'):
|
||||
return 'parallel'
|
||||
if config.get('BRAVE_API_KEY'):
|
||||
return 'brave'
|
||||
if config.get('OPENROUTER_API_KEY'):
|
||||
return 'openrouter'
|
||||
return None
|
||||
|
||||
|
||||
def get_missing_keys(config: Dict[str, Any]) -> str:
|
||||
"""Determine which sources are missing (accounting for Bird).
|
||||
|
||||
Returns: 'both', 'reddit', 'x', or 'none'
|
||||
Returns: 'all', 'both', 'reddit', 'x', 'web', or 'none'
|
||||
"""
|
||||
has_openai = bool(config.get('OPENAI_API_KEY'))
|
||||
has_xai = bool(config.get('XAI_API_KEY'))
|
||||
has_web = has_web_search_keys(config)
|
||||
|
||||
# Check if Bird provides X access (import here to avoid circular dependency)
|
||||
from . import bird_x
|
||||
@@ -98,14 +131,16 @@ def get_missing_keys(config: Dict[str, Any]) -> str:
|
||||
|
||||
has_x = has_xai or has_bird
|
||||
|
||||
if has_openai and has_x:
|
||||
if has_openai and has_x and has_web:
|
||||
return 'none'
|
||||
elif has_openai and has_x:
|
||||
return 'web' # Missing web search keys
|
||||
elif has_openai:
|
||||
return 'x' # Missing X source
|
||||
return 'x' # Missing X source (and possibly web)
|
||||
elif has_x:
|
||||
return 'reddit' # Missing OpenAI key
|
||||
return 'reddit' # Missing OpenAI key (and possibly web)
|
||||
else:
|
||||
return 'both' # Missing both
|
||||
return 'all' # Missing everything
|
||||
|
||||
|
||||
def validate_sources(requested: str, available: str, include_web: bool = False) -> tuple[str, Optional[str]]:
|
||||
@@ -119,14 +154,23 @@ def validate_sources(requested: str, available: str, include_web: bool = False)
|
||||
Returns:
|
||||
Tuple of (effective_sources, error_message)
|
||||
"""
|
||||
# WebSearch-only mode (no API keys)
|
||||
# No API keys at all
|
||||
if available == 'none':
|
||||
if requested == 'auto':
|
||||
return 'web', "No API keys configured. The assistant can still search the web if it has a search tool."
|
||||
elif requested == 'web':
|
||||
return 'web', None
|
||||
else:
|
||||
return 'web', f"No API keys configured. Add keys to ~/.config/last30days/.env for Reddit/X."
|
||||
|
||||
# Web-only mode (only web search API keys)
|
||||
if available == 'web':
|
||||
if requested == 'auto':
|
||||
return 'web', None
|
||||
elif requested == 'web':
|
||||
return 'web', None
|
||||
else:
|
||||
return 'web', f"No API keys configured. Using WebSearch fallback. Add keys to ~/.config/last30days/.env for Reddit/X."
|
||||
return 'web', f"Only web search keys configured. Add OPENAI_API_KEY for Reddit, XAI_API_KEY for X."
|
||||
|
||||
if requested == 'auto':
|
||||
# Add web to sources if include_web is set
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Perplexity Sonar Pro web search via OpenRouter for last30days skill.
|
||||
|
||||
Uses OpenRouter's chat completions API with Perplexity's Sonar Pro model,
|
||||
which has built-in web search and returns citations with URLs, titles, and dates.
|
||||
This is the recommended web search backend -- highest quality results.
|
||||
|
||||
API docs: https://openrouter.ai/docs/quickstart
|
||||
Model: perplexity/sonar-pro
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import http
|
||||
|
||||
ENDPOINT = "https://openrouter.ai/api/v1/chat/completions"
|
||||
MODEL = "perplexity/sonar-pro"
|
||||
|
||||
# Domains to exclude (handled by Reddit/X search)
|
||||
EXCLUDED_DOMAINS = {
|
||||
"reddit.com", "www.reddit.com", "old.reddit.com",
|
||||
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
|
||||
}
|
||||
|
||||
|
||||
def search_web(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
api_key: str,
|
||||
depth: str = "default",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web via Perplexity Sonar Pro on OpenRouter.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
api_key: OpenRouter API key
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
|
||||
|
||||
Raises:
|
||||
http.HTTPError: On API errors
|
||||
"""
|
||||
max_tokens = {"quick": 1024, "default": 2048, "deep": 4096}.get(depth, 2048)
|
||||
|
||||
prompt = (
|
||||
f"Find recent blog posts, news articles, tutorials, and discussions "
|
||||
f"about {topic} published between {from_date} and {to_date}. "
|
||||
f"Exclude results from reddit.com, x.com, and twitter.com. "
|
||||
f"For each result, provide the title, URL, publication date, "
|
||||
f"and a brief summary of why it's relevant."
|
||||
)
|
||||
|
||||
payload = {
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
|
||||
sys.stderr.write(f"[Web] Searching Sonar Pro via OpenRouter for: {topic}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
response = http.post(
|
||||
ENDPOINT,
|
||||
json_data=payload,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"HTTP-Referer": "https://github.com/mvanhorn/last30days-openclaw",
|
||||
"X-Title": "last30days",
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
return _normalize_results(response)
|
||||
|
||||
|
||||
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Convert Sonar Pro response to websearch item schema.
|
||||
|
||||
Sonar Pro returns:
|
||||
- search_results: [{title, url, date}] -- structured source metadata
|
||||
- citations: [url, ...] -- flat list of cited URLs
|
||||
- choices[0].message.content -- the synthesized text with [N] references
|
||||
|
||||
We prefer search_results (richer metadata), fall back to citations.
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Try search_results first (has title, url, date)
|
||||
search_results = response.get("search_results", [])
|
||||
if isinstance(search_results, list) and search_results:
|
||||
items = _parse_search_results(search_results)
|
||||
|
||||
# Fall back to citations if no search_results
|
||||
if not items:
|
||||
citations = response.get("citations", [])
|
||||
content = _get_content(response)
|
||||
if isinstance(citations, list) and citations:
|
||||
items = _parse_citations(citations, content)
|
||||
|
||||
sys.stderr.write(f"[Web] Sonar Pro: {len(items)} results\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _parse_search_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Parse the search_results array from Sonar Pro."""
|
||||
items = []
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
|
||||
url = result.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
title = str(result.get("title", "")).strip()
|
||||
if not title:
|
||||
continue
|
||||
|
||||
# Sonar Pro provides dates in search_results
|
||||
date = result.get("date")
|
||||
date_confidence = "med" if date else "low"
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": str(result.get("snippet", result.get("description", ""))).strip()[:500],
|
||||
"date": date,
|
||||
"date_confidence": date_confidence,
|
||||
"relevance": 0.7, # Sonar Pro results are generally high quality
|
||||
"why_relevant": "",
|
||||
})
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _parse_citations(citations: List[str], content: str) -> List[Dict[str, Any]]:
|
||||
"""Parse the flat citations array, enriching with content context."""
|
||||
items = []
|
||||
|
||||
for i, url in enumerate(citations):
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
# Try to extract title from content references like [1] Title...
|
||||
title = _extract_title_for_citation(content, i + 1) or domain
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": "",
|
||||
"date": None,
|
||||
"date_confidence": "low",
|
||||
"relevance": 0.6,
|
||||
"why_relevant": "",
|
||||
})
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _get_content(response: Dict[str, Any]) -> str:
|
||||
"""Extract the text content from the chat completion response."""
|
||||
try:
|
||||
return response["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_title_for_citation(content: str, index: int) -> Optional[str]:
|
||||
"""Try to extract a title near a citation reference [N] in the content."""
|
||||
if not content:
|
||||
return None
|
||||
|
||||
# Look for patterns like [1] Title or [1](url) Title
|
||||
pattern = rf'\[{index}\][)\s]*([^\[\n]{{5,80}})'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
title = match.group(1).strip().rstrip('.')
|
||||
# Clean up markdown artifacts
|
||||
title = re.sub(r'[*_`]', '', title)
|
||||
return title if len(title) > 3 else None
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Parallel AI web search for last30days skill.
|
||||
|
||||
Uses the Parallel AI Search API to find web content (blogs, docs, news, tutorials).
|
||||
This is the preferred web search backend -- it returns LLM-optimized results
|
||||
with extended excerpts ranked by relevance.
|
||||
|
||||
API docs: https://docs.parallel.ai/search-api/search-quickstart
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from . import http
|
||||
|
||||
ENDPOINT = "https://api.parallel.ai/v1beta/search"
|
||||
|
||||
# Domains to exclude (handled by Reddit/X search)
|
||||
EXCLUDED_DOMAINS = {
|
||||
"reddit.com", "www.reddit.com", "old.reddit.com",
|
||||
"twitter.com", "www.twitter.com", "x.com", "www.x.com",
|
||||
}
|
||||
|
||||
|
||||
def search_web(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
api_key: str,
|
||||
depth: str = "default",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search the web via Parallel AI Search API.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
api_key: Parallel AI API key
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
List of result dicts with keys: url, title, snippet, source_domain, date, relevance
|
||||
|
||||
Raises:
|
||||
http.HTTPError: On API errors
|
||||
"""
|
||||
max_results = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
|
||||
|
||||
payload = {
|
||||
"objective": (
|
||||
f"Find recent blog posts, tutorials, news articles, and discussions "
|
||||
f"about {topic} from {from_date} to {to_date}. "
|
||||
f"Exclude reddit.com, x.com, and twitter.com."
|
||||
),
|
||||
"max_results": max_results,
|
||||
"max_chars_per_result": 500,
|
||||
}
|
||||
|
||||
sys.stderr.write(f"[Web] Searching Parallel AI for: {topic}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
response = http.post(
|
||||
ENDPOINT,
|
||||
json_data=payload,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"parallel-beta": "search-extract-2025-10-10",
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
return _normalize_results(response)
|
||||
|
||||
|
||||
def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Convert Parallel AI response to websearch item schema.
|
||||
|
||||
Args:
|
||||
response: Raw API response
|
||||
|
||||
Returns:
|
||||
List of normalized result dicts
|
||||
"""
|
||||
items = []
|
||||
|
||||
# Handle different response shapes
|
||||
results = response.get("results", [])
|
||||
if not isinstance(results, list):
|
||||
return items
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
|
||||
url = result.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Skip excluded domains
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
if domain in EXCLUDED_DOMAINS:
|
||||
continue
|
||||
# Clean domain for display
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
except Exception:
|
||||
domain = ""
|
||||
|
||||
title = str(result.get("title", "")).strip()
|
||||
snippet = str(result.get("excerpt", result.get("snippet", result.get("description", "")))).strip()
|
||||
|
||||
if not title and not snippet:
|
||||
continue
|
||||
|
||||
# Extract relevance score if provided
|
||||
relevance = result.get("relevance_score", result.get("relevance", 0.6))
|
||||
try:
|
||||
relevance = min(1.0, max(0.0, float(relevance)))
|
||||
except (TypeError, ValueError):
|
||||
relevance = 0.6
|
||||
|
||||
items.append({
|
||||
"id": f"W{i+1}",
|
||||
"title": title[:200],
|
||||
"url": url,
|
||||
"source_domain": domain,
|
||||
"snippet": snippet[:500],
|
||||
"date": result.get("published_date", result.get("date")),
|
||||
"date_confidence": "med" if result.get("published_date") or result.get("date") else "low",
|
||||
"relevance": relevance,
|
||||
"why_relevant": str(result.get("summary", "")).strip()[:200],
|
||||
})
|
||||
|
||||
sys.stderr.write(f"[Web] Parallel AI: {len(items)} results\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
return items
|
||||
Reference in New Issue
Block a user