feat(open): Port web search backends, persistence layer, and env merge from openclaw

scripts/lib/openrouter_search.py

@@ -0,0 +1,216 @@
+"""Perplexity Sonar Pro web search via OpenRouter for last30days skill.
+
+Uses OpenRouter's chat completions API with Perplexity's Sonar Pro model,
+which has built-in web search and returns citations with URLs, titles, and dates.
+This is the recommended web search backend -- highest quality results.
+
+API docs: https://openrouter.ai/docs/quickstart
+Model: perplexity/sonar-pro
+"""
+
+import re
+import sys
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
+
+from . import http
+
+ENDPOINT = "https://openrouter.ai/api/v1/chat/completions"
+MODEL = "perplexity/sonar-pro"
+
+# Domains to exclude (handled by Reddit/X search)
+EXCLUDED_DOMAINS = {
+    "reddit.com", "www.reddit.com", "old.reddit.com",
+    "twitter.com", "www.twitter.com", "x.com", "www.x.com",
+}
+
+
+def search_web(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    api_key: str,
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Search the web via Perplexity Sonar Pro on OpenRouter.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        api_key: OpenRouter API key
+        depth: 'quick', 'default', or 'deep'
+
+    Returns:
+        List of result dicts with keys: url, title, snippet, source_domain, date, relevance
+
+    Raises:
+        http.HTTPError: On API errors
+    """
+    max_tokens = {"quick": 1024, "default": 2048, "deep": 4096}.get(depth, 2048)
+
+    prompt = (
+        f"Find recent blog posts, news articles, tutorials, and discussions "
+        f"about {topic} published between {from_date} and {to_date}. "
+        f"Exclude results from reddit.com, x.com, and twitter.com. "
+        f"For each result, provide the title, URL, publication date, "
+        f"and a brief summary of why it's relevant."
+    )
+
+    payload = {
+        "model": MODEL,
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": max_tokens,
+    }
+
+    sys.stderr.write(f"[Web] Searching Sonar Pro via OpenRouter for: {topic}\n")
+    sys.stderr.flush()
+
+    response = http.post(
+        ENDPOINT,
+        json_data=payload,
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "HTTP-Referer": "https://github.com/mvanhorn/last30days-openclaw",
+            "X-Title": "last30days",
+        },
+        timeout=30,
+    )
+
+    return _normalize_results(response)
+
+
+def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Convert Sonar Pro response to websearch item schema.
+
+    Sonar Pro returns:
+    - search_results: [{title, url, date}] -- structured source metadata
+    - citations: [url, ...] -- flat list of cited URLs
+    - choices[0].message.content -- the synthesized text with [N] references
+
+    We prefer search_results (richer metadata), fall back to citations.
+    """
+    items = []
+
+    # Try search_results first (has title, url, date)
+    search_results = response.get("search_results", [])
+    if isinstance(search_results, list) and search_results:
+        items = _parse_search_results(search_results)
+
+    # Fall back to citations if no search_results
+    if not items:
+        citations = response.get("citations", [])
+        content = _get_content(response)
+        if isinstance(citations, list) and citations:
+            items = _parse_citations(citations, content)
+
+    sys.stderr.write(f"[Web] Sonar Pro: {len(items)} results\n")
+    sys.stderr.flush()
+
+    return items
+
+
+def _parse_search_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Parse the search_results array from Sonar Pro."""
+    items = []
+
+    for i, result in enumerate(results):
+        if not isinstance(result, dict):
+            continue
+
+        url = result.get("url", "")
+        if not url:
+            continue
+
+        # Skip excluded domains
+        try:
+            domain = urlparse(url).netloc.lower()
+            if domain in EXCLUDED_DOMAINS:
+                continue
+            if domain.startswith("www."):
+                domain = domain[4:]
+        except Exception:
+            domain = ""
+
+        title = str(result.get("title", "")).strip()
+        if not title:
+            continue
+
+        # Sonar Pro provides dates in search_results
+        date = result.get("date")
+        date_confidence = "med" if date else "low"
+
+        items.append({
+            "id": f"W{i+1}",
+            "title": title[:200],
+            "url": url,
+            "source_domain": domain,
+            "snippet": str(result.get("snippet", result.get("description", ""))).strip()[:500],
+            "date": date,
+            "date_confidence": date_confidence,
+            "relevance": 0.7,  # Sonar Pro results are generally high quality
+            "why_relevant": "",
+        })
+
+    return items
+
+
+def _parse_citations(citations: List[str], content: str) -> List[Dict[str, Any]]:
+    """Parse the flat citations array, enriching with content context."""
+    items = []
+
+    for i, url in enumerate(citations):
+        if not isinstance(url, str) or not url:
+            continue
+
+        # Skip excluded domains
+        try:
+            domain = urlparse(url).netloc.lower()
+            if domain in EXCLUDED_DOMAINS:
+                continue
+            if domain.startswith("www."):
+                domain = domain[4:]
+        except Exception:
+            domain = ""
+
+        # Try to extract title from content references like [1] Title...
+        title = _extract_title_for_citation(content, i + 1) or domain
+
+        items.append({
+            "id": f"W{i+1}",
+            "title": title[:200],
+            "url": url,
+            "source_domain": domain,
+            "snippet": "",
+            "date": None,
+            "date_confidence": "low",
+            "relevance": 0.6,
+            "why_relevant": "",
+        })
+
+    return items
+
+
+def _get_content(response: Dict[str, Any]) -> str:
+    """Extract the text content from the chat completion response."""
+    try:
+        return response["choices"][0]["message"]["content"]
+    except (KeyError, IndexError, TypeError):
+        return ""
+
+
+def _extract_title_for_citation(content: str, index: int) -> Optional[str]:
+    """Try to extract a title near a citation reference [N] in the content."""
+    if not content:
+        return None
+
+    # Look for patterns like [1] Title or [1](url) Title
+    pattern = rf'\[{index}\][)\s]*([^\[\n]{{5,80}})'
+    match = re.search(pattern, content)
+    if match:
+        title = match.group(1).strip().rstrip('.')
+        # Clean up markdown artifacts
+        title = re.sub(r'[*_`]', '', title)
+        return title if len(title) > 3 else None
+
+    return None