feat: add Xiaohongshu source and Reddit public fallback

- add xiaohongshu/xhs source path via xiaohongshu-mcp HTTP API\n- add Reddit public JSON fallback when OpenAI auth is unavailable\n- update diagnostics/UI rendering for new source availability states\n- harden Xiaohongshu availability probe to reduce false negatives\n- include source status reporting for Xiaohongshu

scripts/lib/env.py

@@ -199,6 +199,7 @@ def get_config() -> Dict[str, Any]:
         ('OPENROUTER_API_KEY', None),
         ('PARALLEL_API_KEY', None),
         ('BRAVE_API_KEY', None),
+        ('XIAOHONGSHU_API_BASE', None),
         ('OPENAI_MODEL_POLICY', 'auto'),
         ('OPENAI_MODEL_PIN', None),
         ('XAI_MODEL_POLICY', 'latest'),
@@ -221,24 +222,20 @@ def config_exists() -> bool:
 
 
 def get_available_sources(config: Dict[str, Any]) -> str:
-    """Determine which sources are available based on API keys.
+    """Determine which sources are available.
 
     Returns: 'all', 'both', 'reddit', 'reddit-web', 'x', 'x-web', 'web', or 'none'
     """
-    has_openai = bool(config.get('OPENAI_API_KEY')) and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK
+    # Reddit is available via public JSON fallback even without OpenAI auth.
+    has_reddit = True
     has_xai = bool(config.get('XAI_API_KEY'))
     has_web = has_web_search_keys(config)
 
-    if has_openai and has_xai:
+    if has_reddit and has_xai:
         return 'all' if has_web else 'both'
-    elif has_openai:
+    elif has_reddit:
         return 'reddit-web' if has_web else 'reddit'
-    elif has_xai:
-        return 'x-web' if has_web else 'x'
-    elif has_web:
-        return 'web'
-    else:
-        return 'web'  # Fallback: assistant WebSearch (no API keys needed)
+    return 'web' if has_web else 'none'
 
 
 def has_web_search_keys(config: Dict[str, Any]) -> bool:
@@ -267,7 +264,7 @@ def get_missing_keys(config: Dict[str, Any]) -> str:
 
     Returns: 'all', 'both', 'reddit', 'x', 'web', or 'none'
     """
-    has_openai = bool(config.get('OPENAI_API_KEY')) and config.get('OPENAI_AUTH_STATUS') == AUTH_STATUS_OK
+    has_reddit = True
     has_xai = bool(config.get('XAI_API_KEY'))
     has_web = has_web_search_keys(config)
 
@@ -277,16 +274,15 @@ def get_missing_keys(config: Dict[str, Any]) -> str:
 
     has_x = has_xai or has_bird
 
-    if has_openai and has_x and has_web:
+    if has_reddit and has_x and has_web:
         return 'none'
-    elif has_openai and has_x:
+    elif has_reddit and has_x:
         return 'web'  # Missing web search keys
-    elif has_openai:
+    elif has_reddit and has_web:
+        return 'x'  # Missing X source
+    elif has_reddit:
         return 'x'  # Missing X source (and possibly web)
-    elif has_x:
-        return 'reddit'  # Missing OpenAI key (and possibly web)
-    else:
-        return 'all'  # Missing everything
+    return 'all'
 
 
 def validate_sources(requested: str, available: str, include_web: bool = False) -> tuple[str, Optional[str]]:
@@ -300,56 +296,51 @@ def validate_sources(requested: str, available: str, include_web: bool = False)
     Returns:
         Tuple of (effective_sources, error_message)
     """
-    # No API keys at all
-    if available == 'none':
-        if requested == 'auto':
-            return 'web', "No API keys configured. The assistant can still search the web if it has a search tool."
-        elif requested == 'web':
-            return 'web', None
-        else:
-            return 'web', f"No API keys configured. Add keys to ~/.config/last30days/.env for Reddit/X."
-
-    # Web-only mode (only web search API keys)
-    if available == 'web':
-        if requested == 'auto':
-            return 'web', None
-        elif requested == 'web':
-            return 'web', None
-        else:
-            return 'web', "Only web search keys configured. Add OPENAI_API_KEY (or run codex login) for Reddit, XAI_API_KEY for X."
+    has_reddit = available in ('reddit', 'both', 'reddit-web', 'all')
+    has_x = available in ('x', 'both', 'x-web', 'all')
+    has_web = available in ('web', 'reddit-web', 'x-web', 'all')
 
     if requested == 'auto':
-        # Add web to sources if include_web is set
+        if has_reddit and has_x:
+            base = 'both'
+        elif has_reddit:
+            base = 'reddit'
+        elif has_x:
+            base = 'x'
+        elif has_web:
+            base = 'web'
+        else:
+            return 'none', "No sources are available."
+
         if include_web:
-            if available == 'both':
-                return 'all', None  # reddit + x + web
-            elif available == 'reddit':
+            if base == 'both':
+                return 'all', None
+            if base == 'reddit':
                 return 'reddit-web', None
-            elif available == 'x':
+            if base == 'x':
                 return 'x-web', None
-        return available, None
+        return base, None
 
     if requested == 'web':
         return 'web', None
 
     if requested == 'both':
-        if available not in ('both',):
-            missing = 'xAI' if available == 'reddit' else 'OpenAI'
-            return 'none', f"Requested both sources but {missing} key is missing. Use --sources=auto to use available keys."
+        if not (has_reddit and has_x):
+            return 'none', "Requested both sources but X source is missing."
         if include_web:
             return 'all', None
         return 'both', None
 
     if requested == 'reddit':
-        if available == 'x':
+        if not has_reddit:
             return 'none', "Requested Reddit but only xAI key is available."
         if include_web:
             return 'reddit-web', None
         return 'reddit', None
 
     if requested == 'x':
-        if available == 'reddit':
-            return 'none', "Requested X but only OpenAI key is available."
+        if not has_x:
+            return 'none', "Requested X but no X source is available (need Bird auth or XAI_API_KEY)."
         if include_web:
             return 'x-web', None
         return 'x', None
@@ -435,6 +426,40 @@ def get_instagram_token(config: Dict[str, Any]) -> str:
     return config.get('SCRAPECREATORS_API_KEY') or ''
 
 
+def get_xiaohongshu_api_base(config: Dict[str, Any]) -> str:
+    """Get Xiaohongshu HTTP API base URL.
+
+    Defaults to host.docker.internal so OpenClaw Docker can reach host service.
+    """
+    return (config.get('XIAOHONGSHU_API_BASE') or "http://host.docker.internal:18060").rstrip("/")
+
+
+def is_xiaohongshu_available(config: Dict[str, Any]) -> bool:
+    """Check whether Xiaohongshu HTTP API is reachable and logged in."""
+    # Import here to avoid heavy imports at module load.
+    from . import http
+
+    base = get_xiaohongshu_api_base(config)
+    try:
+        # Keep health probe snappy, but allow one retry for transient hiccups.
+        health = http.get(f"{base}/health", timeout=3, retries=2)
+        if not isinstance(health, dict):
+            return False
+        if not health.get("success"):
+            return False
+
+        # Login probe can be slower on some deployments (browser/session checks),
+        # so use a slightly longer timeout to avoid false negatives.
+        login = http.get(f"{base}/api/v1/login/status", timeout=8, retries=2)
+        is_logged_in = (
+            login.get("data", {}).get("is_logged_in")
+            if isinstance(login, dict) else False
+        )
+        return bool(is_logged_in)
+    except Exception:
+        return False
+
+
 # Backward compat alias
 is_apify_available = is_tiktok_available
 

scripts/lib/openai_reddit.py

@@ -355,6 +355,105 @@ def search_reddit(
     raise http.HTTPError("No models available")
 
 
+def _public_relevance(score: int, num_comments: int) -> float:
+    """Estimate relevance for public Reddit search results."""
+    # Lightweight heuristic: blend normalized score + comments.
+    score_component = min(1.0, max(0.0, score / 500.0))
+    comments_component = min(1.0, max(0.0, num_comments / 200.0))
+    return round((score_component * 0.6) + (comments_component * 0.4), 3)
+
+
+def search_reddit_public(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Search Reddit directly via public JSON endpoint (no OpenAI key required).
+
+    This is a fallback mode for environments where OpenAI auth is unavailable.
+    It uses reddit.com/search/.json with recency filter (t=month).
+    """
+    _, max_items = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    limit = min(100, max(20, max_items))
+
+    core = _extract_core_subject(topic)
+    queries = [topic]
+    if core and core.lower() != topic.lower():
+        queries.append(core)
+        queries.append(f'"{core}"')
+
+    seen_urls = set()
+    all_items: List[Dict[str, Any]] = []
+
+    headers = {
+        "User-Agent": http.USER_AGENT,
+        "Accept": "application/json",
+    }
+
+    for query in queries:
+        try:
+            url = (
+                "https://www.reddit.com/search/.json"
+                f"?q={_url_encode(query)}&sort=new&t=month&limit={limit}&raw_json=1"
+            )
+            data = http.get(url, headers=headers, timeout=20, retries=2)
+            children = data.get("data", {}).get("children", [])
+            for child in children:
+                if child.get("kind") != "t3":
+                    continue
+                post = child.get("data", {})
+                permalink = str(post.get("permalink", "")).strip()
+                if not permalink or "/comments/" not in permalink:
+                    continue
+
+                full_url = f"https://www.reddit.com{permalink}"
+                if full_url in seen_urls:
+                    continue
+                seen_urls.add(full_url)
+
+                score = int(post.get("score", 0) or 0)
+                num_comments = int(post.get("num_comments", 0) or 0)
+
+                # Parse date from created_utc
+                created_utc = post.get("created_utc")
+                date_value = None
+                if created_utc:
+                    from . import dates as dates_mod
+                    date_value = dates_mod.timestamp_to_date(created_utc)
+
+                all_items.append({
+                    "id": f"R{len(all_items)+1}",
+                    "title": str(post.get("title", "")).strip(),
+                    "url": full_url,
+                    "subreddit": str(post.get("subreddit", "")).strip(),
+                    "date": date_value,
+                    "why_relevant": "Found via Reddit public search",
+                    "relevance": _public_relevance(score, num_comments),
+                    "engagement": {
+                        "score": score,
+                        "num_comments": num_comments,
+                        "upvote_ratio": post.get("upvote_ratio"),
+                    },
+                })
+
+        except http.HTTPError as e:
+            _log_info(f"Public Reddit search failed for query '{query}': {e}")
+            # Continue with next query; partial results are still useful.
+            continue
+        except Exception as e:
+            _log_info(f"Public Reddit search error for query '{query}': {e}")
+            continue
+
+    # Sort by date (desc, unknown dates last), then relevance desc
+    def _sort_key(item: Dict[str, Any]):
+        date_str = item.get("date") or ""
+        return (date_str, float(item.get("relevance", 0.0)))
+
+    all_items.sort(key=_sort_key, reverse=True)
+    return all_items[: max_items * 2]
+
+
 def search_subreddits(
     subreddits: List[str],
     topic: str,

scripts/lib/render.py

@@ -502,6 +502,20 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
         lines.append(f"  ✅ Instagram: {len(report.instagram)} reels ({with_captions} with captions)")
     # Hide when zero results
 
+    # Xiaohongshu (from Web source bucket)
+    xhs_count = 0
+    if report.web:
+        xhs_count = sum(
+            1 for w in report.web
+            if getattr(w, "source_domain", "").lower().endswith("xiaohongshu.com")
+        )
+    if xhs_count > 0:
+        lines.append(f"  ✅ Xiaohongshu: {xhs_count} notes")
+    else:
+        reason = source_info.get("xiaohongshu_skip_reason")
+        if reason:
+            lines.append(f"  ⚡ Xiaohongshu: {reason}")
+
     # Hacker News
     if report.hackernews_error:
         lines.append(f"  ❌ HN: error - {report.hackernews_error}")

scripts/lib/ui.py

@@ -426,12 +426,15 @@ def show_diagnostic_banner(diag: dict):
             bird_username, youtube, web_search_backend
     """
     has_openai = diag.get("openai", False)
+    has_reddit_public = diag.get("reddit_public", False)
+    has_reddit = has_openai or has_reddit_public
     has_x = diag.get("x_source") is not None
     has_youtube = diag.get("youtube", False)
+    has_xiaohongshu = diag.get("xiaohongshu", False)
     has_web = diag.get("web_search_backend") is not None
 
     # If everything is available, no banner needed
-    if has_openai and has_x and has_youtube and has_web:
+    if has_reddit and has_x and has_youtube and has_web:
         return
 
     lines = []
@@ -443,7 +446,9 @@ def show_diagnostic_banner(diag: dict):
 
         # Reddit
         if has_openai:
-            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ Reddit{Colors.RESET}    — OPENAI_API_KEY found                {Colors.DIM}│{Colors.RESET}")
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ Reddit{Colors.RESET}    — OpenAI/Codex auth found             {Colors.DIM}│{Colors.RESET}")
+        elif has_reddit_public:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ Reddit{Colors.RESET}    — Public Reddit search (no key)       {Colors.DIM}│{Colors.RESET}")
         else:
             lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.RED}❌ Reddit{Colors.RESET}    — No OPENAI_API_KEY                    {Colors.DIM}│{Colors.RESET}")
             lines.append(f"{Colors.DIM}│{Colors.RESET}     └─ Add to ~/.config/last30days/.env            {Colors.DIM}│{Colors.RESET}")
@@ -469,6 +474,12 @@ def show_diagnostic_banner(diag: dict):
             lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.RED}❌ YouTube{Colors.RESET}   — yt-dlp not installed                {Colors.DIM}│{Colors.RESET}")
             lines.append(f"{Colors.DIM}│{Colors.RESET}     └─ Fix: brew install yt-dlp (free)                {Colors.DIM}│{Colors.RESET}")
 
+        # Xiaohongshu
+        if has_xiaohongshu:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ Xiaohongshu{Colors.RESET} — API connected + logged in         {Colors.DIM}│{Colors.RESET}")
+        else:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.YELLOW}⚡ Xiaohongshu{Colors.RESET} — API not connected/logged in      {Colors.DIM}│{Colors.RESET}")
+
         # Web
         if has_web:
             backend = diag.get("web_search_backend", "")
@@ -486,7 +497,9 @@ def show_diagnostic_banner(diag: dict):
         lines.append("│                                                     │")
 
         if has_openai:
-            lines.append("│  ✅ Reddit    — OPENAI_API_KEY found                │")
+            lines.append("│  ✅ Reddit    — OpenAI/Codex auth found             │")
+        elif has_reddit_public:
+            lines.append("│  ✅ Reddit    — Public Reddit search (no key)       │")
         else:
             lines.append("│  ❌ Reddit    — No OPENAI_API_KEY                    │")
             lines.append("│     └─ Add to ~/.config/last30days/.env            │")
@@ -506,6 +519,11 @@ def show_diagnostic_banner(diag: dict):
             lines.append("│  ❌ YouTube   — yt-dlp not installed                │")
             lines.append("│     └─ Fix: brew install yt-dlp (free)                │")
 
+        if has_xiaohongshu:
+            lines.append("│  ✅ Xiaohongshu — API connected + logged in         │")
+        else:
+            lines.append("│  ⚡ Xiaohongshu — API not connected/logged in       │")
+
         if has_web:
             lines.append("│  ✅ Web       — API search available                │")
         else:

scripts/lib/xiaohongshu_api.py

@@ -0,0 +1,162 @@
+"""Xiaohongshu HTTP API search client for last30days.
+
+Uses xpzouying/xiaohongshu-mcp REST endpoints:
+- GET/POST /api/v1/feeds/search
+- GET /api/v1/login/status
+"""
+
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from . import http
+
+
+def _to_int(value: Any) -> int:
+    """Convert Xiaohongshu count strings to int.
+
+    Supports plain ints and Chinese suffixes like 1.2万 / 3亿.
+    """
+    if value is None:
+        return 0
+    if isinstance(value, (int, float)):
+        return int(value)
+
+    text = str(value).strip().lower().replace(",", "")
+    if not text:
+        return 0
+
+    try:
+        if text.endswith("万"):
+            return int(float(text[:-1]) * 10000)
+        if text.endswith("亿"):
+            return int(float(text[:-1]) * 100000000)
+        return int(float(text))
+    except (TypeError, ValueError):
+        return 0
+
+
+def _timestamp_to_date_ms(ts: Any) -> Optional[str]:
+    """Convert millisecond timestamp to YYYY-MM-DD."""
+    try:
+        iv = int(ts)
+        if iv <= 0:
+            return None
+        # API examples use milliseconds.
+        dt = datetime.fromtimestamp(iv / 1000.0, tz=timezone.utc)
+        return dt.strftime("%Y-%m-%d")
+    except (TypeError, ValueError, OSError):
+        return None
+
+
+def _relevance_from_interactions(likes: int, comments: int, favorites: int) -> float:
+    """Heuristic relevance score from engagement metrics."""
+    # Weighted engagement with soft caps to [0, 1].
+    weighted = (likes * 1.0) + (comments * 2.5) + (favorites * 1.5)
+    # 5000 weighted engagement ~= strong relevance.
+    score = min(1.0, max(0.05, weighted / 5000.0))
+    return round(score, 3)
+
+
+def _build_note_url(feed_id: str, xsec_token: str) -> str:
+    """Build a stable Xiaohongshu note URL."""
+    if xsec_token:
+        return f"https://www.xiaohongshu.com/explore/{feed_id}?xsec_token={xsec_token}"
+    return f"https://www.xiaohongshu.com/explore/{feed_id}"
+
+
+def search_feeds(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    base_url: str,
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Search Xiaohongshu feeds and normalize to web-item shape."""
+    base = (base_url or "").rstrip("/")
+    if not base:
+        raise ValueError("Missing Xiaohongshu API base URL")
+
+    # Quick login sanity check.
+    login = http.get(f"{base}/api/v1/login/status", timeout=8, retries=1)
+    is_logged_in = (
+        login.get("data", {}).get("is_logged_in")
+        if isinstance(login, dict) else False
+    )
+    if not is_logged_in:
+        raise http.HTTPError("Xiaohongshu API reachable but not logged in")
+
+    # API supports filters; use recency-oriented defaults.
+    publish_time = "一天内" if depth == "quick" else "一周内" if depth == "default" else "半年内"
+    payload = {
+        "keyword": topic,
+        "filters": {
+            "sort_by": "综合",
+            "note_type": "不限",
+            "publish_time": publish_time,
+            "search_scope": "不限",
+            "location": "不限",
+        },
+    }
+
+    resp = http.post(f"{base}/api/v1/feeds/search", payload, timeout=20, retries=1)
+    feeds = resp.get("data", {}).get("feeds", []) if isinstance(resp, dict) else []
+    if not isinstance(feeds, list):
+        feeds = []
+
+    # Cap source volume similarly to other web sources.
+    limit = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
+    items: List[Dict[str, Any]] = []
+
+    for i, feed in enumerate(feeds[:limit]):
+        if not isinstance(feed, dict):
+            continue
+        note = feed.get("noteCard") or {}
+        if not isinstance(note, dict):
+            note = {}
+        interact = note.get("interactInfo") or {}
+        if not isinstance(interact, dict):
+            interact = {}
+
+        feed_id = str(feed.get("id") or note.get("noteId") or "").strip()
+        if not feed_id:
+            continue
+
+        xsec_token = str(feed.get("xsecToken") or note.get("xsecToken") or "").strip()
+        title = str(
+            note.get("displayTitle")
+            or note.get("title")
+            or ""
+        ).strip()
+        snippet = str(
+            note.get("desc")
+            or note.get("displayDesc")
+            or title
+            or ""
+        ).strip()
+
+        likes = _to_int(interact.get("likedCount"))
+        comments = _to_int(interact.get("commentCount"))
+        favorites = _to_int(interact.get("collectedCount"))
+
+        date_value = _timestamp_to_date_ms(note.get("time"))
+        why = f"Xiaohongshu engagement: likes={likes}, comments={comments}, favorites={favorites}"
+
+        items.append({
+            "id": f"XHS{i+1}",
+            "title": title[:200] if title else f"Xiaohongshu note {feed_id}",
+            "url": _build_note_url(feed_id, xsec_token),
+            "source_domain": "xiaohongshu.com",
+            "snippet": snippet[:500],
+            "date": date_value,
+            "date_confidence": "high" if date_value else "low",
+            "relevance": _relevance_from_interactions(likes, comments, favorites),
+            "why_relevant": why,
+            # Keep raw engagement for debugging/possible future rendering.
+            "engagement": {
+                "likes": likes,
+                "comments": comments,
+                "favorites": favorites,
+            },
+        })
+
+    return items