From 1b8b202e1ed0367aca76ef898266452bda84e092 Mon Sep 17 00:00:00 2001
From: Panniantong <panniantong@users.noreply.github.com>
Date: Tue, 24 Feb 2026 06:31:48 +0100
Subject: [PATCH] feat: complete channel menu install flow + XiaoHongShu
 channel
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Major install flow polish:
- Channels organized by setup difficulty:
  🟢 Zero-config (Web, GitHub, RSS) — works immediately
  🔵 Cookie-based (Twitter, YouTube, Bilibili, XiaoHongShu) — free, ~2min
  🟡 Free API key (Exa Search) — one key, 30 seconds
  🟠 Proxy-based (Reddit, Bilibili on server) — $1/month
- Every channel explains: what it does, what's needed, what you miss without it
- Server vs local affects which channels need proxy

New: XiaoHongShu channel (cookie-based, falls back to Jina Reader)
New configure commands: twitter-cookies, xhs-cookie, youtube-cookies
Each command auto-tests after saving.
---
 agent_eyes/channels/__init__.py    |   2 +
 agent_eyes/channels/xiaohongshu.py | 110 ++++++++++++++
 agent_eyes/cli.py                  | 109 ++++++++++----
 docs/install.md                    | 232 ++++++++++++++++++-----------
 4 files changed, 336 insertions(+), 117 deletions(-)
 create mode 100644 agent_eyes/channels/xiaohongshu.py

diff --git a/agent_eyes/channels/__init__.py b/agent_eyes/channels/__init__.py
index 999eb68..37e964e 100644
--- a/agent_eyes/channels/__init__.py
+++ b/agent_eyes/channels/__init__.py
@@ -19,6 +19,7 @@ from .reddit import RedditChannel
 from .rss import RSSChannel
 from .bilibili import BilibiliChannel
 from .exa_search import ExaSearchChannel
+from .xiaohongshu import XiaoHongShuChannel
 
 
 # Channel registry — order matters (first match wins, web is last as fallback)
@@ -28,6 +29,7 @@ ALL_CHANNELS: List[Channel] = [
     YouTubeChannel(),
     RedditChannel(),
     BilibiliChannel(),
+    XiaoHongShuChannel(),
     RSSChannel(),
     ExaSearchChannel(),
     WebChannel(),        # Fallback — handles any URL
diff --git a/agent_eyes/channels/xiaohongshu.py b/agent_eyes/channels/xiaohongshu.py
new file mode 100644
index 0000000..e0e0a75
--- /dev/null
+++ b/agent_eyes/channels/xiaohongshu.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+"""XiaoHongShu (小红书) — via cookie-based API access.
+
+Backend: XHS web API + cookies
+Swap to: any XHS access method
+"""
+
+import re
+import json
+import requests
+from urllib.parse import urlparse
+from .base import Channel, ReadResult
+
+
+class XiaoHongShuChannel(Channel):
+    name = "xiaohongshu"
+    description = "XiaoHongShu (小红书) notes"
+    backends = ["XHS Web API"]
+    requires_config = ["xhs_cookie"]
+    tier = 2
+
+    def can_handle(self, url: str) -> bool:
+        domain = urlparse(url).netloc.lower()
+        return "xiaohongshu.com" in domain or "xhslink.com" in domain
+
+    async def read(self, url: str, config=None) -> ReadResult:
+        cookie = config.get("xhs_cookie") if config else None
+
+        if not cookie:
+            # Fallback to Jina Reader (works for some public notes)
+            from agent_eyes.channels.web import WebChannel
+            return await WebChannel().read(url, config)
+
+        # Extract note ID from URL
+        note_id = self._extract_note_id(url)
+        if not note_id:
+            from agent_eyes.channels.web import WebChannel
+            return await WebChannel().read(url, config)
+
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+            "Cookie": cookie,
+            "Referer": "https://www.xiaohongshu.com/",
+        }
+
+        # Fetch note page
+        resp = requests.get(
+            f"https://www.xiaohongshu.com/explore/{note_id}",
+            headers=headers,
+            timeout=15,
+        )
+        resp.raise_for_status()
+        html = resp.text
+
+        # Extract note data from HTML
+        title, content, author = self._parse_html(html)
+
+        return ReadResult(
+            title=title or f"XHS Note {note_id}",
+            content=content or "Could not extract content. Cookie may be expired.",
+            url=url,
+            author=author,
+            platform="xiaohongshu",
+        )
+
+    def _extract_note_id(self, url: str) -> str:
+        """Extract note ID from various XHS URL formats."""
+        # https://www.xiaohongshu.com/explore/xxxxx
+        # https://www.xiaohongshu.com/discovery/item/xxxxx
+        # https://xhslink.com/xxxxx
+        path = urlparse(url).path
+        parts = path.strip("/").split("/")
+        if parts:
+            return parts[-1]
+        return ""
+
+    def _parse_html(self, html: str):
+        """Extract title, content, author from XHS HTML."""
+        title = ""
+        content = ""
+        author = ""
+
+        # Try to find JSON data in page
+        match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.*?})\s*</script>', html, re.DOTALL)
+        if match:
+            try:
+                # XHS embeds note data in initial state
+                state = json.loads(match.group(1).replace('undefined', 'null'))
+                note_data = state.get("note", {}).get("noteDetailMap", {})
+                if note_data:
+                    first_note = list(note_data.values())[0]
+                    note = first_note.get("note", {})
+                    title = note.get("title", "")
+                    content = note.get("desc", "")
+                    author = note.get("user", {}).get("nickname", "")
+            except (json.JSONDecodeError, KeyError, IndexError):
+                pass
+
+        # Fallback: extract from meta tags
+        if not title:
+            m = re.search(r'<title>(.*?)</title>', html)
+            if m:
+                title = m.group(1)
+
+        if not content:
+            m = re.search(r'<meta name="description" content="(.*?)"', html)
+            if m:
+                content = m.group(1)
+
+        return title, content, author
diff --git a/agent_eyes/cli.py b/agent_eyes/cli.py
index 50e05a4..f87f454 100644
--- a/agent_eyes/cli.py
+++ b/agent_eyes/cli.py
@@ -81,9 +81,10 @@ def main():
 
     # ── configure ──
     p_conf = sub.add_parser("configure", help="Set a config value")
-    p_conf.add_argument("key", choices=["exa-key", "proxy", "github-token", "groq-key"],
+    p_conf.add_argument("key", choices=["exa-key", "proxy", "github-token", "groq-key",
+                                        "twitter-cookies", "xhs-cookie", "youtube-cookies"],
                         help="What to configure")
-    p_conf.add_argument("value", help="The value to set")
+    p_conf.add_argument("value", nargs="+", help="The value(s) to set")
 
     # ── doctor ──
     sub.add_parser("doctor", help="Check platform availability")
@@ -178,28 +179,36 @@ def _cmd_install(args):
 def _cmd_configure(args):
     """Set a config value and test it."""
     from agent_eyes.config import Config
-    import subprocess
 
     config = Config()
+    value = " ".join(args.value) if isinstance(args.value, list) else args.value
 
-    key_map = {
-        "exa-key": "exa_api_key",
-        "proxy": ("reddit_proxy", "bilibili_proxy"),
-        "github-token": "github_token",
-        "groq-key": "groq_api_key",
-    }
+    if args.key == "proxy":
+        config.set("reddit_proxy", value)
+        config.set("bilibili_proxy", value)
+        print(f"✅ Proxy configured for Reddit + Bilibili!")
 
-    config_key = key_map.get(args.key)
-    if isinstance(config_key, tuple):
-        for k in config_key:
-            config.set(k, args.value)
-    else:
-        config.set(config_key, args.value)
+        # Auto-test
+        print("Testing Reddit access...", end=" ")
+        try:
+            import requests
+            resp = requests.get(
+                "https://www.reddit.com/r/test.json?limit=1",
+                headers={"User-Agent": "Mozilla/5.0"},
+                proxies={"http": value, "https": value},
+                timeout=10,
+            )
+            if resp.status_code == 200:
+                print("✅ Reddit works!")
+            else:
+                print(f"⚠️ Reddit returned {resp.status_code}")
+        except Exception as e:
+            print(f"❌ Failed: {e}")
 
-    print(f"✅ {args.key} configured!")
+    elif args.key == "exa-key":
+        config.set("exa_api_key", value)
+        print(f"✅ Exa key configured!")
 
-    # Auto-test
-    if args.key == "exa-key":
         print("Testing search...", end=" ")
         try:
             import asyncio
@@ -209,27 +218,73 @@ def _cmd_configure(args):
             if results:
                 print("✅ Search works!")
             else:
-                print("⚠️  No results, but API connected.")
+                print("⚠️ No results, but API connected.")
         except Exception as e:
             print(f"❌ Failed: {e}")
 
-    elif args.key == "proxy":
-        print("Testing Reddit access...", end=" ")
+    elif args.key == "twitter-cookies":
+        # Expect: auth_token ct0
+        parts = value.split()
+        if len(parts) == 2:
+            config.set("twitter_auth_token", parts[0])
+            config.set("twitter_ct0", parts[1])
+            print(f"✅ Twitter cookies configured!")
+
+            print("Testing Twitter access...", end=" ")
+            try:
+                import subprocess
+                result = subprocess.run(
+                    ["birdx", "search", "test", "-n", "1",
+                     "--auth-token", parts[0], "--ct0", parts[1]],
+                    capture_output=True, text=True, timeout=15,
+                )
+                if result.returncode == 0 and result.stdout.strip():
+                    print("✅ Twitter Advanced works!")
+                else:
+                    print(f"⚠️ Test returned no results (cookies might be wrong)")
+            except FileNotFoundError:
+                print("⚠️ birdx not installed. Run: pip install birdx")
+            except Exception as e:
+                print(f"❌ Failed: {e}")
+        else:
+            print("❌ Usage: agent-eyes configure twitter-cookies AUTH_TOKEN CT0")
+            print("   (two values separated by space)")
+
+    elif args.key == "xhs-cookie":
+        config.set("xhs_cookie", value)
+        print(f"✅ XiaoHongShu cookie configured!")
+
+        print("Testing XHS access...", end=" ")
         try:
             import requests
             resp = requests.get(
-                "https://www.reddit.com/r/test.json?limit=1",
-                headers={"User-Agent": "Mozilla/5.0"},
-                proxies={"http": args.value, "https": args.value},
+                "https://www.xiaohongshu.com/",
+                headers={
+                    "User-Agent": "Mozilla/5.0",
+                    "Cookie": value,
+                },
                 timeout=10,
             )
-            if resp.status_code == 200:
-                print("✅ Reddit accessible!")
+            if resp.status_code == 200 and "xiaohongshu" in resp.text.lower():
+                print("✅ XiaoHongShu works!")
             else:
-                print(f"❌ Reddit returned {resp.status_code}")
+                print(f"⚠️ Got status {resp.status_code}, cookie might be expired")
         except Exception as e:
             print(f"❌ Failed: {e}")
 
+    elif args.key == "youtube-cookies":
+        config.set("youtube_cookies_from", value)
+        print(f"✅ YouTube cookie source configured: {value}")
+        print("   yt-dlp will use cookies from this browser for age-restricted/member videos.")
+
+    elif args.key == "github-token":
+        config.set("github_token", value)
+        print(f"✅ GitHub token configured!")
+
+    elif args.key == "groq-key":
+        config.set("groq_api_key", value)
+        print(f"✅ Groq key configured!")
+
 
 def _cmd_doctor():
     from agent_eyes.config import Config
diff --git a/docs/install.md b/docs/install.md
index 7cc87c1..19d39ed 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -16,7 +16,7 @@ Your Agent will handle everything. You just answer a few questions.
 ## For AI Agents
 
 You are installing Agent Eyes — giving yourself eyes to see the entire internet.
-Follow this guide step by step. Do not skip steps.
+Follow this guide step by step. Do not improvise.
 
 ### Step 1: Install
 
@@ -31,148 +31,197 @@ Verify:
 agent-eyes version
 ```
 
-### Step 2: Ask environment
+### Step 2: Detect environment
 
 Ask the user:
 
-> "You're installing Agent Eyes! First question: are you running on a **local computer** or a **server/VPS**?"
+> "First question: are you running on a **local computer** or a **server/VPS**?"
 
-Remember their answer — it affects which channels need extra setup.
+This determines what works out of the box and what needs extra setup.
 
 ### Step 3: Show channel menu
 
-Show the user this menu. Explain each option clearly.
+Present this menu to the user. Adjust the details based on their environment (local vs server).
 
-**Present it like this** (adjust the Bilibili line based on Step 2 answer):
+**IMPORTANT**: Make sure the user understands what each tier means and what they get/miss.
 
 ---
 
-> Here are all the channels Agent Eyes supports. I'll help you set up whichever ones you want.
+> **Agent Eyes supports these channels. Let me help you choose what to set up.**
 >
-> **Already active — no setup needed ✅**
-> These work right now, nothing to configure:
-> - ✅ **Web pages** — read any URL
-> - ✅ **GitHub** — read repos, issues, PRs + search repos
-> - ✅ **RSS** — read any RSS/Atom feed
-> - ✅ **YouTube** — video transcripts via yt-dlp
-> - ✅ **Twitter basic** — read individual tweets
+> ---
 >
-> **Recommended — free, takes 30 seconds each 🔍**
-> - 🔍 **Exa Search** (STRONGLY RECOMMENDED)
->   - What it unlocks: **full web search + Reddit search + Twitter search**
->   - What you need: one free API key from https://exa.ai (1000 searches/month free)
->   - Without it: I can read URLs you give me, but can't search the internet for you
+> **🟢 Already active — works right now, nothing needed:**
 >
-> **Optional — easy setup 🔧**
-> - 🐦 **Twitter Advanced** — search timelines, read threads, deep search
->   - What you need: export cookies from your browser (free, 2 minutes)
->   - Without it: I can still search Twitter via Exa and read individual tweets
+> | Channel | What it does |
+> |---------|-------------|
+> | ✅ Web pages | Read any URL on the internet |
+> | ✅ GitHub | Read repos, issues, PRs + search repositories |
+> | ✅ RSS | Read any RSS/Atom feed |
 >
-> - 📺 **Bilibili** [IF LOCAL: "already works ✅"] [IF SERVER: "⚠️ needs proxy — Bilibili blocks server IPs. Proxy costs ~$1/month at webshare.io. Without it, Bilibili won't work from your server."]
+> These 3 channels are always available. No setup needed.
 >
-> **Optional — more setup 🔨**
-> - 📖 **Reddit Full Reader** — read complete posts + all comments
->   - What you need: residential proxy (~$1/month at https://webshare.io)
->   - Without it: I can still **search** Reddit content via Exa (free!), just can't read full posts with all comments
->   - [IF SERVER: "Same proxy works for both Reddit and Bilibili"]
+> ---
 >
-> - 💬 **WeChat** — read WeChat public articles
->   - What you need: install browser component (~150MB download)
+> **🔵 Cookie-based — free, just need to export cookies from your browser (~2 min each):**
 >
-> - 📕 **XiaoHongShu** — read XHS notes
->   - What you need: browser component + scan QR code once to login
+> | Channel | What it does | Without cookies |
+> |---------|-------------|-----------------|
+> | 🐦 Twitter/X | Read tweets, search timelines, read threads | Can still read individual tweet URLs (basic mode) |
+> | 📺 YouTube | Video transcripts (public + age-restricted) | Public videos work without cookies; age-restricted/member-only videos need cookies |
+> | 📺 Bilibili | Video info + subtitles | [LOCAL: works without cookies ✅] [SERVER: ⚠️ blocked — needs proxy, see below] |
+> | 📕 XiaoHongShu | Read XHS notes + comments | Falls back to basic web reading (less content) |
 >
-> **Which ones would you like me to set up?** (I recommend at least Exa Search)
+> **How to export cookies**: open the site in Chrome → F12 → Application → Cookies → copy the values. I'll walk you through it for each one you choose.
+>
+> ---
+>
+> **🟡 Free API key — one key unlocks 3 search capabilities (~30 seconds):**
+>
+> | Channel | What it unlocks |
+> |---------|----------------|
+> | 🔍 Exa Search (RECOMMENDED) | **Web search + Reddit search + Twitter search** — one free key powers all three |
+>
+> Without Exa: I can read URLs you give me, but I cannot search the internet for you.
+> With Exa: I can find information across the entire web, Reddit, and Twitter.
+>
+> Free tier: 1000 searches/month at https://exa.ai — more than enough for most users.
+>
+> ---
+>
+> **🟠 Proxy-based — needed only on servers, $1/month shared across channels:**
+>
+> | Channel | Why proxy needed | Without proxy |
+> |---------|-----------------|---------------|
+> | 📖 Reddit | Reddit blocks server IPs | **Reddit search still works via Exa (free!)** — you just can't read full posts with all comments |
+> | 📺 Bilibili | Bilibili blocks server IPs | Won't work from server |
+>
+> [ONLY SHOW THIS SECTION IF USER SAID "SERVER"]
+> [IF LOCAL: Skip this section — Reddit and Bilibili work directly on local computers]
+>
+> One proxy works for both. Recommend: https://www.webshare.io ($1/month residential proxy).
+> Same proxy also works for Bilibili cookies-based access.
+>
+> ---
+>
+> **Which channels do you want me to set up?**
+> (I recommend: **Exa Search** for sure, plus any cookie-based channels you actually use)
 
 ---
 
-Wait for user to choose. Then proceed to Step 4.
+Wait for the user to choose. Then go to Step 4.
 
-### Step 4: Run installer
+### Step 4: Run base installer
 
 ```bash
 agent-eyes install --env=<local|server>
 ```
 
-### Step 5: Configure selected channels
+### Step 5: Configure each selected channel
 
-For each channel the user selected, run the corresponding configure command.
-Each command auto-tests after configuring.
+Run the corresponding command for each channel the user chose.
+Every configure command auto-tests and gives instant feedback (✅ or ❌).
 
-#### If user selected: Exa Search
+---
 
-Tell the user:
-> "Go to https://exa.ai, sign up (free), and send me your API Key."
+#### Exa Search
+
+Tell user: "Go to https://exa.ai, sign up (free), copy your API Key, and send it to me."
 
-When they provide it:
 ```bash
 agent-eyes configure exa-key THE_KEY
 ```
-Expected: `✅ exa-key configured! Testing search... ✅ Search works!`
 
-#### If user selected: Reddit Full Reader and/or Bilibili (on server)
+---
 
-Tell the user:
-> "Reddit and Bilibili need a residential proxy. Steps:
+#### Twitter/X
+
+Tell user:
+> "Open https://x.com in Chrome (make sure you're logged in).
+> Press F12 → Application → Cookies → https://x.com
+> Find these two values: **auth_token** and **ct0**
+> Copy both and send them to me."
+
+```bash
+agent-eyes configure twitter-cookies AUTH_TOKEN CT0
+```
+
+---
+
+#### YouTube (cookies for age-restricted videos)
+
+Tell user:
+> "Most YouTube videos work without any setup. If you want access to age-restricted or member-only videos:
+> Which browser do you use? (chrome/firefox/edge/safari)"
+
+```bash
+agent-eyes configure youtube-cookies chrome
+```
+(replace `chrome` with whatever browser they use — yt-dlp reads cookies directly from the browser)
+
+---
+
+#### Bilibili
+
+**If local**: Already works. No setup needed.
+
+**If server**: Needs proxy (same proxy as Reddit, see Proxy section below).
+
+If user also wants cookies for member-only content:
+> "Open https://bilibili.com in Chrome (logged in).
+> F12 → Application → Cookies → find **SESSDATA**
+> Send it to me."
+
+(For now, Bilibili public API works without cookies. Cookie support is for future member-only content.)
+
+---
+
+#### XiaoHongShu
+
+Tell user:
+> "Open https://www.xiaohongshu.com in Chrome (make sure you're logged in).
+> Press F12 → Application → Cookies → https://www.xiaohongshu.com
+> Select all cookies, right-click → Copy All
+> Or just copy the full cookie string from a network request header.
+> Send it to me."
+
+```bash
+agent-eyes configure xhs-cookie THE_COOKIE_STRING
+```
+
+---
+
+#### Proxy (Reddit + Bilibili on server)
+
+Tell user:
+> "Reddit and Bilibili block server IPs. You need a residential proxy to access them.
 > 1. Go to https://www.webshare.io
-> 2. Sign up and get a proxy ($1/month plan is enough)
+> 2. Sign up and get a residential proxy ($1/month is enough)
 > 3. Copy your proxy address (format: http://username:password@ip:port)
 > 4. Send it to me"
 
-When they provide it:
 ```bash
 agent-eyes configure proxy THE_PROXY_URL
 ```
-Expected: `✅ proxy configured! Testing Reddit... ✅ Reddit accessible!`
 
-#### If user selected: Twitter Advanced
+This configures the proxy for both Reddit and Bilibili at once.
 
-Tell the user:
-> "Twitter Advanced needs cookies from your browser. Steps:
-> 1. Open https://x.com in Chrome and make sure you're logged in
-> 2. Press F12 → Application tab → Cookies → https://x.com
-> 3. Find and copy these two values: **auth_token** and **ct0**
-> 4. Send them to me"
+---
 
-When they provide cookies:
-```bash
-# birdx cookie setup (if birdx is available)
-which birdx || pip install birdx
-# Configure birdx with cookies (method varies by birdx version)
-```
-
-#### If user selected: WeChat
-
-```bash
-pip install agent-eyes[browser]
-playwright install chromium
-```
-Tell user: "✅ WeChat is ready! Send me any WeChat article URL."
-
-#### If user selected: XiaoHongShu
-
-```bash
-pip install agent-eyes[browser]
-playwright install chromium
-```
-Tell user: "I'll need you to scan a QR code once to log in. Ready?" Then read the guide: read the file at `agent_eyes/guides/setup-xiaohongshu.md` for detailed steps.
-
-### Step 6: Final check
+### Step 6: Final health check
 
 ```bash
 agent-eyes doctor
 ```
 
-Summarize results for the user naturally. Example:
+Summarize what's working for the user. Example:
 
 > "✅ All done! Here's what I can do now:
-> - 📖 Read: Web, GitHub, YouTube, RSS, Tweets, Reddit, Bilibili
-> - 🔍 Search: Web, GitHub, Reddit, Twitter
 >
-> Try asking me things like:
-> - 'Search Reddit for best self-hosted LLM'
-> - 'Read this GitHub repo: [URL]'
-> - 'What's trending on Twitter about AI agents?'"
+> 📖 **Read**: Web, GitHub, YouTube, Bilibili, Twitter, XiaoHongShu, RSS
+> 🔍 **Search**: Web, GitHub, Reddit, Twitter
+>
+> Try it out — ask me to search or read anything!"
 
 ---
 
@@ -181,11 +230,14 @@ Summarize results for the user naturally. Example:
 | Action | Command |
 |--------|---------|
 | Install | `pip install git+https://github.com/Panniantong/agent-eyes.git` |
-| Setup | `agent-eyes install --env=<local\|server>` |
+| Base setup | `agent-eyes install --env=<local\|server>` |
 | Add Exa key | `agent-eyes configure exa-key YOUR_KEY` |
+| Add Twitter | `agent-eyes configure twitter-cookies AUTH_TOKEN CT0` |
+| Add YouTube | `agent-eyes configure youtube-cookies chrome` |
+| Add XiaoHongShu | `agent-eyes configure xhs-cookie COOKIE_STRING` |
 | Add proxy | `agent-eyes configure proxy http://user:pass@ip:port` |
 | Health check | `agent-eyes doctor` |
-| Read any URL | `agent-eyes read <url>` |
+| Read URL | `agent-eyes read <url>` |
 | Search web | `agent-eyes search "query"` |
 | Search GitHub | `agent-eyes search-github "query"` |
 | Search Reddit | `agent-eyes search-reddit "query"` |