fix(reddit,bilibili): switch to rdt-cli and add bili-cli support (#235)
Reddit: Exa crawling had chronic CRAWL_LIVECRAWL_TIMEOUT issues. rdt-cli (304 stars, public-clis) works without login — search, read full posts, and comments all verified. Massive improvement. Bilibili: add bili-cli (590 stars) as optional enhanced backend for hot/rank/search/feed. yt-dlp remains for video metadata + subtitles. Also fix UA string (was "agent-reach/1.0", now proper browser UA). 75 tests passing. Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Pnant
@@ -1,5 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Bilibili — video via yt-dlp, search via /x/web-interface API."""
|
||||
"""Bilibili — video via yt-dlp, search/browse via bili-cli or API."""
|
||||
|
||||
import json
|
||||
import os
|
||||
@@ -8,7 +8,7 @@ import subprocess
|
||||
import urllib.request
|
||||
from .base import Channel
|
||||
|
||||
_UA = "agent-reach/1.0"
|
||||
_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
_TIMEOUT = 10
|
||||
_SEARCH_API = "https://api.bilibili.com/x/web-interface/search/all/v2?keyword=test&page=1"
|
||||
|
||||
@@ -24,23 +24,10 @@ def _search_api_ok() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _bilisearch_ok() -> bool:
|
||||
"""Return True if yt-dlp bilisearch works without 412."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["yt-dlp", "--flat-playlist", "--no-download", "-j",
|
||||
"bilisearch1:test"],
|
||||
capture_output=True, text=True, timeout=_TIMEOUT,
|
||||
)
|
||||
return result.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class BilibiliChannel(Channel):
|
||||
name = "bilibili"
|
||||
description = "B站视频和字幕"
|
||||
backends = ["yt-dlp", "B站搜索 API"]
|
||||
description = "B站视频、字幕和搜索"
|
||||
backends = ["yt-dlp", "bili-cli (可选)", "B站搜索 API"]
|
||||
tier = 1
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
@@ -53,11 +40,7 @@ class BilibiliChannel(Channel):
|
||||
return "off", "yt-dlp 未安装。安装：pip install yt-dlp"
|
||||
|
||||
proxy = (config.get("bilibili_proxy") if config else None) or os.environ.get("BILIBILI_PROXY")
|
||||
|
||||
# 检测搜索 API 连通性
|
||||
api_ok = _search_api_ok()
|
||||
# 检测 yt-dlp bilisearch 是否 412
|
||||
ytdlp_search_ok = _bilisearch_ok()
|
||||
has_bili_cli = bool(shutil.which("bili"))
|
||||
|
||||
parts = []
|
||||
|
||||
@@ -65,16 +48,19 @@ class BilibiliChannel(Channel):
|
||||
if proxy:
|
||||
parts.append("视频读取：yt-dlp（代理已配置）")
|
||||
else:
|
||||
parts.append("视频读取：yt-dlp（本地环境，服务器可能需要代理）")
|
||||
parts.append("视频读取：yt-dlp")
|
||||
|
||||
# 搜索状态
|
||||
if api_ok:
|
||||
parts.append("搜索：B站 API 可用（/x/web-interface/search/all/v2）")
|
||||
# bili-cli 增强
|
||||
if has_bili_cli:
|
||||
parts.append("搜索/热门/排行：bili-cli 可用")
|
||||
else:
|
||||
parts.append("搜索：B站 API 不可达，搜索功能可能受限")
|
||||
# 检测搜索 API 连通性
|
||||
api_ok = _search_api_ok()
|
||||
if api_ok:
|
||||
parts.append("搜索：B站 API 可用")
|
||||
else:
|
||||
parts.append("搜索：B站 API 不可达")
|
||||
parts.append("提示：安装 bili-cli 可解锁热门/排行/动态：pipx install bilibili-cli")
|
||||
|
||||
if not ytdlp_search_ok:
|
||||
parts.append("提示：yt-dlp bilisearch 不可用（可能 HTTP 412 反爬），搜索将走 B站 API")
|
||||
|
||||
status = "ok" if api_ok else "warn"
|
||||
status = "ok" if has_bili_cli or _search_api_ok() else "warn"
|
||||
return status, "。".join(parts)
|
||||
|
||||
@@ -1,30 +1,15 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Reddit — search and read via Exa (no direct Reddit API needed)."""
|
||||
"""Reddit — search and read via rdt-cli (public-clis/rdt-cli)."""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from .base import Channel
|
||||
|
||||
|
||||
def _exa_available() -> bool:
|
||||
"""Return True if mcporter is installed and Exa MCP is configured."""
|
||||
mcporter = shutil.which("mcporter")
|
||||
if not mcporter:
|
||||
return False
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[mcporter, "config", "list"], capture_output=True,
|
||||
encoding="utf-8", errors="replace", timeout=5
|
||||
)
|
||||
return "exa" in r.stdout.lower()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class RedditChannel(Channel):
|
||||
name = "reddit"
|
||||
description = "Reddit 帖子和评论（通过 Exa 搜索和阅读）"
|
||||
backends = ["Exa via mcporter"]
|
||||
description = "Reddit 帖子和评论"
|
||||
backends = ["rdt-cli"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
@@ -33,10 +18,14 @@ class RedditChannel(Channel):
|
||||
return "reddit.com" in d or "redd.it" in d
|
||||
|
||||
def check(self, config=None):
|
||||
if _exa_available():
|
||||
return "ok", "通过 Exa 搜索和阅读 Reddit 内容（免费，无需代理）"
|
||||
rdt = shutil.which("rdt")
|
||||
if rdt:
|
||||
return "ok", (
|
||||
"rdt-cli 可用（搜索帖子、阅读全文、查看评论，无需登录）"
|
||||
)
|
||||
return "off", (
|
||||
"需要 mcporter + Exa MCP。安装：\n"
|
||||
" npm install -g mcporter\n"
|
||||
" mcporter config add exa https://mcp.exa.ai/mcp"
|
||||
"需要安装 rdt-cli：\n"
|
||||
" pipx install rdt-cli\n"
|
||||
"或：\n"
|
||||
" uv tool install rdt-cli"
|
||||
)
|
||||
|
||||
+7
-7
@@ -243,9 +243,9 @@ def _cmd_install(args):
|
||||
# Environment-specific advice
|
||||
if env == "server":
|
||||
print()
|
||||
print("Tip: Reddit and Bilibili block server IPs.")
|
||||
print(" Reddit search still works via Exa (free).")
|
||||
print(" For full access: agent-reach configure proxy http://user:pass@ip:port")
|
||||
print("Tip: Bilibili may block server IPs.")
|
||||
print(" Reddit: rdt-cli works without proxy (pipx install rdt-cli).")
|
||||
print(" For Bilibili full access: agent-reach configure proxy http://user:pass@ip:port")
|
||||
print(" Cheap option: https://www.webshare.io ($1/month)")
|
||||
|
||||
# Test channels
|
||||
@@ -1000,7 +1000,7 @@ def _cmd_configure(args):
|
||||
if args.key == "proxy":
|
||||
config.set("bilibili_proxy", value)
|
||||
print(f"✅ Proxy configured for Bilibili!")
|
||||
print(" Note: Reddit 已改为通过 Exa 访问，无需代理。")
|
||||
print(" Note: Reddit 已改为通过 rdt-cli 访问，无需代理。")
|
||||
|
||||
elif args.key == "twitter-cookies":
|
||||
# Accept two formats:
|
||||
@@ -1433,9 +1433,9 @@ def _cmd_setup():
|
||||
print(" 跳过。公开 API 也能用")
|
||||
print()
|
||||
|
||||
# Step 3: Reddit — no config needed (uses Exa)
|
||||
print("【信息】Reddit — 通过 Exa 搜索和阅读，无需配置")
|
||||
print(" 搜索和阅读 Reddit 内容已通过 Exa 自动完成，免费无需代理。")
|
||||
# Step 3: Reddit — rdt-cli
|
||||
print("【信息】Reddit — 通过 rdt-cli 搜索和阅读，无需配置")
|
||||
print(" 安装：pipx install rdt-cli")
|
||||
print()
|
||||
|
||||
# Step 4: Groq (Whisper)
|
||||
|
||||
@@ -63,11 +63,11 @@ twitter search "query" --limit 10
|
||||
# YouTube/B站字幕
|
||||
yt-dlp --write-sub --skip-download -o "/tmp/%(id)s" "URL"
|
||||
|
||||
# Reddit 搜索（通过 Exa，免费无需代理）
|
||||
mcporter call 'exa.web_search_exa(query: "query", numResults: 5, includeDomains: ["reddit.com"])'
|
||||
# Reddit 搜索
|
||||
rdt search "query" --limit 10
|
||||
|
||||
# Reddit 读帖（通过 Exa）
|
||||
mcporter call 'exa.crawling_exa(urls: ["https://www.reddit.com/r/.../comments/.../"], maxCharacters: 10000)'
|
||||
# Reddit 读帖 + 评论
|
||||
rdt read POST_ID
|
||||
|
||||
# V2EX 热门
|
||||
curl -s "https://www.v2ex.com/api/topics/hot.json" -H "User-Agent: agent-reach/1.0"
|
||||
|
||||
@@ -142,20 +142,23 @@ user = ch.get_user("Livid")
|
||||
|
||||
> **节点列表**: https://www.v2ex.com/planes
|
||||
|
||||
## Reddit (通过 Exa)
|
||||
|
||||
Reddit 封锁了几乎所有非浏览器访问（包括代理 IP）。搜索和阅读全部通过 Exa 完成，免费且无需代理。
|
||||
|
||||
### 搜索 Reddit 内容
|
||||
## Reddit (rdt-cli)
|
||||
|
||||
```bash
|
||||
mcporter call 'exa.web_search_exa(query: "your search query", numResults: 5, includeDomains: ["reddit.com"])'
|
||||
# 搜索帖子
|
||||
rdt search "query" --limit 10
|
||||
|
||||
# 读帖子全文 + 评论
|
||||
rdt read POST_ID
|
||||
|
||||
# 浏览 subreddit
|
||||
rdt sub python --limit 20
|
||||
|
||||
# 浏览热门
|
||||
rdt popular --limit 10
|
||||
|
||||
# 浏览 /r/all
|
||||
rdt all --limit 10
|
||||
```
|
||||
|
||||
### 阅读完整帖子和评论
|
||||
|
||||
```bash
|
||||
mcporter call 'exa.crawling_exa(urls: ["https://www.reddit.com/r/SUBREDDIT/comments/POST_ID/TITLE/"], maxCharacters: 10000)'
|
||||
```
|
||||
|
||||
> **零配置**: 只需安装 Exa MCP（`agent-reach install --env=auto` 自动完成）。无需代理，无需 API Key。
|
||||
> **安装**: `pipx install rdt-cli`。无需登录即可搜索和阅读。
|
||||
|
||||
Reference in New Issue
Block a user