refactor: 统一所有渠道后端,对齐 research 技能

GitHub: REST API → gh CLI(官方工具,认证后完整能力)
Bilibili: 自写 API → yt-dlp(和 YouTube 统一后端,支持搜索 bilisearch)
YouTube: 新增搜索功能(ytsearch via yt-dlp)

README 中英文同步更新:
- 平台表格:小红书/Exa/GitHub/YouTube/B站 描述全部更新
- 选型表格:新增 gh CLI、xiaohongshu-mcp,更新 yt-dlp/Exa 描述
- 按需解锁:去掉 Exa Key 注册步骤(已自动配置)
- 配置难度说明:新增「自动配置」「mcporter」级别
This commit is contained in:
Panniantong
2026-02-24 13:25:57 +01:00
parent bb9ce6f7f1
commit 88eaebe263
5 changed files with 298 additions and 230 deletions
+118 -88
View File
@@ -1,121 +1,151 @@
# -*- coding: utf-8 -*-
"""Bilibili — via public API (free, no config needed).
"""Bilibili — via yt-dlp (same backend as YouTube).
Backend: Bilibili public API
Swap to: any Bilibili access method
Backend: yt-dlp (https://github.com/yt-dlp/yt-dlp)
yt-dlp natively supports Bilibili — video info, subtitles, and search.
"""
import requests
from urllib.parse import urlparse, parse_qs
from .base import Channel, ReadResult
import json
import shutil
import subprocess
from urllib.parse import urlparse
from .base import Channel, ReadResult, SearchResult
from typing import List
class BilibiliChannel(Channel):
name = "bilibili"
description = "B站视频信息和字幕"
backends = ["Bilibili API"]
backends = ["yt-dlp"]
requires_tools = ["yt-dlp"]
tier = 0
def can_handle(self, url: str) -> bool:
domain = urlparse(url).netloc.lower()
return "bilibili.com" in domain or "b23.tv" in domain
d = urlparse(url).netloc.lower()
return "bilibili.com" in d or "b23.tv" in d
def check(self, config=None):
if not shutil.which("yt-dlp"):
return "off", "yt-dlp 未安装。安装:pip install yt-dlp"
proxy = config.get("bilibili_proxy") if config else None
if proxy:
return "ok", "已配置代理,完整可用"
# Detect if we're on a server (same logic as cli._detect_environment)
import os
indicators = [
os.path.exists("/var/run/docker.sock"),
os.path.exists("/etc/cloud"),
"SSH_CONNECTION" in os.environ,
"container" in os.environ.get("container", ""),
]
is_server = any(indicators)
is_server = bool(os.environ.get("SSH_CONNECTION") or os.path.exists("/etc/cloud"))
if is_server:
return "warn", "服务器 IP 可能被封,配置代理即可解决:agent-reach configure proxy URL"
return "ok", "本地直连可用"
async def read(self, url: str, config=None) -> ReadResult:
# Proxy support (Bilibili blocks server IPs)
if not shutil.which("yt-dlp"):
raise RuntimeError("yt-dlp not installed. Install: pip install yt-dlp")
proxy = config.get("bilibili_proxy") if config else None
proxies = {"http": proxy, "https": proxy} if proxy else None
# Extract BV id from URL
path = urlparse(url).path
bv_id = ""
for part in path.split("/"):
if part.startswith("BV"):
bv_id = part
break
if not bv_id:
# Fallback to Jina Reader
from agent_reach.channels.web import WebChannel
return await WebChannel().read(url, config)
# Get video info
resp = requests.get(
"https://api.bilibili.com/x/web-interface/view",
params={"bvid": bv_id},
headers={"User-Agent": "Mozilla/5.0"},
proxies=proxies,
timeout=15,
)
resp.raise_for_status()
api_data = resp.json()
# Check for API errors (IP blocked, video not found, etc.)
if api_data.get("code") != 0:
msg = api_data.get("message", "Unknown error")
# Bilibili returns -404 when server IP is blocked
if api_data.get("code") in (-404, -403, -412):
return ReadResult(
title=f"Bilibili: {bv_id}",
content=f"⚠️ Bilibili blocked this request ({msg}). "
f"This usually means the server IP is blocked. "
f"Try: agent-reach configure proxy http://user:pass@ip:port",
url=url,
platform="bilibili",
)
# Get video info via yt-dlp
info = self._get_info(url, proxy)
if not info:
return ReadResult(
title=f"Bilibili: {bv_id}",
content=f"Bilibili API error: {msg} (code: {api_data.get('code')})",
url=url,
platform="bilibili",
title="Bilibili",
content=f"⚠️ 无法获取视频信息: {url}\n服务器 IP 可能被封,配个代理:agent-reach configure proxy URL",
url=url, platform="bilibili",
)
data = api_data.get("data", {})
title = data.get("title", "")
desc = data.get("desc", "")
author = data.get("owner", {}).get("name", "")
# Try to get subtitles
subtitle_text = ""
subtitle_list = data.get("subtitle", {}).get("list", [])
if subtitle_list:
sub_url = subtitle_list[0].get("subtitle_url", "")
if sub_url:
if sub_url.startswith("//"):
sub_url = "https:" + sub_url
sr = requests.get(sub_url, timeout=10)
if sr.ok:
sub_data = sr.json()
lines = [item.get("content", "") for item in sub_data.get("body", [])]
subtitle_text = "\n".join(lines)
title = info.get("title", url)
author = info.get("uploader", "")
desc = info.get("description", "")
# Try subtitles
subtitle = self._get_subtitles(url, proxy)
content = desc
if subtitle_text:
content += f"\n\n## Transcript\n{subtitle_text}"
if subtitle:
content += f"\n\n## 字幕\n{subtitle}"
return ReadResult(
title=title,
content=content,
url=url,
author=author,
platform="bilibili",
extra={"view": data.get("stat", {}).get("view", 0),
"like": data.get("stat", {}).get("like", 0)},
title=title, content=content, url=url,
author=author, platform="bilibili",
extra={
"view_count": info.get("view_count"),
"like_count": info.get("like_count"),
"duration": info.get("duration_string"),
},
)
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
"""Search Bilibili via yt-dlp's bilisearch."""
if not shutil.which("yt-dlp"):
raise RuntimeError("yt-dlp not installed. Install: pip install yt-dlp")
limit = kwargs.get("limit", 10)
proxy = config.get("bilibili_proxy") if config else None
cmd = [
"yt-dlp", "--dump-json", "--flat-playlist",
f"bilisearch{limit}:{query}",
]
if proxy:
cmd += ["--proxy", proxy]
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
results = []
for line in r.stdout.strip().split("\n"):
if not line.strip():
continue
try:
d = json.loads(line)
results.append(SearchResult(
title=d.get("title", ""),
url=f"https://www.bilibili.com/video/{d.get('id', '')}",
snippet=f"👤 {d.get('uploader', '?')} · 👁 {d.get('view_count', '?')}",
extra={
"view_count": d.get("view_count"),
"uploader": d.get("uploader"),
},
))
except json.JSONDecodeError:
continue
return results
except subprocess.TimeoutExpired:
return []
def _get_info(self, url: str, proxy: str = None) -> dict:
cmd = ["yt-dlp", "--dump-json", "--no-download", url]
if proxy:
cmd += ["--proxy", proxy]
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
if r.returncode == 0:
return json.loads(r.stdout)
except (subprocess.TimeoutExpired, json.JSONDecodeError):
pass
return {}
def _get_subtitles(self, url: str, proxy: str = None) -> str:
import tempfile
from pathlib import Path
with tempfile.TemporaryDirectory() as tmpdir:
cmd = [
"yt-dlp", "--write-sub", "--write-auto-sub",
"--sub-lang", "zh-Hans,zh,en",
"--skip-download", "--sub-format", "vtt",
"-o", f"{tmpdir}/%(id)s.%(ext)s", url,
]
if proxy:
cmd += ["--proxy", proxy]
try:
subprocess.run(cmd, capture_output=True, text=True, timeout=30)
for f in Path(tmpdir).glob("*.vtt"):
text = f.read_text(errors="replace")
lines = []
for line in text.split("\n"):
line = line.strip()
if not line or line.startswith("WEBVTT") or "-->" in line or line.isdigit():
continue
if line not in lines[-1:]:
lines.append(line)
return "\n".join(lines)
except subprocess.TimeoutExpired:
pass
return ""