refactor: 统一所有渠道后端,对齐 research 技能
GitHub: REST API → gh CLI(官方工具,认证后完整能力) Bilibili: 自写 API → yt-dlp(和 YouTube 统一后端,支持搜索 bilisearch) YouTube: 新增搜索功能(ytsearch via yt-dlp) README 中英文同步更新: - 平台表格:小红书/Exa/GitHub/YouTube/B站 描述全部更新 - 选型表格:新增 gh CLI、xiaohongshu-mcp,更新 yt-dlp/Exa 描述 - 按需解锁:去掉 Exa Key 注册步骤(已自动配置) - 配置难度说明:新增「自动配置」「mcporter」级别
This commit is contained in:
@@ -1,121 +1,151 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Bilibili — via public API (free, no config needed).
|
||||
"""Bilibili — via yt-dlp (same backend as YouTube).
|
||||
|
||||
Backend: Bilibili public API
|
||||
Swap to: any Bilibili access method
|
||||
Backend: yt-dlp (https://github.com/yt-dlp/yt-dlp)
|
||||
yt-dlp natively supports Bilibili — video info, subtitles, and search.
|
||||
"""
|
||||
|
||||
import requests
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from .base import Channel, ReadResult
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult, SearchResult
|
||||
from typing import List
|
||||
|
||||
|
||||
class BilibiliChannel(Channel):
|
||||
name = "bilibili"
|
||||
description = "B站视频信息和字幕"
|
||||
backends = ["Bilibili API"]
|
||||
backends = ["yt-dlp"]
|
||||
requires_tools = ["yt-dlp"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "bilibili.com" in domain or "b23.tv" in domain
|
||||
d = urlparse(url).netloc.lower()
|
||||
return "bilibili.com" in d or "b23.tv" in d
|
||||
|
||||
def check(self, config=None):
|
||||
if not shutil.which("yt-dlp"):
|
||||
return "off", "yt-dlp 未安装。安装:pip install yt-dlp"
|
||||
proxy = config.get("bilibili_proxy") if config else None
|
||||
if proxy:
|
||||
return "ok", "已配置代理,完整可用"
|
||||
# Detect if we're on a server (same logic as cli._detect_environment)
|
||||
import os
|
||||
indicators = [
|
||||
os.path.exists("/var/run/docker.sock"),
|
||||
os.path.exists("/etc/cloud"),
|
||||
"SSH_CONNECTION" in os.environ,
|
||||
"container" in os.environ.get("container", ""),
|
||||
]
|
||||
is_server = any(indicators)
|
||||
is_server = bool(os.environ.get("SSH_CONNECTION") or os.path.exists("/etc/cloud"))
|
||||
if is_server:
|
||||
return "warn", "服务器 IP 可能被封,配置代理即可解决:agent-reach configure proxy URL"
|
||||
return "ok", "本地直连可用"
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
# Proxy support (Bilibili blocks server IPs)
|
||||
if not shutil.which("yt-dlp"):
|
||||
raise RuntimeError("yt-dlp not installed. Install: pip install yt-dlp")
|
||||
|
||||
proxy = config.get("bilibili_proxy") if config else None
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
|
||||
# Extract BV id from URL
|
||||
path = urlparse(url).path
|
||||
bv_id = ""
|
||||
for part in path.split("/"):
|
||||
if part.startswith("BV"):
|
||||
bv_id = part
|
||||
break
|
||||
|
||||
if not bv_id:
|
||||
# Fallback to Jina Reader
|
||||
from agent_reach.channels.web import WebChannel
|
||||
return await WebChannel().read(url, config)
|
||||
|
||||
# Get video info
|
||||
resp = requests.get(
|
||||
"https://api.bilibili.com/x/web-interface/view",
|
||||
params={"bvid": bv_id},
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
proxies=proxies,
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
api_data = resp.json()
|
||||
|
||||
# Check for API errors (IP blocked, video not found, etc.)
|
||||
if api_data.get("code") != 0:
|
||||
msg = api_data.get("message", "Unknown error")
|
||||
# Bilibili returns -404 when server IP is blocked
|
||||
if api_data.get("code") in (-404, -403, -412):
|
||||
return ReadResult(
|
||||
title=f"Bilibili: {bv_id}",
|
||||
content=f"⚠️ Bilibili blocked this request ({msg}). "
|
||||
f"This usually means the server IP is blocked. "
|
||||
f"Try: agent-reach configure proxy http://user:pass@ip:port",
|
||||
url=url,
|
||||
platform="bilibili",
|
||||
)
|
||||
# Get video info via yt-dlp
|
||||
info = self._get_info(url, proxy)
|
||||
if not info:
|
||||
return ReadResult(
|
||||
title=f"Bilibili: {bv_id}",
|
||||
content=f"Bilibili API error: {msg} (code: {api_data.get('code')})",
|
||||
url=url,
|
||||
platform="bilibili",
|
||||
title="Bilibili",
|
||||
content=f"⚠️ 无法获取视频信息: {url}\n服务器 IP 可能被封,配个代理:agent-reach configure proxy URL",
|
||||
url=url, platform="bilibili",
|
||||
)
|
||||
|
||||
data = api_data.get("data", {})
|
||||
|
||||
title = data.get("title", "")
|
||||
desc = data.get("desc", "")
|
||||
author = data.get("owner", {}).get("name", "")
|
||||
|
||||
# Try to get subtitles
|
||||
subtitle_text = ""
|
||||
subtitle_list = data.get("subtitle", {}).get("list", [])
|
||||
if subtitle_list:
|
||||
sub_url = subtitle_list[0].get("subtitle_url", "")
|
||||
if sub_url:
|
||||
if sub_url.startswith("//"):
|
||||
sub_url = "https:" + sub_url
|
||||
sr = requests.get(sub_url, timeout=10)
|
||||
if sr.ok:
|
||||
sub_data = sr.json()
|
||||
lines = [item.get("content", "") for item in sub_data.get("body", [])]
|
||||
subtitle_text = "\n".join(lines)
|
||||
title = info.get("title", url)
|
||||
author = info.get("uploader", "")
|
||||
desc = info.get("description", "")
|
||||
|
||||
# Try subtitles
|
||||
subtitle = self._get_subtitles(url, proxy)
|
||||
content = desc
|
||||
if subtitle_text:
|
||||
content += f"\n\n## Transcript\n{subtitle_text}"
|
||||
if subtitle:
|
||||
content += f"\n\n## 字幕\n{subtitle}"
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=content,
|
||||
url=url,
|
||||
author=author,
|
||||
platform="bilibili",
|
||||
extra={"view": data.get("stat", {}).get("view", 0),
|
||||
"like": data.get("stat", {}).get("like", 0)},
|
||||
title=title, content=content, url=url,
|
||||
author=author, platform="bilibili",
|
||||
extra={
|
||||
"view_count": info.get("view_count"),
|
||||
"like_count": info.get("like_count"),
|
||||
"duration": info.get("duration_string"),
|
||||
},
|
||||
)
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
"""Search Bilibili via yt-dlp's bilisearch."""
|
||||
if not shutil.which("yt-dlp"):
|
||||
raise RuntimeError("yt-dlp not installed. Install: pip install yt-dlp")
|
||||
|
||||
limit = kwargs.get("limit", 10)
|
||||
proxy = config.get("bilibili_proxy") if config else None
|
||||
|
||||
cmd = [
|
||||
"yt-dlp", "--dump-json", "--flat-playlist",
|
||||
f"bilisearch{limit}:{query}",
|
||||
]
|
||||
if proxy:
|
||||
cmd += ["--proxy", proxy]
|
||||
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
results = []
|
||||
for line in r.stdout.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
d = json.loads(line)
|
||||
results.append(SearchResult(
|
||||
title=d.get("title", ""),
|
||||
url=f"https://www.bilibili.com/video/{d.get('id', '')}",
|
||||
snippet=f"👤 {d.get('uploader', '?')} · 👁 {d.get('view_count', '?')}",
|
||||
extra={
|
||||
"view_count": d.get("view_count"),
|
||||
"uploader": d.get("uploader"),
|
||||
},
|
||||
))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return results
|
||||
except subprocess.TimeoutExpired:
|
||||
return []
|
||||
|
||||
def _get_info(self, url: str, proxy: str = None) -> dict:
|
||||
cmd = ["yt-dlp", "--dump-json", "--no-download", url]
|
||||
if proxy:
|
||||
cmd += ["--proxy", proxy]
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
if r.returncode == 0:
|
||||
return json.loads(r.stdout)
|
||||
except (subprocess.TimeoutExpired, json.JSONDecodeError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _get_subtitles(self, url: str, proxy: str = None) -> str:
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cmd = [
|
||||
"yt-dlp", "--write-sub", "--write-auto-sub",
|
||||
"--sub-lang", "zh-Hans,zh,en",
|
||||
"--skip-download", "--sub-format", "vtt",
|
||||
"-o", f"{tmpdir}/%(id)s.%(ext)s", url,
|
||||
]
|
||||
if proxy:
|
||||
cmd += ["--proxy", proxy]
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
for f in Path(tmpdir).glob("*.vtt"):
|
||||
text = f.read_text(errors="replace")
|
||||
lines = []
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("WEBVTT") or "-->" in line or line.isdigit():
|
||||
continue
|
||||
if line not in lines[-1:]:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
return ""
|
||||
|
||||
Reference in New Issue
Block a user