fix(reddit,bilibili): switch to rdt-cli and add bili-cli support (#235)

Reddit: Exa crawling had chronic CRAWL_LIVECRAWL_TIMEOUT issues.
rdt-cli (304 stars, public-clis) works without login — search, read
full posts, and comments all verified. Massive improvement.

Bilibili: add bili-cli (590 stars) as optional enhanced backend for
hot/rank/search/feed. yt-dlp remains for video metadata + subtitles.

Also fix UA string (was "agent-reach/1.0", now proper browser UA).

75 tests passing.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Pnant
2026-03-31 17:04:58 +08:00
committed by GitHub
parent 794455cc9f
commit 15f161e5b5
6 changed files with 1696 additions and 78 deletions
+17 -31
View File
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
"""Bilibili — video via yt-dlp, search via /x/web-interface API."""
"""Bilibili — video via yt-dlp, search/browse via bili-cli or API."""
import json
import os
@@ -8,7 +8,7 @@ import subprocess
import urllib.request
from .base import Channel
_UA = "agent-reach/1.0"
_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
_TIMEOUT = 10
_SEARCH_API = "https://api.bilibili.com/x/web-interface/search/all/v2?keyword=test&page=1"
@@ -24,23 +24,10 @@ def _search_api_ok() -> bool:
return False
def _bilisearch_ok() -> bool:
"""Return True if yt-dlp bilisearch works without 412."""
try:
result = subprocess.run(
["yt-dlp", "--flat-playlist", "--no-download", "-j",
"bilisearch1:test"],
capture_output=True, text=True, timeout=_TIMEOUT,
)
return result.returncode == 0
except Exception:
return False
class BilibiliChannel(Channel):
name = "bilibili"
description = "B站视频字幕"
backends = ["yt-dlp", "B站搜索 API"]
description = "B站视频字幕和搜索"
backends = ["yt-dlp", "bili-cli (可选)", "B站搜索 API"]
tier = 1
def can_handle(self, url: str) -> bool:
@@ -53,11 +40,7 @@ class BilibiliChannel(Channel):
return "off", "yt-dlp 未安装。安装:pip install yt-dlp"
proxy = (config.get("bilibili_proxy") if config else None) or os.environ.get("BILIBILI_PROXY")
# 检测搜索 API 连通性
api_ok = _search_api_ok()
# 检测 yt-dlp bilisearch 是否 412
ytdlp_search_ok = _bilisearch_ok()
has_bili_cli = bool(shutil.which("bili"))
parts = []
@@ -65,16 +48,19 @@ class BilibiliChannel(Channel):
if proxy:
parts.append("视频读取:yt-dlp(代理已配置)")
else:
parts.append("视频读取:yt-dlp(本地环境,服务器可能需要代理)")
parts.append("视频读取:yt-dlp")
# 搜索状态
if api_ok:
parts.append("搜索B站 API 可用(/x/web-interface/search/all/v2")
# bili-cli 增强
if has_bili_cli:
parts.append("搜索/热门/排行:bili-cli 可用")
else:
parts.append("搜索:B站 API 不可达,搜索功能可能受限")
# 检测搜索 API 连通性
api_ok = _search_api_ok()
if api_ok:
parts.append("搜索:B站 API 可用")
else:
parts.append("搜索:B站 API 不可达")
parts.append("提示:安装 bili-cli 可解锁热门/排行/动态:pipx install bilibili-cli")
if not ytdlp_search_ok:
parts.append("提示:yt-dlp bilisearch 不可用(可能 HTTP 412 反爬),搜索将走 B站 API")
status = "ok" if api_ok else "warn"
status = "ok" if has_bili_cli or _search_api_ok() else "warn"
return status, "".join(parts)