From c912051173c1d541b9ce2095aa8cec23a9c25e49 Mon Sep 17 00:00:00 2001 From: Pnant <73925474+Panniantong@users.noreply.github.com> Date: Mon, 9 Mar 2026 00:34:22 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=AE=87?= =?UTF-8?q?=E5=AE=99=E6=92=AD=E5=AE=A2=E8=BD=AC=E6=96=87=E5=AD=97=E6=B8=A0?= =?UTF-8?q?=E9=81=93=20(#124)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Xiaoyuzhou podcast transcription channel - New channel: 小宇宙播客 (xiaoyuzhoufm.com) → full text transcript - Uses Groq Whisper API (free, no credit card needed) - Auto-downloads audio, converts to low-bitrate MP3, splits by 25MB limit - Supports any length podcast with automatic chunking - Script installed to ~/.agent-reach/tools/xiaoyuzhou/transcribe.sh - User just needs: agent-reach configure groq-key gsk_xxxxx - Updated README (CN/EN), install.md, pyproject.toml * docs: clarify Xiaoyuzhou setup — free key, limitations, step-by-step --------- Co-authored-by: Panniantong --- README.md | 1 + agent_reach/channels/__init__.py | 2 + agent_reach/channels/xiaoyuzhou.py | 58 +++++++ agent_reach/cli.py | 50 ++++++ agent_reach/scripts/transcribe_xiaoyuzhou.sh | 167 +++++++++++++++++++ docs/README_en.md | 2 + docs/install.md | 31 ++++ pyproject.toml | 1 + 8 files changed, 312 insertions(+) create mode 100644 agent_reach/channels/xiaoyuzhou.py create mode 100755 agent_reach/scripts/transcribe_xiaoyuzhou.sh diff --git a/README.md b/README.md index b34703b..2abd9d5 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ AI Agent 已经能帮你写代码、改文档、管项目——但你让它去 | 🏢 **Boss直聘** | Jina Reader 读职位页 | 搜索职位、向 HR 打招呼 | 告诉 Agent「帮我配 Boss直聘」 | | 💬 **微信公众号** | 搜索 + 阅读公众号文章（全文 Markdown） | — | 无需配置 | | 📰 **微博** | 热搜、搜索内容/用户/话题、用户动态、评论 | — | 无需配置 | +| 🎙️ **小宇宙播客** | — | 播客音频转文字（Whisper 转录，免费 Key） | 告诉 Agent「帮我配小宇宙播客」 | > **不知道怎么配？不用查文档。** 直接告诉 Agent「帮我配 XXX」，它知道需要什么、会一步一步引导你。 > diff --git a/agent_reach/channels/__init__.py b/agent_reach/channels/__init__.py index 627cb66..db12feb 100644 --- a/agent_reach/channels/__init__.py +++ b/agent_reach/channels/__init__.py @@ -21,6 +21,7 @@ from .linkedin import LinkedInChannel from .bosszhipin import BossZhipinChannel from .wechat import WeChatChannel from .weibo import WeiboChannel +from .xiaoyuzhou import XiaoyuzhouChannel # Channel registry @@ -36,6 +37,7 @@ ALL_CHANNELS: List[Channel] = [ BossZhipinChannel(), WeChatChannel(), WeiboChannel(), + XiaoyuzhouChannel(), RSSChannel(), ExaSearchChannel(), WebChannel(), diff --git a/agent_reach/channels/xiaoyuzhou.py b/agent_reach/channels/xiaoyuzhou.py new file mode 100644 index 0000000..3eb9e12 --- /dev/null +++ b/agent_reach/channels/xiaoyuzhou.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +"""Xiaoyuzhou Podcast (小宇宙播客) — transcribe podcasts via Groq Whisper API.""" + +import os +import shutil +from .base import Channel + + +class XiaoyuzhouChannel(Channel): + name = "xiaoyuzhou" + description = "小宇宙播客转文字" + backends = ["groq-whisper", "ffmpeg"] + tier = 1 + + def can_handle(self, url: str) -> bool: + from urllib.parse import urlparse + d = urlparse(url).netloc.lower() + return "xiaoyuzhoufm.com" in d + + def check(self, config=None): + # Check ffmpeg + if not shutil.which("ffmpeg"): + return "off", ( + "需要 ffmpeg（音频转码和切片）。安装：\n" + " Ubuntu/Debian: apt install -y ffmpeg\n" + " macOS: brew install ffmpeg" + ) + + # Check script exists + script = os.path.expanduser("~/.agent-reach/tools/xiaoyuzhou/transcribe.sh") + if not os.path.isfile(script): + return "off", ( + "转录脚本未安装。运行：\n" + " agent-reach install --env=auto\n" + " 或手动复制 transcribe.sh 到 ~/.agent-reach/tools/xiaoyuzhou/" + ) + + # Check GROQ_API_KEY + if not os.environ.get("GROQ_API_KEY"): + # Check if saved in config + config_path = os.path.expanduser("~/.agent-reach/config.json") + has_key = False + if os.path.isfile(config_path): + try: + import json + with open(config_path) as f: + cfg = json.load(f) + has_key = bool(cfg.get("groq_api_key")) + except Exception: + pass + if not has_key: + return "warn", ( + "需要配置 Groq API Key（免费）。步骤：\n" + " 1. 注册 https://console.groq.com\n" + " 2. 运行: agent-reach configure groq-api-key gsk_xxxxx" + ) + + return "ok", "完整可用（播客下载 + Whisper 转录）" diff --git a/agent_reach/cli.py b/agent_reach/cli.py index fc92a19..7c7b91a 100644 --- a/agent_reach/cli.py +++ b/agent_reach/cli.py @@ -450,10 +450,60 @@ def _install_system_deps(): # ── Weibo (mcp-server-weibo fork with visitor passport fix) ── _install_weibo_deps() + # ── Xiaoyuzhou Podcast (transcribe.sh + ffmpeg) ── + _install_xiaoyuzhou_deps() + # ── WeChat Articles (miku_ai + camoufox + wechat-article-for-ai) ── _install_wechat_deps() +def _install_xiaoyuzhou_deps(): + """Install Xiaoyuzhou podcast transcription script.""" + print("Setting up Xiaoyuzhou podcast transcription...") + + tools_dir = os.path.expanduser("~/.agent-reach/tools/xiaoyuzhou") + script_dst = os.path.join(tools_dir, "transcribe.sh") + + if os.path.isfile(script_dst): + print(" ✅ Xiaoyuzhou transcription script already installed") + else: + # Copy script from package + script_src = os.path.join(os.path.dirname(__file__), "scripts", "transcribe_xiaoyuzhou.sh") + if os.path.isfile(script_src): + try: + os.makedirs(tools_dir, exist_ok=True) + import shutil as _shutil + _shutil.copy2(script_src, script_dst) + os.chmod(script_dst, 0o755) + print(" ✅ Xiaoyuzhou transcription script installed") + except Exception as e: + print(f" [!] Failed to install script: {e}") + else: + print(" [!] Script source not found in package") + + # Check ffmpeg + if shutil.which("ffmpeg"): + print(" ✅ ffmpeg available") + else: + print(" -- ffmpeg not found. Install: apt install -y ffmpeg (or brew install ffmpeg)") + + # Check GROQ_API_KEY + config_path = os.path.expanduser("~/.agent-reach/config.json") + has_key = bool(os.environ.get("GROQ_API_KEY")) + if not has_key and os.path.isfile(config_path): + try: + with open(config_path) as f: + cfg = json.load(f) + has_key = bool(cfg.get("groq_api_key")) + except Exception: + pass + if has_key: + print(" ✅ Groq API key configured") + else: + print(" -- Groq API key not set. Get free key at https://console.groq.com") + print(" Then run: agent-reach configure groq-api-key gsk_xxxxx") + + def _install_weibo_deps(): """Install Weibo MCP server (Panniantong fork with visitor passport auth).""" import subprocess diff --git a/agent_reach/scripts/transcribe_xiaoyuzhou.sh b/agent_reach/scripts/transcribe_xiaoyuzhou.sh new file mode 100755 index 0000000..2f5ef65 --- /dev/null +++ b/agent_reach/scripts/transcribe_xiaoyuzhou.sh @@ -0,0 +1,167 @@ +#!/bin/bash +# 小宇宙播客转文字脚本 +# 用法: bash transcribe.sh <小宇宙链接> [输出文件路径] +# 环境变量: GROQ_API_KEY (必须) + +set -e + +URL="${1:?用法: bash transcribe.sh <小宇宙链接> [输出文件路径]}" +OUTPUT="${2:-/tmp/podcast_transcript.txt}" +TMPDIR="/tmp/xiaoyuzhou_$$" + +# Try env var first, then agent-reach config +if [ -z "$GROQ_API_KEY" ]; then + CONFIG_FILE="$HOME/.agent-reach/config.json" + if [ -f "$CONFIG_FILE" ]; then + GROQ_API_KEY=$(python3 -c "import json; print(json.load(open('$CONFIG_FILE')).get('groq_api_key',''))" 2>/dev/null || true) + fi +fi +GROQ_API_KEY="${GROQ_API_KEY:?请设置 GROQ_API_KEY 环境变量或运行 agent-reach configure groq-key}" + +# Groq API 限制: 25MB per file +MAX_CHUNK_SIZE_MB=20 +AUDIO_BITRATE="64k" + +cleanup() { + rm -rf "$TMPDIR" +} +trap cleanup EXIT + +mkdir -p "$TMPDIR" + +echo "📻 小宇宙播客转文字" +echo "====================" + +# Step 1: 提取音频 URL 和标题 +echo "🔍 正在解析页面..." +PAGE=$(curl -s "$URL") +AUDIO_URL=$(echo "$PAGE" | grep -oP 'https://media\.xyzcdn\.net/[^"]*\.(m4a|mp3)' | head -1) +TITLE=$(echo "$PAGE" | grep -oP '"title":"[^"]*"' | head -1 | sed 's/"title":"//;s/"//') + +if [ -z "$AUDIO_URL" ]; then + echo "❌ 无法从页面提取音频链接" + exit 1 +fi + +echo "📝 标题: $TITLE" +echo "🔗 音频: $AUDIO_URL" + +# Step 2: 下载音频 +echo "⬇️ 正在下载音频..." +EXT="${AUDIO_URL##*.}" +curl -sL -o "$TMPDIR/original.$EXT" "$AUDIO_URL" +FILE_SIZE=$(ls -lh "$TMPDIR/original.$EXT" | awk '{print $5}') +echo "📦 文件大小: $FILE_SIZE" + +# Step 3: 获取时长 +DURATION=$(ffprobe -v quiet -show_entries format=duration -of csv=p=0 "$TMPDIR/original.$EXT" 2>/dev/null | cut -d. -f1) +DURATION_MIN=$((DURATION / 60)) +DURATION_SEC=$((DURATION % 60)) +echo "⏱️ 时长: ${DURATION_MIN}分${DURATION_SEC}秒" + +# Step 4: 转为低码率单声道 MP3 +echo "🔄 正在转码..." +ffmpeg -y -i "$TMPDIR/original.$EXT" -b:a "$AUDIO_BITRATE" -ac 1 "$TMPDIR/mono.mp3" 2>/dev/null +MONO_SIZE=$(stat -c%s "$TMPDIR/mono.mp3" 2>/dev/null || stat -f%z "$TMPDIR/mono.mp3") +echo "📦 转码后: $(echo "$MONO_SIZE / 1024 / 1024" | bc)MB" + +# Step 5: 按大小切片 +MAX_BYTES=$((MAX_CHUNK_SIZE_MB * 1024 * 1024)) + +if [ "$MONO_SIZE" -le "$MAX_BYTES" ]; then + # 不需要切片 + cp "$TMPDIR/mono.mp3" "$TMPDIR/chunk_0.mp3" + NUM_CHUNKS=1 + echo "📎 无需切片" +else + # 计算需要几个 chunk + NUM_CHUNKS=$(( (MONO_SIZE / MAX_BYTES) + 1 )) + CHUNK_DURATION=$(( DURATION / NUM_CHUNKS + 10 )) # 加 10 秒缓冲 + echo "✂️ 切分为 $NUM_CHUNKS 段 (每段约 $((CHUNK_DURATION / 60)) 分钟)..." + + for i in $(seq 0 $((NUM_CHUNKS - 1))); do + START=$((i * CHUNK_DURATION)) + ffmpeg -y -i "$TMPDIR/mono.mp3" -ss "$START" -t "$CHUNK_DURATION" -c copy "$TMPDIR/chunk_${i}.mp3" 2>/dev/null + CHUNK_SIZE=$(ls -lh "$TMPDIR/chunk_${i}.mp3" | awk '{print $5}') + echo " 段 $((i+1))/$NUM_CHUNKS: $CHUNK_SIZE" + done +fi + +# Step 6: 调用 Groq Whisper API 转录 +echo "🎙️ 正在转录 (Groq Whisper large-v3)..." + +for i in $(seq 0 $((NUM_CHUNKS - 1))); do + echo -n " 段 $((i+1))/$NUM_CHUNKS... " + + RESPONSE=$(curl -s -w "\n%{http_code}" \ + https://api.groq.com/openai/v1/audio/transcriptions \ + -H "Authorization: Bearer $GROQ_API_KEY" \ + -F file="@$TMPDIR/chunk_${i}.mp3" \ + -F model="whisper-large-v3" \ + -F language="zh" \ + -F response_format="text") + + HTTP_CODE=$(echo "$RESPONSE" | tail -1) + BODY=$(echo "$RESPONSE" | sed '$d') + + if [ "$HTTP_CODE" != "200" ]; then + echo "❌ API 错误 (HTTP $HTTP_CODE)" + echo "$BODY" + + # 如果是速率限制，等待后重试 + if [ "$HTTP_CODE" = "429" ]; then + # 从错误信息中提取等待时间，默认 120 秒 + WAIT_SEC=$(echo "$BODY" | grep -oP 'in \K[0-9]+m' | sed 's/m//' | head -1) + WAIT_SEC=${WAIT_SEC:-2} + WAIT_SEC=$((WAIT_SEC * 60 + 30)) + echo " ⏳ 速率限制，等待 ${WAIT_SEC} 秒后重试..." + sleep "$WAIT_SEC" + RESPONSE=$(curl -s -w "\n%{http_code}" \ + https://api.groq.com/openai/v1/audio/transcriptions \ + -H "Authorization: Bearer $GROQ_API_KEY" \ + -F file="@$TMPDIR/chunk_${i}.mp3" \ + -F model="whisper-large-v3" \ + -F language="zh" \ + -F response_format="text") + HTTP_CODE=$(echo "$RESPONSE" | tail -1) + BODY=$(echo "$RESPONSE" | sed '$d') + + if [ "$HTTP_CODE" != "200" ]; then + echo " ❌ 重试失败" + exit 1 + fi + else + exit 1 + fi + fi + + echo "$BODY" > "$TMPDIR/transcript_${i}.txt" + CHARS=$(wc -m < "$TMPDIR/transcript_${i}.txt") + echo "✅ ($CHARS 字)" +done + +# Step 7: 合并输出 +echo "📄 正在合并文字稿..." + +{ + echo "# $TITLE" + echo "" + echo "来源: $URL" + echo "时长: ${DURATION_MIN}分${DURATION_SEC}秒" + echo "转录时间: $(date '+%Y-%m-%d %H:%M')" + echo "" + echo "---" + echo "" + + for i in $(seq 0 $((NUM_CHUNKS - 1))); do + cat "$TMPDIR/transcript_${i}.txt" + echo "" + done +} > "$OUTPUT" + +TOTAL_CHARS=$(wc -m < "$OUTPUT") +echo "" +echo "✅ 完成！" +echo "📄 输出: $OUTPUT" +echo "📊 总字数: $TOTAL_CHARS" +echo "====================" diff --git a/docs/README_en.md b/docs/README_en.md index 906413b..3e50cca 100644 --- a/docs/README_en.md +++ b/docs/README_en.md @@ -69,6 +69,7 @@ Update Agent Reach: https://raw.githubusercontent.com/Panniantong/agent-reach/ma | 🏢 **Boss直聘** | Jina Reader (job pages) | Job search, greet recruiters | Tell your Agent "help me set up Boss直聘" | | 💬 **WeChat Articles** | Search + Read | Zero config | Search + read WeChat Official Account articles (full Markdown) ([wechat-article-for-ai](https://github.com/bzd6661/wechat-article-for-ai) + [miku_ai](https://github.com/GobinFan/Miku_Spider)) | | 📰 **Weibo** | Trending · Search · Feeds · Comments | Zero config | Hot search, content/user/topic search, feeds, comments ([mcp-server-weibo](https://github.com/Panniantong/mcp-server-weibo)) | +| 🎙️ **Xiaoyuzhou Podcast** | Transcription | Free API key | Podcast audio → full text transcript via Groq Whisper (free) | | 🔍 **Web Search** | Search | Auto-configured | Auto-configured during install, free, no API key ([Exa](https://exa.ai) via [mcporter](https://github.com/nicepkg/mcporter)) | | 📦 **GitHub** | Read · Search | Zero config | [gh CLI](https://cli.github.com) powered. Public repos work immediately. `gh auth login` unlocks Fork, Issue, PR | | 📺 **YouTube** | Read · **Search** | Zero config | Subtitles + search across 1800+ video sites ([yt-dlp](https://github.com/yt-dlp/yt-dlp) ⭐148K) | @@ -226,6 +227,7 @@ Each channel file only checks whether its upstream tool is installed and working | Boss直聘 | [mcp-bosszp](https://github.com/mucsbr/mcp-bosszp) | MCP server, job search + recruiter greeting | | WeChat Articles | [wechat-article-for-ai](https://github.com/bzd6661/wechat-article-for-ai) + [miku_ai](https://github.com/GobinFan/Miku_Spider) | Stealth browser for full article reading + Sogou search | | Weibo | `mcporter` | `mcporter call 'weibo.get_trendings(limit: 10)'` | +| Xiaoyuzhou Podcast | `transcribe.sh` | `bash ~/.agent-reach/tools/xiaoyuzhou/transcribe.sh ` | > 📌 These are the *current* choices. Don't like one? Swap out the file. That's the whole point of scaffolding. diff --git a/docs/install.md b/docs/install.md index 244af9b..049e316 100644 --- a/docs/install.md +++ b/docs/install.md @@ -174,6 +174,35 @@ mcporter config add weibo --command 'mcp-server-weibo' > 无需登录、无需 Cookie、无需代理。海外服务器也可以直接访问。 +**小宇宙播客 / Xiaoyuzhou Podcast (Groq Whisper):** +> "小宇宙播客转文字已默认安装，只需要一个免费的 Groq API Key。" + +脚本已随 Agent Reach 自动安装，用户只需提供 Key： + +```bash +agent-reach configure groq-key gsk_xxxxx +``` + +> **获取 Groq API Key（免费、无需信用卡、30 秒搞定）：** +> 1. 打开 https://console.groq.com +> 2. 用 Google/GitHub 账号登录（或注册） +> 3. 左侧菜单 → API Keys → Create API Key +> 4. 复制 Key（以 `gsk_` 开头），发给 Agent 即可 +> +> **使用方式：** +> 用户发一个小宇宙链接给 Agent，Agent 自动调用： +> ```bash +> bash ~/.agent-reach/tools/xiaoyuzhou/transcribe.sh https://www.xiaoyuzhoufm.com/episode/xxxxx +> ``` +> +> 自动下载音频 → 转码切片 → Groq Whisper 转录 → 输出完整中文文字稿。 +> +> **免费额度和限制：** +> - 每小时约 2 小时音频（7200 秒），超出后等 15 分钟自动恢复 +> - 日常听几期播客完全够用 +> - 转录质量高（Whisper large-v3），但不区分说话人 +> - 2 小时以上的播客建议分批处理 + **抖音 / Douyin (douyin-mcp-server):** > "抖音视频解析需要一个 MCP 服务。安装 douyin-mcp-server 后即可解析视频、获取无水印下载链接。" @@ -308,6 +337,7 @@ If the user wants a different agent to handle it, let them choose. | `agent-reach check-update` | Check for new versions | | `agent-reach configure twitter-cookies "..."` | Unlock Twitter search + posting | | `agent-reach configure proxy URL` | Unlock Reddit + Bilibili on servers | +| `agent-reach configure groq-key gsk_xxx` | Unlock Xiaoyuzhou podcast transcription | After installation, use upstream tools directly. See SKILL.md for the full command reference: @@ -322,6 +352,7 @@ After installation, use upstream tools directly. See SKILL.md for the full comma | Exa Search | `mcporter` | `mcporter call 'exa.web_search_exa(...)'` | | 小红书 | `mcporter` | `mcporter call 'xiaohongshu.search_feeds(...)'` | | 微博 | `mcporter` | `mcporter call 'weibo.get_trendings(limit: 10)'` | +| 小宇宙播客 | `transcribe.sh` | `bash ~/.agent-reach/tools/xiaoyuzhou/transcribe.sh ` | | 抖音 | `mcporter` | `mcporter call 'douyin.parse_douyin_video_info(...)'` | | LinkedIn | `mcporter` | `mcporter call 'linkedin.get_person_profile(...)'` | | Boss直聘 | `mcporter` | `mcporter call 'bosszhipin.search_jobs_tool(...)'` | diff --git a/pyproject.toml b/pyproject.toml index ebfa2b7..6f09e52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ packages = ["agent_reach"] [tool.hatch.build.targets.wheel.force-include] "agent_reach/guides" = "agent_reach/guides" "agent_reach/skill" = "agent_reach/skill" +"agent_reach/scripts" = "agent_reach/scripts" [tool.ruff] target-version = "py310"