feat: 新增小宇宙播客转文字渠道 (#124)

* feat: add Xiaoyuzhou podcast transcription channel - New channel: 小宇宙播客 (xiaoyuzhoufm.com) → full text transcript - Uses Groq Whisper API (free, no credit card needed) - Auto-downloads audio, converts to low-bitrate MP3, splits by 25MB limit - Supports any length podcast with automatic chunking - Script installed to ~/.agent-reach/tools/xiaoyuzhou/transcribe.sh - User just needs: agent-reach configure groq-key gsk_xxxxx - Updated README (CN/EN), install.md, pyproject.toml * docs: clarify Xiaoyuzhou setup — free key, limitations, step-by-step --------- Co-authored-by: Panniantong <panniantong@users.noreply.github.com>
2026-03-09 00:34:22 +08:00
parent 22d7ad766a
commit c912051173
8 changed files with 312 additions and 0 deletions
@@ -21,6 +21,7 @@ from .linkedin import LinkedInChannel
 from .bosszhipin import BossZhipinChannel
 from .wechat import WeChatChannel
 from .weibo import WeiboChannel
+from .xiaoyuzhou import XiaoyuzhouChannel


 # Channel registry
@@ -36,6 +37,7 @@ ALL_CHANNELS: List[Channel] = [
    BossZhipinChannel(),
    WeChatChannel(),
    WeiboChannel(),
+    XiaoyuzhouChannel(),
    RSSChannel(),
    ExaSearchChannel(),
    WebChannel(),
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""Xiaoyuzhou Podcast (小宇宙播客) — transcribe podcasts via Groq Whisper API."""
+
+import os
+import shutil
+from .base import Channel
+
+
+class XiaoyuzhouChannel(Channel):
+    name = "xiaoyuzhou"
+    description = "小宇宙播客转文字"
+    backends = ["groq-whisper", "ffmpeg"]
+    tier = 1
+
+    def can_handle(self, url: str) -> bool:
+        from urllib.parse import urlparse
+        d = urlparse(url).netloc.lower()
+        return "xiaoyuzhoufm.com" in d
+
+    def check(self, config=None):
+        # Check ffmpeg
+        if not shutil.which("ffmpeg"):
+            return "off", (
+                "需要 ffmpeg（音频转码和切片）。安装：\n"
+                "  Ubuntu/Debian: apt install -y ffmpeg\n"
+                "  macOS: brew install ffmpeg"
+            )
+
+        # Check script exists
+        script = os.path.expanduser("~/.agent-reach/tools/xiaoyuzhou/transcribe.sh")
+        if not os.path.isfile(script):
+            return "off", (
+                "转录脚本未安装。运行：\n"
+                "  agent-reach install --env=auto\n"
+                "  或手动复制 transcribe.sh 到 ~/.agent-reach/tools/xiaoyuzhou/"
+            )
+
+        # Check GROQ_API_KEY
+        if not os.environ.get("GROQ_API_KEY"):
+            # Check if saved in config
+            config_path = os.path.expanduser("~/.agent-reach/config.json")
+            has_key = False
+            if os.path.isfile(config_path):
+                try:
+                    import json
+                    with open(config_path) as f:
+                        cfg = json.load(f)
+                    has_key = bool(cfg.get("groq_api_key"))
+                except Exception:
+                    pass
+            if not has_key:
+                return "warn", (
+                    "需要配置 Groq API Key（免费）。步骤：\n"
+                    "  1. 注册 https://console.groq.com\n"
+                    "  2. 运行: agent-reach configure groq-api-key gsk_xxxxx"
+                )
+
+        return "ok", "完整可用（播客下载 + Whisper 转录）"
@@ -450,10 +450,60 @@ def _install_system_deps():
    # ── Weibo (mcp-server-weibo fork with visitor passport fix) ──
    _install_weibo_deps()

+    # ── Xiaoyuzhou Podcast (transcribe.sh + ffmpeg) ──
+    _install_xiaoyuzhou_deps()
+
    # ── WeChat Articles (miku_ai + camoufox + wechat-article-for-ai) ──
    _install_wechat_deps()


+def _install_xiaoyuzhou_deps():
+    """Install Xiaoyuzhou podcast transcription script."""
+    print("Setting up Xiaoyuzhou podcast transcription...")
+
+    tools_dir = os.path.expanduser("~/.agent-reach/tools/xiaoyuzhou")
+    script_dst = os.path.join(tools_dir, "transcribe.sh")
+
+    if os.path.isfile(script_dst):
+        print("  ✅ Xiaoyuzhou transcription script already installed")
+    else:
+        # Copy script from package
+        script_src = os.path.join(os.path.dirname(__file__), "scripts", "transcribe_xiaoyuzhou.sh")
+        if os.path.isfile(script_src):
+            try:
+                os.makedirs(tools_dir, exist_ok=True)
+                import shutil as _shutil
+                _shutil.copy2(script_src, script_dst)
+                os.chmod(script_dst, 0o755)
+                print("  ✅ Xiaoyuzhou transcription script installed")
+            except Exception as e:
+                print(f"  [!]  Failed to install script: {e}")
+        else:
+            print("  [!]  Script source not found in package")
+
+    # Check ffmpeg
+    if shutil.which("ffmpeg"):
+        print("  ✅ ffmpeg available")
+    else:
+        print("  -- ffmpeg not found. Install: apt install -y ffmpeg (or brew install ffmpeg)")
+
+    # Check GROQ_API_KEY
+    config_path = os.path.expanduser("~/.agent-reach/config.json")
+    has_key = bool(os.environ.get("GROQ_API_KEY"))
+    if not has_key and os.path.isfile(config_path):
+        try:
+            with open(config_path) as f:
+                cfg = json.load(f)
+            has_key = bool(cfg.get("groq_api_key"))
+        except Exception:
+            pass
+    if has_key:
+        print("  ✅ Groq API key configured")
+    else:
+        print("  -- Groq API key not set. Get free key at https://console.groq.com")
+        print("     Then run: agent-reach configure groq-api-key gsk_xxxxx")
+
+
 def _install_weibo_deps():
    """Install Weibo MCP server (Panniantong fork with visitor passport auth)."""
    import subprocess
@@ -0,0 +1,167 @@
+#!/bin/bash
+# 小宇宙播客转文字脚本
+# 用法: bash transcribe.sh <小宇宙链接> [输出文件路径]
+# 环境变量: GROQ_API_KEY (必须)
+
+set -e
+
+URL="${1:?用法: bash transcribe.sh <小宇宙链接> [输出文件路径]}"
+OUTPUT="${2:-/tmp/podcast_transcript.txt}"
+TMPDIR="/tmp/xiaoyuzhou_$$"
+
+# Try env var first, then agent-reach config
+if [ -z "$GROQ_API_KEY" ]; then
+    CONFIG_FILE="$HOME/.agent-reach/config.json"
+    if [ -f "$CONFIG_FILE" ]; then
+        GROQ_API_KEY=$(python3 -c "import json; print(json.load(open('$CONFIG_FILE')).get('groq_api_key',''))" 2>/dev/null || true)
+    fi
+fi
+GROQ_API_KEY="${GROQ_API_KEY:?请设置 GROQ_API_KEY 环境变量或运行 agent-reach configure groq-key}"
+
+# Groq API 限制: 25MB per file
+MAX_CHUNK_SIZE_MB=20
+AUDIO_BITRATE="64k"
+
+cleanup() {
+    rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
+mkdir -p "$TMPDIR"
+
+echo "📻 小宇宙播客转文字"
+echo "===================="
+
+# Step 1: 提取音频 URL 和标题
+echo "🔍 正在解析页面..."
+PAGE=$(curl -s "$URL")
+AUDIO_URL=$(echo "$PAGE" | grep -oP 'https://media\.xyzcdn\.net/[^"]*\.(m4a|mp3)' | head -1)
+TITLE=$(echo "$PAGE" | grep -oP '"title":"[^"]*"' | head -1 | sed 's/"title":"//;s/"//')
+
+if [ -z "$AUDIO_URL" ]; then
+    echo "❌ 无法从页面提取音频链接"
+    exit 1
+fi
+
+echo "📝 标题: $TITLE"
+echo "🔗 音频: $AUDIO_URL"
+
+# Step 2: 下载音频
+echo "⬇️  正在下载音频..."
+EXT="${AUDIO_URL##*.}"
+curl -sL -o "$TMPDIR/original.$EXT" "$AUDIO_URL"
+FILE_SIZE=$(ls -lh "$TMPDIR/original.$EXT" | awk '{print $5}')
+echo "📦 文件大小: $FILE_SIZE"
+
+# Step 3: 获取时长
+DURATION=$(ffprobe -v quiet -show_entries format=duration -of csv=p=0 "$TMPDIR/original.$EXT" 2>/dev/null | cut -d. -f1)
+DURATION_MIN=$((DURATION / 60))
+DURATION_SEC=$((DURATION % 60))
+echo "⏱️  时长: ${DURATION_MIN}分${DURATION_SEC}秒"
+
+# Step 4: 转为低码率单声道 MP3
+echo "🔄 正在转码..."
+ffmpeg -y -i "$TMPDIR/original.$EXT" -b:a "$AUDIO_BITRATE" -ac 1 "$TMPDIR/mono.mp3" 2>/dev/null
+MONO_SIZE=$(stat -c%s "$TMPDIR/mono.mp3" 2>/dev/null || stat -f%z "$TMPDIR/mono.mp3")
+echo "📦 转码后: $(echo "$MONO_SIZE / 1024 / 1024" | bc)MB"
+
+# Step 5: 按大小切片
+MAX_BYTES=$((MAX_CHUNK_SIZE_MB * 1024 * 1024))
+
+if [ "$MONO_SIZE" -le "$MAX_BYTES" ]; then
+    # 不需要切片
+    cp "$TMPDIR/mono.mp3" "$TMPDIR/chunk_0.mp3"
+    NUM_CHUNKS=1
+    echo "📎 无需切片"
+else
+    # 计算需要几个 chunk
+    NUM_CHUNKS=$(( (MONO_SIZE / MAX_BYTES) + 1 ))
+    CHUNK_DURATION=$(( DURATION / NUM_CHUNKS + 10 ))  # 加 10 秒缓冲
+    echo "✂️  切分为 $NUM_CHUNKS 段 (每段约 $((CHUNK_DURATION / 60)) 分钟)..."
+    
+    for i in $(seq 0 $((NUM_CHUNKS - 1))); do
+        START=$((i * CHUNK_DURATION))
+        ffmpeg -y -i "$TMPDIR/mono.mp3" -ss "$START" -t "$CHUNK_DURATION" -c copy "$TMPDIR/chunk_${i}.mp3" 2>/dev/null
+        CHUNK_SIZE=$(ls -lh "$TMPDIR/chunk_${i}.mp3" | awk '{print $5}')
+        echo "   段 $((i+1))/$NUM_CHUNKS: $CHUNK_SIZE"
+    done
+fi
+
+# Step 6: 调用 Groq Whisper API 转录
+echo "🎙️  正在转录 (Groq Whisper large-v3)..."
+
+for i in $(seq 0 $((NUM_CHUNKS - 1))); do
+    echo -n "   段 $((i+1))/$NUM_CHUNKS... "
+    
+    RESPONSE=$(curl -s -w "\n%{http_code}" \
+        https://api.groq.com/openai/v1/audio/transcriptions \
+        -H "Authorization: Bearer $GROQ_API_KEY" \
+        -F file="@$TMPDIR/chunk_${i}.mp3" \
+        -F model="whisper-large-v3" \
+        -F language="zh" \
+        -F response_format="text")
+    
+    HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+    BODY=$(echo "$RESPONSE" | sed '$d')
+    
+    if [ "$HTTP_CODE" != "200" ]; then
+        echo "❌ API 错误 (HTTP $HTTP_CODE)"
+        echo "$BODY"
+        
+        # 如果是速率限制，等待后重试
+        if [ "$HTTP_CODE" = "429" ]; then
+            # 从错误信息中提取等待时间，默认 120 秒
+            WAIT_SEC=$(echo "$BODY" | grep -oP 'in \K[0-9]+m' | sed 's/m//' | head -1)
+            WAIT_SEC=${WAIT_SEC:-2}
+            WAIT_SEC=$((WAIT_SEC * 60 + 30))
+            echo "   ⏳ 速率限制，等待 ${WAIT_SEC} 秒后重试..."
+            sleep "$WAIT_SEC"
+            RESPONSE=$(curl -s -w "\n%{http_code}" \
+                https://api.groq.com/openai/v1/audio/transcriptions \
+                -H "Authorization: Bearer $GROQ_API_KEY" \
+                -F file="@$TMPDIR/chunk_${i}.mp3" \
+                -F model="whisper-large-v3" \
+                -F language="zh" \
+                -F response_format="text")
+            HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+            BODY=$(echo "$RESPONSE" | sed '$d')
+            
+            if [ "$HTTP_CODE" != "200" ]; then
+                echo "   ❌ 重试失败"
+                exit 1
+            fi
+        else
+            exit 1
+        fi
+    fi
+    
+    echo "$BODY" > "$TMPDIR/transcript_${i}.txt"
+    CHARS=$(wc -m < "$TMPDIR/transcript_${i}.txt")
+    echo "✅ ($CHARS 字)"
+done
+
+# Step 7: 合并输出
+echo "📄 正在合并文字稿..."
+
+{
+    echo "# $TITLE"
+    echo ""
+    echo "来源: $URL"
+    echo "时长: ${DURATION_MIN}分${DURATION_SEC}秒"
+    echo "转录时间: $(date '+%Y-%m-%d %H:%M')"
+    echo ""
+    echo "---"
+    echo ""
+    
+    for i in $(seq 0 $((NUM_CHUNKS - 1))); do
+        cat "$TMPDIR/transcript_${i}.txt"
+        echo ""
+    done
+} > "$OUTPUT"
+
+TOTAL_CHARS=$(wc -m < "$OUTPUT")
+echo ""
+echo "✅ 完成！"
+echo "📄 输出: $OUTPUT"
+echo "📊 总字数: $TOTAL_CHARS"
+echo "===================="