From 5b07d95e7e3c65909f49d3cdac1bb5b1f59dd1b9 Mon Sep 17 00:00:00 2001 From: hymansun <10735997+hymansun@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:56:38 +0800 Subject: [PATCH] feat(xiaoyuzhou): Chinese punctuation prompt + optional --polish flag (#291) Adds a punctuation-aware Whisper prompt and an optional --polish step (free Llama 3.3 70B on Groq) for Chinese podcast transcripts. Maintainer follow-up on the branch: replaced ASCII quotes that bash swallowed inside the prompt (hexdump-verified), added the same prompt to the 429-retry call, and documented --polish as optional. bash -n passes. --- agent_reach/scripts/transcribe_xiaoyuzhou.sh | 110 ++++++++++++++++++- agent_reach/skill/references/video.md | 8 +- 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/agent_reach/scripts/transcribe_xiaoyuzhou.sh b/agent_reach/scripts/transcribe_xiaoyuzhou.sh index 19a8f42..3909ca7 100755 --- a/agent_reach/scripts/transcribe_xiaoyuzhou.sh +++ b/agent_reach/scripts/transcribe_xiaoyuzhou.sh @@ -1,11 +1,29 @@ #!/bin/bash # 小宇宙播客转文字脚本 -# 用法: bash transcribe.sh <小宇宙链接> [输出文件路径] +# 用法: bash transcribe.sh [--polish] <小宇宙链接> [输出文件路径] # 环境变量: GROQ_API_KEY (必须) +# +# --polish: 转录后调用 Groq Llama 3.3 70B 给文稿补中文标点+合理分段 +# （Whisper 对中文标点支持较弱，开启后阅读体验显著更好） set -e -URL="${1:?用法: bash transcribe.sh <小宇宙链接> [输出文件路径]}" +POLISH=0 +while [ $# -gt 0 ]; do + case "$1" in + --polish) POLISH=1; shift ;; + --) shift; break ;; + -h|--help) + echo "用法: bash transcribe.sh [--polish] <小宇宙链接> [输出文件路径]" + exit 0 ;; + --*) + echo "未知选项: $1" >&2 + exit 1 ;; + *) break ;; + esac +done + +URL="${1:?用法: bash transcribe.sh [--polish] <小宇宙链接> [输出文件路径]}" OUTPUT="${2:-/tmp/podcast_transcript.txt}" TMPDIR="/tmp/xiaoyuzhou_$$" @@ -99,6 +117,7 @@ for i in $(seq 0 $((NUM_CHUNKS - 1))); do -F file="@$TMPDIR/chunk_${i}.mp3" \ -F model="whisper-large-v3" \ -F language="zh" \ + -F prompt="以下是一段中文普通话播客录音，请输出包含完整中文标点（，。？！：；“”‘’）的转写文本。" \ -F response_format="text") HTTP_CODE=$(echo "$RESPONSE" | tail -1) @@ -122,6 +141,7 @@ for i in $(seq 0 $((NUM_CHUNKS - 1))); do -F file="@$TMPDIR/chunk_${i}.mp3" \ -F model="whisper-large-v3" \ -F language="zh" \ + -F prompt="以下是一段中文普通话播客录音，请输出包含完整中文标点（，。？！：；“”‘’）的转写文本。" \ -F response_format="text") HTTP_CODE=$(echo "$RESPONSE" | tail -1) BODY=$(echo "$RESPONSE" | sed '$d') @@ -140,6 +160,81 @@ for i in $(seq 0 $((NUM_CHUNKS - 1))); do echo "✅ ($CHARS 字)" done +# Step 6.5 (可选): 用 Llama 3.3 70B 给文稿补标点+分段 +if [ "$POLISH" = "1" ]; then + echo "✨ 正在润色（Llama 3.3 70B 加标点+分段）..." + for i in $(seq 0 $((NUM_CHUNKS - 1))); do + echo -n " 段 $((i+1))/$NUM_CHUNKS... " + IN_FILE="$TMPDIR/transcript_${i}.txt" \ + OUT_FILE="$TMPDIR/polished_${i}.txt" \ + GROQ_API_KEY="$GROQ_API_KEY" \ + python3 <<'PY' +import json, os, sys, urllib.request, urllib.error + +KEY = os.environ["GROQ_API_KEY"] +IN = os.environ["IN_FILE"] +OUT = os.environ["OUT_FILE"] + +MODEL = "llama-3.3-70b-versatile" +MAX_DEPTH = 3 +PROMPT_TMPL = ( + "以下是一段中文普通话播客的语音转写片段，由于 Whisper 对中文标点支持较弱，" + "整段几乎没有标点。请你**只做一件事**：在合适位置补充中文标点（，。！？：；），" + "可以适度分段。\n\n" + "**严格要求**：\n" + "- 不得修改、删除、增加任何汉字或英文/数字\n" + "- 不得改写、润色、总结\n" + "- 不得添加任何解释、前言、后记\n" + "- 直接输出加好标点+合理分段后的全文\n\n" + "原文：\n{}" +) + +def call_groq(text): + body = json.dumps({ + "model": MODEL, + "temperature": 0.2, + "max_completion_tokens": 8192, + "messages": [{"role": "user", "content": PROMPT_TMPL.format(text)}], + }).encode() + req = urllib.request.Request( + "https://api.groq.com/openai/v1/chat/completions", + data=body, + headers={ + "Authorization": f"Bearer {KEY}", + "Content-Type": "application/json", + "User-Agent": "agent-reach-xiaoyuzhou/1.0", + }, + ) + with urllib.request.urlopen(req, timeout=180) as r: + resp = json.load(r) + return ( + resp["choices"][0]["message"]["content"].strip(), + resp["choices"][0].get("finish_reason"), + ) + +def polish(text, depth=0): + try: + out, fr = call_groq(text) + except urllib.error.HTTPError as e: + sys.stderr.write(f"polish HTTP {e.code}: {e.read().decode(errors='replace')[:200]}\n") + return text # fallback to raw + except Exception as e: + sys.stderr.write(f"polish error: {e}\n") + return text + if fr != "length" or depth >= MAX_DEPTH: + return out + # 输出被截断：从中点切两半递归处理 + mid = len(text) // 2 + return polish(text[:mid], depth + 1) + polish(text[mid:], depth + 1) + +content = open(IN, encoding="utf-8").read().strip() +result = polish(content) +open(OUT, "w", encoding="utf-8").write(result + "\n") +print(f"✅ ({len(result)} 字)") +PY + done +fi + # Step 7: 合并输出 echo "📄 正在合并文字稿..." @@ -149,12 +244,19 @@ echo "📄 正在合并文字稿..." echo "来源: $URL" echo "时长: ${DURATION_MIN}分${DURATION_SEC}秒" echo "转录时间: $(date '+%Y-%m-%d %H:%M')" + if [ "$POLISH" = "1" ]; then + echo "润色: Groq Llama 3.3 70B" + fi echo "" echo "---" echo "" - + for i in $(seq 0 $((NUM_CHUNKS - 1))); do - cat "$TMPDIR/transcript_${i}.txt" + if [ "$POLISH" = "1" ] && [ -f "$TMPDIR/polished_${i}.txt" ]; then + cat "$TMPDIR/polished_${i}.txt" + else + cat "$TMPDIR/transcript_${i}.txt" + fi echo "" done } > "$OUTPUT" diff --git a/agent_reach/skill/references/video.md b/agent_reach/skill/references/video.md index 7ce12b4..9c27238 100644 --- a/agent_reach/skill/references/video.md +++ b/agent_reach/skill/references/video.md @@ -71,13 +71,15 @@ bili rank -n 10 ## 小宇宙播客 / Xiaoyuzhou Podcast -### 转录单集播客 +### 转录单集播客（可选 --polish 增强标点） ```bash -# 输出 Markdown 文件到 /tmp/ -~/.agent-reach/tools/xiaoyuzhou/transcribe.sh "https://www.xiaoyuzhoufm.com/episode/EPISODE_ID" +# 输出 Markdown 文件到 /tmp/。--polish 让 Llama 3.3 70B 给文稿补中文标点+合理分段 +~/.agent-reach/tools/xiaoyuzhou/transcribe.sh --polish "https://www.xiaoyuzhoufm.com/episode/EPISODE_ID" ``` +> 转写 prompt 已要求 Whisper 输出中文标点；若标点效果仍不理想，可加 `--polish` 用 Groq 上免费的 Llama 3.3 70B 补标点+合理分段（9 分钟播客约多 ~7 秒）。每次转写多一轮 LLM 调用，按需使用。 + ### 前置要求 1. **ffmpeg**: `brew install ffmpeg`