From 7162eb6b3629239a9c9b3700004ee14d4cff3889 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Sun, 15 Feb 2026 00:02:16 -0800 Subject: [PATCH] fix: YouTube display and search quality - Remove 2>&1 from SKILL.md so stderr doesn't pollute model input - Run script in foreground (not background) with 5min timeout - Add explicit YouTube synthesis instruction for Claude - Remove --flat-playlist which broke date filtering (all dates were None) - Move date filtering to Python with soft fallback for evergreen topics - Keep 'tips', 'tutorial', 'review', 'guide' in YouTube search queries - Increase yt-dlp timeout from 60s to 120s for full metadata fetch Co-Authored-By: Claude Opus 4.6 --- SKILL.md | 21 +++++++--- scripts/lib/youtube_yt.py | 82 ++++++++++++++++++++++++++++++--------- 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/SKILL.md b/SKILL.md index 2b8329a..671e00c 100644 --- a/SKILL.md +++ b/SKILL.md @@ -59,7 +59,10 @@ This text MUST appear before you call any tools. It confirms to the user that yo ## Research Execution -**Step 1: Run the research script** +**Step 1: Run the research script (FOREGROUND — do NOT background this)** + +**CRITICAL: Run this command in the FOREGROUND with a 5-minute timeout. Do NOT use run_in_background. The full output contains Reddit, X, AND YouTube data that you need to read completely.** + ```bash # Find skill root — works in repo checkout, Claude Code, or Codex install for dir in \ @@ -76,19 +79,25 @@ if [ -z "${SKILL_ROOT:-}" ]; then exit 1 fi -python3 "${SKILL_ROOT}/scripts/last30days.py" "$ARGUMENTS" --emit=compact 2>&1 +python3 "${SKILL_ROOT}/scripts/last30days.py" "$ARGUMENTS" --emit=compact ``` +Use a **timeout of 300000** (5 minutes) on the Bash call. The script typically takes 1-3 minutes. + The script will automatically: - Detect available API keys -- Run Reddit/X searches if keys exist -- Signal if WebSearch is needed +- Run Reddit/X/YouTube searches +- Output ALL results including YouTube transcripts + +**Read the ENTIRE output.** It contains THREE data sections in this order: Reddit items, X items, and YouTube items. If you miss the YouTube section, you will produce incomplete stats. + +**YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, and optional transcript snippet. Count them and include them in your synthesis and stats block. --- -## STEP 2: DO WEBSEARCH WHILE SCRIPT RUNS +## STEP 2: DO WEBSEARCH AFTER SCRIPT COMPLETES -The script auto-detects sources (Bird CLI, API keys, etc). While waiting for it, do WebSearch. +After the script finishes, do WebSearch to supplement with blogs, tutorials, and news. For **ALL modes**, do WebSearch to supplement (or provide all data in web-only mode). diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py index 6be7a0c..5f174bd 100644 --- a/scripts/lib/youtube_yt.py +++ b/scripts/lib/youtube_yt.py @@ -8,7 +8,9 @@ Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI). import json import math +import os import re +import signal import shutil import subprocess import sys @@ -65,19 +67,22 @@ def _extract_core_subject(topic: str) -> str: text = text[len(p):].strip() # Strip individual noise words + # NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews' + # are intentionally KEPT — they're YouTube content types that improve search noise = { 'best', 'top', 'good', 'great', 'awesome', 'killer', 'latest', 'new', 'news', 'update', 'updates', 'trending', 'hottest', 'popular', 'viral', - 'practices', 'features', 'guide', 'tutorial', - 'recommendations', 'advice', 'review', 'reviews', - 'prompt', 'prompts', 'prompting', 'techniques', 'tips', - 'tricks', 'methods', 'strategies', 'approaches', + 'practices', 'features', + 'recommendations', 'advice', + 'prompt', 'prompts', 'prompting', + 'methods', 'strategies', 'approaches', } words = text.split() filtered = [w for w in words if w not in noise] - return ' '.join(filtered) if filtered else text + result = ' '.join(filtered) if filtered else text + return result.rstrip('?!.') def search_youtube( @@ -102,36 +107,51 @@ def search_youtube( count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) core_topic = _extract_core_subject(topic) - date_filter = from_date.replace("-", "") # YYYYMMDD format _log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})") - # yt-dlp search with metadata extraction via JSON + # yt-dlp search with full metadata (no --flat-playlist so dates are real). + # No --dateafter — we filter by date in Python with a soft fallback, + # because YouTube search returns relevance-sorted results and strict date + # filtering returns 0 for evergreen topics like "thumbnail tips". cmd = [ "yt-dlp", f"ytsearch{count}:{core_topic}", - "--dateafter", date_filter, - "--flat-playlist", "--dump-json", + "--no-warnings", + "--no-download", ] + preexec = os.setsid if hasattr(os, 'setsid') else None + try: - result = subprocess.run( - cmd, capture_output=True, text=True, timeout=60, + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + preexec_fn=preexec, ) - except subprocess.TimeoutExpired: - _log("YouTube search timed out (60s)") - return {"items": [], "error": "Search timed out"} + try: + stdout, stderr = proc.communicate(timeout=120) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + proc.kill() + proc.wait(timeout=5) + _log("YouTube search timed out (120s)") + return {"items": [], "error": "Search timed out"} except FileNotFoundError: return {"items": [], "error": "yt-dlp not found"} - if not result.stdout.strip(): + if not (stdout or "").strip(): _log("YouTube search returned 0 results") return {"items": []} # Parse JSON-per-line output items = [] - for line in result.stdout.strip().split("\n"): + for line in stdout.strip().split("\n"): line = line.strip() if not line: continue @@ -167,10 +187,17 @@ def search_youtube( "why_relevant": f"YouTube video about {core_topic}", }) + # Soft date filter: prefer recent items but fall back to all if too few + recent = [i for i in items if i["date"] and i["date"] >= from_date] + if len(recent) >= 3: + items = recent + _log(f"Found {len(items)} videos within date range") + else: + _log(f"Found {len(items)} videos ({len(recent)} within date range, keeping all)") + # Sort by views descending items.sort(key=lambda x: x["engagement"]["views"], reverse=True) - _log(f"Found {len(items)} videos") return {"items": items} @@ -217,9 +244,26 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: f"https://www.youtube.com/watch?v={video_id}", ] + preexec = os.setsid if hasattr(os, 'setsid') else None + try: - subprocess.run(cmd, capture_output=True, text=True, timeout=30) - except (subprocess.TimeoutExpired, FileNotFoundError): + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + preexec_fn=preexec, + ) + try: + proc.communicate(timeout=30) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + proc.kill() + proc.wait(timeout=5) + return None + except FileNotFoundError: return None # yt-dlp may save as .en.vtt or .en-orig.vtt