feat: Add YouTube as 4th research source via yt-dlp
YouTube search and transcript extraction runs automatically when yt-dlp is installed. Searches for topic videos from the last N days, fetches auto-generated transcripts for top results, and feeds them through the same scoring pipeline (relevance + recency + engagement) as Reddit/X. New files: - youtube_yt.py: search, transcript extraction, VTT cleanup Modified files: - schema.py: YouTubeItem dataclass, updated Report - normalize.py: normalize_youtube_items() - score.py: YouTube engagement scoring (views-dominated) - dedupe.py: YouTube deduplication - render.py: YouTube section in compact output - env.py: is_ytdlp_available() check - ui.py: YouTube progress messages - last30days.py: _search_youtube(), parallel execution with Reddit/X - SKILL.md: YouTube in stats box, citation priority - README.md: YouTube docs, yt-dlp requirement, Peter shoutout Inspired by Peter Steinberger's yt-dlp + summarize toolchain approach. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
+11
-1
@@ -36,10 +36,12 @@ def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def get_item_text(item: Union[schema.RedditItem, schema.XItem]) -> str:
|
||||
def get_item_text(item: Union[schema.RedditItem, schema.XItem, schema.YouTubeItem]) -> str:
|
||||
"""Get comparable text from an item."""
|
||||
if isinstance(item, schema.RedditItem):
|
||||
return item.title
|
||||
elif isinstance(item, schema.YouTubeItem):
|
||||
return f"{item.title} {item.channel_name}"
|
||||
else:
|
||||
return item.text
|
||||
|
||||
@@ -118,3 +120,11 @@ def dedupe_x(
|
||||
) -> List[schema.XItem]:
|
||||
"""Dedupe X items."""
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_youtube(
|
||||
items: List[schema.YouTubeItem],
|
||||
threshold: float = 0.7,
|
||||
) -> List[schema.YouTubeItem]:
|
||||
"""Dedupe YouTube items."""
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
@@ -196,6 +196,12 @@ def get_x_source(config: Dict[str, Any]) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def is_ytdlp_available() -> bool:
|
||||
"""Check if yt-dlp is installed for YouTube search."""
|
||||
from . import youtube_yt
|
||||
return youtube_yt.is_ytdlp_installed()
|
||||
|
||||
|
||||
def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Get detailed X source status for UI decisions.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem)
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem)
|
||||
|
||||
|
||||
def filter_by_date_range(
|
||||
@@ -155,6 +155,51 @@ def normalize_x_items(
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_youtube_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.YouTubeItem]:
|
||||
"""Normalize raw YouTube items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw YouTube items from yt-dlp
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of YouTubeItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for item in items:
|
||||
# Parse engagement
|
||||
eng_raw = item.get("engagement", {})
|
||||
engagement = schema.Engagement(
|
||||
views=eng_raw.get("views"),
|
||||
likes=eng_raw.get("likes"),
|
||||
num_comments=eng_raw.get("comments"),
|
||||
)
|
||||
|
||||
# YouTube dates are reliable (always YYYY-MM-DD from yt-dlp)
|
||||
date_str = item.get("date")
|
||||
|
||||
normalized.append(schema.YouTubeItem(
|
||||
id=item.get("video_id", ""),
|
||||
title=item.get("title", ""),
|
||||
url=item.get("url", ""),
|
||||
channel_name=item.get("channel_name", ""),
|
||||
date=date_str,
|
||||
date_confidence="high",
|
||||
engagement=engagement,
|
||||
transcript_snippet=item.get("transcript_snippet", ""),
|
||||
relevance=item.get("relevance", 0.7),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
|
||||
"""Convert schema items to dicts for JSON serialization."""
|
||||
return [item.to_dict() for item in items]
|
||||
|
||||
@@ -170,6 +170,40 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
# YouTube items
|
||||
if report.youtube_error:
|
||||
lines.append("### YouTube Videos")
|
||||
lines.append("")
|
||||
lines.append(f"**ERROR:** {report.youtube_error}")
|
||||
lines.append("")
|
||||
elif report.youtube:
|
||||
lines.append("### YouTube Videos")
|
||||
lines.append("")
|
||||
for item in report.youtube[:limit]:
|
||||
eng_str = ""
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
parts = []
|
||||
if eng.views is not None:
|
||||
parts.append(f"{eng.views:,} views")
|
||||
if eng.likes is not None:
|
||||
parts.append(f"{eng.likes:,} likes")
|
||||
if parts:
|
||||
eng_str = f" [{', '.join(parts)}]"
|
||||
|
||||
date_str = f" ({item.date})" if item.date else ""
|
||||
|
||||
lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}")
|
||||
lines.append(f" {item.title}")
|
||||
lines.append(f" {item.url}")
|
||||
if item.transcript_snippet:
|
||||
snippet = item.transcript_snippet[:200]
|
||||
if len(item.transcript_snippet) > 200:
|
||||
snippet += "..."
|
||||
lines.append(f" Transcript: {snippet}")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
# Web items (if any - populated by Claude)
|
||||
if report.web_error:
|
||||
lines.append("### Web Results")
|
||||
|
||||
@@ -19,6 +19,9 @@ class Engagement:
|
||||
replies: Optional[int] = None
|
||||
quotes: Optional[int] = None
|
||||
|
||||
# YouTube fields
|
||||
views: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {}
|
||||
if self.score is not None:
|
||||
@@ -35,6 +38,8 @@ class Engagement:
|
||||
d['replies'] = self.replies
|
||||
if self.quotes is not None:
|
||||
d['quotes'] = self.quotes
|
||||
if self.views is not None:
|
||||
d['views'] = self.views
|
||||
return d if d else None
|
||||
|
||||
|
||||
@@ -169,6 +174,39 @@ class WebSearchItem:
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class YouTubeItem:
|
||||
"""Normalized YouTube item."""
|
||||
id: str # video_id
|
||||
title: str
|
||||
url: str
|
||||
channel_name: str
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "high" # YouTube dates are always reliable
|
||||
engagement: Optional[Engagement] = None
|
||||
transcript_snippet: str = ""
|
||||
relevance: float = 0.7
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'url': self.url,
|
||||
'channel_name': self.channel_name,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||
'transcript_snippet': self.transcript_snippet,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
"""Full research report."""
|
||||
@@ -182,6 +220,7 @@ class Report:
|
||||
reddit: List[RedditItem] = field(default_factory=list)
|
||||
x: List[XItem] = field(default_factory=list)
|
||||
web: List[WebSearchItem] = field(default_factory=list)
|
||||
youtube: List[YouTubeItem] = field(default_factory=list)
|
||||
best_practices: List[str] = field(default_factory=list)
|
||||
prompt_pack: List[str] = field(default_factory=list)
|
||||
context_snippet_md: str = ""
|
||||
@@ -189,6 +228,7 @@ class Report:
|
||||
reddit_error: Optional[str] = None
|
||||
x_error: Optional[str] = None
|
||||
web_error: Optional[str] = None
|
||||
youtube_error: Optional[str] = None
|
||||
# Cache info
|
||||
from_cache: bool = False
|
||||
cache_age_hours: Optional[float] = None
|
||||
@@ -207,6 +247,7 @@ class Report:
|
||||
'reddit': [r.to_dict() for r in self.reddit],
|
||||
'x': [x.to_dict() for x in self.x],
|
||||
'web': [w.to_dict() for w in self.web],
|
||||
'youtube': [y.to_dict() for y in self.youtube],
|
||||
'best_practices': self.best_practices,
|
||||
'prompt_pack': self.prompt_pack,
|
||||
'context_snippet_md': self.context_snippet_md,
|
||||
@@ -217,6 +258,8 @@ class Report:
|
||||
d['x_error'] = self.x_error
|
||||
if self.web_error:
|
||||
d['web_error'] = self.web_error
|
||||
if self.youtube_error:
|
||||
d['youtube_error'] = self.youtube_error
|
||||
if self.from_cache:
|
||||
d['from_cache'] = self.from_cache
|
||||
if self.cache_age_hours is not None:
|
||||
@@ -294,6 +337,28 @@ class Report:
|
||||
score=w.get('score', 0),
|
||||
))
|
||||
|
||||
# Reconstruct YouTube items
|
||||
youtube_items = []
|
||||
for y in data.get('youtube', []):
|
||||
eng = None
|
||||
if y.get('engagement'):
|
||||
eng = Engagement(**y['engagement'])
|
||||
subs = SubScores(**y.get('subs', {})) if y.get('subs') else SubScores()
|
||||
youtube_items.append(YouTubeItem(
|
||||
id=y['id'],
|
||||
title=y['title'],
|
||||
url=y['url'],
|
||||
channel_name=y.get('channel_name', ''),
|
||||
date=y.get('date'),
|
||||
date_confidence=y.get('date_confidence', 'high'),
|
||||
engagement=eng,
|
||||
transcript_snippet=y.get('transcript_snippet', ''),
|
||||
relevance=y.get('relevance', 0.7),
|
||||
why_relevant=y.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=y.get('score', 0),
|
||||
))
|
||||
|
||||
return cls(
|
||||
topic=data['topic'],
|
||||
range_from=range_from,
|
||||
@@ -305,12 +370,14 @@ class Report:
|
||||
reddit=reddit_items,
|
||||
x=x_items,
|
||||
web=web_items,
|
||||
youtube=youtube_items,
|
||||
best_practices=data.get('best_practices', []),
|
||||
prompt_pack=data.get('prompt_pack', []),
|
||||
context_snippet_md=data.get('context_snippet_md', ''),
|
||||
reddit_error=data.get('reddit_error'),
|
||||
x_error=data.get('x_error'),
|
||||
web_error=data.get('web_error'),
|
||||
youtube_error=data.get('youtube_error'),
|
||||
from_cache=data.get('from_cache', False),
|
||||
cache_age_hours=data.get('cache_age_hours'),
|
||||
)
|
||||
|
||||
+64
-3
@@ -221,6 +221,65 @@ def score_x_items(items: List[schema.XItem]) -> List[schema.XItem]:
|
||||
return items
|
||||
|
||||
|
||||
def compute_youtube_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
"""Compute raw engagement score for YouTube item.
|
||||
|
||||
Formula: 0.50*log1p(views) + 0.35*log1p(likes) + 0.15*log1p(comments)
|
||||
Views dominate on YouTube — they're the primary discovery signal.
|
||||
"""
|
||||
if engagement is None:
|
||||
return None
|
||||
|
||||
if engagement.views is None and engagement.likes is None:
|
||||
return None
|
||||
|
||||
views = log1p_safe(engagement.views)
|
||||
likes = log1p_safe(engagement.likes)
|
||||
comments = log1p_safe(engagement.num_comments)
|
||||
|
||||
return 0.50 * views + 0.35 * likes + 0.15 * comments
|
||||
|
||||
|
||||
def score_youtube_items(items: List[schema.YouTubeItem]) -> List[schema.YouTubeItem]:
|
||||
"""Compute scores for YouTube items.
|
||||
|
||||
Uses same weight structure as Reddit/X (relevance + recency + engagement).
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
|
||||
eng_raw = [compute_youtube_engagement_raw(item.engagement) for item in items]
|
||||
eng_normalized = normalize_to_100(eng_raw)
|
||||
|
||||
for i, item in enumerate(items):
|
||||
rel_score = int(item.relevance * 100)
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
if eng_normalized[i] is not None:
|
||||
eng_score = int(eng_normalized[i])
|
||||
else:
|
||||
eng_score = DEFAULT_ENGAGEMENT
|
||||
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=eng_score,
|
||||
)
|
||||
|
||||
overall = (
|
||||
WEIGHT_RELEVANCE * rel_score +
|
||||
WEIGHT_RECENCY * rec_score +
|
||||
WEIGHT_ENGAGEMENT * eng_score
|
||||
)
|
||||
|
||||
if eng_raw[i] is None:
|
||||
overall -= UNKNOWN_ENGAGEMENT_PENALTY
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
|
||||
"""Compute scores for WebSearch items WITHOUT engagement metrics.
|
||||
|
||||
@@ -278,7 +337,7 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS
|
||||
return items
|
||||
|
||||
|
||||
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem]]) -> List:
|
||||
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem]]) -> List:
|
||||
"""Sort items by score (descending), then date, then source priority.
|
||||
|
||||
Args:
|
||||
@@ -295,13 +354,15 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
|
||||
date = item.date or "0000-00-00"
|
||||
date_key = -int(date.replace("-", ""))
|
||||
|
||||
# Tertiary: source priority (Reddit > X > WebSearch)
|
||||
# Tertiary: source priority (Reddit > X > YouTube > WebSearch)
|
||||
if isinstance(item, schema.RedditItem):
|
||||
source_priority = 0
|
||||
elif isinstance(item, schema.XItem):
|
||||
source_priority = 1
|
||||
else: # WebSearchItem
|
||||
elif isinstance(item, schema.YouTubeItem):
|
||||
source_priority = 2
|
||||
else: # WebSearchItem
|
||||
source_priority = 3
|
||||
|
||||
# Quaternary: title/text for stability
|
||||
text = getattr(item, "title", "") or getattr(item, "text", "")
|
||||
|
||||
+26
-3
@@ -64,6 +64,14 @@ ENRICHING_MESSAGES = [
|
||||
"Analyzing discussions...",
|
||||
]
|
||||
|
||||
YOUTUBE_MESSAGES = [
|
||||
"Searching YouTube for videos...",
|
||||
"Finding relevant video content...",
|
||||
"Scanning YouTube channels...",
|
||||
"Discovering video discussions...",
|
||||
"Fetching transcripts...",
|
||||
]
|
||||
|
||||
PROCESSING_MESSAGES = [
|
||||
"Crunching the data...",
|
||||
"Scoring and ranking...",
|
||||
@@ -280,6 +288,15 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.CYAN}X{Colors.RESET} Found {count} posts")
|
||||
|
||||
def start_youtube(self):
|
||||
msg = random.choice(YOUTUBE_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.RED}YouTube{Colors.RESET} {msg}", Colors.RED)
|
||||
self.spinner.start()
|
||||
|
||||
def end_youtube(self, count: int):
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.RED}YouTube{Colors.RESET} Found {count} videos")
|
||||
|
||||
def start_processing(self):
|
||||
msg = random.choice(PROCESSING_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.PURPLE}Processing{Colors.RESET} {msg}", Colors.PURPLE)
|
||||
@@ -289,15 +306,21 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop()
|
||||
|
||||
def show_complete(self, reddit_count: int, x_count: int):
|
||||
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0):
|
||||
elapsed = time.time() - self.start_time
|
||||
if IS_TTY:
|
||||
sys.stderr.write(f"\n{Colors.GREEN}{Colors.BOLD}✓ Research complete{Colors.RESET} ")
|
||||
sys.stderr.write(f"{Colors.DIM}({elapsed:.1f}s){Colors.RESET}\n")
|
||||
sys.stderr.write(f" {Colors.YELLOW}Reddit:{Colors.RESET} {reddit_count} threads ")
|
||||
sys.stderr.write(f"{Colors.CYAN}X:{Colors.RESET} {x_count} posts\n\n")
|
||||
sys.stderr.write(f"{Colors.CYAN}X:{Colors.RESET} {x_count} posts")
|
||||
if youtube_count:
|
||||
sys.stderr.write(f" {Colors.RED}YouTube:{Colors.RESET} {youtube_count} videos")
|
||||
sys.stderr.write("\n\n")
|
||||
else:
|
||||
sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - Reddit: {reddit_count} threads, X: {x_count} posts\n")
|
||||
parts = [f"Reddit: {reddit_count} threads", f"X: {x_count} posts"]
|
||||
if youtube_count:
|
||||
parts.append(f"YouTube: {youtube_count} videos")
|
||||
sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - {', '.join(parts)}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
def show_cached(self, age_hours: float = None):
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
"""YouTube search and transcript extraction via yt-dlp for /last30days v2.1.
|
||||
|
||||
Uses yt-dlp (https://github.com/yt-dlp/yt-dlp) for both YouTube search and
|
||||
transcript extraction. No API keys needed — just have yt-dlp installed.
|
||||
|
||||
Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI).
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
# Depth configurations: how many videos to search / transcribe
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 10,
|
||||
"default": 20,
|
||||
"deep": 40,
|
||||
}
|
||||
|
||||
TRANSCRIPT_LIMITS = {
|
||||
"quick": 3,
|
||||
"default": 5,
|
||||
"deep": 8,
|
||||
}
|
||||
|
||||
# Max words to keep from each transcript
|
||||
TRANSCRIPT_MAX_WORDS = 500
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
"""Log to stderr."""
|
||||
sys.stderr.write(f"[YouTube] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def is_ytdlp_installed() -> bool:
|
||||
"""Check if yt-dlp is available in PATH."""
|
||||
return shutil.which("yt-dlp") is not None
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for YouTube search.
|
||||
|
||||
Strips meta/research words to keep only the core product/concept name,
|
||||
similar to bird_x.py's approach.
|
||||
"""
|
||||
text = topic.lower().strip()
|
||||
|
||||
# Strip multi-word prefixes
|
||||
prefixes = [
|
||||
'what are the best', 'what is the best', 'what are the latest',
|
||||
'what are people saying about', 'what do people think about',
|
||||
'how do i use', 'how to use', 'how to',
|
||||
'what are', 'what is', 'tips for', 'best practices for',
|
||||
]
|
||||
for p in prefixes:
|
||||
if text.startswith(p + ' '):
|
||||
text = text[len(p):].strip()
|
||||
|
||||
# Strip individual noise words
|
||||
noise = {
|
||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
||||
'latest', 'new', 'news', 'update', 'updates',
|
||||
'trending', 'hottest', 'popular', 'viral',
|
||||
'practices', 'features', 'guide', 'tutorial',
|
||||
'recommendations', 'advice', 'review', 'reviews',
|
||||
'prompt', 'prompts', 'prompting', 'techniques', 'tips',
|
||||
'tricks', 'methods', 'strategies', 'approaches',
|
||||
}
|
||||
words = text.split()
|
||||
filtered = [w for w in words if w not in noise]
|
||||
|
||||
return ' '.join(filtered) if filtered else text
|
||||
|
||||
|
||||
def search_youtube(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search YouTube via yt-dlp. No API key needed.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
Dict with 'items' list of video metadata dicts.
|
||||
"""
|
||||
if not is_ytdlp_installed():
|
||||
return {"items": [], "error": "yt-dlp not installed"}
|
||||
|
||||
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
date_filter = from_date.replace("-", "") # YYYYMMDD format
|
||||
|
||||
_log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})")
|
||||
|
||||
# yt-dlp search with metadata extraction via JSON
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
f"ytsearch{count}:{core_topic}",
|
||||
"--dateafter", date_filter,
|
||||
"--flat-playlist",
|
||||
"--dump-json",
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
_log("YouTube search timed out (60s)")
|
||||
return {"items": [], "error": "Search timed out"}
|
||||
except FileNotFoundError:
|
||||
return {"items": [], "error": "yt-dlp not found"}
|
||||
|
||||
if not result.stdout.strip():
|
||||
_log("YouTube search returned 0 results")
|
||||
return {"items": []}
|
||||
|
||||
# Parse JSON-per-line output
|
||||
items = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
video = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
video_id = video.get("id", "")
|
||||
view_count = video.get("view_count") or 0
|
||||
like_count = video.get("like_count") or 0
|
||||
comment_count = video.get("comment_count") or 0
|
||||
upload_date = video.get("upload_date", "") # YYYYMMDD
|
||||
|
||||
# Convert YYYYMMDD to YYYY-MM-DD
|
||||
date_str = None
|
||||
if upload_date and len(upload_date) == 8:
|
||||
date_str = f"{upload_date[:4]}-{upload_date[4:6]}-{upload_date[6:8]}"
|
||||
|
||||
items.append({
|
||||
"video_id": video_id,
|
||||
"title": video.get("title", ""),
|
||||
"url": f"https://www.youtube.com/watch?v={video_id}",
|
||||
"channel_name": video.get("channel", video.get("uploader", "")),
|
||||
"date": date_str,
|
||||
"engagement": {
|
||||
"views": view_count,
|
||||
"likes": like_count,
|
||||
"comments": comment_count,
|
||||
},
|
||||
"duration": video.get("duration"),
|
||||
"relevance": 0.7, # Default; no LLM relevance scoring for YouTube
|
||||
"why_relevant": f"YouTube video about {core_topic}",
|
||||
})
|
||||
|
||||
# Sort by views descending
|
||||
items.sort(key=lambda x: x["engagement"]["views"], reverse=True)
|
||||
|
||||
_log(f"Found {len(items)} videos")
|
||||
return {"items": items}
|
||||
|
||||
|
||||
def _clean_vtt(vtt_text: str) -> str:
|
||||
"""Convert VTT subtitle format to clean plaintext."""
|
||||
# Strip VTT header
|
||||
text = re.sub(r'^WEBVTT.*?\n\n', '', vtt_text, flags=re.DOTALL)
|
||||
# Strip timestamps
|
||||
text = re.sub(r'\d{2}:\d{2}:\d{2}\.\d{3}\s*-->\s*\d{2}:\d{2}:\d{2}\.\d{3}.*\n', '', text)
|
||||
# Strip position/alignment tags
|
||||
text = re.sub(r'<[^>]+>', '', text)
|
||||
# Strip cue numbers
|
||||
text = re.sub(r'^\d+\s*$', '', text, flags=re.MULTILINE)
|
||||
# Deduplicate overlapping lines
|
||||
lines = text.strip().split('\n')
|
||||
seen = set()
|
||||
unique = []
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped and stripped not in seen:
|
||||
seen.add(stripped)
|
||||
unique.append(stripped)
|
||||
return re.sub(r'\s+', ' ', ' '.join(unique)).strip()
|
||||
|
||||
|
||||
def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
|
||||
"""Fetch auto-generated transcript for a YouTube video.
|
||||
|
||||
Args:
|
||||
video_id: YouTube video ID
|
||||
temp_dir: Temporary directory for subtitle files
|
||||
|
||||
Returns:
|
||||
Plaintext transcript string, or None if no captions available.
|
||||
"""
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"--write-auto-subs",
|
||||
"--sub-lang", "en",
|
||||
"--sub-format", "vtt",
|
||||
"--skip-download",
|
||||
"--no-warnings",
|
||||
"-o", f"{temp_dir}/%(id)s",
|
||||
f"https://www.youtube.com/watch?v={video_id}",
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return None
|
||||
|
||||
# yt-dlp may save as .en.vtt or .en-orig.vtt
|
||||
vtt_path = Path(temp_dir) / f"{video_id}.en.vtt"
|
||||
if not vtt_path.exists():
|
||||
# Try alternate naming
|
||||
for p in Path(temp_dir).glob(f"{video_id}*.vtt"):
|
||||
vtt_path = p
|
||||
break
|
||||
else:
|
||||
return None
|
||||
|
||||
try:
|
||||
raw = vtt_path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
transcript = _clean_vtt(raw)
|
||||
|
||||
# Truncate to max words
|
||||
words = transcript.split()
|
||||
if len(words) > TRANSCRIPT_MAX_WORDS:
|
||||
transcript = ' '.join(words[:TRANSCRIPT_MAX_WORDS]) + '...'
|
||||
|
||||
return transcript if transcript else None
|
||||
|
||||
|
||||
def fetch_transcripts_parallel(
|
||||
video_ids: List[str],
|
||||
max_workers: int = 5,
|
||||
) -> Dict[str, Optional[str]]:
|
||||
"""Fetch transcripts for multiple videos in parallel.
|
||||
|
||||
Args:
|
||||
video_ids: List of YouTube video IDs
|
||||
max_workers: Max parallel fetches
|
||||
|
||||
Returns:
|
||||
Dict mapping video_id to transcript text (or None).
|
||||
"""
|
||||
if not video_ids:
|
||||
return {}
|
||||
|
||||
_log(f"Fetching transcripts for {len(video_ids)} videos")
|
||||
|
||||
results = {}
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {
|
||||
executor.submit(fetch_transcript, vid, temp_dir): vid
|
||||
for vid in video_ids
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
vid = futures[future]
|
||||
try:
|
||||
results[vid] = future.result()
|
||||
except Exception:
|
||||
results[vid] = None
|
||||
|
||||
got = sum(1 for v in results.values() if v)
|
||||
_log(f"Got transcripts for {got}/{len(video_ids)} videos")
|
||||
return results
|
||||
|
||||
|
||||
def search_and_transcribe(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""Full YouTube search: find videos, then fetch transcripts for top results.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
Dict with 'items' list. Each item has a 'transcript_snippet' field.
|
||||
"""
|
||||
# Step 1: Search
|
||||
search_result = search_youtube(topic, from_date, to_date, depth)
|
||||
items = search_result.get("items", [])
|
||||
|
||||
if not items:
|
||||
return search_result
|
||||
|
||||
# Step 2: Fetch transcripts for top N by views
|
||||
transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
|
||||
top_ids = [item["video_id"] for item in items[:transcript_limit]]
|
||||
transcripts = fetch_transcripts_parallel(top_ids)
|
||||
|
||||
# Step 3: Attach transcripts to items
|
||||
for item in items:
|
||||
vid = item["video_id"]
|
||||
transcript = transcripts.get(vid)
|
||||
item["transcript_snippet"] = transcript or ""
|
||||
|
||||
return {"items": items}
|
||||
|
||||
|
||||
def parse_youtube_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Parse YouTube search response to normalized format.
|
||||
|
||||
Returns:
|
||||
List of item dicts ready for normalization.
|
||||
"""
|
||||
return response.get("items", [])
|
||||
Reference in New Issue
Block a user