feat: Add YouTube as 4th research source via yt-dlp

YouTube search and transcript extraction runs automatically when yt-dlp
is installed. Searches for topic videos from the last N days, fetches
auto-generated transcripts for top results, and feeds them through the
same scoring pipeline (relevance + recency + engagement) as Reddit/X.

New files:
- youtube_yt.py: search, transcript extraction, VTT cleanup

Modified files:
- schema.py: YouTubeItem dataclass, updated Report
- normalize.py: normalize_youtube_items()
- score.py: YouTube engagement scoring (views-dominated)
- dedupe.py: YouTube deduplication
- render.py: YouTube section in compact output
- env.py: is_ytdlp_available() check
- ui.py: YouTube progress messages
- last30days.py: _search_youtube(), parallel execution with Reddit/X
- SKILL.md: YouTube in stats box, citation priority
- README.md: YouTube docs, yt-dlp requirement, Peter shoutout

Inspired by Peter Steinberger's yt-dlp + summarize toolchain approach.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-02-14 21:38:04 -08:00
parent 31313c69ac
commit c66ca7f43d
12 changed files with 1017 additions and 33 deletions
+46 -1
View File
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
from . import dates, schema
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem)
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem)
def filter_by_date_range(
@@ -155,6 +155,51 @@ def normalize_x_items(
return normalized
def normalize_youtube_items(
items: List[Dict[str, Any]],
from_date: str,
to_date: str,
) -> List[schema.YouTubeItem]:
"""Normalize raw YouTube items to schema.
Args:
items: Raw YouTube items from yt-dlp
from_date: Start of date range
to_date: End of date range
Returns:
List of YouTubeItem objects
"""
normalized = []
for item in items:
# Parse engagement
eng_raw = item.get("engagement", {})
engagement = schema.Engagement(
views=eng_raw.get("views"),
likes=eng_raw.get("likes"),
num_comments=eng_raw.get("comments"),
)
# YouTube dates are reliable (always YYYY-MM-DD from yt-dlp)
date_str = item.get("date")
normalized.append(schema.YouTubeItem(
id=item.get("video_id", ""),
title=item.get("title", ""),
url=item.get("url", ""),
channel_name=item.get("channel_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
transcript_snippet=item.get("transcript_snippet", ""),
relevance=item.get("relevance", 0.7),
why_relevant=item.get("why_relevant", ""),
))
return normalized
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
"""Convert schema items to dicts for JSON serialization."""
return [item.to_dict() for item in items]