feat: Add YouTube as 4th research source via yt-dlp

YouTube search and transcript extraction runs automatically when yt-dlp
is installed. Searches for topic videos from the last N days, fetches
auto-generated transcripts for top results, and feeds them through the
same scoring pipeline (relevance + recency + engagement) as Reddit/X.

New files:
- youtube_yt.py: search, transcript extraction, VTT cleanup

Modified files:
- schema.py: YouTubeItem dataclass, updated Report
- normalize.py: normalize_youtube_items()
- score.py: YouTube engagement scoring (views-dominated)
- dedupe.py: YouTube deduplication
- render.py: YouTube section in compact output
- env.py: is_ytdlp_available() check
- ui.py: YouTube progress messages
- last30days.py: _search_youtube(), parallel execution with Reddit/X
- SKILL.md: YouTube in stats box, citation priority
- README.md: YouTube docs, yt-dlp requirement, Peter shoutout

Inspired by Peter Steinberger's yt-dlp + summarize toolchain approach.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-02-14 21:38:04 -08:00
parent 31313c69ac
commit c66ca7f43d
12 changed files with 1017 additions and 33 deletions
+64 -3
View File
@@ -221,6 +221,65 @@ def score_x_items(items: List[schema.XItem]) -> List[schema.XItem]:
return items
def compute_youtube_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for YouTube item.
Formula: 0.50*log1p(views) + 0.35*log1p(likes) + 0.15*log1p(comments)
Views dominate on YouTube — they're the primary discovery signal.
"""
if engagement is None:
return None
if engagement.views is None and engagement.likes is None:
return None
views = log1p_safe(engagement.views)
likes = log1p_safe(engagement.likes)
comments = log1p_safe(engagement.num_comments)
return 0.50 * views + 0.35 * likes + 0.15 * comments
def score_youtube_items(items: List[schema.YouTubeItem]) -> List[schema.YouTubeItem]:
"""Compute scores for YouTube items.
Uses same weight structure as Reddit/X (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_youtube_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
"""Compute scores for WebSearch items WITHOUT engagement metrics.
@@ -278,7 +337,7 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS
return items
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem]]) -> List:
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem]]) -> List:
"""Sort items by score (descending), then date, then source priority.
Args:
@@ -295,13 +354,15 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
date = item.date or "0000-00-00"
date_key = -int(date.replace("-", ""))
# Tertiary: source priority (Reddit > X > WebSearch)
# Tertiary: source priority (Reddit > X > YouTube > WebSearch)
if isinstance(item, schema.RedditItem):
source_priority = 0
elif isinstance(item, schema.XItem):
source_priority = 1
else: # WebSearchItem
elif isinstance(item, schema.YouTubeItem):
source_priority = 2
else: # WebSearchItem
source_priority = 3
# Quaternary: title/text for stability
text = getattr(item, "title", "") or getattr(item, "text", "")