feat: Add YouTube as 4th research source via yt-dlp
YouTube search and transcript extraction runs automatically when yt-dlp is installed. Searches for topic videos from the last N days, fetches auto-generated transcripts for top results, and feeds them through the same scoring pipeline (relevance + recency + engagement) as Reddit/X. New files: - youtube_yt.py: search, transcript extraction, VTT cleanup Modified files: - schema.py: YouTubeItem dataclass, updated Report - normalize.py: normalize_youtube_items() - score.py: YouTube engagement scoring (views-dominated) - dedupe.py: YouTube deduplication - render.py: YouTube section in compact output - env.py: is_ytdlp_available() check - ui.py: YouTube progress messages - last30days.py: _search_youtube(), parallel execution with Reddit/X - SKILL.md: YouTube in stats box, citation priority - README.md: YouTube docs, yt-dlp requirement, Peter shoutout Inspired by Peter Steinberger's yt-dlp + summarize toolchain approach. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
@@ -19,6 +19,9 @@ class Engagement:
|
||||
replies: Optional[int] = None
|
||||
quotes: Optional[int] = None
|
||||
|
||||
# YouTube fields
|
||||
views: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {}
|
||||
if self.score is not None:
|
||||
@@ -35,6 +38,8 @@ class Engagement:
|
||||
d['replies'] = self.replies
|
||||
if self.quotes is not None:
|
||||
d['quotes'] = self.quotes
|
||||
if self.views is not None:
|
||||
d['views'] = self.views
|
||||
return d if d else None
|
||||
|
||||
|
||||
@@ -169,6 +174,39 @@ class WebSearchItem:
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class YouTubeItem:
|
||||
"""Normalized YouTube item."""
|
||||
id: str # video_id
|
||||
title: str
|
||||
url: str
|
||||
channel_name: str
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "high" # YouTube dates are always reliable
|
||||
engagement: Optional[Engagement] = None
|
||||
transcript_snippet: str = ""
|
||||
relevance: float = 0.7
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'url': self.url,
|
||||
'channel_name': self.channel_name,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||
'transcript_snippet': self.transcript_snippet,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
"""Full research report."""
|
||||
@@ -182,6 +220,7 @@ class Report:
|
||||
reddit: List[RedditItem] = field(default_factory=list)
|
||||
x: List[XItem] = field(default_factory=list)
|
||||
web: List[WebSearchItem] = field(default_factory=list)
|
||||
youtube: List[YouTubeItem] = field(default_factory=list)
|
||||
best_practices: List[str] = field(default_factory=list)
|
||||
prompt_pack: List[str] = field(default_factory=list)
|
||||
context_snippet_md: str = ""
|
||||
@@ -189,6 +228,7 @@ class Report:
|
||||
reddit_error: Optional[str] = None
|
||||
x_error: Optional[str] = None
|
||||
web_error: Optional[str] = None
|
||||
youtube_error: Optional[str] = None
|
||||
# Cache info
|
||||
from_cache: bool = False
|
||||
cache_age_hours: Optional[float] = None
|
||||
@@ -207,6 +247,7 @@ class Report:
|
||||
'reddit': [r.to_dict() for r in self.reddit],
|
||||
'x': [x.to_dict() for x in self.x],
|
||||
'web': [w.to_dict() for w in self.web],
|
||||
'youtube': [y.to_dict() for y in self.youtube],
|
||||
'best_practices': self.best_practices,
|
||||
'prompt_pack': self.prompt_pack,
|
||||
'context_snippet_md': self.context_snippet_md,
|
||||
@@ -217,6 +258,8 @@ class Report:
|
||||
d['x_error'] = self.x_error
|
||||
if self.web_error:
|
||||
d['web_error'] = self.web_error
|
||||
if self.youtube_error:
|
||||
d['youtube_error'] = self.youtube_error
|
||||
if self.from_cache:
|
||||
d['from_cache'] = self.from_cache
|
||||
if self.cache_age_hours is not None:
|
||||
@@ -294,6 +337,28 @@ class Report:
|
||||
score=w.get('score', 0),
|
||||
))
|
||||
|
||||
# Reconstruct YouTube items
|
||||
youtube_items = []
|
||||
for y in data.get('youtube', []):
|
||||
eng = None
|
||||
if y.get('engagement'):
|
||||
eng = Engagement(**y['engagement'])
|
||||
subs = SubScores(**y.get('subs', {})) if y.get('subs') else SubScores()
|
||||
youtube_items.append(YouTubeItem(
|
||||
id=y['id'],
|
||||
title=y['title'],
|
||||
url=y['url'],
|
||||
channel_name=y.get('channel_name', ''),
|
||||
date=y.get('date'),
|
||||
date_confidence=y.get('date_confidence', 'high'),
|
||||
engagement=eng,
|
||||
transcript_snippet=y.get('transcript_snippet', ''),
|
||||
relevance=y.get('relevance', 0.7),
|
||||
why_relevant=y.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=y.get('score', 0),
|
||||
))
|
||||
|
||||
return cls(
|
||||
topic=data['topic'],
|
||||
range_from=range_from,
|
||||
@@ -305,12 +370,14 @@ class Report:
|
||||
reddit=reddit_items,
|
||||
x=x_items,
|
||||
web=web_items,
|
||||
youtube=youtube_items,
|
||||
best_practices=data.get('best_practices', []),
|
||||
prompt_pack=data.get('prompt_pack', []),
|
||||
context_snippet_md=data.get('context_snippet_md', ''),
|
||||
reddit_error=data.get('reddit_error'),
|
||||
x_error=data.get('x_error'),
|
||||
web_error=data.get('web_error'),
|
||||
youtube_error=data.get('youtube_error'),
|
||||
from_cache=data.get('from_cache', False),
|
||||
cache_age_hours=data.get('cache_age_hours'),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user