feat: Add YouTube as 4th research source via yt-dlp
YouTube search and transcript extraction runs automatically when yt-dlp is installed. Searches for topic videos from the last N days, fetches auto-generated transcripts for top results, and feeds them through the same scoring pipeline (relevance + recency + engagement) as Reddit/X. New files: - youtube_yt.py: search, transcript extraction, VTT cleanup Modified files: - schema.py: YouTubeItem dataclass, updated Report - normalize.py: normalize_youtube_items() - score.py: YouTube engagement scoring (views-dominated) - dedupe.py: YouTube deduplication - render.py: YouTube section in compact output - env.py: is_ytdlp_available() check - ui.py: YouTube progress messages - last30days.py: _search_youtube(), parallel execution with Reddit/X - SKILL.md: YouTube in stats box, citation priority - README.md: YouTube docs, yt-dlp requirement, Peter shoutout Inspired by Peter Steinberger's yt-dlp + summarize toolchain approach. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+26
-3
@@ -64,6 +64,14 @@ ENRICHING_MESSAGES = [
|
||||
"Analyzing discussions...",
|
||||
]
|
||||
|
||||
YOUTUBE_MESSAGES = [
|
||||
"Searching YouTube for videos...",
|
||||
"Finding relevant video content...",
|
||||
"Scanning YouTube channels...",
|
||||
"Discovering video discussions...",
|
||||
"Fetching transcripts...",
|
||||
]
|
||||
|
||||
PROCESSING_MESSAGES = [
|
||||
"Crunching the data...",
|
||||
"Scoring and ranking...",
|
||||
@@ -280,6 +288,15 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.CYAN}X{Colors.RESET} Found {count} posts")
|
||||
|
||||
def start_youtube(self):
|
||||
msg = random.choice(YOUTUBE_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.RED}YouTube{Colors.RESET} {msg}", Colors.RED)
|
||||
self.spinner.start()
|
||||
|
||||
def end_youtube(self, count: int):
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.RED}YouTube{Colors.RESET} Found {count} videos")
|
||||
|
||||
def start_processing(self):
|
||||
msg = random.choice(PROCESSING_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.PURPLE}Processing{Colors.RESET} {msg}", Colors.PURPLE)
|
||||
@@ -289,15 +306,21 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop()
|
||||
|
||||
def show_complete(self, reddit_count: int, x_count: int):
|
||||
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0):
|
||||
elapsed = time.time() - self.start_time
|
||||
if IS_TTY:
|
||||
sys.stderr.write(f"\n{Colors.GREEN}{Colors.BOLD}✓ Research complete{Colors.RESET} ")
|
||||
sys.stderr.write(f"{Colors.DIM}({elapsed:.1f}s){Colors.RESET}\n")
|
||||
sys.stderr.write(f" {Colors.YELLOW}Reddit:{Colors.RESET} {reddit_count} threads ")
|
||||
sys.stderr.write(f"{Colors.CYAN}X:{Colors.RESET} {x_count} posts\n\n")
|
||||
sys.stderr.write(f"{Colors.CYAN}X:{Colors.RESET} {x_count} posts")
|
||||
if youtube_count:
|
||||
sys.stderr.write(f" {Colors.RED}YouTube:{Colors.RESET} {youtube_count} videos")
|
||||
sys.stderr.write("\n\n")
|
||||
else:
|
||||
sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - Reddit: {reddit_count} threads, X: {x_count} posts\n")
|
||||
parts = [f"Reddit: {reddit_count} threads", f"X: {x_count} posts"]
|
||||
if youtube_count:
|
||||
parts.append(f"YouTube: {youtube_count} videos")
|
||||
sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - {', '.join(parts)}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
def show_cached(self, age_hours: float = None):
|
||||
|
||||
Reference in New Issue
Block a user