feat(podcasts): add YouTube podcast source with transcript-first discovery
New "podcasts" source that discovers podcast content by scanning transcripts from LLM-resolved YouTube channels. Finds content invisible to title-based search — Acquired's "The NFL" episode mentions Taylor Swift 18x, ESPN 117x, Netflix 102x, none in the title. Architecture: - LLM resolves 6-12 podcast channel @handles per topic - Engine fetches recent episodes via yt-dlp (no video download) - Downloads auto-captions and greps for topic keywords - Episodes with 5+ mentions become podcast results with highlights - Runs in parallel, ~15-20s latency, invisible in 3-min research run Pipeline integration: - New source module: scripts/lib/podcast_yt.py - Registered in pipeline, normalizer, signals, planner, render - CLI flag: --podcast-channels=AcquiredFM,lexfridman,... - SOURCE_QUALITY: 0.88 (above YouTube's 0.85) - Opt-in via INCLUDE_SOURCES=podcasts or --search=podcasts Zero new API keys. Zero new dependencies. Reuses yt-dlp + transcript pipeline. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -40,6 +40,7 @@ from . import (
|
||||
xai_x,
|
||||
xiaohongshu_api,
|
||||
xquik,
|
||||
podcast_yt,
|
||||
youtube_yt,
|
||||
)
|
||||
from .cluster import cluster_candidates
|
||||
@@ -77,6 +78,7 @@ MOCK_AVAILABLE_SOURCES = [
|
||||
"github",
|
||||
"perplexity",
|
||||
"xquik",
|
||||
"podcasts",
|
||||
]
|
||||
|
||||
|
||||
@@ -122,6 +124,8 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
|
||||
available.append("pinterest")
|
||||
if env.is_xquik_available(config):
|
||||
available.append("xquik")
|
||||
if podcast_yt.is_available() and ("podcasts" in include_sources or (requested_sources and "podcasts" in requested_sources)):
|
||||
available.append("podcasts")
|
||||
return available
|
||||
|
||||
|
||||
@@ -177,6 +181,7 @@ def run(
|
||||
lookback_days: int = 30,
|
||||
github_user: str | None = None,
|
||||
github_repos: list[str] | None = None,
|
||||
podcast_channels: list[str] | None = None,
|
||||
) -> schema.Report:
|
||||
settings = DEPTH_SETTINGS[depth]
|
||||
requested_sources = normalize_requested_sources(requested_sources)
|
||||
@@ -318,6 +323,7 @@ def run(
|
||||
tiktok_hashtags=tiktok_hashtags,
|
||||
tiktok_creators=tiktok_creators,
|
||||
ig_creators=ig_creators,
|
||||
podcast_channels=podcast_channels,
|
||||
)
|
||||
] = (subquery, source)
|
||||
|
||||
@@ -348,6 +354,7 @@ def run(
|
||||
tiktok_hashtags=tiktok_hashtags,
|
||||
tiktok_creators=tiktok_creators,
|
||||
ig_creators=ig_creators,
|
||||
podcast_channels=podcast_channels,
|
||||
)
|
||||
except Exception as retry_exc:
|
||||
bundle.errors_by_source[source] = f"{exc} (retried once, still failed: {retry_exc})"
|
||||
@@ -787,6 +794,7 @@ def _retrieve_stream(
|
||||
tiktok_hashtags: list[str] | None = None,
|
||||
tiktok_creators: list[str] | None = None,
|
||||
ig_creators: list[str] | None = None,
|
||||
podcast_channels: list[str] | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
# Early exit if source was rate-limited by a sibling future
|
||||
if rate_limited_sources is not None and source in rate_limited_sources:
|
||||
@@ -874,6 +882,13 @@ def _retrieve_stream(
|
||||
sc_token = config.get("SCRAPECREATORS_API_KEY", "")
|
||||
youtube_yt.enrich_with_comments(items, token=sc_token)
|
||||
return items, {}
|
||||
if source == "podcasts":
|
||||
podcast_query = raw_topic or subquery.search_query
|
||||
result = podcast_yt.search_podcast_youtube(
|
||||
podcast_query, from_date, to_date,
|
||||
depth=depth, channels=podcast_channels,
|
||||
)
|
||||
return result.get("items", []), {}
|
||||
if source == "tiktok":
|
||||
# Use raw_topic so expand_tiktok_queries() generates diverse variants
|
||||
# from the original user topic, not the planner's narrowed search_query.
|
||||
|
||||
Reference in New Issue
Block a user