From 0a5102e1931870b7dfc9a88f55bb90e7a7cd0f21 Mon Sep 17 00:00:00 2001 From: Daniel Zivkovic Date: Sat, 2 May 2026 22:02:45 -0400 Subject: [PATCH] fix(youtube): surface transcript-fetch ratio in footer + add degraded nudge When yt-dlp is installed but stale (or otherwise unable to fetch transcripts for any returned videos), runs previously reported YouTube as fully successful in two user-facing surfaces: 1. Footer (render.py): showed "N videos | M views" with no indication that zero transcripts were captured. The "with transcripts" segment was conditionally suppressed when the count was zero - converting the canonical stale-binary failure mode into a silent absence at the very surface users read for "did this work?". 2. Quality nudge (quality_nudge.py): classified YouTube as "active" based purely on yt-dlp installation + absence of a top-level error. Per-video transcript-fetch ratio was never inspected. A run that returned N videos with 0 transcripts (canonical stale-binary failure) was reported as fully active. The engine itself logs the failure correctly at default stderr level (`[YouTube] Got transcripts for 0/N videos (N failed)`), but that line gets buried in 100+ lines of parallel-source progress output and is contradicted by the success-shaped footer and nudge that follow. This change makes both conclusion surfaces honest: * render.py footer always renders "M/N with transcripts" so the ratio is visible regardless of value. Zero is no longer hidden. Format is M/N (not bare M) so the denominator is in the message and the user does not have to cross-reference the "videos" count. * quality_nudge.py adds a third tier between "active" and "missing": "degraded". Triggered when yt-dlp is installed AND videos were returned AND transcript-fetch ratio is below threshold (default 50%, tunable via DEGRADED_TRANSCRIPT_THRESHOLD env var). Emits an actionable nudge: "YouTube returned N videos but only M transcripts captured. The most common cause is a stale yt-dlp binary - YouTube's caption format changes frequently and old binaries silently fail every transcript. Update via your package manager: scoop update yt-dlp (Windows), brew upgrade yt-dlp (macOS), or pip install -U yt-dlp." * last30days.py populates youtube_videos_count and youtube_transcripts_count in the research_results dict it passes to compute_quality_score, enabling the new degraded check at the call site. Threshold rationale: 50% accommodates a few legitimate caption-disabled videos in a multi-video result, but a stale-binary run that fails every transcript trips the nudge cleanly. Score impact: degradation is informational, not score-affecting. YouTube still counts as "active" in score_pct so users do not see their score drop for a fixable client-side issue. The nudge directs them to their own package manager. Tests: * tests/test_quality_nudge.py: 6 new TestYouTubeDegraded cases cover zero-transcripts-flags-degraded, partial-above-threshold-does-not-flag, zero-videos-does-not-flag (no false positives on absence), one-of-three-flags-degraded, threshold-tunable-via-config, and degraded-does-not-affect-score. * tests/test_render_v3.py: 4 new YoutubeFooterTranscriptRatioTests cases cover zero-transcripts-with-videos-renders-zero-over-total (the regression repro), partial-renders-ratio, full-renders-ratio, and no-videos-suppresses-entire-segment. All 29 new test cases verified GREEN with the fix and RED without it (temp-reverted both files separately to confirm each test catches the specific regression it asserts). Integration validation: ran the engine against an intentionally stale yt-dlp 2025.03.31 binary placed first on PATH. Pre-fix the footer showed `YouTube: 3 videos | 386,815 views` (no transcript signal). Post-fix the footer shows `YouTube: 3 videos | 386,815 views | 0/3 with transcripts` and stderr emits "Degraded: YouTube" plus the actionable update-yt-dlp nudge. Out of scope (deserves its own PR): exposing transcripts_captured in the EVIDENCE FOR SYNTHESIS block so the synthesizing model can flag degradation in prose. Larger schema-touching change. --- skills/last30days/scripts/last30days.py | 13 ++- .../last30days/scripts/lib/quality_nudge.py | 76 +++++++++++++-- skills/last30days/scripts/lib/render.py | 9 +- tests/test_quality_nudge.py | 94 +++++++++++++++++++ tests/test_render_v3.py | 85 +++++++++++++++++ 5 files changed, 266 insertions(+), 11 deletions(-) diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 960cb2d..811ea0e 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -880,7 +880,18 @@ def main() -> int: # Show quality nudge if applicable try: from lib import quality_nudge - quality = quality_nudge.compute_quality_score(config, {}) + # Populate transcript-fetch ratio so quality_nudge can detect the + # degraded-YouTube failure mode (videos returned but transcripts + # silently failed - typically a stale yt-dlp binary). + youtube_items = report.items_by_source.get("youtube") or [] + research_results = { + "youtube_videos_count": len(youtube_items), + "youtube_transcripts_count": sum( + 1 for it in youtube_items + if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet")) + ), + } + quality = quality_nudge.compute_quality_score(config, research_results) if quality.get("nudge_text"): sys.stderr.write(f"\n{quality['nudge_text']}\n") sys.stderr.flush() diff --git a/skills/last30days/scripts/lib/quality_nudge.py b/skills/last30days/scripts/lib/quality_nudge.py index 6e84fe4..7ffbb62 100644 --- a/skills/last30days/scripts/lib/quality_nudge.py +++ b/skills/last30days/scripts/lib/quality_nudge.py @@ -45,26 +45,51 @@ def _is_youtube_active(config: dict, research_results: dict) -> bool: return True +# Below this transcript-fetch ratio, YouTube is considered "degraded" rather +# than active. Picked at 50% so a single legitimate caption-disabled video in a +# multi-video result does not trip the nudge, but a stale-yt-dlp run that fails +# every transcript does. Tunable via DEGRADED_TRANSCRIPT_THRESHOLD env var if +# operators need to adjust without code changes. +DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD = 0.5 + + +def _is_youtube_degraded(research_results: dict, threshold: float) -> bool: + """YouTube is degraded when videos were returned but the transcript-fetch + ratio is below threshold. The canonical cause is a stale yt-dlp binary - + YouTube's caption format changes frequently and old binaries silently fail + every transcript while the search itself still succeeds. + """ + videos = int(research_results.get("youtube_videos_count") or 0) + transcripts = int(research_results.get("youtube_transcripts_count") or 0) + if videos <= 0: + return False + return (transcripts / videos) < threshold + + def compute_quality_score(config: dict, research_results: dict) -> dict: """Compute research quality score based on 5 core sources. Args: config: Configuration dict from env.get_config() research_results: Dict with keys like x_error, youtube_error, - reddit_error reflecting what happened this run. + reddit_error reflecting what happened this run. Optional keys + ``youtube_videos_count`` and ``youtube_transcripts_count`` enable + degraded-YouTube detection (transcript-fetch ratio below threshold). Returns: { "score_pct": 40-100, "core_active": ["hn", "polymarket", ...], "core_missing": ["x", "youtube"], - "core_errored": [], # configured but errored + "core_errored": [], # configured but errored at top level + "core_degraded": [], # configured and returned items but quality below threshold "nudge_text": "..." or None if 100% } """ core_active: List[str] = [] core_missing: List[str] = [] core_errored: List[str] = [] + core_degraded: List[str] = [] # HN, Polymarket, and Reddit are always active core_active.append("hn") @@ -84,6 +109,13 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: yt_active = _is_youtube_active(config, research_results) if yt_active: core_active.append("youtube") + # Active means yt-dlp is installed and search did not error at the top + # level. But search-success + transcript-failure is the canonical + # stale-binary failure mode that the footer used to hide. Flag as + # degraded so the user gets an actionable nudge to update the binary. + threshold = float(config.get("DEGRADED_TRANSCRIPT_THRESHOLD") or DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD) + if _is_youtube_degraded(research_results, threshold): + core_degraded.append("youtube") else: core_missing.append("youtube") # Check if configured but errored (yt-dlp installed but failed this run) @@ -99,24 +131,41 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: has_sc = bool(config.get("SCRAPECREATORS_API_KEY")) active_sources = research_results.get("active_sources") or [] - nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None + nudge_text = _build_nudge_text( + core_missing, + core_errored, + core_degraded, + research_results, + has_sc=has_sc, + active_sources=active_sources, + ) if (core_missing or core_degraded) else None return { "score_pct": score_pct, "core_active": core_active, "core_missing": core_missing, "core_errored": core_errored, + "core_degraded": core_degraded, "nudge_text": nudge_text, } -def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str: - """Build human-readable nudge text describing what was missed. +def _build_nudge_text( + core_missing: List[str], + core_errored: List[str], + core_degraded: List[str] = None, + research_results: dict = None, + has_sc: bool = False, + active_sources: list = None, +) -> str: + """Build human-readable nudge text describing what was missed or degraded. Prioritizes free suggestions. Optionally mentions bonus sources (TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured. """ lines: List[str] = [] + core_degraded = core_degraded or [] + research_results = research_results or {} # Describe what was missed missed_parts: List[str] = [] @@ -129,7 +178,11 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: active_count = 5 - len(core_missing) lines.append(f"Research quality: {active_count}/5 core sources.") - lines.append(f"Missing: {', '.join(missed_parts)}.") + if missed_parts: + lines.append(f"Missing: {', '.join(missed_parts)}.") + if core_degraded: + degraded_labels = ", ".join(SOURCE_LABELS[s] for s in core_degraded) + lines.append(f"Degraded: {degraded_labels}.") lines.append("") # Free suggestions @@ -159,6 +212,17 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: "explanations on any topic. Install yt-dlp: brew install yt-dlp (free)" ) + if "youtube" in core_degraded: + videos = int(research_results.get("youtube_videos_count") or 0) + transcripts = int(research_results.get("youtube_transcripts_count") or 0) + free_suggestions.append( + f"YouTube returned {videos} videos but only {transcripts} transcripts " + "captured. The most common cause is a stale yt-dlp binary - YouTube's " + "caption format changes frequently and old binaries silently fail every " + "transcript. Update via your package manager: scoop update yt-dlp " + "(Windows), brew upgrade yt-dlp (macOS), or pip install -U yt-dlp." + ) + # Mention bonus opt-in sources when SC key is present if has_sc: bonus_hints = [] diff --git a/skills/last30days/scripts/lib/render.py b/skills/last30days/scripts/lib/render.py index 7f21aa2..a89fb8f 100644 --- a/skills/last30days/scripts/lib/render.py +++ b/skills/last30days/scripts/lib/render.py @@ -1285,15 +1285,16 @@ def _build_source_footer_lines(report: schema.Report) -> list[str]: if total > 0: total_str = f"{total:,}" if total >= 1000 else str(total) parts.append(f"{total_str} {word}") - # YouTube: append "N with transcripts" instead of a third likes-based column. - # Transcripts are a more meaningful research-depth signal than likes. + # YouTube: always append "M/N with transcripts" so a zero-transcript run + # (typically caused by a stale yt-dlp binary) is visible at the conclusion + # surface. Hiding zero converts a problem signal into an absence; the very + # case that needs to be loud is the one previously omitted from the footer. if source_key == "youtube": with_transcripts = sum( 1 for it in items if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet")) ) - if with_transcripts > 0: - parts.append(f"{with_transcripts} with transcripts") + parts.append(f"{with_transcripts}/{len(items)} with transcripts") stats = " │ ".join(parts) out.append(_footer_line_for_source(emoji, label, len(items), item_word, stats)) diff --git a/tests/test_quality_nudge.py b/tests/test_quality_nudge.py index 25bad17..a2592e9 100644 --- a/tests/test_quality_nudge.py +++ b/tests/test_quality_nudge.py @@ -199,3 +199,97 @@ class TestRedditNeverInCoreErrored: # Reddit is always-active in core (public path), error doesn't demote it assert "reddit" in q["core_active"] assert q["score_pct"] == 100 + + +class TestYouTubeDegraded: + """YouTube is `degraded` when videos returned but transcripts below threshold. + + Canonical failure mode: a stale yt-dlp binary still finds videos via search + but silently fails every transcript fetch because YouTube's caption format + has moved on. Pre-fix the user got no signal of this; the footer hid zero, + and quality_nudge only checked top-level errors. + """ + + def test_zero_of_six_transcripts_flags_degraded(self): + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 0, + }, + ) + assert "youtube" in q["core_degraded"] + assert q["nudge_text"] is not None + # Counts surface in the message so the user sees the actual ratio + assert "6 videos" in q["nudge_text"] + assert "0 transcripts" in q["nudge_text"] + assert "stale yt-dlp" in q["nudge_text"].lower() + # Updates path mentions all three common package managers + assert "scoop" in q["nudge_text"].lower() + assert "brew" in q["nudge_text"].lower() + assert "pip install" in q["nudge_text"].lower() + + def test_five_of_six_transcripts_does_not_flag_degraded(self): + # 83% transcript success - well above the 50% threshold + # X is also enabled so all 5 cores are active and no nudge should fire + q = _compute( + config_overrides={"AUTH_TOKEN": "tok123"}, + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 5, + }, + ) + assert "youtube" not in q["core_degraded"] + assert q["nudge_text"] is None # All 5 core sources active, no degradation + + def test_zero_videos_does_not_flag_degraded(self): + # No videos returned -> degraded check is meaningless and must not fire + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 0, + "youtube_transcripts_count": 0, + }, + ) + assert "youtube" not in q["core_degraded"] + + def test_one_of_three_transcripts_flags_degraded(self): + # 33% - below 50% threshold; the canonical "yt-dlp partially working" case + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 3, + "youtube_transcripts_count": 1, + }, + ) + assert "youtube" in q["core_degraded"] + assert "Degraded: YouTube" in q["nudge_text"] + + def test_threshold_tunable_via_config(self): + # Operator overrides threshold via env-style config to be more permissive + q = _compute( + config_overrides={"DEGRADED_TRANSCRIPT_THRESHOLD": "0.1"}, + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 10, + "youtube_transcripts_count": 2, # 20%, below default 50% but above override 10% + }, + ) + assert "youtube" not in q["core_degraded"] + + def test_degraded_does_not_affect_score(self): + # Degradation is informational, not score-affecting; YouTube still counts as active + q = _compute( + config_overrides={"AUTH_TOKEN": "tok123"}, + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 0, + }, + ) + assert "youtube" in q["core_active"] + assert q["score_pct"] == 100 # Full active count regardless of degradation + # But nudge still fires + assert q["nudge_text"] is not None + assert "Degraded: YouTube" in q["nudge_text"] diff --git a/tests/test_render_v3.py b/tests/test_render_v3.py index 22514c0..2862c18 100644 --- a/tests/test_render_v3.py +++ b/tests/test_render_v3.py @@ -552,5 +552,90 @@ class DegradedRunBannerTests(unittest.TestCase): self.assertIn("--plan", text) +class YoutubeFooterTranscriptRatioTests(unittest.TestCase): + """The YouTube footer line must surface the transcript-fetch ratio in all + cases where videos were returned. Pre-fix the segment was suppressed when + transcripts == 0, which converted the canonical stale-yt-dlp failure mode + into a silent absence at the footer (the very surface users read for + 'did this work?'). Always-render the ratio so zero is loud. + """ + + def _build_youtube_report(self, transcript_flags: list[bool]) -> schema.Report: + """Build a Report with one YouTube item per entry in transcript_flags. + True means the item has transcript data; False means it does not. + """ + items = [] + for idx, has_transcript in enumerate(transcript_flags): + metadata = {"views": 1000} + if has_transcript: + metadata["transcript_highlights"] = ["Some pre-extracted quote."] + items.append(schema.SourceItem( + item_id=f"yt{idx}", + source="youtube", + title=f"Video {idx}", + body=f"Description for video {idx}.", + url=f"https://youtube.com/watch?v=v{idx}", + container="some-channel", + published_at="2026-04-15", + date_confidence="high", + engagement={"views": 1000, "likes": 100}, + metadata=metadata, + )) + return schema.Report( + topic="test topic", + range_from="2026-04-01", + range_to="2026-05-01", + generated_at="2026-05-01T00:00:00+00:00", + provider_runtime=schema.ProviderRuntime( + reasoning_provider="gemini", + planner_model="gemini", + rerank_model="gemini", + ), + query_plan=schema.QueryPlan( + intent="general", + freshness_mode="balanced_recent", + cluster_mode="none", + raw_topic="test topic", + subqueries=[schema.SubQuery( + label="primary", search_query="test topic", + ranking_query="What about test topic?", sources=["youtube"], + )], + source_weights={"youtube": 1.0}, + ), + clusters=[], + ranked_candidates=[], + items_by_source={"youtube": items}, + errors_by_source={}, + ) + + def test_zero_transcripts_with_videos_present_renders_zero_over_total(self): + # The canonical stale-yt-dlp case: 6 videos found, 0 transcripts captured. + # Pre-fix the footer hid this entirely; post-fix it must say "0/6 with transcripts". + report = self._build_youtube_report([False] * 6) + text = render.render_compact(report) + self.assertIn("0/6 with transcripts", text) + + def test_partial_transcripts_renders_ratio(self): + # 5 of 6 transcripts captured - shows ratio so user knows one was missed. + report = self._build_youtube_report([True] * 5 + [False]) + text = render.render_compact(report) + self.assertIn("5/6 with transcripts", text) + + def test_full_transcripts_renders_ratio(self): + # All 3 transcripts captured - still shows ratio for consistency. + report = self._build_youtube_report([True] * 3) + text = render.render_compact(report) + self.assertIn("3/3 with transcripts", text) + + def test_no_videos_no_transcript_segment(self): + # When YouTube has no items at all, the YouTube footer line is + # suppressed entirely (existing behavior) - the transcript segment + # should not appear without a parent line. + report = self._build_youtube_report([]) + text = render.render_compact(report) + # No YouTube footer line at all - so no transcript segment either + self.assertNotIn("with transcripts", text) + + if __name__ == "__main__": unittest.main()