Merge pull request #340 from dzivkovi/fix/youtube-transcript-observability

fix(youtube): surface transcript-fetch ratio + add degraded nudge for stale yt-dlp

tests/test_quality_nudge.py

@@ -199,3 +199,97 @@ class TestRedditNeverInCoreErrored:
         # Reddit is always-active in core (public path), error doesn't demote it
         assert "reddit" in q["core_active"]
         assert q["score_pct"] == 100
+
+
+class TestYouTubeDegraded:
+    """YouTube is `degraded` when videos returned but transcripts below threshold.
+
+    Canonical failure mode: a stale yt-dlp binary still finds videos via search
+    but silently fails every transcript fetch because YouTube's caption format
+    has moved on. Pre-fix the user got no signal of this; the footer hid zero,
+    and quality_nudge only checked top-level errors.
+    """
+
+    def test_zero_of_six_transcripts_flags_degraded(self):
+        q = _compute(
+            ytdlp_installed=True,
+            result_overrides={
+                "youtube_videos_count": 6,
+                "youtube_transcripts_count": 0,
+            },
+        )
+        assert "youtube" in q["core_degraded"]
+        assert q["nudge_text"] is not None
+        # Counts surface in the message so the user sees the actual ratio
+        assert "6 videos" in q["nudge_text"]
+        assert "0 transcripts" in q["nudge_text"]
+        assert "stale yt-dlp" in q["nudge_text"].lower()
+        # Updates path mentions all three common package managers
+        assert "scoop" in q["nudge_text"].lower()
+        assert "brew" in q["nudge_text"].lower()
+        assert "pip install" in q["nudge_text"].lower()
+
+    def test_five_of_six_transcripts_does_not_flag_degraded(self):
+        # 83% transcript success - well above the 50% threshold
+        # X is also enabled so all 5 cores are active and no nudge should fire
+        q = _compute(
+            config_overrides={"AUTH_TOKEN": "tok123"},
+            ytdlp_installed=True,
+            result_overrides={
+                "youtube_videos_count": 6,
+                "youtube_transcripts_count": 5,
+            },
+        )
+        assert "youtube" not in q["core_degraded"]
+        assert q["nudge_text"] is None  # All 5 core sources active, no degradation
+
+    def test_zero_videos_does_not_flag_degraded(self):
+        # No videos returned -> degraded check is meaningless and must not fire
+        q = _compute(
+            ytdlp_installed=True,
+            result_overrides={
+                "youtube_videos_count": 0,
+                "youtube_transcripts_count": 0,
+            },
+        )
+        assert "youtube" not in q["core_degraded"]
+
+    def test_one_of_three_transcripts_flags_degraded(self):
+        # 33% - below 50% threshold; the canonical "yt-dlp partially working" case
+        q = _compute(
+            ytdlp_installed=True,
+            result_overrides={
+                "youtube_videos_count": 3,
+                "youtube_transcripts_count": 1,
+            },
+        )
+        assert "youtube" in q["core_degraded"]
+        assert "Degraded: YouTube" in q["nudge_text"]
+
+    def test_threshold_tunable_via_config(self):
+        # Operator overrides threshold via env-style config to be more permissive
+        q = _compute(
+            config_overrides={"DEGRADED_TRANSCRIPT_THRESHOLD": "0.1"},
+            ytdlp_installed=True,
+            result_overrides={
+                "youtube_videos_count": 10,
+                "youtube_transcripts_count": 2,  # 20%, below default 50% but above override 10%
+            },
+        )
+        assert "youtube" not in q["core_degraded"]
+
+    def test_degraded_does_not_affect_score(self):
+        # Degradation is informational, not score-affecting; YouTube still counts as active
+        q = _compute(
+            config_overrides={"AUTH_TOKEN": "tok123"},
+            ytdlp_installed=True,
+            result_overrides={
+                "youtube_videos_count": 6,
+                "youtube_transcripts_count": 0,
+            },
+        )
+        assert "youtube" in q["core_active"]
+        assert q["score_pct"] == 100  # Full active count regardless of degradation
+        # But nudge still fires
+        assert q["nudge_text"] is not None
+        assert "Degraded: YouTube" in q["nudge_text"]

tests/test_render_v3.py

@@ -552,5 +552,90 @@ class DegradedRunBannerTests(unittest.TestCase):
         self.assertIn("--plan", text)
 
 
+class YoutubeFooterTranscriptRatioTests(unittest.TestCase):
+    """The YouTube footer line must surface the transcript-fetch ratio in all
+    cases where videos were returned. Pre-fix the segment was suppressed when
+    transcripts == 0, which converted the canonical stale-yt-dlp failure mode
+    into a silent absence at the footer (the very surface users read for
+    'did this work?'). Always-render the ratio so zero is loud.
+    """
+
+    def _build_youtube_report(self, transcript_flags: list[bool]) -> schema.Report:
+        """Build a Report with one YouTube item per entry in transcript_flags.
+        True means the item has transcript data; False means it does not.
+        """
+        items = []
+        for idx, has_transcript in enumerate(transcript_flags):
+            metadata = {"views": 1000}
+            if has_transcript:
+                metadata["transcript_highlights"] = ["Some pre-extracted quote."]
+            items.append(schema.SourceItem(
+                item_id=f"yt{idx}",
+                source="youtube",
+                title=f"Video {idx}",
+                body=f"Description for video {idx}.",
+                url=f"https://youtube.com/watch?v=v{idx}",
+                container="some-channel",
+                published_at="2026-04-15",
+                date_confidence="high",
+                engagement={"views": 1000, "likes": 100},
+                metadata=metadata,
+            ))
+        return schema.Report(
+            topic="test topic",
+            range_from="2026-04-01",
+            range_to="2026-05-01",
+            generated_at="2026-05-01T00:00:00+00:00",
+            provider_runtime=schema.ProviderRuntime(
+                reasoning_provider="gemini",
+                planner_model="gemini",
+                rerank_model="gemini",
+            ),
+            query_plan=schema.QueryPlan(
+                intent="general",
+                freshness_mode="balanced_recent",
+                cluster_mode="none",
+                raw_topic="test topic",
+                subqueries=[schema.SubQuery(
+                    label="primary", search_query="test topic",
+                    ranking_query="What about test topic?", sources=["youtube"],
+                )],
+                source_weights={"youtube": 1.0},
+            ),
+            clusters=[],
+            ranked_candidates=[],
+            items_by_source={"youtube": items},
+            errors_by_source={},
+        )
+
+    def test_zero_transcripts_with_videos_present_renders_zero_over_total(self):
+        # The canonical stale-yt-dlp case: 6 videos found, 0 transcripts captured.
+        # Pre-fix the footer hid this entirely; post-fix it must say "0/6 with transcripts".
+        report = self._build_youtube_report([False] * 6)
+        text = render.render_compact(report)
+        self.assertIn("0/6 with transcripts", text)
+
+    def test_partial_transcripts_renders_ratio(self):
+        # 5 of 6 transcripts captured - shows ratio so user knows one was missed.
+        report = self._build_youtube_report([True] * 5 + [False])
+        text = render.render_compact(report)
+        self.assertIn("5/6 with transcripts", text)
+
+    def test_full_transcripts_renders_ratio(self):
+        # All 3 transcripts captured - still shows ratio for consistency.
+        report = self._build_youtube_report([True] * 3)
+        text = render.render_compact(report)
+        self.assertIn("3/3 with transcripts", text)
+
+    def test_no_videos_no_transcript_segment(self):
+        # When YouTube has no items at all, the YouTube footer line is
+        # suppressed entirely (existing behavior) - the transcript segment
+        # should not appear without a parent line.
+        report = self._build_youtube_report([])
+        text = render.render_compact(report)
+        # No YouTube footer line at all - so no transcript segment either
+        self.assertNotIn("with transcripts", text)
+
+
 if __name__ == "__main__":
     unittest.main()