diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 960cb2d..811ea0e 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -880,7 +880,18 @@ def main() -> int: # Show quality nudge if applicable try: from lib import quality_nudge - quality = quality_nudge.compute_quality_score(config, {}) + # Populate transcript-fetch ratio so quality_nudge can detect the + # degraded-YouTube failure mode (videos returned but transcripts + # silently failed - typically a stale yt-dlp binary). + youtube_items = report.items_by_source.get("youtube") or [] + research_results = { + "youtube_videos_count": len(youtube_items), + "youtube_transcripts_count": sum( + 1 for it in youtube_items + if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet")) + ), + } + quality = quality_nudge.compute_quality_score(config, research_results) if quality.get("nudge_text"): sys.stderr.write(f"\n{quality['nudge_text']}\n") sys.stderr.flush() diff --git a/skills/last30days/scripts/lib/quality_nudge.py b/skills/last30days/scripts/lib/quality_nudge.py index 6e84fe4..7ffbb62 100644 --- a/skills/last30days/scripts/lib/quality_nudge.py +++ b/skills/last30days/scripts/lib/quality_nudge.py @@ -45,26 +45,51 @@ def _is_youtube_active(config: dict, research_results: dict) -> bool: return True +# Below this transcript-fetch ratio, YouTube is considered "degraded" rather +# than active. Picked at 50% so a single legitimate caption-disabled video in a +# multi-video result does not trip the nudge, but a stale-yt-dlp run that fails +# every transcript does. Tunable via DEGRADED_TRANSCRIPT_THRESHOLD env var if +# operators need to adjust without code changes. +DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD = 0.5 + + +def _is_youtube_degraded(research_results: dict, threshold: float) -> bool: + """YouTube is degraded when videos were returned but the transcript-fetch + ratio is below threshold. The canonical cause is a stale yt-dlp binary - + YouTube's caption format changes frequently and old binaries silently fail + every transcript while the search itself still succeeds. + """ + videos = int(research_results.get("youtube_videos_count") or 0) + transcripts = int(research_results.get("youtube_transcripts_count") or 0) + if videos <= 0: + return False + return (transcripts / videos) < threshold + + def compute_quality_score(config: dict, research_results: dict) -> dict: """Compute research quality score based on 5 core sources. Args: config: Configuration dict from env.get_config() research_results: Dict with keys like x_error, youtube_error, - reddit_error reflecting what happened this run. + reddit_error reflecting what happened this run. Optional keys + ``youtube_videos_count`` and ``youtube_transcripts_count`` enable + degraded-YouTube detection (transcript-fetch ratio below threshold). Returns: { "score_pct": 40-100, "core_active": ["hn", "polymarket", ...], "core_missing": ["x", "youtube"], - "core_errored": [], # configured but errored + "core_errored": [], # configured but errored at top level + "core_degraded": [], # configured and returned items but quality below threshold "nudge_text": "..." or None if 100% } """ core_active: List[str] = [] core_missing: List[str] = [] core_errored: List[str] = [] + core_degraded: List[str] = [] # HN, Polymarket, and Reddit are always active core_active.append("hn") @@ -84,6 +109,13 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: yt_active = _is_youtube_active(config, research_results) if yt_active: core_active.append("youtube") + # Active means yt-dlp is installed and search did not error at the top + # level. But search-success + transcript-failure is the canonical + # stale-binary failure mode that the footer used to hide. Flag as + # degraded so the user gets an actionable nudge to update the binary. + threshold = float(config.get("DEGRADED_TRANSCRIPT_THRESHOLD") or DEFAULT_DEGRADED_TRANSCRIPT_THRESHOLD) + if _is_youtube_degraded(research_results, threshold): + core_degraded.append("youtube") else: core_missing.append("youtube") # Check if configured but errored (yt-dlp installed but failed this run) @@ -99,24 +131,41 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: has_sc = bool(config.get("SCRAPECREATORS_API_KEY")) active_sources = research_results.get("active_sources") or [] - nudge_text = _build_nudge_text(core_missing, core_errored, has_sc=has_sc, active_sources=active_sources) if core_missing else None + nudge_text = _build_nudge_text( + core_missing, + core_errored, + core_degraded, + research_results, + has_sc=has_sc, + active_sources=active_sources, + ) if (core_missing or core_degraded) else None return { "score_pct": score_pct, "core_active": core_active, "core_missing": core_missing, "core_errored": core_errored, + "core_degraded": core_degraded, "nudge_text": nudge_text, } -def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: bool = False, active_sources: list = None) -> str: - """Build human-readable nudge text describing what was missed. +def _build_nudge_text( + core_missing: List[str], + core_errored: List[str], + core_degraded: List[str] = None, + research_results: dict = None, + has_sc: bool = False, + active_sources: list = None, +) -> str: + """Build human-readable nudge text describing what was missed or degraded. Prioritizes free suggestions. Optionally mentions bonus sources (TikTok, Instagram, Threads, Pinterest) if ScrapeCreators key is configured. """ lines: List[str] = [] + core_degraded = core_degraded or [] + research_results = research_results or {} # Describe what was missed missed_parts: List[str] = [] @@ -129,7 +178,11 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: active_count = 5 - len(core_missing) lines.append(f"Research quality: {active_count}/5 core sources.") - lines.append(f"Missing: {', '.join(missed_parts)}.") + if missed_parts: + lines.append(f"Missing: {', '.join(missed_parts)}.") + if core_degraded: + degraded_labels = ", ".join(SOURCE_LABELS[s] for s in core_degraded) + lines.append(f"Degraded: {degraded_labels}.") lines.append("") # Free suggestions @@ -159,6 +212,17 @@ def _build_nudge_text(core_missing: List[str], core_errored: List[str], has_sc: "explanations on any topic. Install yt-dlp: brew install yt-dlp (free)" ) + if "youtube" in core_degraded: + videos = int(research_results.get("youtube_videos_count") or 0) + transcripts = int(research_results.get("youtube_transcripts_count") or 0) + free_suggestions.append( + f"YouTube returned {videos} videos but only {transcripts} transcripts " + "captured. The most common cause is a stale yt-dlp binary - YouTube's " + "caption format changes frequently and old binaries silently fail every " + "transcript. Update via your package manager: scoop update yt-dlp " + "(Windows), brew upgrade yt-dlp (macOS), or pip install -U yt-dlp." + ) + # Mention bonus opt-in sources when SC key is present if has_sc: bonus_hints = [] diff --git a/skills/last30days/scripts/lib/render.py b/skills/last30days/scripts/lib/render.py index 7f21aa2..a89fb8f 100644 --- a/skills/last30days/scripts/lib/render.py +++ b/skills/last30days/scripts/lib/render.py @@ -1285,15 +1285,16 @@ def _build_source_footer_lines(report: schema.Report) -> list[str]: if total > 0: total_str = f"{total:,}" if total >= 1000 else str(total) parts.append(f"{total_str} {word}") - # YouTube: append "N with transcripts" instead of a third likes-based column. - # Transcripts are a more meaningful research-depth signal than likes. + # YouTube: always append "M/N with transcripts" so a zero-transcript run + # (typically caused by a stale yt-dlp binary) is visible at the conclusion + # surface. Hiding zero converts a problem signal into an absence; the very + # case that needs to be loud is the one previously omitted from the footer. if source_key == "youtube": with_transcripts = sum( 1 for it in items if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet")) ) - if with_transcripts > 0: - parts.append(f"{with_transcripts} with transcripts") + parts.append(f"{with_transcripts}/{len(items)} with transcripts") stats = " │ ".join(parts) out.append(_footer_line_for_source(emoji, label, len(items), item_word, stats)) diff --git a/tests/test_quality_nudge.py b/tests/test_quality_nudge.py index 25bad17..a2592e9 100644 --- a/tests/test_quality_nudge.py +++ b/tests/test_quality_nudge.py @@ -199,3 +199,97 @@ class TestRedditNeverInCoreErrored: # Reddit is always-active in core (public path), error doesn't demote it assert "reddit" in q["core_active"] assert q["score_pct"] == 100 + + +class TestYouTubeDegraded: + """YouTube is `degraded` when videos returned but transcripts below threshold. + + Canonical failure mode: a stale yt-dlp binary still finds videos via search + but silently fails every transcript fetch because YouTube's caption format + has moved on. Pre-fix the user got no signal of this; the footer hid zero, + and quality_nudge only checked top-level errors. + """ + + def test_zero_of_six_transcripts_flags_degraded(self): + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 0, + }, + ) + assert "youtube" in q["core_degraded"] + assert q["nudge_text"] is not None + # Counts surface in the message so the user sees the actual ratio + assert "6 videos" in q["nudge_text"] + assert "0 transcripts" in q["nudge_text"] + assert "stale yt-dlp" in q["nudge_text"].lower() + # Updates path mentions all three common package managers + assert "scoop" in q["nudge_text"].lower() + assert "brew" in q["nudge_text"].lower() + assert "pip install" in q["nudge_text"].lower() + + def test_five_of_six_transcripts_does_not_flag_degraded(self): + # 83% transcript success - well above the 50% threshold + # X is also enabled so all 5 cores are active and no nudge should fire + q = _compute( + config_overrides={"AUTH_TOKEN": "tok123"}, + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 5, + }, + ) + assert "youtube" not in q["core_degraded"] + assert q["nudge_text"] is None # All 5 core sources active, no degradation + + def test_zero_videos_does_not_flag_degraded(self): + # No videos returned -> degraded check is meaningless and must not fire + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 0, + "youtube_transcripts_count": 0, + }, + ) + assert "youtube" not in q["core_degraded"] + + def test_one_of_three_transcripts_flags_degraded(self): + # 33% - below 50% threshold; the canonical "yt-dlp partially working" case + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 3, + "youtube_transcripts_count": 1, + }, + ) + assert "youtube" in q["core_degraded"] + assert "Degraded: YouTube" in q["nudge_text"] + + def test_threshold_tunable_via_config(self): + # Operator overrides threshold via env-style config to be more permissive + q = _compute( + config_overrides={"DEGRADED_TRANSCRIPT_THRESHOLD": "0.1"}, + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 10, + "youtube_transcripts_count": 2, # 20%, below default 50% but above override 10% + }, + ) + assert "youtube" not in q["core_degraded"] + + def test_degraded_does_not_affect_score(self): + # Degradation is informational, not score-affecting; YouTube still counts as active + q = _compute( + config_overrides={"AUTH_TOKEN": "tok123"}, + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 0, + }, + ) + assert "youtube" in q["core_active"] + assert q["score_pct"] == 100 # Full active count regardless of degradation + # But nudge still fires + assert q["nudge_text"] is not None + assert "Degraded: YouTube" in q["nudge_text"] diff --git a/tests/test_render_v3.py b/tests/test_render_v3.py index 22514c0..2862c18 100644 --- a/tests/test_render_v3.py +++ b/tests/test_render_v3.py @@ -552,5 +552,90 @@ class DegradedRunBannerTests(unittest.TestCase): self.assertIn("--plan", text) +class YoutubeFooterTranscriptRatioTests(unittest.TestCase): + """The YouTube footer line must surface the transcript-fetch ratio in all + cases where videos were returned. Pre-fix the segment was suppressed when + transcripts == 0, which converted the canonical stale-yt-dlp failure mode + into a silent absence at the footer (the very surface users read for + 'did this work?'). Always-render the ratio so zero is loud. + """ + + def _build_youtube_report(self, transcript_flags: list[bool]) -> schema.Report: + """Build a Report with one YouTube item per entry in transcript_flags. + True means the item has transcript data; False means it does not. + """ + items = [] + for idx, has_transcript in enumerate(transcript_flags): + metadata = {"views": 1000} + if has_transcript: + metadata["transcript_highlights"] = ["Some pre-extracted quote."] + items.append(schema.SourceItem( + item_id=f"yt{idx}", + source="youtube", + title=f"Video {idx}", + body=f"Description for video {idx}.", + url=f"https://youtube.com/watch?v=v{idx}", + container="some-channel", + published_at="2026-04-15", + date_confidence="high", + engagement={"views": 1000, "likes": 100}, + metadata=metadata, + )) + return schema.Report( + topic="test topic", + range_from="2026-04-01", + range_to="2026-05-01", + generated_at="2026-05-01T00:00:00+00:00", + provider_runtime=schema.ProviderRuntime( + reasoning_provider="gemini", + planner_model="gemini", + rerank_model="gemini", + ), + query_plan=schema.QueryPlan( + intent="general", + freshness_mode="balanced_recent", + cluster_mode="none", + raw_topic="test topic", + subqueries=[schema.SubQuery( + label="primary", search_query="test topic", + ranking_query="What about test topic?", sources=["youtube"], + )], + source_weights={"youtube": 1.0}, + ), + clusters=[], + ranked_candidates=[], + items_by_source={"youtube": items}, + errors_by_source={}, + ) + + def test_zero_transcripts_with_videos_present_renders_zero_over_total(self): + # The canonical stale-yt-dlp case: 6 videos found, 0 transcripts captured. + # Pre-fix the footer hid this entirely; post-fix it must say "0/6 with transcripts". + report = self._build_youtube_report([False] * 6) + text = render.render_compact(report) + self.assertIn("0/6 with transcripts", text) + + def test_partial_transcripts_renders_ratio(self): + # 5 of 6 transcripts captured - shows ratio so user knows one was missed. + report = self._build_youtube_report([True] * 5 + [False]) + text = render.render_compact(report) + self.assertIn("5/6 with transcripts", text) + + def test_full_transcripts_renders_ratio(self): + # All 3 transcripts captured - still shows ratio for consistency. + report = self._build_youtube_report([True] * 3) + text = render.render_compact(report) + self.assertIn("3/3 with transcripts", text) + + def test_no_videos_no_transcript_segment(self): + # When YouTube has no items at all, the YouTube footer line is + # suppressed entirely (existing behavior) - the transcript segment + # should not appear without a parent line. + report = self._build_youtube_report([]) + text = render.render_compact(report) + # No YouTube footer line at all - so no transcript segment either + self.assertNotIn("with transcripts", text) + + if __name__ == "__main__": unittest.main()