feat: v2.9.6 — free-first NUX, cookie extraction, quality scoring

Setup wizard with consent-first cookie extraction (Chrome/Firefox/Safari), yt-dlp auto-install, ScrapeCreators push, quality scoring (5 core sources), status banner redesign, honest Reddit labeling, inline YouTube transcripts, Exa free web search, Reddit public fallback, and post-research quality nudge. Co-authored-by: Matt Van Horn <mvanhorn@MacBook-Pro.local> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-29 14:33:17 -07:00
parent 4d6224f79a
commit 775596ce21
34 changed files with 6238 additions and 320 deletions
@@ -1,8 +1,10 @@
 """Tests for yt-dlp invocation safety flags."""

+import json
 import sys
 import tempfile
 import unittest
+import urllib.error
 from pathlib import Path
 from unittest import mock

@@ -37,6 +39,7 @@ class TestYtDlpFlags(unittest.TestCase):
    def test_transcript_fetch_ignores_global_config_and_browser_cookies(self):
        proc = _DummyProc()
        with tempfile.TemporaryDirectory() as temp_dir, \
+             mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
             mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock:
            youtube_yt.fetch_transcript("abc123", temp_dir)

@@ -76,5 +79,140 @@ class TestExtractTranscriptHighlights(unittest.TestCase):
        self.assertEqual(len(highlights), 3)


+class TestFetchTranscriptDirect(unittest.TestCase):
+    """Tests for _fetch_transcript_direct() — direct HTTP transcript fetching."""
+
+    # Minimal ytInitialPlayerResponse JSON with a caption track
+    _PLAYER_RESPONSE = json.dumps({
+        "captions": {
+            "playerCaptionsTracklistRenderer": {
+                "captionTracks": [
+                    {
+                        "baseUrl": "https://www.youtube.com/api/timedtext?v=abc123&lang=en",
+                        "languageCode": "en",
+                    }
+                ]
+            }
+        }
+    })
+
+    _WATCH_HTML = (
+        '<html><script>var ytInitialPlayerResponse = '
+        + _PLAYER_RESPONSE
+        + ';</script></html>'
+    )
+
+    _SAMPLE_VTT = (
+        "WEBVTT\n\n"
+        "00:00:00.000 --> 00:00:02.000\n"
+        "Hello world this is a test sentence with enough words to pass.\n\n"
+        "00:00:02.000 --> 00:00:04.000\n"
+        "Another line of transcript text here for testing purposes.\n"
+    )
+
+    def _mock_urlopen(self, url_or_req, *, timeout=None):
+        """Return watch HTML or VTT depending on URL."""
+        url = url_or_req.full_url if hasattr(url_or_req, 'full_url') else url_or_req
+
+        class _Resp:
+            def __init__(self, data):
+                self._data = data.encode("utf-8")
+            def read(self):
+                return self._data
+            def __enter__(self):
+                return self
+            def __exit__(self, *a):
+                pass
+
+        if "watch?" in url:
+            return _Resp(self._WATCH_HTML)
+        elif "timedtext" in url:
+            return _Resp(self._SAMPLE_VTT)
+        raise urllib.error.URLError("unexpected URL")
+
+    def test_extracts_vtt_from_mock_page(self):
+        """Happy path: extracts VTT text from a page with captions."""
+        with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=self._mock_urlopen):
+            result = youtube_yt._fetch_transcript_direct("abc123")
+        self.assertIsNotNone(result)
+        self.assertIn("WEBVTT", result)
+        self.assertIn("Hello world", result)
+
+    def test_no_captions_returns_none(self):
+        """Video with no caption tracks returns None."""
+        no_captions_response = json.dumps({"captions": {"playerCaptionsTracklistRenderer": {"captionTracks": []}}})
+        html = f'<html><script>var ytInitialPlayerResponse = {no_captions_response};</script></html>'
+
+        class _Resp:
+            def __init__(self, data):
+                self._data = data.encode("utf-8")
+            def read(self):
+                return self._data
+            def __enter__(self):
+                return self
+            def __exit__(self, *a):
+                pass
+
+        def mock_open(req, *, timeout=None):
+            return _Resp(html)
+
+        with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=mock_open):
+            result = youtube_yt._fetch_transcript_direct("nocaps")
+        self.assertIsNone(result)
+
+    def test_http_timeout_returns_none(self):
+        """HTTP timeout on watch page returns None."""
+        def timeout_open(req, *, timeout=None):
+            raise TimeoutError("timed out")
+
+        with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=timeout_open):
+            result = youtube_yt._fetch_transcript_direct("timeout_vid")
+        self.assertIsNone(result)
+
+    def test_direct_vtt_feeds_into_clean_vtt(self):
+        """VTT from direct fetch produces clean plaintext via _clean_vtt()."""
+        cleaned = youtube_yt._clean_vtt(self._SAMPLE_VTT)
+        self.assertNotIn("WEBVTT", cleaned)
+        self.assertNotIn("-->", cleaned)
+        self.assertIn("Hello world", cleaned)
+        self.assertIn("Another line", cleaned)
+
+
+class TestFetchTranscriptFallback(unittest.TestCase):
+    """Tests that fetch_transcript picks yt-dlp or direct path correctly."""
+
+    def test_uses_ytdlp_when_installed(self):
+        """When yt-dlp is installed, uses _fetch_transcript_ytdlp."""
+        with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
+             mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", return_value="WEBVTT\n\nfake") as yt_mock, \
+             mock.patch.object(youtube_yt, "_fetch_transcript_direct") as direct_mock:
+            result = youtube_yt.fetch_transcript("vid1", "/tmp/test")
+        yt_mock.assert_called_once_with("vid1", "/tmp/test")
+        direct_mock.assert_not_called()
+
+    def test_uses_direct_when_ytdlp_missing(self):
+        """When yt-dlp is NOT installed, falls back to _fetch_transcript_direct."""
+        sample_vtt = (
+            "WEBVTT\n\n"
+            "00:00:00.000 --> 00:00:02.000\n"
+            "Direct transcript content with enough words for testing.\n"
+        )
+        with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \
+             mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp") as yt_mock, \
+             mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=sample_vtt) as direct_mock:
+            result = youtube_yt.fetch_transcript("vid2", "/tmp/test")
+        yt_mock.assert_not_called()
+        direct_mock.assert_called_once_with("vid2")
+        self.assertIsNotNone(result)
+        self.assertIn("Direct transcript content", result)
+
+    def test_returns_none_when_both_fail(self):
+        """Returns None when the chosen path returns None."""
+        with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \
+             mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=None):
+            result = youtube_yt.fetch_transcript("novid", "/tmp/test")
+        self.assertIsNone(result)
+
+
 if __name__ == "__main__":
    unittest.main()