feat(youtube): route yt-dlp through SSH host for residential IP egress
Adds LAST30DAYS_YT_SSH_HOST env var (or `~/.config/last30days/.env` key). When set, yt-dlp YouTube search invocations are wrapped as `ssh <host> "yt-dlp ..."` so they run on a residential-IP machine. Motivation: when last30days runs on a datacenter VPS (Hetzner, DigitalOcean, AWS, etc.), `ytsearch:` queries return 0 results because YouTube's bot-wall fingerprints datacenter IP ranges before any cookie check runs. Cookies alone don't fix this — the IP reputation is checked first. Verified across yt-dlp stable 2026.03.17 and nightly builds. The existing fallbacks (browser cookies, residential proxy services, excluding YouTube) all have downsides: cookies expire, proxies cost money, exclusion loses signal. Many users with a Mac mini, Pi, or home server can host yt-dlp on their own residential IP — this just needs an SSH alias and a one-line env var to wire it up. Behaviour: - Default (env var unset): identical to before, no shape change. - Env var set: search command list is wrapped with `ssh -o BatchMode=yes <host> "<shell-quoted yt-dlp invocation>"`. is_ytdlp_installed() returns True without a local PATH check (the binary lives on the remote host). - Transcript path: when SSH-routing is on, skips the yt-dlp transcript path (which writes a VTT file we couldn't easily read back over SSH) and uses the existing _fetch_transcript_direct HTTP fallback. The timedtext API isn't bot-walled, so this works fine on datacenter IPs. Setup pitfall documented in the function docstring: on macOS hosts with Homebrew, `eval "$(/opt/homebrew/bin/brew shellenv zsh)"` must live in ~/.zshenv (not just ~/.zprofile) — non-login SSH shells don't source .zprofile, so without this `ssh macmini "yt-dlp ..."` returns "command not found" while interactive SSH works fine. Tests: 10 new cases covering env var read, whitespace stripping, empty-value handling, command wrapping passthrough/active modes, shlex quoting, is_ytdlp_installed short-circuit, and end-to-end search_youtube wrapping. Full test suite: 0 new failures (the 14 pre-existing failures in test_store, test_watchlist, test_setup_openclaw, test_safari_cookies, test_version_consistency are unchanged on main). Verified live: 0 results → 4 real hits for "claude code" search from a Hetzner VPS routed through a Mac mini exit node on Tailscale.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
"""Tests for YouTube transcript highlights and yt-dlp safety flags."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -407,5 +408,93 @@ class TestSearchAndTranscribe(unittest.TestCase):
|
||||
ft_mock.assert_not_called()
|
||||
|
||||
|
||||
class TestYtdlpSSHRouting(unittest.TestCase):
|
||||
"""LAST30DAYS_YT_SSH_HOST routes yt-dlp invocations through SSH for residential IP."""
|
||||
|
||||
def setUp(self):
|
||||
# Ensure clean env for each test
|
||||
self._saved_env = os.environ.pop("LAST30DAYS_YT_SSH_HOST", None)
|
||||
|
||||
def tearDown(self):
|
||||
os.environ.pop("LAST30DAYS_YT_SSH_HOST", None)
|
||||
if self._saved_env is not None:
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = self._saved_env
|
||||
|
||||
def test_no_env_var_returns_none(self):
|
||||
"""Without the env var set, _ytdlp_ssh_host returns None."""
|
||||
self.assertIsNone(youtube_yt._ytdlp_ssh_host())
|
||||
|
||||
def test_env_var_returns_host(self):
|
||||
"""With LAST30DAYS_YT_SSH_HOST set, _ytdlp_ssh_host returns it."""
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini"
|
||||
self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini")
|
||||
|
||||
def test_env_var_whitespace_stripped(self):
|
||||
"""Whitespace around the host alias is stripped."""
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = " macmini "
|
||||
self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini")
|
||||
|
||||
def test_empty_env_var_falls_back_to_none(self):
|
||||
"""An empty env var is treated as unset."""
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = ""
|
||||
self.assertIsNone(youtube_yt._ytdlp_ssh_host())
|
||||
|
||||
def test_wrap_cmd_passthrough_when_unset(self):
|
||||
"""_wrap_ytdlp_cmd returns input unchanged when SSH routing is off."""
|
||||
cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"]
|
||||
self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd)
|
||||
|
||||
def test_wrap_cmd_prepends_ssh_when_set(self):
|
||||
"""_wrap_ytdlp_cmd prepends ssh <host> when SSH routing is on."""
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini"
|
||||
cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"]
|
||||
wrapped = youtube_yt._wrap_ytdlp_cmd(cmd)
|
||||
self.assertEqual(wrapped[0], "ssh")
|
||||
self.assertEqual(wrapped[1], "-o")
|
||||
self.assertEqual(wrapped[2], "BatchMode=yes")
|
||||
self.assertEqual(wrapped[3], "macmini")
|
||||
# Final arg is the shell-quoted command string
|
||||
self.assertIn("yt-dlp", wrapped[4])
|
||||
self.assertIn("ytsearch5:test", wrapped[4])
|
||||
|
||||
def test_wrap_cmd_quotes_args_with_spaces(self):
|
||||
"""Args containing spaces or special chars are shell-quoted."""
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini"
|
||||
cmd = ["yt-dlp", "ytsearch5:hello world", "--dump-json"]
|
||||
wrapped = youtube_yt._wrap_ytdlp_cmd(cmd)
|
||||
# shlex.quote wraps the whole arg in single quotes when it contains spaces
|
||||
self.assertIn("'ytsearch5:hello world'", wrapped[4])
|
||||
|
||||
def test_is_ytdlp_installed_short_circuits_with_ssh(self):
|
||||
"""is_ytdlp_installed returns True without local check when SSH routing is on."""
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini"
|
||||
with mock.patch("lib.youtube_yt.shutil.which", return_value=None) as which_mock:
|
||||
self.assertTrue(youtube_yt.is_ytdlp_installed())
|
||||
which_mock.assert_not_called()
|
||||
|
||||
def test_is_ytdlp_installed_falls_through_without_ssh(self):
|
||||
"""is_ytdlp_installed checks PATH normally when SSH routing is off."""
|
||||
with mock.patch("lib.youtube_yt.shutil.which", return_value="/usr/bin/yt-dlp"):
|
||||
self.assertTrue(youtube_yt.is_ytdlp_installed())
|
||||
with mock.patch("lib.youtube_yt.shutil.which", return_value=None):
|
||||
self.assertFalse(youtube_yt.is_ytdlp_installed())
|
||||
|
||||
def test_search_call_routes_through_ssh(self):
|
||||
"""search_youtube wraps the yt-dlp invocation when SSH routing is on."""
|
||||
os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini"
|
||||
from lib.subproc import SubprocResult
|
||||
fake_result = SubprocResult(returncode=0, stdout="", stderr="")
|
||||
with mock.patch.object(youtube_yt.subproc, "run_with_timeout",
|
||||
return_value=fake_result) as run_mock:
|
||||
youtube_yt.search_youtube("test", "2026-02-01", "2026-03-01")
|
||||
cmd = run_mock.call_args.args[0]
|
||||
self.assertEqual(cmd[0], "ssh")
|
||||
self.assertEqual(cmd[3], "macmini")
|
||||
# The shell-quoted yt-dlp invocation lives at index 4
|
||||
self.assertIn("yt-dlp", cmd[4])
|
||||
self.assertIn("--ignore-config", cmd[4])
|
||||
self.assertIn("--no-cookies-from-browser", cmd[4])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user