Merge pull request #376 from shoobee/feat/yt-dlp-ssh-routing

feat(youtube): route yt-dlp through SSH host for residential IP egress

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `LAST30DAYS_YOUTUBE_SSH_HOST` env var: when set, yt-dlp YouTube search invocations are routed through `ssh <host>` for residential-IP egress. Bypasses YouTube's bot-wall on datacenter IPs (Hetzner/DigitalOcean/AWS) where `ytsearch:` returns 0 results regardless of cookies (the IP fingerprint is checked first). The named host must be configured in `~/.ssh/config` and have yt-dlp installed. Host value is validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection (e.g. a leading `-` masquerading as a flag). The transcript path is unchanged (uses the existing HTTP fallback when SSH-routing is on, since the timedtext API isn't bot-walled).
+
 ### Changed
 
 - Replace the SKILL_ROOT resolver loops in Step 1 and comparison-mode with a single `SKILL_DIR` substitution pattern. The model templates the absolute path of the SKILL.md's own directory (which it always knows from the Read tool result); the bash block just validates that `scripts/last30days.py` lives there. Removes ~80 lines of bash across the two locations. Fixes a real bug: the previous resolver could pick a different install than the SKILL.md the model loaded from (spec-vs-engine divergence) and didn't enumerate harnesses like Hermes at all. The simplification works for any harness without enumeration because it just uses wherever SKILL.md was loaded from. STEP 0's marketplaces-stale-clone hop is unchanged.

skills/last30days/scripts/last30days.py

@@ -541,6 +541,13 @@ def main() -> int:
 
     config = env.get_config()
 
+    # Surface SSH-routing config as an env var so library modules (e.g.
+    # youtube_yt) can read it without taking a config dependency. This
+    # routes yt-dlp through `ssh <host>` to bypass YouTube's bot-wall on
+    # datacenter IPs (see lib/youtube_yt.py for details).
+    if config.get("LAST30DAYS_YOUTUBE_SSH_HOST") and "LAST30DAYS_YOUTUBE_SSH_HOST" not in os.environ:
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"]
+
     # Handle setup subcommand
     topic = " ".join(args.topic).strip()
     if topic.lower() == "setup":

skills/last30days/scripts/lib/env.py

@@ -329,6 +329,7 @@ def get_config() -> dict[str, Any]:
         ('SETUP_COMPLETE', None),
         ('INCLUDE_SOURCES', ''),
         ('EXCLUDE_SOURCES', ''),
+        ('LAST30DAYS_YOUTUBE_SSH_HOST', None),
     ]
 
     for key, default in keys:

skills/last30days/scripts/lib/youtube_yt.py

@@ -8,7 +8,9 @@ Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI).
 
 import json
 import math
+import os
 import re
+import shlex
 import shutil
 import sys
 import tempfile
@@ -96,10 +98,76 @@ def _log(msg: str):
 
 
 def is_ytdlp_installed() -> bool:
-    """Check if yt-dlp is available in PATH."""
+    """Check if yt-dlp is available locally, or if SSH routing is configured.
+
+    When LAST30DAYS_YOUTUBE_SSH_HOST is set, returns True without a local check —
+    yt-dlp lives on the remote host. Failures surface naturally on first use.
+    """
+    if _ytdlp_ssh_host():
+        return True
     return shutil.which("yt-dlp") is not None
 
 
+# Host aliases must be plain hostnames / SSH config aliases — no flags, no
+# shell metacharacters. Rejects any value that could be reinterpreted by ssh
+# (or the surrounding shell) as something other than a destination.
+_SSH_HOST_ALIAS_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
+
+
+def _ytdlp_ssh_host() -> Optional[str]:
+    """Return SSH host alias if yt-dlp should be routed via SSH, else None.
+
+    Set LAST30DAYS_YOUTUBE_SSH_HOST=<ssh-alias> (e.g. 'macmini') in the environment
+    to route yt-dlp through SSH for residential IP egress. This bypasses
+    YouTube's bot-wall on datacenter IPs (Hetzner, DigitalOcean, AWS, etc.)
+    where ytsearch returns 0 results regardless of cookies.
+
+    The remote host must have yt-dlp installed and reachable via the named
+    SSH alias (configured in ~/.ssh/config). On macOS hosts with Homebrew,
+    add brew shellenv to ~/.zshenv (not just ~/.zprofile) so non-login SSH
+    shells find yt-dlp on PATH.
+
+    Validation: host value must match ``[A-Za-z0-9._-]+``. Anything starting
+    with ``-`` or containing shell/SSH metacharacters is rejected with a
+    stderr warning and treated as unset, so a misconfigured or attacker-
+    controlled value can't slip through as an SSH option flag or proxy command.
+    The ``--`` option terminator in ``_wrap_ytdlp_cmd`` is a second line of
+    defense; this regex closes the door on the env var ever reaching ssh
+    in the first place.
+
+    To use a value from ~/.config/last30days/.env, export it into the
+    environment before invoking the engine, e.g. in a wrapper:
+        set -a; source ~/.config/last30days/.env; set +a
+        python3 last30days.py "..."
+    """
+    host = os.environ.get("LAST30DAYS_YOUTUBE_SSH_HOST", "").strip()
+    if not host:
+        return None
+    if not _SSH_HOST_ALIAS_RE.match(host):
+        sys.stderr.write(
+            f"[youtube_yt] WARNING: LAST30DAYS_YOUTUBE_SSH_HOST={host!r} "
+            "does not look like a plain hostname/alias; ignoring. "
+            "Expected pattern: letters, digits, dot, underscore, hyphen.\n"
+        )
+        return None
+    return host
+
+
+def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]:
+    """Wrap a yt-dlp command list with `ssh <host>` when SSH routing is set.
+
+    Args are shell-quoted to survive the remote shell. Uses BatchMode=yes so
+    a misconfigured key fails fast instead of hanging on a password prompt.
+    The `--` option terminator prevents an SSH option-injection if
+    LAST30DAYS_YOUTUBE_SSH_HOST were ever set to a value starting with `-`.
+    """
+    host = _ytdlp_ssh_host()
+    if not host:
+        return cmd
+    remote_cmd = " ".join(shlex.quote(a) for a in cmd)
+    return ["ssh", "-o", "BatchMode=yes", "--", host, remote_cmd]
+
+
 def _extract_core_subject(topic: str) -> str:
     """Extract core subject from verbose query for YouTube search.
 
@@ -223,6 +291,7 @@ def search_youtube(
         "--no-warnings",
         "--no-download",
     ]
+    cmd = _wrap_ytdlp_cmd(cmd)
 
     try:
         result = subproc.run_with_timeout(cmd, timeout=120)
@@ -472,11 +541,20 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
         Plaintext transcript string, or None if no captions available.
     """
     raw_vtt = None
-    if is_ytdlp_installed():
+    # When SSH-routing is on, the yt-dlp transcript path would write a VTT
+    # file on the remote host that we can't easily read back. Skip it and
+    # use the HTTP transcript fallback (different YouTube endpoint, less
+    # bot-walled, works fine from datacenter IPs).
+    ssh_host = _ytdlp_ssh_host()
+    use_ytdlp = is_ytdlp_installed() and not ssh_host
+    if use_ytdlp:
         raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
         if not raw_vtt:
             _log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback")
             raw_vtt = _fetch_transcript_direct(video_id)
+    else:
+        if ssh_host:
+            _log("SSH-routing active, using direct HTTP transcript fetch")
         else:
             _log("yt-dlp not installed, using direct HTTP transcript fetch")
         raw_vtt = _fetch_transcript_direct(video_id)

tests/test_youtube_yt.py

@@ -1,6 +1,7 @@
 """Tests for YouTube transcript highlights and yt-dlp safety flags."""
 
 import json
+import os
 import sys
 import tempfile
 import unittest
@@ -407,5 +408,134 @@ class TestSearchAndTranscribe(unittest.TestCase):
         ft_mock.assert_not_called()
 
 
+class TestYtdlpSSHRouting(unittest.TestCase):
+    """LAST30DAYS_YOUTUBE_SSH_HOST routes yt-dlp invocations through SSH for residential IP."""
+
+    def setUp(self):
+        # Ensure clean env for each test
+        self._saved_env = os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None)
+
+    def tearDown(self):
+        os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None)
+        if self._saved_env is not None:
+            os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = self._saved_env
+
+    def test_no_env_var_returns_none(self):
+        """Without the env var set, _ytdlp_ssh_host returns None."""
+        self.assertIsNone(youtube_yt._ytdlp_ssh_host())
+
+    def test_env_var_returns_host(self):
+        """With LAST30DAYS_YOUTUBE_SSH_HOST set, _ytdlp_ssh_host returns it."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini"
+        self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini")
+
+    def test_env_var_whitespace_stripped(self):
+        """Whitespace around the host alias is stripped."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "  macmini  "
+        self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini")
+
+    def test_empty_env_var_falls_back_to_none(self):
+        """An empty env var is treated as unset."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = ""
+        self.assertIsNone(youtube_yt._ytdlp_ssh_host())
+
+    def test_wrap_cmd_passthrough_when_unset(self):
+        """_wrap_ytdlp_cmd returns input unchanged when SSH routing is off."""
+        cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"]
+        self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd)
+
+    def test_wrap_cmd_prepends_ssh_when_set(self):
+        """_wrap_ytdlp_cmd prepends ssh <host> when SSH routing is on."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini"
+        cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"]
+        wrapped = youtube_yt._wrap_ytdlp_cmd(cmd)
+        self.assertEqual(wrapped[0], "ssh")
+        self.assertEqual(wrapped[1], "-o")
+        self.assertEqual(wrapped[2], "BatchMode=yes")
+        # `--` terminates SSH option parsing so a host starting with `-`
+        # (e.g. `-oProxyCommand=...`) cannot be reinterpreted as a flag.
+        self.assertEqual(wrapped[3], "--")
+        self.assertEqual(wrapped[4], "macmini")
+        # Final arg is the shell-quoted command string
+        self.assertIn("yt-dlp", wrapped[5])
+        self.assertIn("ytsearch5:test", wrapped[5])
+
+    def test_wrap_cmd_quotes_args_with_spaces(self):
+        """Args containing spaces or special chars are shell-quoted."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini"
+        cmd = ["yt-dlp", "ytsearch5:hello world", "--dump-json"]
+        wrapped = youtube_yt._wrap_ytdlp_cmd(cmd)
+        # shlex.quote wraps the whole arg in single quotes when it contains spaces
+        self.assertIn("'ytsearch5:hello world'", wrapped[5])
+
+    def test_wrap_cmd_uses_option_terminator(self):
+        """`--` is inserted before host as defense-in-depth even for valid hosts."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini"
+        cmd = ["yt-dlp", "--version"]
+        wrapped = youtube_yt._wrap_ytdlp_cmd(cmd)
+        dash_idx = wrapped.index("--")
+        self.assertEqual(wrapped[dash_idx + 1], "macmini")
+
+    def test_host_alias_with_dash_prefix_is_rejected(self):
+        """A host value starting with `-` is rejected by the alias validator.
+
+        Without validation, ssh could parse `-oProxyCommand=...` as a flag
+        instead of a hostname. The `--` terminator in _wrap_ytdlp_cmd is
+        defense-in-depth; this regex on _ytdlp_ssh_host() rejects the value
+        before it ever reaches the ssh command line.
+        """
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "-oProxyCommand=evil"
+        self.assertIsNone(youtube_yt._ytdlp_ssh_host())
+        # And the wrap function falls back to the local-execution path.
+        cmd = ["yt-dlp", "--version"]
+        self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd)
+
+    def test_host_alias_with_shell_metacharacters_is_rejected(self):
+        """Host values containing spaces, semicolons, $, etc. are rejected."""
+        for bad in ("host;rm -rf /", "host name", "host$IFS", "host`whoami`", "host&cmd"):
+            os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = bad
+            self.assertIsNone(
+                youtube_yt._ytdlp_ssh_host(),
+                msg=f"validator should reject {bad!r}",
+            )
+
+    def test_host_alias_validator_accepts_realistic_aliases(self):
+        """Valid SSH config aliases are accepted: bare names, FQDNs, IPs."""
+        for good in ("macmini", "home-server", "pi5.local", "192.168.1.10", "homelab_box"):
+            os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = good
+            self.assertEqual(youtube_yt._ytdlp_ssh_host(), good)
+
+    def test_is_ytdlp_installed_short_circuits_with_ssh(self):
+        """is_ytdlp_installed returns True without local check when SSH routing is on."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini"
+        with mock.patch("lib.youtube_yt.shutil.which", return_value=None) as which_mock:
+            self.assertTrue(youtube_yt.is_ytdlp_installed())
+            which_mock.assert_not_called()
+
+    def test_is_ytdlp_installed_falls_through_without_ssh(self):
+        """is_ytdlp_installed checks PATH normally when SSH routing is off."""
+        with mock.patch("lib.youtube_yt.shutil.which", return_value="/usr/bin/yt-dlp"):
+            self.assertTrue(youtube_yt.is_ytdlp_installed())
+        with mock.patch("lib.youtube_yt.shutil.which", return_value=None):
+            self.assertFalse(youtube_yt.is_ytdlp_installed())
+
+    def test_search_call_routes_through_ssh(self):
+        """search_youtube wraps the yt-dlp invocation when SSH routing is on."""
+        os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini"
+        from lib.subproc import SubprocResult
+        fake_result = SubprocResult(returncode=0, stdout="", stderr="")
+        with mock.patch.object(youtube_yt.subproc, "run_with_timeout",
+                               return_value=fake_result) as run_mock:
+            youtube_yt.search_youtube("test", "2026-02-01", "2026-03-01")
+        cmd = run_mock.call_args.args[0]
+        self.assertEqual(cmd[0], "ssh")
+        self.assertEqual(cmd[3], "--")
+        self.assertEqual(cmd[4], "macmini")
+        # The shell-quoted yt-dlp invocation lives at index 5
+        self.assertIn("yt-dlp", cmd[5])
+        self.assertIn("--ignore-config", cmd[5])
+        self.assertIn("--no-cookies-from-browser", cmd[5])
+
+
 if __name__ == "__main__":
     unittest.main()