diff --git a/CHANGELOG.md b/CHANGELOG.md index bb9b03d..b4104b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `LAST30DAYS_YOUTUBE_SSH_HOST` env var: when set, yt-dlp YouTube search invocations are routed through `ssh ` for residential-IP egress. Bypasses YouTube's bot-wall on datacenter IPs (Hetzner/DigitalOcean/AWS) where `ytsearch:` returns 0 results regardless of cookies (the IP fingerprint is checked first). The named host must be configured in `~/.ssh/config` and have yt-dlp installed. Host value is validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection (e.g. a leading `-` masquerading as a flag). The transcript path is unchanged (uses the existing HTTP fallback when SSH-routing is on, since the timedtext API isn't bot-walled). + ### Changed - Replace the SKILL_ROOT resolver loops in Step 1 and comparison-mode with a single `SKILL_DIR` substitution pattern. The model templates the absolute path of the SKILL.md's own directory (which it always knows from the Read tool result); the bash block just validates that `scripts/last30days.py` lives there. Removes ~80 lines of bash across the two locations. Fixes a real bug: the previous resolver could pick a different install than the SKILL.md the model loaded from (spec-vs-engine divergence) and didn't enumerate harnesses like Hermes at all. The simplification works for any harness without enumeration because it just uses wherever SKILL.md was loaded from. STEP 0's marketplaces-stale-clone hop is unchanged. diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 554c7c4..da026c8 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -541,6 +541,13 @@ def main() -> int: config = env.get_config() + # Surface SSH-routing config as an env var so library modules (e.g. + # youtube_yt) can read it without taking a config dependency. This + # routes yt-dlp through `ssh ` to bypass YouTube's bot-wall on + # datacenter IPs (see lib/youtube_yt.py for details). + if config.get("LAST30DAYS_YOUTUBE_SSH_HOST") and "LAST30DAYS_YOUTUBE_SSH_HOST" not in os.environ: + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"] + # Handle setup subcommand topic = " ".join(args.topic).strip() if topic.lower() == "setup": diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index 870b75a..2ca64e7 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -329,6 +329,7 @@ def get_config() -> dict[str, Any]: ('SETUP_COMPLETE', None), ('INCLUDE_SOURCES', ''), ('EXCLUDE_SOURCES', ''), + ('LAST30DAYS_YOUTUBE_SSH_HOST', None), ] for key, default in keys: diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index 2a907b5..fe033d1 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -8,7 +8,9 @@ Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI). import json import math +import os import re +import shlex import shutil import sys import tempfile @@ -96,10 +98,76 @@ def _log(msg: str): def is_ytdlp_installed() -> bool: - """Check if yt-dlp is available in PATH.""" + """Check if yt-dlp is available locally, or if SSH routing is configured. + + When LAST30DAYS_YOUTUBE_SSH_HOST is set, returns True without a local check — + yt-dlp lives on the remote host. Failures surface naturally on first use. + """ + if _ytdlp_ssh_host(): + return True return shutil.which("yt-dlp") is not None +# Host aliases must be plain hostnames / SSH config aliases — no flags, no +# shell metacharacters. Rejects any value that could be reinterpreted by ssh +# (or the surrounding shell) as something other than a destination. +_SSH_HOST_ALIAS_RE = re.compile(r"^[a-zA-Z0-9._-]+$") + + +def _ytdlp_ssh_host() -> Optional[str]: + """Return SSH host alias if yt-dlp should be routed via SSH, else None. + + Set LAST30DAYS_YOUTUBE_SSH_HOST= (e.g. 'macmini') in the environment + to route yt-dlp through SSH for residential IP egress. This bypasses + YouTube's bot-wall on datacenter IPs (Hetzner, DigitalOcean, AWS, etc.) + where ytsearch returns 0 results regardless of cookies. + + The remote host must have yt-dlp installed and reachable via the named + SSH alias (configured in ~/.ssh/config). On macOS hosts with Homebrew, + add brew shellenv to ~/.zshenv (not just ~/.zprofile) so non-login SSH + shells find yt-dlp on PATH. + + Validation: host value must match ``[A-Za-z0-9._-]+``. Anything starting + with ``-`` or containing shell/SSH metacharacters is rejected with a + stderr warning and treated as unset, so a misconfigured or attacker- + controlled value can't slip through as an SSH option flag or proxy command. + The ``--`` option terminator in ``_wrap_ytdlp_cmd`` is a second line of + defense; this regex closes the door on the env var ever reaching ssh + in the first place. + + To use a value from ~/.config/last30days/.env, export it into the + environment before invoking the engine, e.g. in a wrapper: + set -a; source ~/.config/last30days/.env; set +a + python3 last30days.py "..." + """ + host = os.environ.get("LAST30DAYS_YOUTUBE_SSH_HOST", "").strip() + if not host: + return None + if not _SSH_HOST_ALIAS_RE.match(host): + sys.stderr.write( + f"[youtube_yt] WARNING: LAST30DAYS_YOUTUBE_SSH_HOST={host!r} " + "does not look like a plain hostname/alias; ignoring. " + "Expected pattern: letters, digits, dot, underscore, hyphen.\n" + ) + return None + return host + + +def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]: + """Wrap a yt-dlp command list with `ssh ` when SSH routing is set. + + Args are shell-quoted to survive the remote shell. Uses BatchMode=yes so + a misconfigured key fails fast instead of hanging on a password prompt. + The `--` option terminator prevents an SSH option-injection if + LAST30DAYS_YOUTUBE_SSH_HOST were ever set to a value starting with `-`. + """ + host = _ytdlp_ssh_host() + if not host: + return cmd + remote_cmd = " ".join(shlex.quote(a) for a in cmd) + return ["ssh", "-o", "BatchMode=yes", "--", host, remote_cmd] + + def _extract_core_subject(topic: str) -> str: """Extract core subject from verbose query for YouTube search. @@ -223,6 +291,7 @@ def search_youtube( "--no-warnings", "--no-download", ] + cmd = _wrap_ytdlp_cmd(cmd) try: result = subproc.run_with_timeout(cmd, timeout=120) @@ -472,13 +541,22 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: Plaintext transcript string, or None if no captions available. """ raw_vtt = None - if is_ytdlp_installed(): + # When SSH-routing is on, the yt-dlp transcript path would write a VTT + # file on the remote host that we can't easily read back. Skip it and + # use the HTTP transcript fallback (different YouTube endpoint, less + # bot-walled, works fine from datacenter IPs). + ssh_host = _ytdlp_ssh_host() + use_ytdlp = is_ytdlp_installed() and not ssh_host + if use_ytdlp: raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir) if not raw_vtt: _log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback") raw_vtt = _fetch_transcript_direct(video_id) else: - _log("yt-dlp not installed, using direct HTTP transcript fetch") + if ssh_host: + _log("SSH-routing active, using direct HTTP transcript fetch") + else: + _log("yt-dlp not installed, using direct HTTP transcript fetch") raw_vtt = _fetch_transcript_direct(video_id) if not raw_vtt: diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py index 6128d6a..be55b83 100644 --- a/tests/test_youtube_yt.py +++ b/tests/test_youtube_yt.py @@ -1,6 +1,7 @@ """Tests for YouTube transcript highlights and yt-dlp safety flags.""" import json +import os import sys import tempfile import unittest @@ -407,5 +408,134 @@ class TestSearchAndTranscribe(unittest.TestCase): ft_mock.assert_not_called() +class TestYtdlpSSHRouting(unittest.TestCase): + """LAST30DAYS_YOUTUBE_SSH_HOST routes yt-dlp invocations through SSH for residential IP.""" + + def setUp(self): + # Ensure clean env for each test + self._saved_env = os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) + + def tearDown(self): + os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) + if self._saved_env is not None: + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = self._saved_env + + def test_no_env_var_returns_none(self): + """Without the env var set, _ytdlp_ssh_host returns None.""" + self.assertIsNone(youtube_yt._ytdlp_ssh_host()) + + def test_env_var_returns_host(self): + """With LAST30DAYS_YOUTUBE_SSH_HOST set, _ytdlp_ssh_host returns it.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" + self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") + + def test_env_var_whitespace_stripped(self): + """Whitespace around the host alias is stripped.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = " macmini " + self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") + + def test_empty_env_var_falls_back_to_none(self): + """An empty env var is treated as unset.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "" + self.assertIsNone(youtube_yt._ytdlp_ssh_host()) + + def test_wrap_cmd_passthrough_when_unset(self): + """_wrap_ytdlp_cmd returns input unchanged when SSH routing is off.""" + cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"] + self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd) + + def test_wrap_cmd_prepends_ssh_when_set(self): + """_wrap_ytdlp_cmd prepends ssh when SSH routing is on.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" + cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"] + wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) + self.assertEqual(wrapped[0], "ssh") + self.assertEqual(wrapped[1], "-o") + self.assertEqual(wrapped[2], "BatchMode=yes") + # `--` terminates SSH option parsing so a host starting with `-` + # (e.g. `-oProxyCommand=...`) cannot be reinterpreted as a flag. + self.assertEqual(wrapped[3], "--") + self.assertEqual(wrapped[4], "macmini") + # Final arg is the shell-quoted command string + self.assertIn("yt-dlp", wrapped[5]) + self.assertIn("ytsearch5:test", wrapped[5]) + + def test_wrap_cmd_quotes_args_with_spaces(self): + """Args containing spaces or special chars are shell-quoted.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" + cmd = ["yt-dlp", "ytsearch5:hello world", "--dump-json"] + wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) + # shlex.quote wraps the whole arg in single quotes when it contains spaces + self.assertIn("'ytsearch5:hello world'", wrapped[5]) + + def test_wrap_cmd_uses_option_terminator(self): + """`--` is inserted before host as defense-in-depth even for valid hosts.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" + cmd = ["yt-dlp", "--version"] + wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) + dash_idx = wrapped.index("--") + self.assertEqual(wrapped[dash_idx + 1], "macmini") + + def test_host_alias_with_dash_prefix_is_rejected(self): + """A host value starting with `-` is rejected by the alias validator. + + Without validation, ssh could parse `-oProxyCommand=...` as a flag + instead of a hostname. The `--` terminator in _wrap_ytdlp_cmd is + defense-in-depth; this regex on _ytdlp_ssh_host() rejects the value + before it ever reaches the ssh command line. + """ + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "-oProxyCommand=evil" + self.assertIsNone(youtube_yt._ytdlp_ssh_host()) + # And the wrap function falls back to the local-execution path. + cmd = ["yt-dlp", "--version"] + self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd) + + def test_host_alias_with_shell_metacharacters_is_rejected(self): + """Host values containing spaces, semicolons, $, etc. are rejected.""" + for bad in ("host;rm -rf /", "host name", "host$IFS", "host`whoami`", "host&cmd"): + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = bad + self.assertIsNone( + youtube_yt._ytdlp_ssh_host(), + msg=f"validator should reject {bad!r}", + ) + + def test_host_alias_validator_accepts_realistic_aliases(self): + """Valid SSH config aliases are accepted: bare names, FQDNs, IPs.""" + for good in ("macmini", "home-server", "pi5.local", "192.168.1.10", "homelab_box"): + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = good + self.assertEqual(youtube_yt._ytdlp_ssh_host(), good) + + def test_is_ytdlp_installed_short_circuits_with_ssh(self): + """is_ytdlp_installed returns True without local check when SSH routing is on.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" + with mock.patch("lib.youtube_yt.shutil.which", return_value=None) as which_mock: + self.assertTrue(youtube_yt.is_ytdlp_installed()) + which_mock.assert_not_called() + + def test_is_ytdlp_installed_falls_through_without_ssh(self): + """is_ytdlp_installed checks PATH normally when SSH routing is off.""" + with mock.patch("lib.youtube_yt.shutil.which", return_value="/usr/bin/yt-dlp"): + self.assertTrue(youtube_yt.is_ytdlp_installed()) + with mock.patch("lib.youtube_yt.shutil.which", return_value=None): + self.assertFalse(youtube_yt.is_ytdlp_installed()) + + def test_search_call_routes_through_ssh(self): + """search_youtube wraps the yt-dlp invocation when SSH routing is on.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" + from lib.subproc import SubprocResult + fake_result = SubprocResult(returncode=0, stdout="", stderr="") + with mock.patch.object(youtube_yt.subproc, "run_with_timeout", + return_value=fake_result) as run_mock: + youtube_yt.search_youtube("test", "2026-02-01", "2026-03-01") + cmd = run_mock.call_args.args[0] + self.assertEqual(cmd[0], "ssh") + self.assertEqual(cmd[3], "--") + self.assertEqual(cmd[4], "macmini") + # The shell-quoted yt-dlp invocation lives at index 5 + self.assertIn("yt-dlp", cmd[5]) + self.assertIn("--ignore-config", cmd[5]) + self.assertIn("--no-cookies-from-browser", cmd[5]) + + if __name__ == "__main__": unittest.main()