From 79b5d049ceb34aa3c78e63fd49a7808670eb978d Mon Sep 17 00:00:00 2001 From: shoobee <417196+shoobee@users.noreply.github.com> Date: Sun, 10 May 2026 20:29:24 +0000 Subject: [PATCH 1/3] feat(youtube): route yt-dlp through SSH host for residential IP egress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds LAST30DAYS_YT_SSH_HOST env var (or `~/.config/last30days/.env` key). When set, yt-dlp YouTube search invocations are wrapped as `ssh "yt-dlp ..."` so they run on a residential-IP machine. Motivation: when last30days runs on a datacenter VPS (Hetzner, DigitalOcean, AWS, etc.), `ytsearch:` queries return 0 results because YouTube's bot-wall fingerprints datacenter IP ranges before any cookie check runs. Cookies alone don't fix this — the IP reputation is checked first. Verified across yt-dlp stable 2026.03.17 and nightly builds. The existing fallbacks (browser cookies, residential proxy services, excluding YouTube) all have downsides: cookies expire, proxies cost money, exclusion loses signal. Many users with a Mac mini, Pi, or home server can host yt-dlp on their own residential IP — this just needs an SSH alias and a one-line env var to wire it up. Behaviour: - Default (env var unset): identical to before, no shape change. - Env var set: search command list is wrapped with `ssh -o BatchMode=yes ""`. is_ytdlp_installed() returns True without a local PATH check (the binary lives on the remote host). - Transcript path: when SSH-routing is on, skips the yt-dlp transcript path (which writes a VTT file we couldn't easily read back over SSH) and uses the existing _fetch_transcript_direct HTTP fallback. The timedtext API isn't bot-walled, so this works fine on datacenter IPs. Setup pitfall documented in the function docstring: on macOS hosts with Homebrew, `eval "$(/opt/homebrew/bin/brew shellenv zsh)"` must live in ~/.zshenv (not just ~/.zprofile) — non-login SSH shells don't source .zprofile, so without this `ssh macmini "yt-dlp ..."` returns "command not found" while interactive SSH works fine. Tests: 10 new cases covering env var read, whitespace stripping, empty-value handling, command wrapping passthrough/active modes, shlex quoting, is_ytdlp_installed short-circuit, and end-to-end search_youtube wrapping. Full test suite: 0 new failures (the 14 pre-existing failures in test_store, test_watchlist, test_setup_openclaw, test_safari_cookies, test_version_consistency are unchanged on main). Verified live: 0 results → 4 real hits for "claude code" search from a Hetzner VPS routed through a Mac mini exit node on Tailscale. --- CHANGELOG.md | 4 + skills/last30days/scripts/last30days.py | 7 ++ skills/last30days/scripts/lib/env.py | 1 + skills/last30days/scripts/lib/youtube_yt.py | 58 +++++++++++++- tests/test_youtube_yt.py | 89 +++++++++++++++++++++ 5 files changed, 156 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb9b03d..a52e566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `LAST30DAYS_YT_SSH_HOST` env var: when set, yt-dlp YouTube search invocations are routed through `ssh ` for residential-IP egress. Bypasses YouTube's bot-wall on datacenter IPs (Hetzner/DigitalOcean/AWS) where `ytsearch:` returns 0 results regardless of cookies (the IP fingerprint is checked first). The named host must be configured in `~/.ssh/config` and have yt-dlp installed. The transcript path is unchanged (uses the existing HTTP fallback when SSH-routing is on, since the timedtext API isn't bot-walled). + ### Changed - Replace the SKILL_ROOT resolver loops in Step 1 and comparison-mode with a single `SKILL_DIR` substitution pattern. The model templates the absolute path of the SKILL.md's own directory (which it always knows from the Read tool result); the bash block just validates that `scripts/last30days.py` lives there. Removes ~80 lines of bash across the two locations. Fixes a real bug: the previous resolver could pick a different install than the SKILL.md the model loaded from (spec-vs-engine divergence) and didn't enumerate harnesses like Hermes at all. The simplification works for any harness without enumeration because it just uses wherever SKILL.md was loaded from. STEP 0's marketplaces-stale-clone hop is unchanged. diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 554c7c4..a873f05 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -541,6 +541,13 @@ def main() -> int: config = env.get_config() + # Surface SSH-routing config as an env var so library modules (e.g. + # youtube_yt) can read it without taking a config dependency. This + # routes yt-dlp through `ssh ` to bypass YouTube's bot-wall on + # datacenter IPs (see lib/youtube_yt.py for details). + if config.get("LAST30DAYS_YT_SSH_HOST") and "LAST30DAYS_YT_SSH_HOST" not in os.environ: + os.environ["LAST30DAYS_YT_SSH_HOST"] = config["LAST30DAYS_YT_SSH_HOST"] + # Handle setup subcommand topic = " ".join(args.topic).strip() if topic.lower() == "setup": diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index 58751a4..a90dc72 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -329,6 +329,7 @@ def get_config() -> dict[str, Any]: ('SETUP_COMPLETE', None), ('INCLUDE_SOURCES', ''), ('EXCLUDE_SOURCES', ''), + ('LAST30DAYS_YOUTUBE_SSH_HOST', None), ] for key, default in keys: diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index 2a907b5..a34068c 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -8,6 +8,7 @@ Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI). import json import math +import os import re import shutil import sys @@ -96,10 +97,52 @@ def _log(msg: str): def is_ytdlp_installed() -> bool: - """Check if yt-dlp is available in PATH.""" + """Check if yt-dlp is available locally, or if SSH routing is configured. + + When LAST30DAYS_YT_SSH_HOST is set, returns True without a local check — + yt-dlp lives on the remote host. Failures surface naturally on first use. + """ + if _ytdlp_ssh_host(): + return True return shutil.which("yt-dlp") is not None +def _ytdlp_ssh_host() -> Optional[str]: + """Return SSH host alias if yt-dlp should be routed via SSH, else None. + + Set LAST30DAYS_YT_SSH_HOST= (e.g. 'macmini') in the environment + to route yt-dlp through SSH for residential IP egress. This bypasses + YouTube's bot-wall on datacenter IPs (Hetzner, DigitalOcean, AWS, etc.) + where ytsearch returns 0 results regardless of cookies. + + The remote host must have yt-dlp installed and reachable via the named + SSH alias (configured in ~/.ssh/config). On macOS hosts with Homebrew, + add brew shellenv to ~/.zshenv (not just ~/.zprofile) so non-login SSH + shells find yt-dlp on PATH. + + To use a value from ~/.config/last30days/.env, export it into the + environment before invoking the engine, e.g. in a wrapper: + set -a; source ~/.config/last30days/.env; set +a + python3 last30days.py "..." + """ + host = os.environ.get("LAST30DAYS_YT_SSH_HOST", "").strip() + return host or None + + +def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]: + """Wrap a yt-dlp command list with `ssh ` when SSH routing is set. + + Args are shell-quoted to survive the remote shell. Uses BatchMode=yes so + a misconfigured key fails fast instead of hanging on a password prompt. + """ + host = _ytdlp_ssh_host() + if not host: + return cmd + import shlex + remote_cmd = " ".join(shlex.quote(a) for a in cmd) + return ["ssh", "-o", "BatchMode=yes", host, remote_cmd] + + def _extract_core_subject(topic: str) -> str: """Extract core subject from verbose query for YouTube search. @@ -223,6 +266,7 @@ def search_youtube( "--no-warnings", "--no-download", ] + cmd = _wrap_ytdlp_cmd(cmd) try: result = subproc.run_with_timeout(cmd, timeout=120) @@ -472,13 +516,21 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: Plaintext transcript string, or None if no captions available. """ raw_vtt = None - if is_ytdlp_installed(): + # When SSH-routing is on, the yt-dlp transcript path would write a VTT + # file on the remote host that we can't easily read back. Skip it and + # use the HTTP transcript fallback (different YouTube endpoint, less + # bot-walled, works fine from datacenter IPs). + use_ytdlp = is_ytdlp_installed() and not _ytdlp_ssh_host() + if use_ytdlp: raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir) if not raw_vtt: _log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback") raw_vtt = _fetch_transcript_direct(video_id) else: - _log("yt-dlp not installed, using direct HTTP transcript fetch") + if _ytdlp_ssh_host(): + _log("SSH-routing active, using direct HTTP transcript fetch") + else: + _log("yt-dlp not installed, using direct HTTP transcript fetch") raw_vtt = _fetch_transcript_direct(video_id) if not raw_vtt: diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py index 6128d6a..2071749 100644 --- a/tests/test_youtube_yt.py +++ b/tests/test_youtube_yt.py @@ -1,6 +1,7 @@ """Tests for YouTube transcript highlights and yt-dlp safety flags.""" import json +import os import sys import tempfile import unittest @@ -407,5 +408,93 @@ class TestSearchAndTranscribe(unittest.TestCase): ft_mock.assert_not_called() +class TestYtdlpSSHRouting(unittest.TestCase): + """LAST30DAYS_YT_SSH_HOST routes yt-dlp invocations through SSH for residential IP.""" + + def setUp(self): + # Ensure clean env for each test + self._saved_env = os.environ.pop("LAST30DAYS_YT_SSH_HOST", None) + + def tearDown(self): + os.environ.pop("LAST30DAYS_YT_SSH_HOST", None) + if self._saved_env is not None: + os.environ["LAST30DAYS_YT_SSH_HOST"] = self._saved_env + + def test_no_env_var_returns_none(self): + """Without the env var set, _ytdlp_ssh_host returns None.""" + self.assertIsNone(youtube_yt._ytdlp_ssh_host()) + + def test_env_var_returns_host(self): + """With LAST30DAYS_YT_SSH_HOST set, _ytdlp_ssh_host returns it.""" + os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") + + def test_env_var_whitespace_stripped(self): + """Whitespace around the host alias is stripped.""" + os.environ["LAST30DAYS_YT_SSH_HOST"] = " macmini " + self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") + + def test_empty_env_var_falls_back_to_none(self): + """An empty env var is treated as unset.""" + os.environ["LAST30DAYS_YT_SSH_HOST"] = "" + self.assertIsNone(youtube_yt._ytdlp_ssh_host()) + + def test_wrap_cmd_passthrough_when_unset(self): + """_wrap_ytdlp_cmd returns input unchanged when SSH routing is off.""" + cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"] + self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd) + + def test_wrap_cmd_prepends_ssh_when_set(self): + """_wrap_ytdlp_cmd prepends ssh when SSH routing is on.""" + os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"] + wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) + self.assertEqual(wrapped[0], "ssh") + self.assertEqual(wrapped[1], "-o") + self.assertEqual(wrapped[2], "BatchMode=yes") + self.assertEqual(wrapped[3], "macmini") + # Final arg is the shell-quoted command string + self.assertIn("yt-dlp", wrapped[4]) + self.assertIn("ytsearch5:test", wrapped[4]) + + def test_wrap_cmd_quotes_args_with_spaces(self): + """Args containing spaces or special chars are shell-quoted.""" + os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + cmd = ["yt-dlp", "ytsearch5:hello world", "--dump-json"] + wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) + # shlex.quote wraps the whole arg in single quotes when it contains spaces + self.assertIn("'ytsearch5:hello world'", wrapped[4]) + + def test_is_ytdlp_installed_short_circuits_with_ssh(self): + """is_ytdlp_installed returns True without local check when SSH routing is on.""" + os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + with mock.patch("lib.youtube_yt.shutil.which", return_value=None) as which_mock: + self.assertTrue(youtube_yt.is_ytdlp_installed()) + which_mock.assert_not_called() + + def test_is_ytdlp_installed_falls_through_without_ssh(self): + """is_ytdlp_installed checks PATH normally when SSH routing is off.""" + with mock.patch("lib.youtube_yt.shutil.which", return_value="/usr/bin/yt-dlp"): + self.assertTrue(youtube_yt.is_ytdlp_installed()) + with mock.patch("lib.youtube_yt.shutil.which", return_value=None): + self.assertFalse(youtube_yt.is_ytdlp_installed()) + + def test_search_call_routes_through_ssh(self): + """search_youtube wraps the yt-dlp invocation when SSH routing is on.""" + os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + from lib.subproc import SubprocResult + fake_result = SubprocResult(returncode=0, stdout="", stderr="") + with mock.patch.object(youtube_yt.subproc, "run_with_timeout", + return_value=fake_result) as run_mock: + youtube_yt.search_youtube("test", "2026-02-01", "2026-03-01") + cmd = run_mock.call_args.args[0] + self.assertEqual(cmd[0], "ssh") + self.assertEqual(cmd[3], "macmini") + # The shell-quoted yt-dlp invocation lives at index 4 + self.assertIn("yt-dlp", cmd[4]) + self.assertIn("--ignore-config", cmd[4]) + self.assertIn("--no-cookies-from-browser", cmd[4]) + + if __name__ == "__main__": unittest.main() From f4eb0af10405730d0234895d9127d1dca7e45d98 Mon Sep 17 00:00:00 2001 From: shoobee <417196+shoobee@users.noreply.github.com> Date: Sun, 10 May 2026 20:48:01 +0000 Subject: [PATCH 2/3] fix(youtube): address Greptile review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes from automated review on PR #376: 1. Add `--` option terminator before host in _wrap_ytdlp_cmd (P1 security) Prevents SSH option injection if LAST30DAYS_YT_SSH_HOST were ever set to a value starting with `-` (e.g. `-oProxyCommand=...`). Low exploitability since the env var is user-controlled config — but the fix is a single arg and turns a self-harm footgun into no footgun. 2. Hoist `import shlex` to module-level (P2 style) Pure stdlib import, no reason for the deferred form. Cleaner. 3. Cache _ytdlp_ssh_host() result in fetch_transcript (P2 style) Was being called 2-3x per video; the function is cheap (env lookup + strip) so this is purely about readability. Adds test_wrap_cmd_uses_option_terminator covering the security fix explicitly with a `-oFoo=bar` host value. Updates index assertions in the two existing tests that check command shape (host is now at index 4, command string at 5, with `--` at 3). --- skills/last30days/scripts/lib/youtube_yt.py | 11 ++++--- tests/test_youtube_yt.py | 36 +++++++++++++++------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index a34068c..d805124 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -10,6 +10,7 @@ import json import math import os import re +import shlex import shutil import sys import tempfile @@ -134,13 +135,14 @@ def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]: Args are shell-quoted to survive the remote shell. Uses BatchMode=yes so a misconfigured key fails fast instead of hanging on a password prompt. + The `--` option terminator prevents an SSH option-injection if + LAST30DAYS_YT_SSH_HOST were ever set to a value starting with `-`. """ host = _ytdlp_ssh_host() if not host: return cmd - import shlex remote_cmd = " ".join(shlex.quote(a) for a in cmd) - return ["ssh", "-o", "BatchMode=yes", host, remote_cmd] + return ["ssh", "-o", "BatchMode=yes", "--", host, remote_cmd] def _extract_core_subject(topic: str) -> str: @@ -520,14 +522,15 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: # file on the remote host that we can't easily read back. Skip it and # use the HTTP transcript fallback (different YouTube endpoint, less # bot-walled, works fine from datacenter IPs). - use_ytdlp = is_ytdlp_installed() and not _ytdlp_ssh_host() + ssh_host = _ytdlp_ssh_host() + use_ytdlp = is_ytdlp_installed() and not ssh_host if use_ytdlp: raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir) if not raw_vtt: _log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback") raw_vtt = _fetch_transcript_direct(video_id) else: - if _ytdlp_ssh_host(): + if ssh_host: _log("SSH-routing active, using direct HTTP transcript fetch") else: _log("yt-dlp not installed, using direct HTTP transcript fetch") diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py index 2071749..ed8d1d4 100644 --- a/tests/test_youtube_yt.py +++ b/tests/test_youtube_yt.py @@ -452,10 +452,13 @@ class TestYtdlpSSHRouting(unittest.TestCase): self.assertEqual(wrapped[0], "ssh") self.assertEqual(wrapped[1], "-o") self.assertEqual(wrapped[2], "BatchMode=yes") - self.assertEqual(wrapped[3], "macmini") + # `--` terminates SSH option parsing so a host starting with `-` + # (e.g. `-oProxyCommand=...`) cannot be reinterpreted as a flag. + self.assertEqual(wrapped[3], "--") + self.assertEqual(wrapped[4], "macmini") # Final arg is the shell-quoted command string - self.assertIn("yt-dlp", wrapped[4]) - self.assertIn("ytsearch5:test", wrapped[4]) + self.assertIn("yt-dlp", wrapped[5]) + self.assertIn("ytsearch5:test", wrapped[5]) def test_wrap_cmd_quotes_args_with_spaces(self): """Args containing spaces or special chars are shell-quoted.""" @@ -463,7 +466,21 @@ class TestYtdlpSSHRouting(unittest.TestCase): cmd = ["yt-dlp", "ytsearch5:hello world", "--dump-json"] wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) # shlex.quote wraps the whole arg in single quotes when it contains spaces - self.assertIn("'ytsearch5:hello world'", wrapped[4]) + self.assertIn("'ytsearch5:hello world'", wrapped[5]) + + def test_wrap_cmd_uses_option_terminator(self): + """`--` is inserted before host to prevent SSH option injection. + + Without `--`, an env var like `LAST30DAYS_YT_SSH_HOST=-oProxyCommand=...` + would be parsed by ssh as an option flag. The terminator forces it + to be treated as a hostname (which will then fail clean if invalid). + """ + os.environ["LAST30DAYS_YT_SSH_HOST"] = "-oFoo=bar" + cmd = ["yt-dlp", "--version"] + wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) + # Find the `--` terminator and verify the host comes immediately after + dash_idx = wrapped.index("--") + self.assertEqual(wrapped[dash_idx + 1], "-oFoo=bar") def test_is_ytdlp_installed_short_circuits_with_ssh(self): """is_ytdlp_installed returns True without local check when SSH routing is on.""" @@ -489,11 +506,12 @@ class TestYtdlpSSHRouting(unittest.TestCase): youtube_yt.search_youtube("test", "2026-02-01", "2026-03-01") cmd = run_mock.call_args.args[0] self.assertEqual(cmd[0], "ssh") - self.assertEqual(cmd[3], "macmini") - # The shell-quoted yt-dlp invocation lives at index 4 - self.assertIn("yt-dlp", cmd[4]) - self.assertIn("--ignore-config", cmd[4]) - self.assertIn("--no-cookies-from-browser", cmd[4]) + self.assertEqual(cmd[3], "--") + self.assertEqual(cmd[4], "macmini") + # The shell-quoted yt-dlp invocation lives at index 5 + self.assertIn("yt-dlp", cmd[5]) + self.assertIn("--ignore-config", cmd[5]) + self.assertIn("--no-cookies-from-browser", cmd[5]) if __name__ == "__main__": From 27c90504c08ab1f75be5fe313b6179ab9d154b3f Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Sat, 16 May 2026 22:19:35 -0700 Subject: [PATCH 3/3] review: validate SSH host alias + rename LAST30DAYS_YT_SSH_HOST -> LAST30DAYS_YOUTUBE_SSH_HOST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses two concerns surfaced during PR #376 review: 1. **SSH option-injection on the host value.** The original PR uses shlex.quote() on the remote command and added a `--` option terminator in front of the host, but neither one stops a hostile env var like `LAST30DAYS_YT_SSH_HOST=-oProxyCommand=...` from being read in the first place. Tighten `_ytdlp_ssh_host()` to validate the host against `^[a-zA-Z0-9._-]+$` (plain hostname/SSH-config-alias shape: letters, digits, dot, underscore, hyphen). Any value that doesn't match logs a warning to stderr and returns None, so the wrap function falls back to local execution. The `--` terminator stays as defense-in-depth for the case where a valid host happens to start with `-`, but the regex closes the door on the env var reaching ssh at all. 2. **Env var naming consistency.** Existing skill-internal config knobs spell out their domain: `LAST30DAYS_X_BACKEND`, `LAST30DAYS_X_MODEL`, `LAST30DAYS_PLANNER_MODEL`, `LAST30DAYS_RERANK_MODEL`, etc. The module is `youtube_yt.py`, the source key is `youtube`, the function family is `is_youtube_*()` — `YT` was the odd abbreviation out. Rename to `LAST30DAYS_YOUTUBE_SSH_HOST` so the variable matches the user mental model ("route YouTube fetches via residential IP") and the codebase's spelled-out convention. Adds three new tests: - test_host_alias_with_dash_prefix_is_rejected (validator rejects `-o...`) - test_host_alias_with_shell_metacharacters_is_rejected (rejects spaces, ;, $, `, &) - test_host_alias_validator_accepts_realistic_aliases (allows FQDNs, IPs, bare aliases) The existing test_wrap_cmd_uses_option_terminator is rewritten to use a valid host value (since an invalid one is now filtered upstream) and continues to assert the `--` terminator placement as defense-in-depth. 44/44 youtube_yt tests pass (40 prior + 4 net new validator tests). --- CHANGELOG.md | 2 +- skills/last30days/scripts/last30days.py | 4 +- skills/last30days/scripts/lib/youtube_yt.py | 33 +++++++++-- tests/test_youtube_yt.py | 65 ++++++++++++++------- 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a52e566..b4104b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `LAST30DAYS_YT_SSH_HOST` env var: when set, yt-dlp YouTube search invocations are routed through `ssh ` for residential-IP egress. Bypasses YouTube's bot-wall on datacenter IPs (Hetzner/DigitalOcean/AWS) where `ytsearch:` returns 0 results regardless of cookies (the IP fingerprint is checked first). The named host must be configured in `~/.ssh/config` and have yt-dlp installed. The transcript path is unchanged (uses the existing HTTP fallback when SSH-routing is on, since the timedtext API isn't bot-walled). +- `LAST30DAYS_YOUTUBE_SSH_HOST` env var: when set, yt-dlp YouTube search invocations are routed through `ssh ` for residential-IP egress. Bypasses YouTube's bot-wall on datacenter IPs (Hetzner/DigitalOcean/AWS) where `ytsearch:` returns 0 results regardless of cookies (the IP fingerprint is checked first). The named host must be configured in `~/.ssh/config` and have yt-dlp installed. Host value is validated against `^[a-zA-Z0-9._-]+$` to reject SSH option-injection (e.g. a leading `-` masquerading as a flag). The transcript path is unchanged (uses the existing HTTP fallback when SSH-routing is on, since the timedtext API isn't bot-walled). ### Changed diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index a873f05..da026c8 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -545,8 +545,8 @@ def main() -> int: # youtube_yt) can read it without taking a config dependency. This # routes yt-dlp through `ssh ` to bypass YouTube's bot-wall on # datacenter IPs (see lib/youtube_yt.py for details). - if config.get("LAST30DAYS_YT_SSH_HOST") and "LAST30DAYS_YT_SSH_HOST" not in os.environ: - os.environ["LAST30DAYS_YT_SSH_HOST"] = config["LAST30DAYS_YT_SSH_HOST"] + if config.get("LAST30DAYS_YOUTUBE_SSH_HOST") and "LAST30DAYS_YOUTUBE_SSH_HOST" not in os.environ: + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = config["LAST30DAYS_YOUTUBE_SSH_HOST"] # Handle setup subcommand topic = " ".join(args.topic).strip() diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index d805124..fe033d1 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -100,7 +100,7 @@ def _log(msg: str): def is_ytdlp_installed() -> bool: """Check if yt-dlp is available locally, or if SSH routing is configured. - When LAST30DAYS_YT_SSH_HOST is set, returns True without a local check — + When LAST30DAYS_YOUTUBE_SSH_HOST is set, returns True without a local check — yt-dlp lives on the remote host. Failures surface naturally on first use. """ if _ytdlp_ssh_host(): @@ -108,10 +108,16 @@ def is_ytdlp_installed() -> bool: return shutil.which("yt-dlp") is not None +# Host aliases must be plain hostnames / SSH config aliases — no flags, no +# shell metacharacters. Rejects any value that could be reinterpreted by ssh +# (or the surrounding shell) as something other than a destination. +_SSH_HOST_ALIAS_RE = re.compile(r"^[a-zA-Z0-9._-]+$") + + def _ytdlp_ssh_host() -> Optional[str]: """Return SSH host alias if yt-dlp should be routed via SSH, else None. - Set LAST30DAYS_YT_SSH_HOST= (e.g. 'macmini') in the environment + Set LAST30DAYS_YOUTUBE_SSH_HOST= (e.g. 'macmini') in the environment to route yt-dlp through SSH for residential IP egress. This bypasses YouTube's bot-wall on datacenter IPs (Hetzner, DigitalOcean, AWS, etc.) where ytsearch returns 0 results regardless of cookies. @@ -121,13 +127,30 @@ def _ytdlp_ssh_host() -> Optional[str]: add brew shellenv to ~/.zshenv (not just ~/.zprofile) so non-login SSH shells find yt-dlp on PATH. + Validation: host value must match ``[A-Za-z0-9._-]+``. Anything starting + with ``-`` or containing shell/SSH metacharacters is rejected with a + stderr warning and treated as unset, so a misconfigured or attacker- + controlled value can't slip through as an SSH option flag or proxy command. + The ``--`` option terminator in ``_wrap_ytdlp_cmd`` is a second line of + defense; this regex closes the door on the env var ever reaching ssh + in the first place. + To use a value from ~/.config/last30days/.env, export it into the environment before invoking the engine, e.g. in a wrapper: set -a; source ~/.config/last30days/.env; set +a python3 last30days.py "..." """ - host = os.environ.get("LAST30DAYS_YT_SSH_HOST", "").strip() - return host or None + host = os.environ.get("LAST30DAYS_YOUTUBE_SSH_HOST", "").strip() + if not host: + return None + if not _SSH_HOST_ALIAS_RE.match(host): + sys.stderr.write( + f"[youtube_yt] WARNING: LAST30DAYS_YOUTUBE_SSH_HOST={host!r} " + "does not look like a plain hostname/alias; ignoring. " + "Expected pattern: letters, digits, dot, underscore, hyphen.\n" + ) + return None + return host def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]: @@ -136,7 +159,7 @@ def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]: Args are shell-quoted to survive the remote shell. Uses BatchMode=yes so a misconfigured key fails fast instead of hanging on a password prompt. The `--` option terminator prevents an SSH option-injection if - LAST30DAYS_YT_SSH_HOST were ever set to a value starting with `-`. + LAST30DAYS_YOUTUBE_SSH_HOST were ever set to a value starting with `-`. """ host = _ytdlp_ssh_host() if not host: diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py index ed8d1d4..be55b83 100644 --- a/tests/test_youtube_yt.py +++ b/tests/test_youtube_yt.py @@ -409,34 +409,34 @@ class TestSearchAndTranscribe(unittest.TestCase): class TestYtdlpSSHRouting(unittest.TestCase): - """LAST30DAYS_YT_SSH_HOST routes yt-dlp invocations through SSH for residential IP.""" + """LAST30DAYS_YOUTUBE_SSH_HOST routes yt-dlp invocations through SSH for residential IP.""" def setUp(self): # Ensure clean env for each test - self._saved_env = os.environ.pop("LAST30DAYS_YT_SSH_HOST", None) + self._saved_env = os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) def tearDown(self): - os.environ.pop("LAST30DAYS_YT_SSH_HOST", None) + os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) if self._saved_env is not None: - os.environ["LAST30DAYS_YT_SSH_HOST"] = self._saved_env + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = self._saved_env def test_no_env_var_returns_none(self): """Without the env var set, _ytdlp_ssh_host returns None.""" self.assertIsNone(youtube_yt._ytdlp_ssh_host()) def test_env_var_returns_host(self): - """With LAST30DAYS_YT_SSH_HOST set, _ytdlp_ssh_host returns it.""" - os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + """With LAST30DAYS_YOUTUBE_SSH_HOST set, _ytdlp_ssh_host returns it.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") def test_env_var_whitespace_stripped(self): """Whitespace around the host alias is stripped.""" - os.environ["LAST30DAYS_YT_SSH_HOST"] = " macmini " + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = " macmini " self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") def test_empty_env_var_falls_back_to_none(self): """An empty env var is treated as unset.""" - os.environ["LAST30DAYS_YT_SSH_HOST"] = "" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "" self.assertIsNone(youtube_yt._ytdlp_ssh_host()) def test_wrap_cmd_passthrough_when_unset(self): @@ -446,7 +446,7 @@ class TestYtdlpSSHRouting(unittest.TestCase): def test_wrap_cmd_prepends_ssh_when_set(self): """_wrap_ytdlp_cmd prepends ssh when SSH routing is on.""" - os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"] wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) self.assertEqual(wrapped[0], "ssh") @@ -462,29 +462,52 @@ class TestYtdlpSSHRouting(unittest.TestCase): def test_wrap_cmd_quotes_args_with_spaces(self): """Args containing spaces or special chars are shell-quoted.""" - os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" cmd = ["yt-dlp", "ytsearch5:hello world", "--dump-json"] wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) # shlex.quote wraps the whole arg in single quotes when it contains spaces self.assertIn("'ytsearch5:hello world'", wrapped[5]) def test_wrap_cmd_uses_option_terminator(self): - """`--` is inserted before host to prevent SSH option injection. - - Without `--`, an env var like `LAST30DAYS_YT_SSH_HOST=-oProxyCommand=...` - would be parsed by ssh as an option flag. The terminator forces it - to be treated as a hostname (which will then fail clean if invalid). - """ - os.environ["LAST30DAYS_YT_SSH_HOST"] = "-oFoo=bar" + """`--` is inserted before host as defense-in-depth even for valid hosts.""" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" cmd = ["yt-dlp", "--version"] wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) - # Find the `--` terminator and verify the host comes immediately after dash_idx = wrapped.index("--") - self.assertEqual(wrapped[dash_idx + 1], "-oFoo=bar") + self.assertEqual(wrapped[dash_idx + 1], "macmini") + + def test_host_alias_with_dash_prefix_is_rejected(self): + """A host value starting with `-` is rejected by the alias validator. + + Without validation, ssh could parse `-oProxyCommand=...` as a flag + instead of a hostname. The `--` terminator in _wrap_ytdlp_cmd is + defense-in-depth; this regex on _ytdlp_ssh_host() rejects the value + before it ever reaches the ssh command line. + """ + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "-oProxyCommand=evil" + self.assertIsNone(youtube_yt._ytdlp_ssh_host()) + # And the wrap function falls back to the local-execution path. + cmd = ["yt-dlp", "--version"] + self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd) + + def test_host_alias_with_shell_metacharacters_is_rejected(self): + """Host values containing spaces, semicolons, $, etc. are rejected.""" + for bad in ("host;rm -rf /", "host name", "host$IFS", "host`whoami`", "host&cmd"): + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = bad + self.assertIsNone( + youtube_yt._ytdlp_ssh_host(), + msg=f"validator should reject {bad!r}", + ) + + def test_host_alias_validator_accepts_realistic_aliases(self): + """Valid SSH config aliases are accepted: bare names, FQDNs, IPs.""" + for good in ("macmini", "home-server", "pi5.local", "192.168.1.10", "homelab_box"): + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = good + self.assertEqual(youtube_yt._ytdlp_ssh_host(), good) def test_is_ytdlp_installed_short_circuits_with_ssh(self): """is_ytdlp_installed returns True without local check when SSH routing is on.""" - os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" with mock.patch("lib.youtube_yt.shutil.which", return_value=None) as which_mock: self.assertTrue(youtube_yt.is_ytdlp_installed()) which_mock.assert_not_called() @@ -498,7 +521,7 @@ class TestYtdlpSSHRouting(unittest.TestCase): def test_search_call_routes_through_ssh(self): """search_youtube wraps the yt-dlp invocation when SSH routing is on.""" - os.environ["LAST30DAYS_YT_SSH_HOST"] = "macmini" + os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" from lib.subproc import SubprocResult fake_result = SubprocResult(returncode=0, stdout="", stderr="") with mock.patch.object(youtube_yt.subproc, "run_with_timeout",