refactor: extract subprocess cleanup into shared subproc helper (#210)

bird_x.py and youtube_yt.py had four near-identical copies of the same
subprocess cleanup dance (Popen + os.setsid + communicate(timeout) +
SIGTERM via killpg + proc.kill() fallback + wait(5)). Extract to
lib.subproc.run_with_timeout(), which:

- runs the child in its own process group via os.setsid where available
- raises SubprocTimeout on timeout
- on timeout: SIGTERM the group, fall back to proc.kill(), wait up to 5s
- accepts an on_pid callback so bird_x can still register child PIDs
  with last30days.register_child_pid for whole-process cleanup
- captures stdout/stderr as strings in a SubprocResult dataclass

Migrated call sites: _run_bird_search, search_handles inner worker,
search_youtube, fetch_transcript. With the helper in place, the signal
and subprocess imports became dead in both files (plus os in
youtube_yt) and went with them.

Tests: 9 new subproc tests cover success, non-zero exit, stderr capture,
timeout-raises, timeout-kills-group, missing-command, env passthrough,
PID callback, and callback-exception suppression. test_env_v3 and
test_youtube_yt patch subproc.run_with_timeout instead of the removed
bird_x.subprocess and yt-dlp subprocess.
This commit is contained in:
Ilia Alshanetsky
2026-04-25 17:17:47 -04:00
committed by GitHub
parent 2acbf8a869
commit bbf892aecc
6 changed files with 268 additions and 130 deletions
+49 -80
View File
@@ -7,13 +7,11 @@ See scripts/lib/vendor/bird-search/package.json for authoritative version.
import json
import os
import signal
import shutil
import subprocess
import sys
from pathlib import Path
from . import http, log
from . import http, log, subproc
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
@@ -168,62 +166,51 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
"--json",
]
# Use process groups for clean cleanup on timeout/kill
preexec = os.setsid if hasattr(os, 'setsid') else None
pid_holder: list[int] = []
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
preexec_fn=preexec,
env=_subprocess_env(),
)
# Register for cleanup tracking (if available)
def _register(pid: int) -> None:
pid_holder.append(pid)
try:
from last30days import register_child_pid, unregister_child_pid
register_child_pid(proc.pid)
from last30days import register_child_pid
register_child_pid(pid)
except ImportError:
pass
try:
stdout, stderr = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
# Kill the entire process group
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except (ProcessLookupError, PermissionError, OSError):
proc.kill()
proc.wait(timeout=5)
return {"error": f"Search timed out after {timeout}s", "items": []}
finally:
try:
result = subproc.run_with_timeout(
cmd,
timeout=timeout,
env=_subprocess_env(),
on_pid=_register,
)
except subproc.SubprocTimeout:
return {"error": f"Search timed out after {timeout}s", "items": []}
except Exception as e:
return {"error": str(e), "items": []}
finally:
if pid_holder:
try:
from last30days import unregister_child_pid
unregister_child_pid(proc.pid)
unregister_child_pid(pid_holder[0])
except Exception:
pass
if proc.returncode != 0:
error = stderr.strip() if stderr else "Bird search failed"
return {"error": error, "items": []}
if result.returncode != 0:
error = result.stderr.strip() or "Bird search failed"
return {"error": error, "items": []}
output = stdout.strip() if stdout else ""
if not output:
return {"items": []}
output = result.stdout.strip()
if not output:
return {"items": []}
try:
parsed = json.loads(output)
if isinstance(parsed, list):
return {"items": parsed}
return parsed
except json.JSONDecodeError as e:
return {"error": f"Invalid JSON response: {e}", "items": []}
except Exception as e:
return {"error": str(e), "items": []}
if isinstance(parsed, list):
return {"items": parsed}
return parsed
def search_x(
@@ -330,47 +317,29 @@ def search_handles(
"--json",
]
preexec = os.setsid if hasattr(os, 'setsid') else None
try:
result = subproc.run_with_timeout(cmd, timeout=15, env=_subprocess_env())
except subproc.SubprocTimeout:
_log(f"Handle search timed out for @{handle}")
return []
except OSError as e:
_log(f"Handle search error for @{handle}: {e}")
return []
if result.returncode != 0:
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
return []
output = result.stdout.strip()
if not output:
return []
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
preexec_fn=preexec,
env=_subprocess_env(),
)
try:
stdout, stderr = proc.communicate(timeout=15)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except (ProcessLookupError, PermissionError, OSError):
proc.kill()
proc.wait(timeout=5)
_log(f"Handle search timed out for @{handle}")
return []
if proc.returncode != 0:
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
return []
output = (stdout or "").strip()
if not output:
return []
response = json.loads(output)
return parse_bird_response(response, query=core_topic)
except json.JSONDecodeError:
_log(f"Invalid JSON from handle search for @{handle}")
except (OSError, subprocess.SubprocessError) as e:
_log(f"Handle search error for @{handle}: {e}")
return []
return []
return parse_bird_response(response, query=core_topic)
from concurrent.futures import ThreadPoolExecutor, as_completed