refactor: extract subprocess cleanup into shared subproc helper (#210)

bird_x.py and youtube_yt.py had four near-identical copies of the same
subprocess cleanup dance (Popen + os.setsid + communicate(timeout) +
SIGTERM via killpg + proc.kill() fallback + wait(5)). Extract to
lib.subproc.run_with_timeout(), which:

- runs the child in its own process group via os.setsid where available
- raises SubprocTimeout on timeout
- on timeout: SIGTERM the group, fall back to proc.kill(), wait up to 5s
- accepts an on_pid callback so bird_x can still register child PIDs
  with last30days.register_child_pid for whole-process cleanup
- captures stdout/stderr as strings in a SubprocResult dataclass

Migrated call sites: _run_bird_search, search_handles inner worker,
search_youtube, fetch_transcript. With the helper in place, the signal
and subprocess imports became dead in both files (plus os in
youtube_yt) and went with them.

Tests: 9 new subproc tests cover success, non-zero exit, stderr capture,
timeout-raises, timeout-kills-group, missing-command, env passthrough,
PID callback, and callback-exception suppression. test_env_v3 and
test_youtube_yt patch subproc.run_with_timeout instead of the removed
bird_x.subprocess and yt-dlp subprocess.
This commit is contained in:
Ilia Alshanetsky
2026-04-25 17:17:47 -04:00
committed by GitHub
parent 2acbf8a869
commit bbf892aecc
6 changed files with 268 additions and 130 deletions
+11 -43
View File
@@ -8,11 +8,8 @@ Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI).
import json
import math
import os
import re
import signal
import shutil
import subprocess
import sys
import tempfile
import urllib.error
@@ -37,7 +34,7 @@ TRANSCRIPT_LIMITS = {
# Max words to keep from each transcript
TRANSCRIPT_MAX_WORDS = 5000
from . import http, log
from . import http, log, subproc
from .relevance import token_overlap_relevance as _compute_relevance
@@ -227,30 +224,16 @@ def search_youtube(
"--no-download",
]
preexec = os.setsid if hasattr(os, 'setsid') else None
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
preexec_fn=preexec,
)
try:
stdout, stderr = proc.communicate(timeout=120)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except (ProcessLookupError, PermissionError, OSError):
proc.kill()
proc.wait(timeout=5)
_log("YouTube search timed out (120s)")
return {"items": [], "error": "Search timed out"}
result = subproc.run_with_timeout(cmd, timeout=120)
except subproc.SubprocTimeout:
_log("YouTube search timed out (120s)")
return {"items": [], "error": "Search timed out"}
except FileNotFoundError:
return {"items": [], "error": "yt-dlp not found"}
if not (stdout or "").strip():
stdout = result.stdout
if not stdout.strip():
_log("YouTube search returned 0 results")
return {"items": []}
@@ -452,25 +435,10 @@ def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:
f"https://www.youtube.com/watch?v={video_id}",
]
preexec = os.setsid if hasattr(os, 'setsid') else None
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
preexec_fn=preexec,
)
try:
proc.communicate(timeout=30)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except (ProcessLookupError, PermissionError, OSError):
proc.kill()
proc.wait(timeout=5)
return None
subproc.run_with_timeout(cmd, timeout=30)
except subproc.SubprocTimeout:
return None
except FileNotFoundError:
return None
@@ -556,7 +524,7 @@ def fetch_transcripts_parallel(
vid = futures[future]
try:
results[vid] = future.result()
except (OSError, subprocess.SubprocessError) as exc:
except OSError as exc:
_log(f"Transcript fetch error for {vid}: {exc}")
results[vid] = None
except Exception as exc: