fix: add timeouts, process cleanup, and source diagnostics

Script was hanging indefinitely when API sources were slow or unresponsive. Now enforces bounded execution: - Global timeout watchdog (180s default, 90s --quick, 300s --deep) - Per-source future.result() timeouts (60s/30s/90s by depth) - Parallel Reddit enrichment capped at 15 items / 45s total - Subprocess process-group isolation (os.setsid + killpg) - atexit cleanup kills all tracked child processes - --timeout=N flag for user override Also fixes the UX gap where missing sources were silently skipped: - Pre-flight diagnostic banner shows source status before research - Source status footer in compact output shows used/skipped/why - Actionable fix commands for each missing source Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 00:05:36 -08:00
parent 7162eb6b36
commit 06f74a4d0c
4 changed files with 413 additions and 41 deletions
@@ -5,6 +5,8 @@ via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js 2
 """

 import json
+import os
+import signal
 import shutil
 import subprocess
 import sys
@@ -175,26 +177,52 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
        "--json",
    ]

+    # Use process groups for clean cleanup on timeout/kill
+    preexec = os.setsid if hasattr(os, 'setsid') else None
+
    try:
-        result = subprocess.run(
+        proc = subprocess.Popen(
            cmd,
-            capture_output=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
            text=True,
-            timeout=timeout,
+            preexec_fn=preexec,
        )

-        if result.returncode != 0:
-            error = result.stderr.strip() or "Bird search failed"
+        # Register for cleanup tracking (if available)
+        try:
+            from last30days import register_child_pid, unregister_child_pid
+            register_child_pid(proc.pid)
+        except ImportError:
+            pass
+
+        try:
+            stdout, stderr = proc.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            # Kill the entire process group
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+            except (ProcessLookupError, PermissionError, OSError):
+                proc.kill()
+            proc.wait(timeout=5)
+            return {"error": f"Search timed out after {timeout}s", "items": []}
+        finally:
+            try:
+                from last30days import unregister_child_pid
+                unregister_child_pid(proc.pid)
+            except (ImportError, Exception):
+                pass
+
+        if proc.returncode != 0:
+            error = stderr.strip() if stderr else "Bird search failed"
            return {"error": error, "items": []}

-        output = result.stdout.strip()
+        output = stdout.strip() if stdout else ""
        if not output:
            return {"items": []}

        return json.loads(output)

-    except subprocess.TimeoutExpired:
-        return {"error": "Search timed out", "items": []}
    except json.JSONDecodeError as e:
        return {"error": f"Invalid JSON response: {e}", "items": []}
    except Exception as e:
@@ -276,19 +304,33 @@ def search_handles(
            "--json",
        ]

+        preexec = os.setsid if hasattr(os, 'setsid') else None
+
        try:
-            result = subprocess.run(
+            proc = subprocess.Popen(
                cmd,
-                capture_output=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
                text=True,
-                timeout=15,  # Short timeout per handle
+                preexec_fn=preexec,
            )

-            if result.returncode != 0:
-                _log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
+            try:
+                stdout, stderr = proc.communicate(timeout=15)
+            except subprocess.TimeoutExpired:
+                try:
+                    os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+                except (ProcessLookupError, PermissionError, OSError):
+                    proc.kill()
+                proc.wait(timeout=5)
+                _log(f"Handle search timed out for @{handle}")
                continue

-            output = result.stdout.strip()
+            if proc.returncode != 0:
+                _log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
+                continue
+
+            output = (stdout or "").strip()
            if not output:
                continue

@@ -296,8 +338,6 @@ def search_handles(
            items = parse_bird_response(response)
            all_items.extend(items)

-        except subprocess.TimeoutExpired:
-            _log(f"Handle search timed out for @{handle}")
        except json.JSONDecodeError:
            _log(f"Invalid JSON from handle search for @{handle}")
        except Exception as e:
@@ -238,6 +238,69 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
    return "\n".join(lines)


+def render_source_status(report: schema.Report, source_info: dict = None) -> str:
+    """Render source status footer showing what was used/skipped and why.
+
+    Args:
+        report: Report data
+        source_info: Dict with source availability info:
+            x_skip_reason, youtube_skip_reason, web_skip_reason
+
+    Returns:
+        Source status markdown string
+    """
+    if source_info is None:
+        source_info = {}
+
+    lines = []
+    lines.append("---")
+    lines.append("**Sources:**")
+
+    # Reddit
+    if report.reddit_error:
+        lines.append(f"  ❌ Reddit: error — {report.reddit_error}")
+    elif report.reddit:
+        lines.append(f"  ✅ Reddit: {len(report.reddit)} threads")
+    elif report.mode in ("both", "reddit-only", "all", "reddit-web"):
+        lines.append("  ⚠️ Reddit: 0 threads found")
+    else:
+        reason = source_info.get("reddit_skip_reason", "not configured")
+        lines.append(f"  ⏭️ Reddit: skipped — {reason}")
+
+    # X
+    if report.x_error:
+        lines.append(f"  ❌ X: error — {report.x_error}")
+    elif report.x:
+        lines.append(f"  ✅ X: {len(report.x)} posts")
+    elif report.mode in ("both", "x-only", "all", "x-web"):
+        lines.append("  ⚠️ X: 0 posts found")
+    else:
+        reason = source_info.get("x_skip_reason", "No Bird CLI or XAI_API_KEY")
+        lines.append(f"  ⏭️ X: skipped — {reason}")
+
+    # YouTube
+    if report.youtube_error:
+        lines.append(f"  ❌ YouTube: error — {report.youtube_error}")
+    elif report.youtube:
+        with_transcripts = sum(1 for v in report.youtube if getattr(v, 'transcript_snippet', None))
+        lines.append(f"  ✅ YouTube: {len(report.youtube)} videos ({with_transcripts} with transcripts)")
+    else:
+        reason = source_info.get("youtube_skip_reason", "yt-dlp not installed (brew install yt-dlp)")
+        lines.append(f"  ⏭️ YouTube: skipped — {reason}")
+
+    # Web
+    if report.web_error:
+        lines.append(f"  ❌ Web: error — {report.web_error}")
+    elif report.web:
+        lines.append(f"  ✅ Web: {len(report.web)} pages")
+    else:
+        reason = source_info.get("web_skip_reason", "assistant will use WebSearch")
+        lines.append(f"  ⚡ Web: {reason}")
+
+    lines.append("")
+    return "\n".join(lines)
+
+
 def render_context_snippet(report: schema.Report) -> str:
    """Render reusable context snippet.

@@ -414,6 +414,108 @@ class ProgressDisplay:
        sys.stderr.flush()


+def show_diagnostic_banner(diag: dict):
+    """Show pre-flight source status banner when sources are missing.
+
+    Args:
+        diag: Dict from env diagnostics with keys:
+            openai, xai, x_source, bird_installed, bird_authenticated,
+            bird_username, youtube, web_search_backend
+    """
+    has_openai = diag.get("openai", False)
+    has_x = diag.get("x_source") is not None
+    has_youtube = diag.get("youtube", False)
+    has_web = diag.get("web_search_backend") is not None
+
+    # If everything is available, no banner needed
+    if has_openai and has_x and has_youtube and has_web:
+        return
+
+    lines = []
+
+    if IS_TTY:
+        lines.append(f"{Colors.DIM}┌─────────────────────────────────────────────────────┐{Colors.RESET}")
+        lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.BOLD}/last30days v2.1 — Source Status{Colors.RESET}                    {Colors.DIM}│{Colors.RESET}")
+        lines.append(f"{Colors.DIM}│{Colors.RESET}                                                     {Colors.DIM}│{Colors.RESET}")
+
+        # Reddit
+        if has_openai:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ Reddit{Colors.RESET}    — OPENAI_API_KEY found                {Colors.DIM}│{Colors.RESET}")
+        else:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.RED}❌ Reddit{Colors.RESET}    — No OPENAI_API_KEY                    {Colors.DIM}│{Colors.RESET}")
+            lines.append(f"{Colors.DIM}│{Colors.RESET}     └─ Add to ~/.config/last30days/.env            {Colors.DIM}│{Colors.RESET}")
+
+        # X/Twitter
+        if has_x:
+            source = diag.get("x_source", "")
+            username = diag.get("bird_username", "")
+            label = f"Bird ({username})" if source == "bird" and username else source.upper()
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ X/Twitter{Colors.RESET} — {label}                          {Colors.DIM}│{Colors.RESET}")
+        else:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.RED}❌ X/Twitter{Colors.RESET} — No Bird CLI or XAI_API_KEY          {Colors.DIM}│{Colors.RESET}")
+            if diag.get("bird_installed"):
+                lines.append(f"{Colors.DIM}│{Colors.RESET}     └─ Bird installed but not authenticated         {Colors.DIM}│{Colors.RESET}")
+                lines.append(f"{Colors.DIM}│{Colors.RESET}     └─ Log into x.com in your browser, then retry   {Colors.DIM}│{Colors.RESET}")
+            else:
+                lines.append(f"{Colors.DIM}│{Colors.RESET}     └─ Needs Node.js 22+ (Bird is bundled)           {Colors.DIM}│{Colors.RESET}")
+
+        # YouTube
+        if has_youtube:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ YouTube{Colors.RESET}   — yt-dlp found                      {Colors.DIM}│{Colors.RESET}")
+        else:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.RED}❌ YouTube{Colors.RESET}   — yt-dlp not installed                {Colors.DIM}│{Colors.RESET}")
+            lines.append(f"{Colors.DIM}│{Colors.RESET}     └─ Fix: brew install yt-dlp (free)                {Colors.DIM}│{Colors.RESET}")
+
+        # Web
+        if has_web:
+            backend = diag.get("web_search_backend", "")
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.GREEN}✅ Web{Colors.RESET}       — {backend} API                       {Colors.DIM}│{Colors.RESET}")
+        else:
+            lines.append(f"{Colors.DIM}│{Colors.RESET}  {Colors.YELLOW}⚡ Web{Colors.RESET}       — Using assistant's search tool       {Colors.DIM}│{Colors.RESET}")
+
+        lines.append(f"{Colors.DIM}│{Colors.RESET}                                                     {Colors.DIM}│{Colors.RESET}")
+        lines.append(f"{Colors.DIM}│{Colors.RESET}  Config: {Colors.BOLD}~/.config/last30days/.env{Colors.RESET}                  {Colors.DIM}│{Colors.RESET}")
+        lines.append(f"{Colors.DIM}└─────────────────────────────────────────────────────┘{Colors.RESET}")
+    else:
+        # Plain text for non-TTY (Claude Code / Codex)
+        lines.append("┌─────────────────────────────────────────────────────┐")
+        lines.append("│ /last30days v2.1 — Source Status                    │")
+        lines.append("│                                                     │")
+
+        if has_openai:
+            lines.append("│  ✅ Reddit    — OPENAI_API_KEY found                │")
+        else:
+            lines.append("│  ❌ Reddit    — No OPENAI_API_KEY                    │")
+            lines.append("│     └─ Add to ~/.config/last30days/.env            │")
+
+        if has_x:
+            lines.append("│  ✅ X/Twitter — available                            │")
+        else:
+            lines.append("│  ❌ X/Twitter — No Bird CLI or XAI_API_KEY          │")
+            if diag.get("bird_installed"):
+                lines.append("│     └─ Log into x.com in your browser, then retry   │")
+            else:
+                lines.append("│     └─ Needs Node.js 22+ (Bird is bundled)           │")
+
+        if has_youtube:
+            lines.append("│  ✅ YouTube   — yt-dlp found                        │")
+        else:
+            lines.append("│  ❌ YouTube   — yt-dlp not installed                │")
+            lines.append("│     └─ Fix: brew install yt-dlp (free)                │")
+
+        if has_web:
+            lines.append("│  ✅ Web       — API search available                │")
+        else:
+            lines.append("│  ⚡ Web       — Using assistant's search tool       │")
+
+        lines.append("│                                                     │")
+        lines.append("│  Config: ~/.config/last30days/.env                  │")
+        lines.append("└─────────────────────────────────────────────────────┘")
+
+    sys.stderr.write("\n".join(lines) + "\n\n")
+    sys.stderr.flush()
+
+
 def print_phase(phase: str, message: str):
    """Print a phase message."""
    colors = {