fix: add timeouts, process cleanup, and source diagnostics
Script was hanging indefinitely when API sources were slow or unresponsive. Now enforces bounded execution: - Global timeout watchdog (180s default, 90s --quick, 300s --deep) - Per-source future.result() timeouts (60s/30s/90s by depth) - Parallel Reddit enrichment capped at 15 items / 45s total - Subprocess process-group isolation (os.setsid + killpg) - atexit cleanup kills all tracked child processes - --timeout=N flag for user override Also fixes the UX gap where missing sources were silently skipped: - Pre-flight diagnostic banner shows source status before research - Source status footer in compact output shows used/skipped/why - Actionable fix commands for each missing source Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+187
-20
@@ -17,9 +17,12 @@ Options:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import atexit
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import signal
|
||||||
import sys
|
import sys
|
||||||
|
import threading
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -28,6 +31,69 @@ from pathlib import Path
|
|||||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||||
sys.path.insert(0, str(SCRIPT_DIR))
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Global timeout & child process management
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
_child_pids: set = set()
|
||||||
|
_child_pids_lock = threading.Lock()
|
||||||
|
|
||||||
|
TIMEOUT_PROFILES = {
|
||||||
|
"quick": {"global": 90, "future": 30, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
|
||||||
|
"default": {"global": 180, "future": 60, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
|
||||||
|
"deep": {"global": 300, "future": 90, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def register_child_pid(pid: int):
|
||||||
|
"""Track a child process for cleanup."""
|
||||||
|
with _child_pids_lock:
|
||||||
|
_child_pids.add(pid)
|
||||||
|
|
||||||
|
|
||||||
|
def unregister_child_pid(pid: int):
|
||||||
|
"""Remove a child process from tracking."""
|
||||||
|
with _child_pids_lock:
|
||||||
|
_child_pids.discard(pid)
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_children():
|
||||||
|
"""Kill all tracked child processes."""
|
||||||
|
with _child_pids_lock:
|
||||||
|
pids = list(_child_pids)
|
||||||
|
for pid in pids:
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(pid), signal.SIGTERM)
|
||||||
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
atexit.register(_cleanup_children)
|
||||||
|
|
||||||
|
|
||||||
|
def _install_global_timeout(timeout_seconds: int):
|
||||||
|
"""Install a global timeout watchdog.
|
||||||
|
|
||||||
|
Uses SIGALRM on Unix, threading.Timer as fallback.
|
||||||
|
"""
|
||||||
|
if hasattr(signal, 'SIGALRM'):
|
||||||
|
def _handler(signum, frame):
|
||||||
|
sys.stderr.write(f"\n[TIMEOUT] Global timeout ({timeout_seconds}s) exceeded. Cleaning up.\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
_cleanup_children()
|
||||||
|
sys.exit(1)
|
||||||
|
signal.signal(signal.SIGALRM, _handler)
|
||||||
|
signal.alarm(timeout_seconds)
|
||||||
|
else:
|
||||||
|
# Windows fallback
|
||||||
|
def _watchdog():
|
||||||
|
sys.stderr.write(f"\n[TIMEOUT] Global timeout ({timeout_seconds}s) exceeded. Cleaning up.\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
_cleanup_children()
|
||||||
|
os._exit(1)
|
||||||
|
timer = threading.Timer(timeout_seconds, _watchdog)
|
||||||
|
timer.daemon = True
|
||||||
|
timer.start()
|
||||||
|
|
||||||
from lib import (
|
from lib import (
|
||||||
bird_x,
|
bird_x,
|
||||||
dates,
|
dates,
|
||||||
@@ -384,22 +450,26 @@ def _run_supplemental(
|
|||||||
|
|
||||||
if reddit_future:
|
if reddit_future:
|
||||||
try:
|
try:
|
||||||
raw_reddit = reddit_future.result()
|
raw_reddit = reddit_future.result(timeout=30)
|
||||||
# Filter out URLs already found in Phase 1
|
# Filter out URLs already found in Phase 1
|
||||||
supplemental_reddit = [
|
supplemental_reddit = [
|
||||||
item for item in raw_reddit
|
item for item in raw_reddit
|
||||||
if item.get("url", "") not in existing_urls
|
if item.get("url", "") not in existing_urls
|
||||||
]
|
]
|
||||||
|
except TimeoutError:
|
||||||
|
sys.stderr.write("[Phase 2] Supplemental Reddit timed out (30s)\n")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
sys.stderr.write(f"[Phase 2] Supplemental Reddit error: {e}\n")
|
sys.stderr.write(f"[Phase 2] Supplemental Reddit error: {e}\n")
|
||||||
|
|
||||||
if x_future:
|
if x_future:
|
||||||
try:
|
try:
|
||||||
raw_x = x_future.result()
|
raw_x = x_future.result(timeout=30)
|
||||||
supplemental_x = [
|
supplemental_x = [
|
||||||
item for item in raw_x
|
item for item in raw_x
|
||||||
if item.get("url", "") not in existing_urls
|
if item.get("url", "") not in existing_urls
|
||||||
]
|
]
|
||||||
|
except TimeoutError:
|
||||||
|
sys.stderr.write("[Phase 2] Supplemental X timed out (30s)\n")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
sys.stderr.write(f"[Phase 2] Supplemental X error: {e}\n")
|
sys.stderr.write(f"[Phase 2] Supplemental X error: {e}\n")
|
||||||
|
|
||||||
@@ -424,6 +494,7 @@ def run_research(
|
|||||||
progress: ui.ProgressDisplay = None,
|
progress: ui.ProgressDisplay = None,
|
||||||
x_source: str = "xai",
|
x_source: str = "xai",
|
||||||
run_youtube: bool = False,
|
run_youtube: bool = False,
|
||||||
|
timeouts: dict = None,
|
||||||
) -> tuple:
|
) -> tuple:
|
||||||
"""Run the research pipeline.
|
"""Run the research pipeline.
|
||||||
|
|
||||||
@@ -436,6 +507,10 @@ def run_research(
|
|||||||
(i.e., no native web search API keys are configured). When native web search
|
(i.e., no native web search API keys are configured). When native web search
|
||||||
runs, web_items will be populated and web_needed will be False.
|
runs, web_items will be populated and web_needed will be False.
|
||||||
"""
|
"""
|
||||||
|
if timeouts is None:
|
||||||
|
timeouts = TIMEOUT_PROFILES[depth]
|
||||||
|
future_timeout = timeouts["future"]
|
||||||
|
|
||||||
reddit_items = []
|
reddit_items = []
|
||||||
x_items = []
|
x_items = []
|
||||||
youtube_items = []
|
youtube_items = []
|
||||||
@@ -533,12 +608,16 @@ def run_research(
|
|||||||
_search_web, topic, config, from_date, to_date, depth
|
_search_web, topic, config, from_date, to_date, depth
|
||||||
)
|
)
|
||||||
|
|
||||||
# Collect results
|
# Collect results (with timeouts to prevent indefinite blocking)
|
||||||
if reddit_future:
|
if reddit_future:
|
||||||
try:
|
try:
|
||||||
reddit_items, raw_openai, reddit_error = reddit_future.result()
|
reddit_items, raw_openai, reddit_error = reddit_future.result(timeout=future_timeout)
|
||||||
if reddit_error and progress:
|
if reddit_error and progress:
|
||||||
progress.show_error(f"Reddit error: {reddit_error}")
|
progress.show_error(f"Reddit error: {reddit_error}")
|
||||||
|
except TimeoutError:
|
||||||
|
reddit_error = f"Reddit search timed out after {future_timeout}s"
|
||||||
|
if progress:
|
||||||
|
progress.show_error(reddit_error)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
reddit_error = f"{type(e).__name__}: {e}"
|
reddit_error = f"{type(e).__name__}: {e}"
|
||||||
if progress:
|
if progress:
|
||||||
@@ -548,9 +627,13 @@ def run_research(
|
|||||||
|
|
||||||
if x_future:
|
if x_future:
|
||||||
try:
|
try:
|
||||||
x_items, raw_xai, x_error = x_future.result()
|
x_items, raw_xai, x_error = x_future.result(timeout=future_timeout)
|
||||||
if x_error and progress:
|
if x_error and progress:
|
||||||
progress.show_error(f"X error: {x_error}")
|
progress.show_error(f"X error: {x_error}")
|
||||||
|
except TimeoutError:
|
||||||
|
x_error = f"X search timed out after {future_timeout}s"
|
||||||
|
if progress:
|
||||||
|
progress.show_error(x_error)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
x_error = f"{type(e).__name__}: {e}"
|
x_error = f"{type(e).__name__}: {e}"
|
||||||
if progress:
|
if progress:
|
||||||
@@ -560,9 +643,13 @@ def run_research(
|
|||||||
|
|
||||||
if youtube_future:
|
if youtube_future:
|
||||||
try:
|
try:
|
||||||
youtube_items, youtube_error = youtube_future.result()
|
youtube_items, youtube_error = youtube_future.result(timeout=future_timeout)
|
||||||
if youtube_error and progress:
|
if youtube_error and progress:
|
||||||
progress.show_error(f"YouTube error: {youtube_error}")
|
progress.show_error(f"YouTube error: {youtube_error}")
|
||||||
|
except TimeoutError:
|
||||||
|
youtube_error = f"YouTube search timed out after {future_timeout}s"
|
||||||
|
if progress:
|
||||||
|
progress.show_error(youtube_error)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
youtube_error = f"{type(e).__name__}: {e}"
|
youtube_error = f"{type(e).__name__}: {e}"
|
||||||
if progress:
|
if progress:
|
||||||
@@ -572,9 +659,13 @@ def run_research(
|
|||||||
|
|
||||||
if web_future:
|
if web_future:
|
||||||
try:
|
try:
|
||||||
web_items, web_error = web_future.result()
|
web_items, web_error = web_future.result(timeout=future_timeout)
|
||||||
if web_error and progress:
|
if web_error and progress:
|
||||||
progress.show_error(f"Web error: {web_error}")
|
progress.show_error(f"Web error: {web_error}")
|
||||||
|
except TimeoutError:
|
||||||
|
web_error = f"Web search timed out after {future_timeout}s"
|
||||||
|
if progress:
|
||||||
|
progress.show_error(web_error)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
web_error = f"{type(e).__name__}: {e}"
|
web_error = f"{type(e).__name__}: {e}"
|
||||||
if progress:
|
if progress:
|
||||||
@@ -582,27 +673,59 @@ def run_research(
|
|||||||
sys.stderr.write(f"[web] {len(web_items)} results\n")
|
sys.stderr.write(f"[web] {len(web_items)} results\n")
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
# Enrich Reddit items with real data (sequential, but with error handling per-item)
|
# Enrich Reddit items with real data (parallel, capped)
|
||||||
if reddit_items:
|
enrich_max = timeouts["enrich_max_items"]
|
||||||
|
enrich_total_timeout = timeouts["enrich_total"]
|
||||||
|
items_to_enrich = reddit_items[:enrich_max]
|
||||||
|
|
||||||
|
if items_to_enrich:
|
||||||
if progress:
|
if progress:
|
||||||
progress.start_reddit_enrich(1, len(reddit_items))
|
progress.start_reddit_enrich(1, len(items_to_enrich))
|
||||||
|
|
||||||
for i, item in enumerate(reddit_items):
|
|
||||||
if progress and i > 0:
|
|
||||||
progress.update_reddit_enrich(i + 1, len(reddit_items))
|
|
||||||
|
|
||||||
try:
|
|
||||||
if mock:
|
if mock:
|
||||||
|
# Sequential mock enrichment (fast, no need for parallelism)
|
||||||
|
for i, item in enumerate(items_to_enrich):
|
||||||
|
if progress and i > 0:
|
||||||
|
progress.update_reddit_enrich(i + 1, len(items_to_enrich))
|
||||||
|
try:
|
||||||
mock_thread = load_fixture("reddit_thread_sample.json")
|
mock_thread = load_fixture("reddit_thread_sample.json")
|
||||||
reddit_items[i] = reddit_enrich.enrich_reddit_item(item, mock_thread)
|
reddit_items[i] = reddit_enrich.enrich_reddit_item(item, mock_thread)
|
||||||
else:
|
|
||||||
reddit_items[i] = reddit_enrich.enrich_reddit_item(item)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Log but don't crash - keep the unenriched item
|
|
||||||
if progress:
|
if progress:
|
||||||
progress.show_error(f"Enrich failed for {item.get('url', 'unknown')}: {e}")
|
progress.show_error(f"Enrich failed for {item.get('url', 'unknown')}: {e}")
|
||||||
|
|
||||||
raw_reddit_enriched.append(reddit_items[i])
|
raw_reddit_enriched.append(reddit_items[i])
|
||||||
|
else:
|
||||||
|
# Parallel enrichment with bounded concurrency and total timeout
|
||||||
|
completed_count = 0
|
||||||
|
with ThreadPoolExecutor(max_workers=5) as enrich_pool:
|
||||||
|
futures = {
|
||||||
|
enrich_pool.submit(reddit_enrich.enrich_reddit_item, item): i
|
||||||
|
for i, item in enumerate(items_to_enrich)
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
for future in as_completed(futures, timeout=enrich_total_timeout):
|
||||||
|
idx = futures[future]
|
||||||
|
completed_count += 1
|
||||||
|
if progress:
|
||||||
|
progress.update_reddit_enrich(completed_count, len(items_to_enrich))
|
||||||
|
try:
|
||||||
|
reddit_items[idx] = future.result(timeout=timeouts["enrich_per"])
|
||||||
|
except Exception as e:
|
||||||
|
if progress:
|
||||||
|
progress.show_error(
|
||||||
|
f"Enrich failed for {items_to_enrich[idx].get('url', 'unknown')}: {e}"
|
||||||
|
)
|
||||||
|
raw_reddit_enriched.append(reddit_items[idx])
|
||||||
|
except TimeoutError:
|
||||||
|
if progress:
|
||||||
|
progress.show_error(
|
||||||
|
f"Enrichment timed out after {enrich_total_timeout}s "
|
||||||
|
f"({completed_count}/{len(items_to_enrich)} done)"
|
||||||
|
)
|
||||||
|
# Keep unenriched items as-is
|
||||||
|
for idx in futures.values():
|
||||||
|
if reddit_items[idx] not in raw_reddit_enriched:
|
||||||
|
raw_reddit_enriched.append(reddit_items[idx])
|
||||||
|
|
||||||
if progress:
|
if progress:
|
||||||
progress.end_reddit_enrich()
|
progress.end_reddit_enrich()
|
||||||
@@ -683,6 +806,13 @@ def main():
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Show source availability diagnostics and exit",
|
help="Show source availability diagnostics and exit",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--timeout",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
metavar="SECS",
|
||||||
|
help="Global timeout in seconds (default: 180, quick: 90, deep: 300)",
|
||||||
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
@@ -704,6 +834,11 @@ def main():
|
|||||||
else:
|
else:
|
||||||
depth = "default"
|
depth = "default"
|
||||||
|
|
||||||
|
# Install global timeout watchdog
|
||||||
|
timeouts = TIMEOUT_PROFILES[depth]
|
||||||
|
global_timeout = args.timeout or timeouts["global"]
|
||||||
|
_install_global_timeout(global_timeout)
|
||||||
|
|
||||||
# Load config
|
# Load config
|
||||||
config = env.get_config()
|
config = env.get_config()
|
||||||
|
|
||||||
@@ -742,6 +877,20 @@ def main():
|
|||||||
# Initialize progress display with topic
|
# Initialize progress display with topic
|
||||||
progress = ui.ProgressDisplay(args.topic, show_banner=True)
|
progress = ui.ProgressDisplay(args.topic, show_banner=True)
|
||||||
|
|
||||||
|
# Show diagnostic banner when sources are missing
|
||||||
|
web_source = env.get_web_search_source(config)
|
||||||
|
diag = {
|
||||||
|
"openai": bool(config.get("OPENAI_API_KEY")),
|
||||||
|
"xai": bool(config.get("XAI_API_KEY")),
|
||||||
|
"x_source": x_source_status["source"],
|
||||||
|
"bird_installed": x_source_status["bird_installed"],
|
||||||
|
"bird_authenticated": x_source_status["bird_authenticated"],
|
||||||
|
"bird_username": x_source_status.get("bird_username"),
|
||||||
|
"youtube": has_ytdlp,
|
||||||
|
"web_search_backend": web_source,
|
||||||
|
}
|
||||||
|
ui.show_diagnostic_banner(diag)
|
||||||
|
|
||||||
# Check available sources (accounting for Bird auto-detection)
|
# Check available sources (accounting for Bird auto-detection)
|
||||||
available = env.get_available_sources(config)
|
available = env.get_available_sources(config)
|
||||||
|
|
||||||
@@ -827,6 +976,7 @@ def main():
|
|||||||
progress,
|
progress,
|
||||||
x_source=x_source or "xai",
|
x_source=x_source or "xai",
|
||||||
run_youtube=has_ytdlp,
|
run_youtube=has_ytdlp,
|
||||||
|
timeouts=timeouts,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Processing phase
|
# Processing phase
|
||||||
@@ -902,8 +1052,22 @@ def main():
|
|||||||
else:
|
else:
|
||||||
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube))
|
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube))
|
||||||
|
|
||||||
|
# Build source info for status footer
|
||||||
|
source_info = {}
|
||||||
|
if not bool(config.get("OPENAI_API_KEY")):
|
||||||
|
source_info["reddit_skip_reason"] = "No OPENAI_API_KEY (add to ~/.config/last30days/.env)"
|
||||||
|
if not x_source:
|
||||||
|
if x_source_status["bird_installed"]:
|
||||||
|
source_info["x_skip_reason"] = "Bird installed but not authenticated — log into x.com in browser"
|
||||||
|
else:
|
||||||
|
source_info["x_skip_reason"] = "No Bird CLI or XAI_API_KEY (Node.js 22+ needed for Bird)"
|
||||||
|
if not has_ytdlp:
|
||||||
|
source_info["youtube_skip_reason"] = "yt-dlp not installed — fix: brew install yt-dlp"
|
||||||
|
if not web_source:
|
||||||
|
source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)"
|
||||||
|
|
||||||
# Output result
|
# Output result
|
||||||
output_result(report, args.emit, web_needed, args.topic, from_date, to_date, missing_keys, args.days)
|
output_result(report, args.emit, web_needed, args.topic, from_date, to_date, missing_keys, args.days, source_info)
|
||||||
|
|
||||||
# Persist findings to SQLite if requested
|
# Persist findings to SQLite if requested
|
||||||
if args.store:
|
if args.store:
|
||||||
@@ -977,10 +1141,13 @@ def output_result(
|
|||||||
to_date: str = "",
|
to_date: str = "",
|
||||||
missing_keys: str = "none",
|
missing_keys: str = "none",
|
||||||
days: int = 30,
|
days: int = 30,
|
||||||
|
source_info: dict = None,
|
||||||
):
|
):
|
||||||
"""Output the result based on emit mode."""
|
"""Output the result based on emit mode."""
|
||||||
if emit_mode == "compact":
|
if emit_mode == "compact":
|
||||||
print(render.render_compact(report, missing_keys=missing_keys))
|
print(render.render_compact(report, missing_keys=missing_keys))
|
||||||
|
# Append source status footer
|
||||||
|
print(render.render_source_status(report, source_info))
|
||||||
elif emit_mode == "json":
|
elif emit_mode == "json":
|
||||||
print(json.dumps(report.to_dict(), indent=2))
|
print(json.dumps(report.to_dict(), indent=2))
|
||||||
elif emit_mode == "md":
|
elif emit_mode == "md":
|
||||||
|
|||||||
+56
-16
@@ -5,6 +5,8 @@ via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js 2
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -175,26 +177,52 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
|||||||
"--json",
|
"--json",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Use process groups for clean cleanup on timeout/kill
|
||||||
|
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
proc = subprocess.Popen(
|
||||||
cmd,
|
cmd,
|
||||||
capture_output=True,
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=timeout,
|
preexec_fn=preexec,
|
||||||
)
|
)
|
||||||
|
|
||||||
if result.returncode != 0:
|
# Register for cleanup tracking (if available)
|
||||||
error = result.stderr.strip() or "Bird search failed"
|
try:
|
||||||
|
from last30days import register_child_pid, unregister_child_pid
|
||||||
|
register_child_pid(proc.pid)
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
stdout, stderr = proc.communicate(timeout=timeout)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
# Kill the entire process group
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||||
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
|
proc.kill()
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
return {"error": f"Search timed out after {timeout}s", "items": []}
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
from last30days import unregister_child_pid
|
||||||
|
unregister_child_pid(proc.pid)
|
||||||
|
except (ImportError, Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
if proc.returncode != 0:
|
||||||
|
error = stderr.strip() if stderr else "Bird search failed"
|
||||||
return {"error": error, "items": []}
|
return {"error": error, "items": []}
|
||||||
|
|
||||||
output = result.stdout.strip()
|
output = stdout.strip() if stdout else ""
|
||||||
if not output:
|
if not output:
|
||||||
return {"items": []}
|
return {"items": []}
|
||||||
|
|
||||||
return json.loads(output)
|
return json.loads(output)
|
||||||
|
|
||||||
except subprocess.TimeoutExpired:
|
|
||||||
return {"error": "Search timed out", "items": []}
|
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError as e:
|
||||||
return {"error": f"Invalid JSON response: {e}", "items": []}
|
return {"error": f"Invalid JSON response: {e}", "items": []}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -276,19 +304,33 @@ def search_handles(
|
|||||||
"--json",
|
"--json",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
preexec = os.setsid if hasattr(os, 'setsid') else None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
proc = subprocess.Popen(
|
||||||
cmd,
|
cmd,
|
||||||
capture_output=True,
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=15, # Short timeout per handle
|
preexec_fn=preexec,
|
||||||
)
|
)
|
||||||
|
|
||||||
if result.returncode != 0:
|
try:
|
||||||
_log(f"Handle search failed for @{handle}: {result.stderr.strip()}")
|
stdout, stderr = proc.communicate(timeout=15)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
||||||
|
except (ProcessLookupError, PermissionError, OSError):
|
||||||
|
proc.kill()
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
_log(f"Handle search timed out for @{handle}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
output = result.stdout.strip()
|
if proc.returncode != 0:
|
||||||
|
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
output = (stdout or "").strip()
|
||||||
if not output:
|
if not output:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -296,8 +338,6 @@ def search_handles(
|
|||||||
items = parse_bird_response(response)
|
items = parse_bird_response(response)
|
||||||
all_items.extend(items)
|
all_items.extend(items)
|
||||||
|
|
||||||
except subprocess.TimeoutExpired:
|
|
||||||
_log(f"Handle search timed out for @{handle}")
|
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
_log(f"Invalid JSON from handle search for @{handle}")
|
_log(f"Invalid JSON from handle search for @{handle}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -238,6 +238,69 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def render_source_status(report: schema.Report, source_info: dict = None) -> str:
|
||||||
|
"""Render source status footer showing what was used/skipped and why.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
report: Report data
|
||||||
|
source_info: Dict with source availability info:
|
||||||
|
x_skip_reason, youtube_skip_reason, web_skip_reason
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Source status markdown string
|
||||||
|
"""
|
||||||
|
if source_info is None:
|
||||||
|
source_info = {}
|
||||||
|
|
||||||
|
lines = []
|
||||||
|
lines.append("---")
|
||||||
|
lines.append("**Sources:**")
|
||||||
|
|
||||||
|
# Reddit
|
||||||
|
if report.reddit_error:
|
||||||
|
lines.append(f" ❌ Reddit: error — {report.reddit_error}")
|
||||||
|
elif report.reddit:
|
||||||
|
lines.append(f" ✅ Reddit: {len(report.reddit)} threads")
|
||||||
|
elif report.mode in ("both", "reddit-only", "all", "reddit-web"):
|
||||||
|
lines.append(" ⚠️ Reddit: 0 threads found")
|
||||||
|
else:
|
||||||
|
reason = source_info.get("reddit_skip_reason", "not configured")
|
||||||
|
lines.append(f" ⏭️ Reddit: skipped — {reason}")
|
||||||
|
|
||||||
|
# X
|
||||||
|
if report.x_error:
|
||||||
|
lines.append(f" ❌ X: error — {report.x_error}")
|
||||||
|
elif report.x:
|
||||||
|
lines.append(f" ✅ X: {len(report.x)} posts")
|
||||||
|
elif report.mode in ("both", "x-only", "all", "x-web"):
|
||||||
|
lines.append(" ⚠️ X: 0 posts found")
|
||||||
|
else:
|
||||||
|
reason = source_info.get("x_skip_reason", "No Bird CLI or XAI_API_KEY")
|
||||||
|
lines.append(f" ⏭️ X: skipped — {reason}")
|
||||||
|
|
||||||
|
# YouTube
|
||||||
|
if report.youtube_error:
|
||||||
|
lines.append(f" ❌ YouTube: error — {report.youtube_error}")
|
||||||
|
elif report.youtube:
|
||||||
|
with_transcripts = sum(1 for v in report.youtube if getattr(v, 'transcript_snippet', None))
|
||||||
|
lines.append(f" ✅ YouTube: {len(report.youtube)} videos ({with_transcripts} with transcripts)")
|
||||||
|
else:
|
||||||
|
reason = source_info.get("youtube_skip_reason", "yt-dlp not installed (brew install yt-dlp)")
|
||||||
|
lines.append(f" ⏭️ YouTube: skipped — {reason}")
|
||||||
|
|
||||||
|
# Web
|
||||||
|
if report.web_error:
|
||||||
|
lines.append(f" ❌ Web: error — {report.web_error}")
|
||||||
|
elif report.web:
|
||||||
|
lines.append(f" ✅ Web: {len(report.web)} pages")
|
||||||
|
else:
|
||||||
|
reason = source_info.get("web_skip_reason", "assistant will use WebSearch")
|
||||||
|
lines.append(f" ⚡ Web: {reason}")
|
||||||
|
|
||||||
|
lines.append("")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
def render_context_snippet(report: schema.Report) -> str:
|
def render_context_snippet(report: schema.Report) -> str:
|
||||||
"""Render reusable context snippet.
|
"""Render reusable context snippet.
|
||||||
|
|
||||||
|
|||||||
@@ -414,6 +414,108 @@ class ProgressDisplay:
|
|||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def show_diagnostic_banner(diag: dict):
|
||||||
|
"""Show pre-flight source status banner when sources are missing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
diag: Dict from env diagnostics with keys:
|
||||||
|
openai, xai, x_source, bird_installed, bird_authenticated,
|
||||||
|
bird_username, youtube, web_search_backend
|
||||||
|
"""
|
||||||
|
has_openai = diag.get("openai", False)
|
||||||
|
has_x = diag.get("x_source") is not None
|
||||||
|
has_youtube = diag.get("youtube", False)
|
||||||
|
has_web = diag.get("web_search_backend") is not None
|
||||||
|
|
||||||
|
# If everything is available, no banner needed
|
||||||
|
if has_openai and has_x and has_youtube and has_web:
|
||||||
|
return
|
||||||
|
|
||||||
|
lines = []
|
||||||
|
|
||||||
|
if IS_TTY:
|
||||||
|
lines.append(f"{Colors.DIM}┌─────────────────────────────────────────────────────┐{Colors.RESET}")
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.BOLD}/last30days v2.1 — Source Status{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
||||||
|
|
||||||
|
# Reddit
|
||||||
|
if has_openai:
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Reddit{Colors.RESET} — OPENAI_API_KEY found {Colors.DIM}│{Colors.RESET}")
|
||||||
|
else:
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ Reddit{Colors.RESET} — No OPENAI_API_KEY {Colors.DIM}│{Colors.RESET}")
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Add to ~/.config/last30days/.env {Colors.DIM}│{Colors.RESET}")
|
||||||
|
|
||||||
|
# X/Twitter
|
||||||
|
if has_x:
|
||||||
|
source = diag.get("x_source", "")
|
||||||
|
username = diag.get("bird_username", "")
|
||||||
|
label = f"Bird ({username})" if source == "bird" and username else source.upper()
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ X/Twitter{Colors.RESET} — {label} {Colors.DIM}│{Colors.RESET}")
|
||||||
|
else:
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ X/Twitter{Colors.RESET} — No Bird CLI or XAI_API_KEY {Colors.DIM}│{Colors.RESET}")
|
||||||
|
if diag.get("bird_installed"):
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Bird installed but not authenticated {Colors.DIM}│{Colors.RESET}")
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Log into x.com in your browser, then retry {Colors.DIM}│{Colors.RESET}")
|
||||||
|
else:
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Needs Node.js 22+ (Bird is bundled) {Colors.DIM}│{Colors.RESET}")
|
||||||
|
|
||||||
|
# YouTube
|
||||||
|
if has_youtube:
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ YouTube{Colors.RESET} — yt-dlp found {Colors.DIM}│{Colors.RESET}")
|
||||||
|
else:
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ YouTube{Colors.RESET} — yt-dlp not installed {Colors.DIM}│{Colors.RESET}")
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Fix: brew install yt-dlp (free) {Colors.DIM}│{Colors.RESET}")
|
||||||
|
|
||||||
|
# Web
|
||||||
|
if has_web:
|
||||||
|
backend = diag.get("web_search_backend", "")
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ Web{Colors.RESET} — {backend} API {Colors.DIM}│{Colors.RESET}")
|
||||||
|
else:
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.YELLOW}⚡ Web{Colors.RESET} — Using assistant's search tool {Colors.DIM}│{Colors.RESET}")
|
||||||
|
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
||||||
|
lines.append(f"{Colors.DIM}│{Colors.RESET} Config: {Colors.BOLD}~/.config/last30days/.env{Colors.RESET} {Colors.DIM}│{Colors.RESET}")
|
||||||
|
lines.append(f"{Colors.DIM}└─────────────────────────────────────────────────────┘{Colors.RESET}")
|
||||||
|
else:
|
||||||
|
# Plain text for non-TTY (Claude Code / Codex)
|
||||||
|
lines.append("┌─────────────────────────────────────────────────────┐")
|
||||||
|
lines.append("│ /last30days v2.1 — Source Status │")
|
||||||
|
lines.append("│ │")
|
||||||
|
|
||||||
|
if has_openai:
|
||||||
|
lines.append("│ ✅ Reddit — OPENAI_API_KEY found │")
|
||||||
|
else:
|
||||||
|
lines.append("│ ❌ Reddit — No OPENAI_API_KEY │")
|
||||||
|
lines.append("│ └─ Add to ~/.config/last30days/.env │")
|
||||||
|
|
||||||
|
if has_x:
|
||||||
|
lines.append("│ ✅ X/Twitter — available │")
|
||||||
|
else:
|
||||||
|
lines.append("│ ❌ X/Twitter — No Bird CLI or XAI_API_KEY │")
|
||||||
|
if diag.get("bird_installed"):
|
||||||
|
lines.append("│ └─ Log into x.com in your browser, then retry │")
|
||||||
|
else:
|
||||||
|
lines.append("│ └─ Needs Node.js 22+ (Bird is bundled) │")
|
||||||
|
|
||||||
|
if has_youtube:
|
||||||
|
lines.append("│ ✅ YouTube — yt-dlp found │")
|
||||||
|
else:
|
||||||
|
lines.append("│ ❌ YouTube — yt-dlp not installed │")
|
||||||
|
lines.append("│ └─ Fix: brew install yt-dlp (free) │")
|
||||||
|
|
||||||
|
if has_web:
|
||||||
|
lines.append("│ ✅ Web — API search available │")
|
||||||
|
else:
|
||||||
|
lines.append("│ ⚡ Web — Using assistant's search tool │")
|
||||||
|
|
||||||
|
lines.append("│ │")
|
||||||
|
lines.append("│ Config: ~/.config/last30days/.env │")
|
||||||
|
lines.append("└─────────────────────────────────────────────────────┘")
|
||||||
|
|
||||||
|
sys.stderr.write("\n".join(lines) + "\n\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
def print_phase(phase: str, message: str):
|
def print_phase(phase: str, message: str):
|
||||||
"""Print a phase message."""
|
"""Print a phase message."""
|
||||||
colors = {
|
colors = {
|
||||||
|
|||||||
Reference in New Issue
Block a user