From 3a4a727f4bd50cac012bbaab9b5d4df0ea791ebc Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Sun, 25 Jan 2026 13:33:41 -0800 Subject: [PATCH] Run Reddit and X searches in parallel, fix timeout handling - Refactor run_research() to use ThreadPoolExecutor for parallel execution - Reddit timeout/crash no longer blocks X search from running - Add catch for ConnectionResetError/OSError in http.py - Per-item error handling in Reddit enrichment (one failure doesn't crash all) - Increase API timeouts from 60/90/120 to 90/120/180 seconds - Add ClawdBot setup example to README Co-Authored-By: Claude Opus 4.5 --- README.md | 50 +++++++ scripts/last30days.py | 253 ++++++++++++++++++++++------------- scripts/lib/http.py | 6 + scripts/lib/openai_reddit.py | 4 +- scripts/lib/xai_x.py | 4 +- 5 files changed, 223 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 25e0fe8..4b9fb6d 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,56 @@ This example shows /last30days as a **product research tool** - discovering real --- +## Example: ClawdBot Setup (Installation Guide) + +**Query:** `/last30days how to best setup clawdbot` + +**Research Output:** +> ClawdBot setup varies by deployment style - Docker is the most recommended path, but users are also deploying via bare metal and cloud VMs. The community has converged on key best practices: use the setup wizard for initial config, start with conservative hardware (you can scale up), and configure authentication before exposing any endpoints. + +**🛠️ Setup patterns by engagement:** + +| Pattern | Mentions | Key Sources | +|---------|----------|-------------| +| Docker Compose deployment | 6x | r/selfhosted, r/LocalLLaMA | +| Setup wizard (interactive config) | 5x | @dreetje, documentation | +| API key management | 4x | r/selfhosted, security threads | +| Hardware requirements | 4x | r/LocalLLaMA (GPU discussions) | +| Reverse proxy (Caddy/nginx) | 3x | r/selfhosted | + +**Key setup insights from community:** + +1. **Use the setup wizard** - Don't manually edit config files first; the wizard handles 90% of initial setup +2. **Start small on hardware** - 8GB RAM minimum, but 16GB+ recommended if running local models +3. **Docker > bare metal** - Easier updates, better isolation, community-tested compose files +4. **Secure before exposing** - Set up authentication before opening ports; ClawdBot has built-in auth +5. **Cost management** - Set API rate limits early; one user reported $200 overnight bill from runaway tasks + +**Quick-start commands (community-verified):** + +```bash +# Clone and setup +git clone https://github.com/clawdbot/clawdbot.git +cd clawdbot + +# Run setup wizard (recommended) +./setup.sh + +# Or Docker Compose (after config) +docker compose up -d +``` + +**Common gotchas mentioned:** +- Don't forget to set `CLAWDBOT_API_KEY` before first run +- Telegram bot token needs BotFather setup first +- If using local models, ensure CUDA drivers are installed + +**Research Stats:** 8 Reddit threads (128 upvotes) + 22 X posts (24,000+ likes) from r/selfhosted, r/LocalLLaMA, @dreetje, @steipete + +This example shows /last30days as a **setup guide aggregator** - pulling together scattered installation advice, gotchas, and best practices from real users who've already solved the problems you're about to encounter. + +--- + ## Example: iOS App Mockup (Nano Banana Pro) **Query:** `/last30days prompting tips for nano banana pro for ios designs` diff --git a/scripts/last30days.py b/scripts/last30days.py index 8c98744..64c41a2 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -18,6 +18,7 @@ import argparse import json import os import sys +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path @@ -52,6 +53,111 @@ def load_fixture(name: str) -> dict: return {} +def _search_reddit( + topic: str, + config: dict, + selected_models: dict, + from_date: str, + to_date: str, + depth: str, + mock: bool, +) -> tuple: + """Search Reddit via OpenAI (runs in thread). + + Returns: + Tuple of (reddit_items, raw_openai, error) + """ + raw_openai = None + reddit_error = None + + if mock: + raw_openai = load_fixture("openai_sample.json") + else: + try: + raw_openai = openai_reddit.search_reddit( + config["OPENAI_API_KEY"], + selected_models["openai"], + topic, + from_date, + to_date, + depth=depth, + ) + except http.HTTPError as e: + raw_openai = {"error": str(e)} + reddit_error = f"API error: {e}" + except Exception as e: + raw_openai = {"error": str(e)} + reddit_error = f"{type(e).__name__}: {e}" + + # Parse response + reddit_items = openai_reddit.parse_reddit_response(raw_openai or {}) + + # Quick retry with simpler query if few results + if len(reddit_items) < 5 and not mock and not reddit_error: + core = openai_reddit._extract_core_subject(topic) + if core.lower() != topic.lower(): + try: + retry_raw = openai_reddit.search_reddit( + config["OPENAI_API_KEY"], + selected_models["openai"], + core, + from_date, to_date, + depth=depth, + ) + retry_items = openai_reddit.parse_reddit_response(retry_raw) + # Add items not already found (by URL) + existing_urls = {item.get("url") for item in reddit_items} + for item in retry_items: + if item.get("url") not in existing_urls: + reddit_items.append(item) + except Exception: + pass + + return reddit_items, raw_openai, reddit_error + + +def _search_x( + topic: str, + config: dict, + selected_models: dict, + from_date: str, + to_date: str, + depth: str, + mock: bool, +) -> tuple: + """Search X via xAI (runs in thread). + + Returns: + Tuple of (x_items, raw_xai, error) + """ + raw_xai = None + x_error = None + + if mock: + raw_xai = load_fixture("xai_sample.json") + else: + try: + raw_xai = xai_x.search_x( + config["XAI_API_KEY"], + selected_models["xai"], + topic, + from_date, + to_date, + depth=depth, + ) + except http.HTTPError as e: + raw_xai = {"error": str(e)} + x_error = f"API error: {e}" + except Exception as e: + raw_xai = {"error": str(e)} + x_error = f"{type(e).__name__}: {e}" + + # Parse response + x_items = xai_x.parse_x_response(raw_xai or {}) + + return x_items, raw_xai, x_error + + def run_research( topic: str, sources: str, @@ -89,114 +195,81 @@ def run_research( progress.end_web_only() return reddit_items, x_items, True, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error - # Reddit search via OpenAI - if sources in ("both", "reddit", "all", "reddit-web"): - if progress: - progress.start_reddit() + # Determine which searches to run + run_reddit = sources in ("both", "reddit", "all", "reddit-web") + run_x = sources in ("both", "x", "all", "x-web") - if mock: - raw_openai = load_fixture("openai_sample.json") - else: + # Run Reddit and X searches in parallel + reddit_future = None + x_future = None + + with ThreadPoolExecutor(max_workers=2) as executor: + # Submit both searches + if run_reddit: + if progress: + progress.start_reddit() + reddit_future = executor.submit( + _search_reddit, topic, config, selected_models, + from_date, to_date, depth, mock + ) + + if run_x: + if progress: + progress.start_x() + x_future = executor.submit( + _search_x, topic, config, selected_models, + from_date, to_date, depth, mock + ) + + # Collect results + if reddit_future: try: - raw_openai = openai_reddit.search_reddit( - config["OPENAI_API_KEY"], - selected_models["openai"], - topic, - from_date, - to_date, - depth=depth, - ) - except http.HTTPError as e: - if progress: - progress.show_error(f"Reddit API failed: {e}") - raw_openai = {"error": str(e)} - reddit_error = f"API error: {e}" + reddit_items, raw_openai, reddit_error = reddit_future.result() + if reddit_error and progress: + progress.show_error(f"Reddit error: {reddit_error}") except Exception as e: + reddit_error = f"{type(e).__name__}: {e}" if progress: progress.show_error(f"Reddit error: {e}") - raw_openai = {"error": str(e)} - reddit_error = f"{type(e).__name__}: {e}" - - # Parse response - reddit_items = openai_reddit.parse_reddit_response(raw_openai) - - # Quick retry with simpler query if few results - if len(reddit_items) < 5 and not mock: - core = openai_reddit._extract_core_subject(topic) - if core.lower() != topic.lower(): - try: - retry_raw = openai_reddit.search_reddit( - config["OPENAI_API_KEY"], - selected_models["openai"], - core, - from_date, to_date, - depth=depth, - ) - retry_items = openai_reddit.parse_reddit_response(retry_raw) - # Add items not already found (by URL) - existing_urls = {item.get("url") for item in reddit_items} - for item in retry_items: - if item.get("url") not in existing_urls: - reddit_items.append(item) - except Exception: - pass - - if progress: - progress.end_reddit(len(reddit_items)) - - # Enrich with real Reddit data - if reddit_items: if progress: - progress.start_reddit_enrich(1, len(reddit_items)) + progress.end_reddit(len(reddit_items)) - for i, item in enumerate(reddit_items): - if progress and i > 0: - progress.update_reddit_enrich(i + 1, len(reddit_items)) + if x_future: + try: + x_items, raw_xai, x_error = x_future.result() + if x_error and progress: + progress.show_error(f"X error: {x_error}") + except Exception as e: + x_error = f"{type(e).__name__}: {e}" + if progress: + progress.show_error(f"X error: {e}") + if progress: + progress.end_x(len(x_items)) + # Enrich Reddit items with real data (sequential, but with error handling per-item) + if reddit_items: + if progress: + progress.start_reddit_enrich(1, len(reddit_items)) + + for i, item in enumerate(reddit_items): + if progress and i > 0: + progress.update_reddit_enrich(i + 1, len(reddit_items)) + + try: if mock: mock_thread = load_fixture("reddit_thread_sample.json") reddit_items[i] = reddit_enrich.enrich_reddit_item(item, mock_thread) else: reddit_items[i] = reddit_enrich.enrich_reddit_item(item) - - raw_reddit_enriched.append(reddit_items[i]) - - if progress: - progress.end_reddit_enrich() - - # X search via xAI - if sources in ("both", "x", "all", "x-web"): - if progress: - progress.start_x() - - if mock: - raw_xai = load_fixture("xai_sample.json") - else: - try: - raw_xai = xai_x.search_x( - config["XAI_API_KEY"], - selected_models["xai"], - topic, - from_date, - to_date, - depth=depth, - ) - except http.HTTPError as e: - if progress: - progress.show_error(f"X API failed: {e}") - raw_xai = {"error": str(e)} - x_error = f"API error: {e}" except Exception as e: + # Log but don't crash - keep the unenriched item if progress: - progress.show_error(f"X error: {e}") - raw_xai = {"error": str(e)} - x_error = f"{type(e).__name__}: {e}" + progress.show_error(f"Enrich failed for {item.get('url', 'unknown')}: {e}") - # Parse response - x_items = xai_x.parse_x_response(raw_xai) + raw_reddit_enriched.append(reddit_items[i]) if progress: - progress.end_x(len(x_items)) + progress.end_reddit_enrich() return reddit_items, x_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error diff --git a/scripts/lib/http.py b/scripts/lib/http.py index a8e15be..ef737a9 100644 --- a/scripts/lib/http.py +++ b/scripts/lib/http.py @@ -102,6 +102,12 @@ def request( log(f"JSON decode error: {e}") last_error = HTTPError(f"Invalid JSON response: {e}") raise last_error + except (OSError, TimeoutError, ConnectionResetError) as e: + # Handle socket-level errors (connection reset, timeout, etc.) + log(f"Connection error: {type(e).__name__}: {e}") + last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}") + if attempt < retries - 1: + time.sleep(RETRY_DELAY * (attempt + 1)) if last_error: raise last_error diff --git a/scripts/lib/openai_reddit.py b/scripts/lib/openai_reddit.py index c7988fa..0d093de 100644 --- a/scripts/lib/openai_reddit.py +++ b/scripts/lib/openai_reddit.py @@ -110,8 +110,8 @@ def search_reddit( "Content-Type": "application/json", } - # Adjust timeout based on depth - timeout = 60 if depth == "quick" else 90 if depth == "default" else 120 + # Adjust timeout based on depth (generous for OpenAI web_search which can be slow) + timeout = 90 if depth == "quick" else 120 if depth == "default" else 180 # Note: allowed_domains accepts base domain, not subdomains # We rely on prompt to filter out developers.reddit.com, etc. diff --git a/scripts/lib/xai_x.py b/scripts/lib/xai_x.py index 4193d58..3642dac 100644 --- a/scripts/lib/xai_x.py +++ b/scripts/lib/xai_x.py @@ -88,8 +88,8 @@ def search_x( "Content-Type": "application/json", } - # Adjust timeout based on depth - timeout = 60 if depth == "quick" else 90 if depth == "default" else 120 + # Adjust timeout based on depth (generous for API response time) + timeout = 90 if depth == "quick" else 120 if depth == "default" else 180 # Use Agent Tools API with x_search tool payload = {