Run Reddit and X searches in parallel, fix timeout handling

- Refactor run_research() to use ThreadPoolExecutor for parallel execution
- Reddit timeout/crash no longer blocks X search from running
- Add catch for ConnectionResetError/OSError in http.py
- Per-item error handling in Reddit enrichment (one failure doesn't crash all)
- Increase API timeouts from 60/90/120 to 90/120/180 seconds
- Add ClawdBot setup example to README

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

README.md

@@ -140,6 +140,56 @@ This example shows /last30days as a **product research tool** - discovering real
 
 ---
 
+## Example: ClawdBot Setup (Installation Guide)
+
+**Query:** `/last30days how to best setup clawdbot`
+
+**Research Output:**
+> ClawdBot setup varies by deployment style - Docker is the most recommended path, but users are also deploying via bare metal and cloud VMs. The community has converged on key best practices: use the setup wizard for initial config, start with conservative hardware (you can scale up), and configure authentication before exposing any endpoints.
+
+**🛠️ Setup patterns by engagement:**
+
+| Pattern | Mentions | Key Sources |
+|---------|----------|-------------|
+| Docker Compose deployment | 6x | r/selfhosted, r/LocalLLaMA |
+| Setup wizard (interactive config) | 5x | @dreetje, documentation |
+| API key management | 4x | r/selfhosted, security threads |
+| Hardware requirements | 4x | r/LocalLLaMA (GPU discussions) |
+| Reverse proxy (Caddy/nginx) | 3x | r/selfhosted |
+
+**Key setup insights from community:**
+
+1. **Use the setup wizard** - Don't manually edit config files first; the wizard handles 90% of initial setup
+2. **Start small on hardware** - 8GB RAM minimum, but 16GB+ recommended if running local models
+3. **Docker > bare metal** - Easier updates, better isolation, community-tested compose files
+4. **Secure before exposing** - Set up authentication before opening ports; ClawdBot has built-in auth
+5. **Cost management** - Set API rate limits early; one user reported $200 overnight bill from runaway tasks
+
+**Quick-start commands (community-verified):**
+
+```bash
+# Clone and setup
+git clone https://github.com/clawdbot/clawdbot.git
+cd clawdbot
+
+# Run setup wizard (recommended)
+./setup.sh
+
+# Or Docker Compose (after config)
+docker compose up -d
+```
+
+**Common gotchas mentioned:**
+- Don't forget to set `CLAWDBOT_API_KEY` before first run
+- Telegram bot token needs BotFather setup first
+- If using local models, ensure CUDA drivers are installed
+
+**Research Stats:** 8 Reddit threads (128 upvotes) + 22 X posts (24,000+ likes) from r/selfhosted, r/LocalLLaMA, @dreetje, @steipete
+
+This example shows /last30days as a **setup guide aggregator** - pulling together scattered installation advice, gotchas, and best practices from real users who've already solved the problems you're about to encounter.
+
+---
+
 ## Example: iOS App Mockup (Nano Banana Pro)
 
 **Query:** `/last30days prompting tips for nano banana pro for ios designs`

scripts/last30days.py

@@ -18,6 +18,7 @@ import argparse
 import json
 import os
 import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -52,6 +53,111 @@ def load_fixture(name: str) -> dict:
     return {}
 
 
+def _search_reddit(
+    topic: str,
+    config: dict,
+    selected_models: dict,
+    from_date: str,
+    to_date: str,
+    depth: str,
+    mock: bool,
+) -> tuple:
+    """Search Reddit via OpenAI (runs in thread).
+
+    Returns:
+        Tuple of (reddit_items, raw_openai, error)
+    """
+    raw_openai = None
+    reddit_error = None
+
+    if mock:
+        raw_openai = load_fixture("openai_sample.json")
+    else:
+        try:
+            raw_openai = openai_reddit.search_reddit(
+                config["OPENAI_API_KEY"],
+                selected_models["openai"],
+                topic,
+                from_date,
+                to_date,
+                depth=depth,
+            )
+        except http.HTTPError as e:
+            raw_openai = {"error": str(e)}
+            reddit_error = f"API error: {e}"
+        except Exception as e:
+            raw_openai = {"error": str(e)}
+            reddit_error = f"{type(e).__name__}: {e}"
+
+    # Parse response
+    reddit_items = openai_reddit.parse_reddit_response(raw_openai or {})
+
+    # Quick retry with simpler query if few results
+    if len(reddit_items) < 5 and not mock and not reddit_error:
+        core = openai_reddit._extract_core_subject(topic)
+        if core.lower() != topic.lower():
+            try:
+                retry_raw = openai_reddit.search_reddit(
+                    config["OPENAI_API_KEY"],
+                    selected_models["openai"],
+                    core,
+                    from_date, to_date,
+                    depth=depth,
+                )
+                retry_items = openai_reddit.parse_reddit_response(retry_raw)
+                # Add items not already found (by URL)
+                existing_urls = {item.get("url") for item in reddit_items}
+                for item in retry_items:
+                    if item.get("url") not in existing_urls:
+                        reddit_items.append(item)
+            except Exception:
+                pass
+
+    return reddit_items, raw_openai, reddit_error
+
+
+def _search_x(
+    topic: str,
+    config: dict,
+    selected_models: dict,
+    from_date: str,
+    to_date: str,
+    depth: str,
+    mock: bool,
+) -> tuple:
+    """Search X via xAI (runs in thread).
+
+    Returns:
+        Tuple of (x_items, raw_xai, error)
+    """
+    raw_xai = None
+    x_error = None
+
+    if mock:
+        raw_xai = load_fixture("xai_sample.json")
+    else:
+        try:
+            raw_xai = xai_x.search_x(
+                config["XAI_API_KEY"],
+                selected_models["xai"],
+                topic,
+                from_date,
+                to_date,
+                depth=depth,
+            )
+        except http.HTTPError as e:
+            raw_xai = {"error": str(e)}
+            x_error = f"API error: {e}"
+        except Exception as e:
+            raw_xai = {"error": str(e)}
+            x_error = f"{type(e).__name__}: {e}"
+
+    # Parse response
+    x_items = xai_x.parse_x_response(raw_xai or {})
+
+    return x_items, raw_xai, x_error
+
+
 def run_research(
     topic: str,
     sources: str,
@@ -89,62 +195,58 @@ def run_research(
             progress.end_web_only()
         return reddit_items, x_items, True, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error
 
-    # Reddit search via OpenAI
+    # Determine which searches to run
-    if sources in ("both", "reddit", "all", "reddit-web"):
+    run_reddit = sources in ("both", "reddit", "all", "reddit-web")
+    run_x = sources in ("both", "x", "all", "x-web")
+
+    # Run Reddit and X searches in parallel
+    reddit_future = None
+    x_future = None
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        # Submit both searches
+        if run_reddit:
             if progress:
                 progress.start_reddit()
-
+            reddit_future = executor.submit(
-        if mock:
+                _search_reddit, topic, config, selected_models,
-            raw_openai = load_fixture("openai_sample.json")
+                from_date, to_date, depth, mock
-        else:
-            try:
-                raw_openai = openai_reddit.search_reddit(
-                    config["OPENAI_API_KEY"],
-                    selected_models["openai"],
-                    topic,
-                    from_date,
-                    to_date,
-                    depth=depth,
             )
-            except http.HTTPError as e:
+
+        if run_x:
             if progress:
-                    progress.show_error(f"Reddit API failed: {e}")
+                progress.start_x()
-                raw_openai = {"error": str(e)}
+            x_future = executor.submit(
-                reddit_error = f"API error: {e}"
+                _search_x, topic, config, selected_models,
+                from_date, to_date, depth, mock
+            )
+
+        # Collect results
+        if reddit_future:
+            try:
+                reddit_items, raw_openai, reddit_error = reddit_future.result()
+                if reddit_error and progress:
+                    progress.show_error(f"Reddit error: {reddit_error}")
             except Exception as e:
+                reddit_error = f"{type(e).__name__}: {e}"
                 if progress:
                     progress.show_error(f"Reddit error: {e}")
-                raw_openai = {"error": str(e)}
-                reddit_error = f"{type(e).__name__}: {e}"
-
-        # Parse response
-        reddit_items = openai_reddit.parse_reddit_response(raw_openai)
-
-        # Quick retry with simpler query if few results
-        if len(reddit_items) < 5 and not mock:
-            core = openai_reddit._extract_core_subject(topic)
-            if core.lower() != topic.lower():
-                try:
-                    retry_raw = openai_reddit.search_reddit(
-                        config["OPENAI_API_KEY"],
-                        selected_models["openai"],
-                        core,
-                        from_date, to_date,
-                        depth=depth,
-                    )
-                    retry_items = openai_reddit.parse_reddit_response(retry_raw)
-                    # Add items not already found (by URL)
-                    existing_urls = {item.get("url") for item in reddit_items}
-                    for item in retry_items:
-                        if item.get("url") not in existing_urls:
-                            reddit_items.append(item)
-                except Exception:
-                    pass
-
             if progress:
                 progress.end_reddit(len(reddit_items))
 
-        # Enrich with real Reddit data
+        if x_future:
+            try:
+                x_items, raw_xai, x_error = x_future.result()
+                if x_error and progress:
+                    progress.show_error(f"X error: {x_error}")
+            except Exception as e:
+                x_error = f"{type(e).__name__}: {e}"
+                if progress:
+                    progress.show_error(f"X error: {e}")
+            if progress:
+                progress.end_x(len(x_items))
+
+    # Enrich Reddit items with real data (sequential, but with error handling per-item)
     if reddit_items:
         if progress:
             progress.start_reddit_enrich(1, len(reddit_items))
@@ -153,51 +255,22 @@ def run_research(
             if progress and i > 0:
                 progress.update_reddit_enrich(i + 1, len(reddit_items))
 
+            try:
                 if mock:
                     mock_thread = load_fixture("reddit_thread_sample.json")
                     reddit_items[i] = reddit_enrich.enrich_reddit_item(item, mock_thread)
                 else:
                     reddit_items[i] = reddit_enrich.enrich_reddit_item(item)
+            except Exception as e:
+                # Log but don't crash - keep the unenriched item
+                if progress:
+                    progress.show_error(f"Enrich failed for {item.get('url', 'unknown')}: {e}")
 
             raw_reddit_enriched.append(reddit_items[i])
 
         if progress:
             progress.end_reddit_enrich()
 
-    # X search via xAI
-    if sources in ("both", "x", "all", "x-web"):
-        if progress:
-            progress.start_x()
-
-        if mock:
-            raw_xai = load_fixture("xai_sample.json")
-        else:
-            try:
-                raw_xai = xai_x.search_x(
-                    config["XAI_API_KEY"],
-                    selected_models["xai"],
-                    topic,
-                    from_date,
-                    to_date,
-                    depth=depth,
-                )
-            except http.HTTPError as e:
-                if progress:
-                    progress.show_error(f"X API failed: {e}")
-                raw_xai = {"error": str(e)}
-                x_error = f"API error: {e}"
-            except Exception as e:
-                if progress:
-                    progress.show_error(f"X error: {e}")
-                raw_xai = {"error": str(e)}
-                x_error = f"{type(e).__name__}: {e}"
-
-        # Parse response
-        x_items = xai_x.parse_x_response(raw_xai)
-
-        if progress:
-            progress.end_x(len(x_items))
-
     return reddit_items, x_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error
 
 

scripts/lib/http.py

@@ -102,6 +102,12 @@ def request(
             log(f"JSON decode error: {e}")
             last_error = HTTPError(f"Invalid JSON response: {e}")
             raise last_error
+        except (OSError, TimeoutError, ConnectionResetError) as e:
+            # Handle socket-level errors (connection reset, timeout, etc.)
+            log(f"Connection error: {type(e).__name__}: {e}")
+            last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
 
     if last_error:
         raise last_error

scripts/lib/openai_reddit.py

@@ -110,8 +110,8 @@ def search_reddit(
         "Content-Type": "application/json",
     }
 
-    # Adjust timeout based on depth
+    # Adjust timeout based on depth (generous for OpenAI web_search which can be slow)
-    timeout = 60 if depth == "quick" else 90 if depth == "default" else 120
+    timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
 
     # Note: allowed_domains accepts base domain, not subdomains
     # We rely on prompt to filter out developers.reddit.com, etc.

scripts/lib/xai_x.py

@@ -88,8 +88,8 @@ def search_x(
         "Content-Type": "application/json",
     }
 
-    # Adjust timeout based on depth
+    # Adjust timeout based on depth (generous for API response time)
-    timeout = 60 if depth == "quick" else 90 if depth == "default" else 120
+    timeout = 90 if depth == "quick" else 120 if depth == "default" else 180
 
     # Use Agent Tools API with x_search tool
     payload = {