feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources
v3 rewrites the search engine from the ground up: - Intelligent pre-research: resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching - GitHub person-mode: PR velocity, top repos by stars, release notes - GitHub project-mode: live star counts, README, releases, top issues - ELI5 mode: plain language synthesis, no jargon - 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, Threads, Pinterest, Perplexity, Bluesky, Web - Free Reddit comments via public JSON (no API key needed) - Fun judge v2: humor scoring baked into narrative - Cookie consent before browser scanning - 10,000 free ScrapeCreators calls - 1,012 tests Thank you to the community contributors whose issues and PRs shaped v3: @uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116), @thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78), @zl190 (#115), @hnshah (#84, #85, #86) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
+350
-1
@@ -5,11 +5,15 @@ and writes configuration. The actual wizard UI is SKILL.md-driven (the LLM
|
||||
presents it), but this module provides the detection and setup actions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -184,3 +188,348 @@ def get_setup_status_text(results: Dict[str, Any]) -> str:
|
||||
lines.append("Configuration saved. Future runs will auto-detect your browsers.")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenClaw server-side setup (no browser, JSON output)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_OPENCLAW_KEY_NAMES = [
|
||||
"SCRAPECREATORS_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"BRAVE_API_KEY",
|
||||
"EXA_API_KEY",
|
||||
"SERPER_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"AUTH_TOKEN",
|
||||
]
|
||||
|
||||
|
||||
def run_openclaw_setup(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Server-side setup probe: no cookies, just tool + key availability.
|
||||
|
||||
Returns a dict suitable for JSON output to stdout so that SKILL.md
|
||||
can present appropriate options to the user.
|
||||
"""
|
||||
yt_dlp = shutil.which("yt-dlp") is not None
|
||||
node = shutil.which("node") is not None
|
||||
python3 = shutil.which("python3") is not None
|
||||
|
||||
keys: Dict[str, bool] = {}
|
||||
for key_name in _OPENCLAW_KEY_NAMES:
|
||||
short = key_name.lower().replace("_api_key", "").replace("_key", "").replace("_token", "")
|
||||
# Normalize: AUTH_TOKEN -> auth, SCRAPECREATORS_API_KEY -> scrapecreators
|
||||
keys[short] = bool(config.get(key_name))
|
||||
|
||||
# Determine x_method
|
||||
if config.get("XAI_API_KEY"):
|
||||
x_method: Optional[str] = "xai"
|
||||
elif config.get("AUTH_TOKEN") and config.get("CT0"):
|
||||
x_method = "cookies"
|
||||
else:
|
||||
x_method = None
|
||||
|
||||
return {
|
||||
"yt_dlp": yt_dlp,
|
||||
"node": node,
|
||||
"python3": python3,
|
||||
"keys": keys,
|
||||
"x_method": x_method,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PAT auth flow (GitHub token via ScrapeCreators)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PAT_BASE = "https://api.scrapecreators.com/v1/github/pat"
|
||||
|
||||
|
||||
def auth_with_pat(github_token: str) -> Optional[Dict[str, Any]]:
|
||||
"""Authenticate with ScrapeCreators using a GitHub PAT.
|
||||
|
||||
POSTs the token to the PAT auth endpoint. ScrapeCreators verifies it
|
||||
against GitHub's API, creates/finds the account, and returns an API key.
|
||||
|
||||
Returns:
|
||||
Dict with api_key, github_username, etc. on success, None on failure.
|
||||
"""
|
||||
try:
|
||||
req = Request(f"{_PAT_BASE}/auth", data=b"", method="POST")
|
||||
req.add_header("Authorization", f"Bearer {github_token}")
|
||||
with urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read())
|
||||
except HTTPError as exc:
|
||||
if exc.code == 422:
|
||||
logger.warning("PAT auth: insufficient scope — user needs user:email")
|
||||
else:
|
||||
logger.warning("PAT auth failed: %s", exc)
|
||||
return None
|
||||
except (URLError, OSError) as exc:
|
||||
logger.warning("PAT auth request failed: %s", exc)
|
||||
return None
|
||||
|
||||
if not data.get("api_key"):
|
||||
logger.warning("PAT auth returned no api_key: %s", data)
|
||||
return None
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Device auth flow (GitHub OAuth via ScrapeCreators)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEVICE_BASE = "https://api.scrapecreators.com/v1/github/device"
|
||||
|
||||
|
||||
def run_device_auth() -> Optional[Tuple[str, str, str, int]]:
|
||||
"""Start the device authorization flow.
|
||||
|
||||
POSTs to the ScrapeCreators device/code endpoint.
|
||||
|
||||
Returns:
|
||||
(device_code, user_code, verification_uri, interval) on success,
|
||||
None on failure.
|
||||
"""
|
||||
try:
|
||||
body = json.dumps({}).encode()
|
||||
req = Request(f"{_DEVICE_BASE}/code", data=body, method="POST")
|
||||
req.add_header("Content-Type", "application/json")
|
||||
with urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read())
|
||||
except (HTTPError, URLError, OSError) as exc:
|
||||
logger.warning("Device auth code request failed: %s", exc)
|
||||
return None
|
||||
|
||||
device_code = data.get("device_code")
|
||||
user_code = data.get("user_code")
|
||||
verification_uri = data.get("verification_uri")
|
||||
interval = data.get("interval", 5)
|
||||
|
||||
if not device_code or not user_code:
|
||||
logger.warning("Device auth returned incomplete response: %s", data)
|
||||
return None
|
||||
|
||||
return (device_code, user_code, verification_uri or "", interval)
|
||||
|
||||
|
||||
def poll_device_auth(
|
||||
device_code: str,
|
||||
interval: int,
|
||||
timeout: int = 300,
|
||||
user_code: str = "",
|
||||
clipboard_ok: bool = False,
|
||||
) -> Optional[str]:
|
||||
"""Poll for an access token after the user authorizes the device.
|
||||
|
||||
Args:
|
||||
device_code: The device_code from run_device_auth().
|
||||
interval: Polling interval in seconds.
|
||||
timeout: Maximum time to poll in seconds.
|
||||
user_code: The user code to remind about during polling.
|
||||
clipboard_ok: Whether the code was copied to clipboard.
|
||||
|
||||
Returns:
|
||||
access_token on success, None on timeout or failure.
|
||||
"""
|
||||
import sys
|
||||
|
||||
deadline = time.time() + timeout
|
||||
last_reminder = time.time()
|
||||
reminder_count = 0
|
||||
max_reminders = 4
|
||||
reminder_interval = 30 # seconds between reminders
|
||||
|
||||
while time.time() < deadline:
|
||||
time.sleep(interval)
|
||||
|
||||
# Periodic reminder of the code while waiting
|
||||
if (
|
||||
user_code
|
||||
and reminder_count < max_reminders
|
||||
and time.time() - last_reminder >= reminder_interval
|
||||
):
|
||||
clipboard_hint = " (on your clipboard)" if clipboard_ok else ""
|
||||
print(
|
||||
f" Still waiting... Your code: {user_code}{clipboard_hint}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
last_reminder = time.time()
|
||||
reminder_count += 1
|
||||
|
||||
try:
|
||||
body = json.dumps({"device_code": device_code}).encode()
|
||||
req = Request(f"{_DEVICE_BASE}/token", data=body, method="POST")
|
||||
req.add_header("Content-Type", "application/json")
|
||||
with urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read())
|
||||
except HTTPError as exc:
|
||||
if exc.code in (400, 403, 428):
|
||||
continue
|
||||
logger.warning("Device auth poll error: %s", exc)
|
||||
return None
|
||||
except (URLError, OSError):
|
||||
continue
|
||||
|
||||
if data.get("access_token"):
|
||||
return data["access_token"]
|
||||
|
||||
error = data.get("error")
|
||||
if error == "slow_down":
|
||||
interval = min(interval + 2, 30)
|
||||
continue
|
||||
if error == "authorization_pending":
|
||||
continue
|
||||
if error in ("expired_token", "access_denied"):
|
||||
logger.warning("Device auth failed: %s", error)
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def fetch_api_key(access_token: str) -> Optional[str]:
|
||||
"""Fetch the ScrapeCreators API key using the GitHub access token.
|
||||
|
||||
GETs the device/profile endpoint with Bearer auth.
|
||||
|
||||
Returns:
|
||||
api_key string on success, None on failure.
|
||||
"""
|
||||
try:
|
||||
req = Request(f"{_DEVICE_BASE}/profile")
|
||||
req.add_header("Authorization", f"Bearer {access_token}")
|
||||
with urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read())
|
||||
except (HTTPError, URLError, OSError) as exc:
|
||||
logger.warning("Failed to fetch API key: %s", exc)
|
||||
return None
|
||||
|
||||
return data.get("api_key")
|
||||
|
||||
|
||||
def run_full_device_auth(timeout: int = 300) -> Dict[str, Any]:
|
||||
"""Run the complete GitHub device auth flow and return JSON-serializable result.
|
||||
|
||||
Chains: start device flow -> open browser -> poll -> fetch API key.
|
||||
Designed to be called from the CLI and have its stdout parsed by the LLM.
|
||||
|
||||
Returns:
|
||||
Dict with status and relevant fields:
|
||||
- {"status": "success", "api_key": "sc_...", "user_code": "ABCD-1234"}
|
||||
- {"status": "error", "message": "..."}
|
||||
- {"status": "timeout", "user_code": "ABCD-1234"}
|
||||
- {"status": "denied"}
|
||||
"""
|
||||
import webbrowser
|
||||
|
||||
# Step 1: Start device flow
|
||||
result = run_device_auth()
|
||||
if result is None:
|
||||
return {"status": "error", "message": "Failed to start device auth flow"}
|
||||
|
||||
device_code, user_code, verification_uri, interval = result
|
||||
|
||||
import sys
|
||||
|
||||
# Step 2: Copy code to clipboard BEFORE opening browser
|
||||
clipboard_ok = False
|
||||
if sys.platform == "darwin":
|
||||
try:
|
||||
subprocess.run(
|
||||
["pbcopy"], input=user_code.encode(), check=True, timeout=5,
|
||||
)
|
||||
clipboard_ok = True
|
||||
except Exception:
|
||||
pass # pbcopy unavailable or failed, fall through
|
||||
|
||||
# Step 3: Show code prominently, then open browser
|
||||
clipboard_hint = " (copied to clipboard)" if clipboard_ok else ""
|
||||
code_line = f" Your code: {user_code}{clipboard_hint}"
|
||||
action_line = " Paste it on the GitHub page that just opened"
|
||||
width = max(len(code_line), len(action_line)) + 2
|
||||
border = "-" * width
|
||||
print(f"\n+{border}+", file=sys.stderr)
|
||||
print(f"|{code_line.ljust(width)}|", file=sys.stderr)
|
||||
print(f"|{action_line.ljust(width)}|", file=sys.stderr)
|
||||
print(f"+{border}+", file=sys.stderr)
|
||||
|
||||
if verification_uri:
|
||||
try:
|
||||
webbrowser.open(verification_uri)
|
||||
except Exception:
|
||||
print(f"Open: {verification_uri}", file=sys.stderr)
|
||||
|
||||
print("Waiting for authorization...", file=sys.stderr, flush=True)
|
||||
|
||||
# Step 4: Poll for token (with periodic code reminders)
|
||||
access_token = poll_device_auth(
|
||||
device_code, interval, timeout=timeout,
|
||||
user_code=user_code, clipboard_ok=clipboard_ok,
|
||||
)
|
||||
if access_token is None:
|
||||
return {"status": "timeout", "user_code": user_code, "clipboard_ok": clipboard_ok}
|
||||
|
||||
# Step 4: Fetch API key
|
||||
api_key = fetch_api_key(access_token)
|
||||
if api_key is None:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Authorized but failed to fetch API key",
|
||||
"clipboard_ok": clipboard_ok,
|
||||
}
|
||||
|
||||
return {"status": "success", "method": "device", "api_key": api_key, "user_code": user_code, "clipboard_ok": clipboard_ok}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unified GitHub auth: PAT first, device flow fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_github_auth(timeout: int = 300) -> Dict[str, Any]:
|
||||
"""Try PAT auth via gh CLI, fall back to device flow.
|
||||
|
||||
1. Check for `gh` CLI
|
||||
2. If found, run `gh auth token` to get a PAT
|
||||
3. POST PAT to ScrapeCreators — if it works, done
|
||||
4. If PAT fails for any reason, fall through to device flow
|
||||
|
||||
Returns JSON-serializable dict with status, method, and api_key.
|
||||
"""
|
||||
import sys
|
||||
|
||||
# Step 1: Try PAT via gh CLI
|
||||
gh_path = shutil.which("gh")
|
||||
if gh_path:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["gh", "auth", "token"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
token = result.stdout.strip()
|
||||
print("Found gh CLI — trying PAT auth...", file=sys.stderr)
|
||||
pat_result = auth_with_pat(token)
|
||||
if pat_result and pat_result.get("api_key"):
|
||||
return {
|
||||
"status": "success",
|
||||
"method": "pat",
|
||||
"api_key": pat_result["api_key"],
|
||||
"github_username": pat_result.get("github_username", ""),
|
||||
}
|
||||
# PAT failed — might be insufficient scope
|
||||
print(
|
||||
"PAT auth didn't work (scope or endpoint issue). "
|
||||
"Falling back to GitHub device flow...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("gh auth token failed: %s", exc)
|
||||
|
||||
# Step 2: Fall back to device flow
|
||||
if not gh_path:
|
||||
print("gh CLI not found — using GitHub device flow...", file=sys.stderr)
|
||||
|
||||
return run_full_device_auth(timeout=timeout)
|
||||
|
||||
Reference in New Issue
Block a user