Restructure as Codex plugin

2026-04-23 20:14:23 -04:00
parent 1f7e85a03f
commit 72495c1c14
156 changed files with 218 additions and 140 deletions
@@ -0,0 +1,204 @@
+"""HTTP utilities for last30days skill (stdlib only)."""
+
+import json
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Dict, Optional, Union
+from urllib.parse import urlencode
+
+from . import log as _log
+
+DEFAULT_TIMEOUT = 30
+
+
+def log(msg: str):
+    """Log debug message to stderr."""
+    _log.debug(msg)
+
+
+MAX_RETRIES = 5
+MAX_429_RETRIES = 2
+RETRY_DELAY = 2.0
+USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
+
+
+class HTTPError(Exception):
+    """HTTP request error with status code."""
+    def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.body = body
+
+
+def request(
+    method: str,
+    url: str,
+    headers: Optional[Dict[str, str]] = None,
+    json_data: Optional[Dict[str, Any]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    timeout: int = DEFAULT_TIMEOUT,
+    retries: int = MAX_RETRIES,
+    max_429_retries: int = MAX_429_RETRIES,
+    raw: bool = False,
+) -> Union[Dict[str, Any], str]:
+    """Make an HTTP request and return JSON response.
+
+    Args:
+        method: HTTP method (GET, POST, etc.)
+        url: Request URL
+        headers: Optional headers dict
+        json_data: Optional JSON body (for POST)
+        params: Optional query-string params. Values are stringified. None values
+            are dropped. If ``url`` already has a query string, ``params`` is appended.
+        timeout: Request timeout in seconds
+        retries: Number of retries on failure
+        max_429_retries: Maximum 429 retries before giving up (separate cap)
+        raw: If True, return raw response text instead of parsed JSON
+
+    Returns:
+        Parsed JSON response as dict, or raw text string if raw=True.
+
+    Raises:
+        HTTPError: On request failure
+    """
+    headers = headers or {}
+    headers.setdefault("User-Agent", USER_AGENT)
+
+    if params:
+        filtered = {k: str(v) for k, v in params.items() if v is not None}
+        if filtered:
+            separator = "&" if ("?" in url) else "?"
+            url = f"{url}{separator}{urlencode(filtered)}"
+
+    data = None
+    if json_data is not None:
+        data = json.dumps(json_data).encode('utf-8')
+        headers.setdefault("Content-Type", "application/json")
+
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+
+    safe_url = re.sub(r'([?&])(key|api_key|token|secret)=[^&]*', r'\1\2=***', url)
+    log(f"{method} {safe_url}")
+
+    last_error = None
+    rate_limit_count = 0
+    for attempt in range(retries):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as response:
+                body = response.read().decode('utf-8')
+                log(f"Response: {response.status} ({len(body)} bytes)")
+                if raw:
+                    return body
+                return json.loads(body) if body else {}
+        except urllib.error.HTTPError as e:
+            body = None
+            try:
+                body = e.read().decode('utf-8')
+            except (OSError, UnicodeDecodeError):
+                pass
+            log(f"HTTP Error {e.code}: {e.reason}")
+            if body:
+                snippet = " ".join(body.split())
+                log(f"Error body: {snippet[:200]}")
+            last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body)
+
+            # Don't retry client errors (4xx) except rate limits
+            if 400 <= e.code < 500 and e.code != 429:
+                raise last_error
+
+            # Cap 429 retries separately to avoid wasting latency
+            if e.code == 429:
+                rate_limit_count += 1
+                if rate_limit_count >= max_429_retries:
+                    raise last_error
+
+            if attempt < retries - 1:
+                if e.code == 429:
+                    # Respect Retry-After header, fall back to exponential backoff
+                    retry_after = e.headers.get("Retry-After") if hasattr(e, 'headers') else None
+                    if retry_after:
+                        try:
+                            delay = float(retry_after)
+                        except ValueError:
+                            delay = RETRY_DELAY * (2 ** attempt) + 1
+                    else:
+                        delay = RETRY_DELAY * (2 ** attempt) + 1  # 3s, 5s, 9s...
+                    log(f"Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 2}/{retries}")
+                else:
+                    delay = RETRY_DELAY * (2 ** attempt)
+                time.sleep(delay)
+        except urllib.error.URLError as e:
+            log(f"URL Error: {e.reason}")
+            last_error = HTTPError(f"URL Error: {e.reason}")
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
+        except json.JSONDecodeError as e:
+            log(f"JSON decode error: {e}")
+            last_error = HTTPError(f"Invalid JSON response: {e}")
+            raise last_error
+        except (OSError, TimeoutError, ConnectionResetError) as e:
+            # Handle socket-level errors (connection reset, timeout, etc.)
+            log(f"Connection error: {type(e).__name__}: {e}")
+            last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
+
+    if last_error:
+        raise last_error
+    raise HTTPError("Request failed with no error details")
+
+
+def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
+    """Make a GET request."""
+    return request("GET", url, headers=headers, **kwargs)
+
+
+def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
+    """Make a POST request with JSON body."""
+    return request("POST", url, headers=headers, json_data=json_data, **kwargs)
+
+
+def post_raw(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> str:
+    """Make a POST request with JSON body and return raw text."""
+    return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs)
+
+
+def scrapecreators_headers(token: str) -> Dict[str, str]:
+    """Build ScrapeCreators request headers (x-api-key + JSON content type)."""
+    return {
+        "x-api-key": token,
+        "Content-Type": "application/json",
+    }
+
+
+def get_reddit_json(path: str, timeout: int = DEFAULT_TIMEOUT, retries: int = MAX_RETRIES) -> Dict[str, Any]:
+    """Fetch Reddit thread JSON.
+
+    Args:
+        path: Reddit path (e.g., /r/subreddit/comments/id/title)
+        timeout: HTTP timeout per attempt in seconds
+        retries: Number of retries on failure
+
+    Returns:
+        Parsed JSON response
+    """
+    # Ensure path starts with /
+    if not path.startswith('/'):
+        path = '/' + path
+
+    # Remove trailing slash and add .json
+    path = path.rstrip('/')
+    if not path.endswith('.json'):
+        path = path + '.json'
+
+    url = f"https://www.reddit.com{path}?raw_json=1"
+
+    headers = {
+        "User-Agent": USER_AGENT,
+        "Accept": "application/json",
+    }
+
+    return get(url, headers=headers, timeout=timeout, retries=retries)