"""HTTP utilities for last30days skill (stdlib only).""" import json import re import sys import time import urllib.error import urllib.request from typing import Any, Dict, Optional, Union from urllib.parse import urlencode from . import log as _log DEFAULT_TIMEOUT = 30 def log(msg: str): """Log debug message to stderr.""" _log.debug(msg) MAX_RETRIES = 5 MAX_429_RETRIES = 2 RETRY_DELAY = 2.0 USER_AGENT = "last30days-skill/3.0 (Assistant Skill)" class HTTPError(Exception): """HTTP request error with status code.""" def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None): super().__init__(message) self.status_code = status_code self.body = body def request( method: str, url: str, headers: Optional[Dict[str, str]] = None, json_data: Optional[Dict[str, Any]] = None, params: Optional[Dict[str, Any]] = None, timeout: int = DEFAULT_TIMEOUT, retries: int = MAX_RETRIES, max_429_retries: int = MAX_429_RETRIES, raw: bool = False, ) -> Union[Dict[str, Any], str]: """Make an HTTP request and return JSON response. Args: method: HTTP method (GET, POST, etc.) url: Request URL headers: Optional headers dict json_data: Optional JSON body (for POST) params: Optional query-string params. Values are stringified. None values are dropped. If ``url`` already has a query string, ``params`` is appended. timeout: Request timeout in seconds retries: Number of retries on failure max_429_retries: Maximum 429 retries before giving up (separate cap) raw: If True, return raw response text instead of parsed JSON Returns: Parsed JSON response as dict, or raw text string if raw=True. Raises: HTTPError: On request failure """ headers = headers or {} headers.setdefault("User-Agent", USER_AGENT) if params: filtered = {k: str(v) for k, v in params.items() if v is not None} if filtered: separator = "&" if ("?" in url) else "?" url = f"{url}{separator}{urlencode(filtered)}" data = None if json_data is not None: data = json.dumps(json_data).encode('utf-8') headers.setdefault("Content-Type", "application/json") req = urllib.request.Request(url, data=data, headers=headers, method=method) safe_url = re.sub(r'([?&])(key|api_key|token|secret)=[^&]*', r'\1\2=***', url) log(f"{method} {safe_url}") last_error = None rate_limit_count = 0 for attempt in range(retries): try: with urllib.request.urlopen(req, timeout=timeout) as response: body = response.read().decode('utf-8') log(f"Response: {response.status} ({len(body)} bytes)") if raw: return body return json.loads(body) if body else {} except urllib.error.HTTPError as e: body = None try: body = e.read().decode('utf-8') except (OSError, UnicodeDecodeError): pass log(f"HTTP Error {e.code}: {e.reason}") if body: snippet = " ".join(body.split()) log(f"Error body: {snippet[:200]}") last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body) # Don't retry client errors (4xx) except rate limits if 400 <= e.code < 500 and e.code != 429: raise last_error # Cap 429 retries separately to avoid wasting latency if e.code == 429: rate_limit_count += 1 if rate_limit_count >= max_429_retries: raise last_error if attempt < retries - 1: if e.code == 429: # Respect Retry-After header, fall back to exponential backoff retry_after = e.headers.get("Retry-After") if hasattr(e, 'headers') else None if retry_after: try: delay = float(retry_after) except ValueError: delay = RETRY_DELAY * (2 ** attempt) + 1 else: delay = RETRY_DELAY * (2 ** attempt) + 1 # 3s, 5s, 9s... log(f"Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 2}/{retries}") else: delay = RETRY_DELAY * (2 ** attempt) time.sleep(delay) except urllib.error.URLError as e: log(f"URL Error: {e.reason}") last_error = HTTPError(f"URL Error: {e.reason}") if attempt < retries - 1: time.sleep(RETRY_DELAY * (attempt + 1)) except json.JSONDecodeError as e: log(f"JSON decode error: {e}") last_error = HTTPError(f"Invalid JSON response: {e}") raise last_error except (OSError, TimeoutError, ConnectionResetError) as e: # Handle socket-level errors (connection reset, timeout, etc.) log(f"Connection error: {type(e).__name__}: {e}") last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}") if attempt < retries - 1: time.sleep(RETRY_DELAY * (attempt + 1)) if last_error: raise last_error raise HTTPError("Request failed with no error details") def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: """Make a GET request.""" return request("GET", url, headers=headers, **kwargs) def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: """Make a POST request with JSON body.""" return request("POST", url, headers=headers, json_data=json_data, **kwargs) def post_raw(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> str: """Make a POST request with JSON body and return raw text.""" return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs) def scrapecreators_headers(token: str) -> Dict[str, str]: """Build ScrapeCreators request headers (x-api-key + JSON content type).""" return { "x-api-key": token, "Content-Type": "application/json", } def get_reddit_json(path: str, timeout: int = DEFAULT_TIMEOUT, retries: int = MAX_RETRIES) -> Dict[str, Any]: """Fetch Reddit thread JSON. Args: path: Reddit path (e.g., /r/subreddit/comments/id/title) timeout: HTTP timeout per attempt in seconds retries: Number of retries on failure Returns: Parsed JSON response """ # Ensure path starts with / if not path.startswith('/'): path = '/' + path # Remove trailing slash and add .json path = path.rstrip('/') if not path.endswith('.json'): path = path + '.json' url = f"https://www.reddit.com{path}?raw_json=1" headers = { "User-Agent": USER_AGENT, "Accept": "application/json", } return get(url, headers=headers, timeout=timeout, retries=retries)