Ilia Alshanetsky
9ef9d38b90
Added params kwarg to http.request()/http.get() that urlencodes a dict into the query string. None values are dropped, ints and bools are stringified, and params append correctly if the URL already has a query string. Migrated reddit.py to use this helper for all three ScrapeCreators call sites (global search, subreddit search, post comments). Deleted the try/import requests/except ImportError fallback and the paired if not _requests: / else: branches. Six new http tests cover the params-encoding behavior. Net: reddit.py -70 lines. Behavior is identical - the existing http.py urllib implementation already had retry logic, 429 handling, and HTTPError types that are strictly better than the ad-hoc requests branches we deleted. 99 reddit tests pass. Live smoke test on a real ScrapeCreators run returned 12 threads with the same engagement data as before.
197 lines
6.8 KiB
Python
197 lines
6.8 KiB
Python
"""HTTP utilities for last30days skill (stdlib only)."""
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from typing import Any, Dict, Optional, Union
|
|
from urllib.parse import urlencode
|
|
|
|
from . import log as _log
|
|
|
|
DEFAULT_TIMEOUT = 30
|
|
|
|
|
|
def log(msg: str):
|
|
"""Log debug message to stderr."""
|
|
_log.debug(msg)
|
|
|
|
|
|
MAX_RETRIES = 5
|
|
MAX_429_RETRIES = 2
|
|
RETRY_DELAY = 2.0
|
|
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
|
|
|
|
|
class HTTPError(Exception):
|
|
"""HTTP request error with status code."""
|
|
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.body = body
|
|
|
|
|
|
def request(
|
|
method: str,
|
|
url: str,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
json_data: Optional[Dict[str, Any]] = None,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
timeout: int = DEFAULT_TIMEOUT,
|
|
retries: int = MAX_RETRIES,
|
|
max_429_retries: int = MAX_429_RETRIES,
|
|
raw: bool = False,
|
|
) -> Union[Dict[str, Any], str]:
|
|
"""Make an HTTP request and return JSON response.
|
|
|
|
Args:
|
|
method: HTTP method (GET, POST, etc.)
|
|
url: Request URL
|
|
headers: Optional headers dict
|
|
json_data: Optional JSON body (for POST)
|
|
params: Optional query-string params. Values are stringified. None values
|
|
are dropped. If ``url`` already has a query string, ``params`` is appended.
|
|
timeout: Request timeout in seconds
|
|
retries: Number of retries on failure
|
|
max_429_retries: Maximum 429 retries before giving up (separate cap)
|
|
raw: If True, return raw response text instead of parsed JSON
|
|
|
|
Returns:
|
|
Parsed JSON response as dict, or raw text string if raw=True.
|
|
|
|
Raises:
|
|
HTTPError: On request failure
|
|
"""
|
|
headers = headers or {}
|
|
headers.setdefault("User-Agent", USER_AGENT)
|
|
|
|
if params:
|
|
filtered = {k: str(v) for k, v in params.items() if v is not None}
|
|
if filtered:
|
|
separator = "&" if ("?" in url) else "?"
|
|
url = f"{url}{separator}{urlencode(filtered)}"
|
|
|
|
data = None
|
|
if json_data is not None:
|
|
data = json.dumps(json_data).encode('utf-8')
|
|
headers.setdefault("Content-Type", "application/json")
|
|
|
|
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
|
|
safe_url = re.sub(r'([?&])(key|api_key|token|secret)=[^&]*', r'\1\2=***', url)
|
|
log(f"{method} {safe_url}")
|
|
|
|
last_error = None
|
|
rate_limit_count = 0
|
|
for attempt in range(retries):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
body = response.read().decode('utf-8')
|
|
log(f"Response: {response.status} ({len(body)} bytes)")
|
|
if raw:
|
|
return body
|
|
return json.loads(body) if body else {}
|
|
except urllib.error.HTTPError as e:
|
|
body = None
|
|
try:
|
|
body = e.read().decode('utf-8')
|
|
except (OSError, UnicodeDecodeError):
|
|
pass
|
|
log(f"HTTP Error {e.code}: {e.reason}")
|
|
if body:
|
|
snippet = " ".join(body.split())
|
|
log(f"Error body: {snippet[:200]}")
|
|
last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body)
|
|
|
|
# Don't retry client errors (4xx) except rate limits
|
|
if 400 <= e.code < 500 and e.code != 429:
|
|
raise last_error
|
|
|
|
# Cap 429 retries separately to avoid wasting latency
|
|
if e.code == 429:
|
|
rate_limit_count += 1
|
|
if rate_limit_count >= max_429_retries:
|
|
raise last_error
|
|
|
|
if attempt < retries - 1:
|
|
if e.code == 429:
|
|
# Respect Retry-After header, fall back to exponential backoff
|
|
retry_after = e.headers.get("Retry-After") if hasattr(e, 'headers') else None
|
|
if retry_after:
|
|
try:
|
|
delay = float(retry_after)
|
|
except ValueError:
|
|
delay = RETRY_DELAY * (2 ** attempt) + 1
|
|
else:
|
|
delay = RETRY_DELAY * (2 ** attempt) + 1 # 3s, 5s, 9s...
|
|
log(f"Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 2}/{retries}")
|
|
else:
|
|
delay = RETRY_DELAY * (2 ** attempt)
|
|
time.sleep(delay)
|
|
except urllib.error.URLError as e:
|
|
log(f"URL Error: {e.reason}")
|
|
last_error = HTTPError(f"URL Error: {e.reason}")
|
|
if attempt < retries - 1:
|
|
time.sleep(RETRY_DELAY * (attempt + 1))
|
|
except json.JSONDecodeError as e:
|
|
log(f"JSON decode error: {e}")
|
|
last_error = HTTPError(f"Invalid JSON response: {e}")
|
|
raise last_error
|
|
except (OSError, TimeoutError, ConnectionResetError) as e:
|
|
# Handle socket-level errors (connection reset, timeout, etc.)
|
|
log(f"Connection error: {type(e).__name__}: {e}")
|
|
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
|
if attempt < retries - 1:
|
|
time.sleep(RETRY_DELAY * (attempt + 1))
|
|
|
|
if last_error:
|
|
raise last_error
|
|
raise HTTPError("Request failed with no error details")
|
|
|
|
|
|
def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
|
|
"""Make a GET request."""
|
|
return request("GET", url, headers=headers, **kwargs)
|
|
|
|
|
|
def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
|
|
"""Make a POST request with JSON body."""
|
|
return request("POST", url, headers=headers, json_data=json_data, **kwargs)
|
|
|
|
|
|
def post_raw(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> str:
|
|
"""Make a POST request with JSON body and return raw text."""
|
|
return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs)
|
|
|
|
|
|
def get_reddit_json(path: str, timeout: int = DEFAULT_TIMEOUT, retries: int = MAX_RETRIES) -> Dict[str, Any]:
|
|
"""Fetch Reddit thread JSON.
|
|
|
|
Args:
|
|
path: Reddit path (e.g., /r/subreddit/comments/id/title)
|
|
timeout: HTTP timeout per attempt in seconds
|
|
retries: Number of retries on failure
|
|
|
|
Returns:
|
|
Parsed JSON response
|
|
"""
|
|
# Ensure path starts with /
|
|
if not path.startswith('/'):
|
|
path = '/' + path
|
|
|
|
# Remove trailing slash and add .json
|
|
path = path.rstrip('/')
|
|
if not path.endswith('.json'):
|
|
path = path + '.json'
|
|
|
|
url = f"https://www.reddit.com{path}?raw_json=1"
|
|
|
|
headers = {
|
|
"User-Agent": USER_AGENT,
|
|
"Accept": "application/json",
|
|
}
|
|
|
|
return get(url, headers=headers, timeout=timeout, retries=retries)
|