Trevin Chow
719cdef2fb
PR #382 introduced an `effective_retries` widening on the first gaierror, but the widening leaked: every non-DNS error path (HTTPError, non-DNS URLError, OSError) was gated on `effective_retries - 1` and so inherited the expanded bound. A caller passing `retries=2` who hit DNS-then-non-DNS got 3 attempts instead of 2 — contrary to the PR description and the fail-fast intent of small retry budgets. Fix: - Gate every non-DNS sleep/retry decision on the caller's original `retries`, not the widened `effective_retries`. - Add an explicit `break` in each non-DNS branch when the original budget is exhausted, so the (possibly widened) outer loop bound can't pull us into an extra attempt. Adds two regression tests covering the DNS-then-non-DNS-URLError and DNS-then-OSError sequences flagged in Greptile review on PR #382.
262 lines
10 KiB
Python
262 lines
10 KiB
Python
"""HTTP utilities for last30days skill (stdlib only)."""
|
|
|
|
import json
|
|
import re
|
|
import socket
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from typing import Any, Dict, Optional, Union
|
|
from urllib.parse import urlencode
|
|
|
|
from . import log as _log
|
|
|
|
DEFAULT_TIMEOUT = 30
|
|
|
|
|
|
def log(msg: str):
|
|
"""Log debug message to stderr."""
|
|
_log.debug(msg)
|
|
|
|
|
|
MAX_RETRIES = 5
|
|
MAX_429_RETRIES = 2
|
|
RETRY_DELAY = 2.0
|
|
# DNS resolution failures (gaierror) are transient — typically resolved by a
|
|
# brief backoff and retry. Use a dedicated minimum attempt count + exponential
|
|
# delays (1s, 2s, 4s) so callers that pass a small `retries` value still get a
|
|
# meaningful chance to recover from a transient resolution failure.
|
|
MIN_DNS_RETRIES = 3
|
|
USER_AGENT = "last30days-skill/3.0 (Assistant Skill)"
|
|
|
|
|
|
def _is_dns_failure(err: urllib.error.URLError) -> bool:
|
|
"""Return True if a URLError was caused by DNS resolution (gaierror)."""
|
|
return isinstance(getattr(err, "reason", None), socket.gaierror)
|
|
|
|
|
|
class HTTPError(Exception):
|
|
"""HTTP request error with status code."""
|
|
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.body = body
|
|
|
|
|
|
def request(
|
|
method: str,
|
|
url: str,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
json_data: Optional[Dict[str, Any]] = None,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
timeout: int = DEFAULT_TIMEOUT,
|
|
retries: int = MAX_RETRIES,
|
|
max_429_retries: int = MAX_429_RETRIES,
|
|
raw: bool = False,
|
|
) -> Union[Dict[str, Any], str]:
|
|
"""Make an HTTP request and return JSON response.
|
|
|
|
Args:
|
|
method: HTTP method (GET, POST, etc.)
|
|
url: Request URL
|
|
headers: Optional headers dict
|
|
json_data: Optional JSON body (for POST)
|
|
params: Optional query-string params. Values are stringified. None values
|
|
are dropped. If ``url`` already has a query string, ``params`` is appended.
|
|
timeout: Request timeout in seconds
|
|
retries: Number of retries on failure
|
|
max_429_retries: Maximum 429 retries before giving up (separate cap)
|
|
raw: If True, return raw response text instead of parsed JSON
|
|
|
|
Returns:
|
|
Parsed JSON response as dict, or raw text string if raw=True.
|
|
|
|
Raises:
|
|
HTTPError: On request failure
|
|
"""
|
|
headers = headers or {}
|
|
headers.setdefault("User-Agent", USER_AGENT)
|
|
|
|
if params:
|
|
filtered = {k: str(v) for k, v in params.items() if v is not None}
|
|
if filtered:
|
|
separator = "&" if ("?" in url) else "?"
|
|
url = f"{url}{separator}{urlencode(filtered)}"
|
|
|
|
data = None
|
|
if json_data is not None:
|
|
data = json.dumps(json_data).encode('utf-8')
|
|
headers.setdefault("Content-Type", "application/json")
|
|
|
|
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
|
|
safe_url = re.sub(r'([?&])(key|api_key|token|secret)=[^&]*', r'\1\2=***', url)
|
|
log(f"{method} {safe_url}")
|
|
|
|
last_error = None
|
|
rate_limit_count = 0
|
|
# DNS failures get a dedicated minimum attempt count + exponential backoff.
|
|
# `effective_retries` is the actual loop bound; we expand it on the first
|
|
# gaierror if the caller passed a smaller `retries` value than MIN_DNS_RETRIES.
|
|
effective_retries = retries
|
|
dns_attempts = 0
|
|
attempt = 0
|
|
while attempt < effective_retries:
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
body = response.read().decode('utf-8')
|
|
log(f"Response: {response.status} ({len(body)} bytes)")
|
|
if raw:
|
|
return body
|
|
return json.loads(body) if body else {}
|
|
except urllib.error.HTTPError as e:
|
|
body = None
|
|
try:
|
|
body = e.read().decode('utf-8')
|
|
except (OSError, UnicodeDecodeError):
|
|
pass
|
|
log(f"HTTP Error {e.code}: {e.reason}")
|
|
if body:
|
|
snippet = " ".join(body.split())
|
|
log(f"Error body: {snippet[:200]}")
|
|
last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body)
|
|
|
|
# Don't retry client errors (4xx) except rate limits
|
|
if 400 <= e.code < 500 and e.code != 429:
|
|
raise last_error
|
|
|
|
# Cap 429 retries separately to avoid wasting latency
|
|
if e.code == 429:
|
|
rate_limit_count += 1
|
|
if rate_limit_count >= max_429_retries:
|
|
raise last_error
|
|
|
|
# HTTP errors respect the caller's original `retries`; only DNS
|
|
# failures get the widened `effective_retries` budget.
|
|
if attempt < retries - 1:
|
|
if e.code == 429:
|
|
# Respect Retry-After header, fall back to exponential backoff
|
|
retry_after = e.headers.get("Retry-After") if hasattr(e, 'headers') else None
|
|
if retry_after:
|
|
try:
|
|
delay = float(retry_after)
|
|
except ValueError:
|
|
delay = RETRY_DELAY * (2 ** attempt) + 1
|
|
else:
|
|
delay = RETRY_DELAY * (2 ** attempt) + 1 # 3s, 5s, 9s...
|
|
log(f"Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 2}/{retries}")
|
|
else:
|
|
delay = RETRY_DELAY * (2 ** attempt)
|
|
time.sleep(delay)
|
|
else:
|
|
# Caller's original retry budget exhausted; an earlier DNS
|
|
# failure may have widened `effective_retries`, but that
|
|
# widening is DNS-only — don't grant extra HTTP attempts.
|
|
break
|
|
except urllib.error.URLError as e:
|
|
log(f"URL Error: {e.reason}")
|
|
last_error = HTTPError(f"URL Error: {e.reason}")
|
|
if _is_dns_failure(e):
|
|
# DNS resolution failures are transient; expand the retry budget
|
|
# to MIN_DNS_RETRIES if the caller passed fewer, and use
|
|
# exponential backoff (1s, 2s, 4s, ...) instead of the linear
|
|
# default. Counts DNS attempts separately so other URLError
|
|
# causes don't bypass the regular retry budget.
|
|
dns_attempts += 1
|
|
if effective_retries < MIN_DNS_RETRIES:
|
|
log(
|
|
f"DNS resolution failed; expanding retry budget from "
|
|
f"{effective_retries} to {MIN_DNS_RETRIES}"
|
|
)
|
|
effective_retries = MIN_DNS_RETRIES
|
|
if attempt < effective_retries - 1:
|
|
delay = 2 ** (dns_attempts - 1) # 1s, 2s, 4s, 8s, ...
|
|
log(
|
|
f"DNS resolution failure (attempt {dns_attempts}); "
|
|
f"retrying in {delay:.1f}s"
|
|
)
|
|
time.sleep(delay)
|
|
elif attempt < retries - 1:
|
|
# Non-DNS URLError (e.g. ConnectionRefused) respects the
|
|
# caller's original retry budget, not the DNS-widened bound.
|
|
time.sleep(RETRY_DELAY * (attempt + 1))
|
|
else:
|
|
# Caller's original retry budget exhausted; an earlier DNS
|
|
# failure widening `effective_retries` does not carry over
|
|
# to non-DNS error paths.
|
|
break
|
|
except json.JSONDecodeError as e:
|
|
log(f"JSON decode error: {e}")
|
|
last_error = HTTPError(f"Invalid JSON response: {e}")
|
|
raise last_error
|
|
except (OSError, TimeoutError, ConnectionResetError) as e:
|
|
# Handle socket-level errors (connection reset, timeout, etc.)
|
|
log(f"Connection error: {type(e).__name__}: {e}")
|
|
last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
|
|
if attempt < retries - 1:
|
|
# Socket errors respect the caller's original retry budget.
|
|
time.sleep(RETRY_DELAY * (attempt + 1))
|
|
else:
|
|
# Original budget exhausted; DNS widening doesn't apply here.
|
|
break
|
|
|
|
attempt += 1
|
|
|
|
if last_error:
|
|
raise last_error
|
|
raise HTTPError("Request failed with no error details")
|
|
|
|
|
|
def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
|
|
"""Make a GET request."""
|
|
return request("GET", url, headers=headers, **kwargs)
|
|
|
|
|
|
def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
|
|
"""Make a POST request with JSON body."""
|
|
return request("POST", url, headers=headers, json_data=json_data, **kwargs)
|
|
|
|
|
|
def post_raw(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> str:
|
|
"""Make a POST request with JSON body and return raw text."""
|
|
return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs)
|
|
|
|
|
|
def scrapecreators_headers(token: str) -> Dict[str, str]:
|
|
"""Build ScrapeCreators request headers (x-api-key + JSON content type)."""
|
|
return {
|
|
"x-api-key": token,
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
|
|
def get_reddit_json(path: str, timeout: int = DEFAULT_TIMEOUT, retries: int = MAX_RETRIES) -> Dict[str, Any]:
|
|
"""Fetch Reddit thread JSON.
|
|
|
|
Args:
|
|
path: Reddit path (e.g., /r/subreddit/comments/id/title)
|
|
timeout: HTTP timeout per attempt in seconds
|
|
retries: Number of retries on failure
|
|
|
|
Returns:
|
|
Parsed JSON response
|
|
"""
|
|
# Ensure path starts with /
|
|
if not path.startswith('/'):
|
|
path = '/' + path
|
|
|
|
# Remove trailing slash and add .json
|
|
path = path.rstrip('/')
|
|
if not path.endswith('.json'):
|
|
path = path + '.json'
|
|
|
|
url = f"https://www.reddit.com{path}?raw_json=1"
|
|
|
|
headers = {
|
|
"User-Agent": USER_AGENT,
|
|
"Accept": "application/json",
|
|
}
|
|
|
|
return get(url, headers=headers, timeout=timeout, retries=retries)
|