refactor(reddit): migrate to http.get(params=...) helper
Added params kwarg to http.request()/http.get() that urlencodes a dict into the query string. None values are dropped, ints and bools are stringified, and params append correctly if the URL already has a query string. Migrated reddit.py to use this helper for all three ScrapeCreators call sites (global search, subreddit search, post comments). Deleted the try/import requests/except ImportError fallback and the paired if not _requests: / else: branches. Six new http tests cover the params-encoding behavior. Net: reddit.py -70 lines. Behavior is identical - the existing http.py urllib implementation already had retry logic, 429 handling, and HTTPError types that are strictly better than the ad-hoc requests branches we deleted. 99 reddit tests pass. Live smoke test on a real ScrapeCreators run returned 12 threads with the same engagement data as before.
This commit is contained in:
@@ -38,6 +38,7 @@ def request(
|
||||
url: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
json_data: Optional[Dict[str, Any]] = None,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = DEFAULT_TIMEOUT,
|
||||
retries: int = MAX_RETRIES,
|
||||
max_429_retries: int = MAX_429_RETRIES,
|
||||
@@ -50,6 +51,8 @@ def request(
|
||||
url: Request URL
|
||||
headers: Optional headers dict
|
||||
json_data: Optional JSON body (for POST)
|
||||
params: Optional query-string params. Values are stringified. None values
|
||||
are dropped. If ``url`` already has a query string, ``params`` is appended.
|
||||
timeout: Request timeout in seconds
|
||||
retries: Number of retries on failure
|
||||
max_429_retries: Maximum 429 retries before giving up (separate cap)
|
||||
@@ -64,6 +67,12 @@ def request(
|
||||
headers = headers or {}
|
||||
headers.setdefault("User-Agent", USER_AGENT)
|
||||
|
||||
if params:
|
||||
filtered = {k: str(v) for k, v in params.items() if v is not None}
|
||||
if filtered:
|
||||
separator = "&" if ("?" in url) else "?"
|
||||
url = f"{url}{separator}{urlencode(filtered)}"
|
||||
|
||||
data = None
|
||||
if json_data is not None:
|
||||
data = json.dumps(json_data).encode('utf-8')
|
||||
|
||||
Reference in New Issue
Block a user