Merge pull request #207 from iliaal/refactor/reddit-http-helper

refactor(reddit): migrate to http.get(params=...) helper

scripts/lib/http.py

@@ -38,6 +38,7 @@ def request(
     url: str,
     headers: Optional[Dict[str, str]] = None,
     json_data: Optional[Dict[str, Any]] = None,
+    params: Optional[Dict[str, Any]] = None,
     timeout: int = DEFAULT_TIMEOUT,
     retries: int = MAX_RETRIES,
     max_429_retries: int = MAX_429_RETRIES,
@@ -50,6 +51,8 @@ def request(
         url: Request URL
         headers: Optional headers dict
         json_data: Optional JSON body (for POST)
+        params: Optional query-string params. Values are stringified. None values
+            are dropped. If ``url`` already has a query string, ``params`` is appended.
         timeout: Request timeout in seconds
         retries: Number of retries on failure
         max_429_retries: Maximum 429 retries before giving up (separate cap)
@@ -64,6 +67,12 @@ def request(
     headers = headers or {}
     headers.setdefault("User-Agent", USER_AGENT)
 
+    if params:
+        filtered = {k: str(v) for k, v in params.items() if v is not None}
+        if filtered:
+            separator = "&" if ("?" in url) else "?"
+            url = f"{url}{separator}{urlencode(filtered)}"
+
     data = None
     if json_data is not None:
         data = json.dumps(json_data).encode('utf-8')

scripts/lib/reddit.py

@@ -15,12 +15,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, wait as futures
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Set
 
-try:
-    import requests as _requests
-except ImportError:
-    _requests = None
-
-
 def _first_of(*values, default=None):
     """Return first value that is not None."""
     for v in values:
@@ -350,39 +344,18 @@ def _global_search(
     Returns:
         List of post dicts
     """
-    if not _requests:
-        _log("requests library not installed, falling back to urllib")
-        # Use stdlib http module as fallback
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"query": query, "sort": sort, "timeframe": timeframe})
-            url = f"{SCRAPECREATORS_BASE}/search?{params}"
-            headers = _sc_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-            return data.get("posts", data.get("data", []))
-        except http.HTTPError as e:
-            if e.status_code and e.status_code in (401, 403):
-                raise
-            _log(f"Global search error (urllib): {e}")
-            return []
-        except Exception as e:
-            _log(f"Global search error (urllib): {e}")
-            return []
-
     try:
-        resp = _requests.get(
+        data = http.get(
             f"{SCRAPECREATORS_BASE}/search",
-            params={"query": query, "sort": sort, "timeframe": timeframe},
             headers=_sc_headers(token),
+            params={"query": query, "sort": sort, "timeframe": timeframe},
             timeout=30,
+            retries=2,
         )
-        resp.raise_for_status()
-        data = resp.json()
         return data.get("posts", data.get("data", []))
-    except _requests.exceptions.HTTPError as e:
-        if e.response is not None and e.response.status_code in (401, 403):
-            raise http.HTTPError(f"Auth error: {e}", e.response.status_code)
+    except http.HTTPError as e:
+        if e.status_code in (401, 403):
+            raise
         _log(f"Global search error: {e}")
         return []
     except Exception as e:
@@ -409,36 +382,19 @@ def _subreddit_search(
     Returns:
         List of post dicts
     """
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({
-                "subreddit": subreddit, "query": query,
-                "sort": sort, "timeframe": timeframe,
-            })
-            url = f"{SCRAPECREATORS_BASE}/subreddit/search?{params}"
-            headers = _sc_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-            return data.get("posts", data.get("data", []))
-        except Exception as e:
-            _log(f"Subreddit search error (urllib) for r/{subreddit}: {e}")
-            return []
-
     try:
-        resp = _requests.get(
+        data = http.get(
             f"{SCRAPECREATORS_BASE}/subreddit/search",
+            headers=_sc_headers(token),
             params={
                 "subreddit": subreddit,
                 "query": query,
                 "sort": sort,
                 "timeframe": timeframe,
             },
-            headers=_sc_headers(token),
             timeout=30,
+            retries=2,
         )
-        resp.raise_for_status()
-        data = resp.json()
         return data.get("posts", data.get("data", []))
     except Exception as e:
         _log(f"Subreddit search error for r/{subreddit}: {e}")
@@ -458,28 +414,14 @@ def fetch_post_comments(
     Returns:
         List of comment dicts with score, author, body, etc.
     """
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"url": url})
-            api_url = f"{SCRAPECREATORS_BASE}/post/comments?{params}"
-            headers = _sc_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(api_url, headers=headers, timeout=30, retries=2)
-            return data.get("comments", data.get("data", []))
-        except Exception as e:
-            _log(f"Comment fetch error (urllib): {e}")
-            return []
-
     try:
-        resp = _requests.get(
+        data = http.get(
             f"{SCRAPECREATORS_BASE}/post/comments",
-            params={"url": url},
             headers=_sc_headers(token),
+            params={"url": url},
             timeout=30,
+            retries=2,
         )
-        resp.raise_for_status()
-        data = resp.json()
         return data.get("comments", data.get("data", []))
     except Exception as e:
         _log(f"Comment fetch error: {e}")