From fa42a5d03156726c8ded0b00af08b930057dd398 Mon Sep 17 00:00:00 2001
From: Jeffrey Sperling <jeffrey.sperling@gmail.com>
Date: Wed, 11 Mar 2026 15:08:29 -0700
Subject: [PATCH] Add urllib fallback for TikTok/Instagram when requests
 unavailable

Previously tiktok.py and instagram.py returned an error when the
requests library was not installed. Reddit already had an http.get()
fallback using stdlib urllib. Apply the same pattern so all three
ScrapeCreators modules work without requests installed.
---
 scripts/lib/instagram.py | 42 ++++++++++++++++++++++++++--------------
 scripts/lib/tiktok.py    | 42 ++++++++++++++++++++++++++--------------
 2 files changed, 54 insertions(+), 30 deletions(-)

diff --git a/scripts/lib/instagram.py b/scripts/lib/instagram.py
index 3861a93..bbf7e39 100644
--- a/scripts/lib/instagram.py
+++ b/scripts/lib/instagram.py
@@ -17,6 +17,8 @@ try:
 except ImportError:
     _requests = None
 
+from . import http
+
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
 
 # Depth configurations: how many results to fetch / captions to extract
@@ -207,26 +209,36 @@ def search_instagram(
     if not token:
         return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
 
-    if not _requests:
-        return {"items": [], "error": "requests library not installed"}
-
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     core_topic = _extract_core_subject(topic)
 
     _log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
-    try:
-        resp = _requests.get(
-            f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
-            params={"query": core_topic},
-            headers=_sc_headers(token),
-            timeout=30,
-        )
-        resp.raise_for_status()
-        data = resp.json()
-    except Exception as e:
-        _log(f"ScrapeCreators error: {e}")
-        return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    if not _requests:
+        _log("requests library not installed, falling back to urllib")
+        try:
+            from urllib.parse import urlencode
+            params = urlencode({"query": core_topic})
+            url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
+            headers = _sc_headers(token)
+            headers["User-Agent"] = http.USER_AGENT
+            data = http.get(url, headers=headers, timeout=30, retries=2)
+        except Exception as e:
+            _log(f"ScrapeCreators error (urllib): {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    else:
+        try:
+            resp = _requests.get(
+                f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
+                params={"query": core_topic},
+                headers=_sc_headers(token),
+                timeout=30,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception as e:
+            _log(f"ScrapeCreators error: {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Items are in the 'reels' array (ScrapeCreators v2 response)
     raw_items = data.get("reels") or data.get("items") or data.get("data") or []
diff --git a/scripts/lib/tiktok.py b/scripts/lib/tiktok.py
index 3f5f704..2459ba2 100644
--- a/scripts/lib/tiktok.py
+++ b/scripts/lib/tiktok.py
@@ -17,6 +17,8 @@ try:
 except ImportError:
     _requests = None
 
+from . import http
+
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
 
 # Depth configurations: how many results to fetch / captions to extract
@@ -204,26 +206,36 @@ def search_tiktok(
     if not token:
         return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
 
-    if not _requests:
-        return {"items": [], "error": "requests library not installed"}
-
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     core_topic = _extract_core_subject(topic)
 
     _log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
-    try:
-        resp = _requests.get(
-            f"{SCRAPECREATORS_BASE}/search/keyword",
-            params={"query": core_topic, "sort_by": "relevance"},
-            headers=_sc_headers(token),
-            timeout=30,
-        )
-        resp.raise_for_status()
-        data = resp.json()
-    except Exception as e:
-        _log(f"ScrapeCreators error: {e}")
-        return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    if not _requests:
+        _log("requests library not installed, falling back to urllib")
+        try:
+            from urllib.parse import urlencode
+            params = urlencode({"query": core_topic, "sort_by": "relevance"})
+            url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
+            headers = _sc_headers(token)
+            headers["User-Agent"] = http.USER_AGENT
+            data = http.get(url, headers=headers, timeout=30, retries=2)
+        except Exception as e:
+            _log(f"ScrapeCreators error (urllib): {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    else:
+        try:
+            resp = _requests.get(
+                f"{SCRAPECREATORS_BASE}/search/keyword",
+                params={"query": core_topic, "sort_by": "relevance"},
+                headers=_sc_headers(token),
+                timeout=30,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception as e:
+            _log(f"ScrapeCreators error: {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Items are nested under aweme_info
     raw_entries = data.get("search_item_list") or data.get("data") or []