Add urllib fallback for TikTok/Instagram when requests unavailable
Previously tiktok.py and instagram.py returned an error when the requests library was not installed. Reddit already had an http.get() fallback using stdlib urllib. Apply the same pattern so all three ScrapeCreators modules work without requests installed.
This commit is contained in:
@@ -17,6 +17,8 @@ try:
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import http
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||
|
||||
# Depth configurations: how many results to fetch / captions to extract
|
||||
@@ -207,14 +209,24 @@ def search_instagram(
|
||||
if not token:
|
||||
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
|
||||
|
||||
if not _requests:
|
||||
return {"items": [], "error": "requests library not installed"}
|
||||
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"query": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
|
||||
headers = _sc_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
|
||||
+15
-3
@@ -17,6 +17,8 @@ try:
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import http
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
|
||||
|
||||
# Depth configurations: how many results to fetch / captions to extract
|
||||
@@ -204,14 +206,24 @@ def search_tiktok(
|
||||
if not token:
|
||||
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
|
||||
|
||||
if not _requests:
|
||||
return {"items": [], "error": "requests library not installed"}
|
||||
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
_log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"query": core_topic, "sort_by": "relevance"})
|
||||
url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
|
||||
headers = _sc_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/search/keyword",
|
||||
|
||||
Reference in New Issue
Block a user