Add urllib fallback for TikTok/Instagram when requests unavailable
Previously tiktok.py and instagram.py returned an error when the requests library was not installed. Reddit already had an http.get() fallback using stdlib urllib. Apply the same pattern so all three ScrapeCreators modules work without requests installed.
This commit is contained in:
+27
-15
@@ -17,6 +17,8 @@ try:
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import http
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||
|
||||
# Depth configurations: how many results to fetch / captions to extract
|
||||
@@ -207,26 +209,36 @@ def search_instagram(
|
||||
if not token:
|
||||
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
|
||||
|
||||
if not _requests:
|
||||
return {"items": [], "error": "requests library not installed"}
|
||||
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=_sc_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"query": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
|
||||
headers = _sc_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=_sc_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Items are in the 'reels' array (ScrapeCreators v2 response)
|
||||
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
||||
|
||||
+27
-15
@@ -17,6 +17,8 @@ try:
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
from . import http
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
|
||||
|
||||
# Depth configurations: how many results to fetch / captions to extract
|
||||
@@ -204,26 +206,36 @@ def search_tiktok(
|
||||
if not token:
|
||||
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
|
||||
|
||||
if not _requests:
|
||||
return {"items": [], "error": "requests library not installed"}
|
||||
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
_log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/search/keyword",
|
||||
params={"query": core_topic, "sort_by": "relevance"},
|
||||
headers=_sc_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"query": core_topic, "sort_by": "relevance"})
|
||||
url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
|
||||
headers = _sc_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/search/keyword",
|
||||
params={"query": core_topic, "sort_by": "relevance"},
|
||||
headers=_sc_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Items are nested under aweme_info
|
||||
raw_entries = data.get("search_item_list") or data.get("data") or []
|
||||
|
||||
Reference in New Issue
Block a user