refactor: drop requests dep, route all providers through lib/http urllib wrapper (#393)
Five provider modules (pinterest, threads, instagram, tiktok, youtube_yt) and watchlist.py each carried a try/except `requests` import with parallel urllib + requests branches. The urllib path already used the stdlib-only wrapper at `lib/http.py` (retries, 429 handling, HTTPError). This collapses every dual-branch into a single `http.get`/`http.post` call and removes the `requests` dependency from `pyproject.toml`. Also drops 4 transitive deps (urllib3, certifi, charset-normalizer, idna) from the lockfile, leaving the skill stdlib-only at runtime. Tests for tiktok comments and watchlist delivery were rewritten to mock `lib.http` directly instead of the now-removed `requests` module. Out of scope but flagged during review: the 13 surviving SC call sites share a near-identical scaffold and would benefit from a `http.scrapecreators_get(url, params, token, ...)` helper. Filed for a follow-up PR rather than expanding scope here.
This commit is contained in:
@@ -152,35 +152,16 @@ def search_threads(
|
||||
_log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
if not _requests:
|
||||
_log("requests library not installed, falling back to urllib")
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
params = urlencode({"keyword": core_topic})
|
||||
url = f"{SCRAPECREATORS_BASE}/search?{params}"
|
||||
headers = http.scrapecreators_headers(token)
|
||||
headers["User-Agent"] = http.USER_AGENT
|
||||
data = http.get(url, headers=headers, timeout=30, retries=2)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error (urllib): {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
else:
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
data = http.get(
|
||||
f"{SCRAPECREATORS_BASE}/search",
|
||||
params={"keyword": core_topic},
|
||||
headers=http.scrapecreators_headers(token),
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Extract items from response (try common SC response shapes)
|
||||
raw_items = (
|
||||
|
||||
Reference in New Issue
Block a user