refactor: drop requests dep, route all providers through lib/http urllib wrapper (#393)

Five provider modules (pinterest, threads, instagram, tiktok, youtube_yt)
and watchlist.py each carried a try/except `requests` import with parallel
urllib + requests branches. The urllib path already used the
stdlib-only wrapper at `lib/http.py` (retries, 429 handling, HTTPError).
This collapses every dual-branch into a single `http.get`/`http.post`
call and removes the `requests` dependency from `pyproject.toml`.

Also drops 4 transitive deps (urllib3, certifi, charset-normalizer, idna)
from the lockfile, leaving the skill stdlib-only at runtime.

Tests for tiktok comments and watchlist delivery were rewritten to mock
`lib.http` directly instead of the now-removed `requests` module.

Out of scope but flagged during review: the 13 surviving SC call sites
share a near-identical scaffold and would benefit from a
`http.scrapecreators_get(url, params, token, ...)` helper. Filed for a
follow-up PR rather than expanding scope here.
This commit is contained in:
Trevin Chow
2026-05-15 08:07:43 -07:00
committed by GitHub
parent c845f483d6
commit 80a1a47eef
10 changed files with 218 additions and 596 deletions
+10 -29
View File
@@ -152,35 +152,16 @@ def search_threads(
_log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
try:
import requests as _requests
except ImportError:
_requests = None
if not _requests:
_log("requests library not installed, falling back to urllib")
try:
from urllib.parse import urlencode
params = urlencode({"keyword": core_topic})
url = f"{SCRAPECREATORS_BASE}/search?{params}"
headers = http.scrapecreators_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search",
params={"keyword": core_topic},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
data = http.get(
f"{SCRAPECREATORS_BASE}/search",
params={"keyword": core_topic},
headers=http.scrapecreators_headers(token),
timeout=30,
retries=2,
)
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Extract items from response (try common SC response shapes)
raw_items = (