refactor: drop requests dep, route all providers through lib/http urllib wrapper (#393)

Five provider modules (pinterest, threads, instagram, tiktok, youtube_yt)
and watchlist.py each carried a try/except `requests` import with parallel
urllib + requests branches. The urllib path already used the
stdlib-only wrapper at `lib/http.py` (retries, 429 handling, HTTPError).
This collapses every dual-branch into a single `http.get`/`http.post`
call and removes the `requests` dependency from `pyproject.toml`.

Also drops 4 transitive deps (urllib3, certifi, charset-normalizer, idna)
from the lockfile, leaving the skill stdlib-only at runtime.

Tests for tiktok comments and watchlist delivery were rewritten to mock
`lib.http` directly instead of the now-removed `requests` module.

Out of scope but flagged during review: the 13 surviving SC call sites
share a near-identical scaffold and would benefit from a
`http.scrapecreators_get(url, params, token, ...)` helper. Filed for a
follow-up PR rather than expanding scope here.
This commit is contained in:
Trevin Chow
2026-05-15 08:07:43 -07:00
committed by GitHub
parent c845f483d6
commit 80a1a47eef
10 changed files with 218 additions and 596 deletions
+3 -18
View File
@@ -154,14 +154,7 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
"total": 3,
}
class FakeResp:
def raise_for_status(self):
pass
def json(self):
return fake_sc_response
with patch.object(tiktok, "_requests") as mock_req:
mock_req.get.return_value = FakeResp()
with patch.object(tiktok.http, "get", return_value=fake_sc_response):
out = tiktok._fetch_post_comments(
"https://www.tiktok.com/@u/video/1",
token="k",
@@ -192,14 +185,7 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
"total": 3,
}
class FakeResp:
def raise_for_status(self):
pass
def json(self):
return fake_sc_response
with patch.object(tiktok, "_requests") as mock_req:
mock_req.get.return_value = FakeResp()
with patch.object(tiktok.http, "get", return_value=fake_sc_response):
out = tiktok._fetch_post_comments(
"https://www.tiktok.com/@u/video/1",
token="k",
@@ -217,8 +203,7 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
from unittest.mock import patch
from lib import tiktok
with patch.object(tiktok, "_requests") as mock_req:
mock_req.get.side_effect = Exception("429 rate limit")
with patch.object(tiktok.http, "get", side_effect=Exception("429 rate limit")):
out = tiktok._fetch_post_comments(
"https://www.tiktok.com/@u/video/1",
token="k",