refactor: drop requests dep, route all providers through lib/http urllib wrapper (#393)
Five provider modules (pinterest, threads, instagram, tiktok, youtube_yt) and watchlist.py each carried a try/except `requests` import with parallel urllib + requests branches. The urllib path already used the stdlib-only wrapper at `lib/http.py` (retries, 429 handling, HTTPError). This collapses every dual-branch into a single `http.get`/`http.post` call and removes the `requests` dependency from `pyproject.toml`. Also drops 4 transitive deps (urllib3, certifi, charset-normalizer, idna) from the lockfile, leaving the skill stdlib-only at runtime. Tests for tiktok comments and watchlist delivery were rewritten to mock `lib.http` directly instead of the now-removed `requests` module. Out of scope but flagged during review: the 13 surviving SC call sites share a near-identical scaffold and would benefit from a `http.scrapecreators_get(url, params, token, ...)` helper. Filed for a follow-up PR rather than expanding scope here.
This commit is contained in:
+3
-18
@@ -154,14 +154,7 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
|
||||
"total": 3,
|
||||
}
|
||||
|
||||
class FakeResp:
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
def json(self):
|
||||
return fake_sc_response
|
||||
|
||||
with patch.object(tiktok, "_requests") as mock_req:
|
||||
mock_req.get.return_value = FakeResp()
|
||||
with patch.object(tiktok.http, "get", return_value=fake_sc_response):
|
||||
out = tiktok._fetch_post_comments(
|
||||
"https://www.tiktok.com/@u/video/1",
|
||||
token="k",
|
||||
@@ -192,14 +185,7 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
|
||||
"total": 3,
|
||||
}
|
||||
|
||||
class FakeResp:
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
def json(self):
|
||||
return fake_sc_response
|
||||
|
||||
with patch.object(tiktok, "_requests") as mock_req:
|
||||
mock_req.get.return_value = FakeResp()
|
||||
with patch.object(tiktok.http, "get", return_value=fake_sc_response):
|
||||
out = tiktok._fetch_post_comments(
|
||||
"https://www.tiktok.com/@u/video/1",
|
||||
token="k",
|
||||
@@ -217,8 +203,7 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
|
||||
from unittest.mock import patch
|
||||
from lib import tiktok
|
||||
|
||||
with patch.object(tiktok, "_requests") as mock_req:
|
||||
mock_req.get.side_effect = Exception("429 rate limit")
|
||||
with patch.object(tiktok.http, "get", side_effect=Exception("429 rate limit")):
|
||||
out = tiktok._fetch_post_comments(
|
||||
"https://www.tiktok.com/@u/video/1",
|
||||
token="k",
|
||||
|
||||
Reference in New Issue
Block a user