refactor: drop requests dep, route all providers through lib/http urllib wrapper (#393)

Five provider modules (pinterest, threads, instagram, tiktok, youtube_yt)
and watchlist.py each carried a try/except `requests` import with parallel
urllib + requests branches. The urllib path already used the
stdlib-only wrapper at `lib/http.py` (retries, 429 handling, HTTPError).
This collapses every dual-branch into a single `http.get`/`http.post`
call and removes the `requests` dependency from `pyproject.toml`.

Also drops 4 transitive deps (urllib3, certifi, charset-normalizer, idna)
from the lockfile, leaving the skill stdlib-only at runtime.

Tests for tiktok comments and watchlist delivery were rewritten to mock
`lib.http` directly instead of the now-removed `requests` module.

Out of scope but flagged during review: the 13 surviving SC call sites
share a near-identical scaffold and would benefit from a
`http.scrapecreators_get(url, params, token, ...)` helper. Filed for a
follow-up PR rather than expanding scope here.

skills/last30days/scripts/lib/instagram.py

@@ -12,11 +12,6 @@ import sys
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Set
 
-try:
-    import requests as _requests
-except ImportError:
-    _requests = None
-
 from . import dates, http, log
 
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
@@ -236,30 +231,17 @@ def _user_reels(
     """
     _log(f"User reels: @{handle}")
     reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"handle": handle})
-            url = f"{reels_url}?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as e:
-            _log(f"User reels error (urllib) for @{handle}: {e}")
-            return []
-    else:
-        try:
-            resp = _requests.get(
-                reels_url,
-                params={"handle": handle},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as e:
-            _log(f"User reels error for @{handle}: {e}")
-            return []
+    try:
+        data = http.get(
+            reels_url,
+            params={"handle": handle},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"User reels error for @{handle}: {e}")
+        return []
 
     raw_items = data.get("items") or data.get("reels") or data.get("data") or []
     _log(f"  -> {len(raw_items)} reels from @{handle}")
@@ -293,31 +275,17 @@ def search_instagram(
 
     _log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
-    if not _requests:
-        _log("requests library not installed, falling back to urllib")
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"query": core_topic})
-            url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as e:
-            _log(f"ScrapeCreators error (urllib): {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
-                params={"query": core_topic},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as e:
-            _log(f"ScrapeCreators error: {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
+            params={"query": core_topic},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"ScrapeCreators error: {e}")
+        return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Items are in the 'reels' array (ScrapeCreators v2 response)
     raw_items = data.get("reels") or data.get("items") or data.get("data") or []
@@ -367,7 +335,7 @@ def fetch_captions(
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     max_captions = config["max_captions"]
 
-    if not video_items or not token or not _requests:
+    if not video_items or not token:
         return {}
 
     top_items = video_items[:max_captions]
@@ -392,26 +360,24 @@ def fetch_captions(
         if not url:
             continue
         try:
-            resp = _requests.get(
+            data = http.get(
                 f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
                 params={"url": url},
                 headers=http.scrapecreators_headers(token),
                 timeout=15,
+                retries=1,
             )
-            if resp.status_code == 200:
-                data = resp.json()
-                transcripts = data.get("transcripts") or []
-                if transcripts and isinstance(transcripts, list):
-                    # Combine all transcript segments
-                    transcript_text = " ".join(
-                        t.get("text", "") for t in transcripts
-                        if isinstance(t, dict) and t.get("text")
-                    )
-                    if transcript_text:
-                        words = transcript_text.split()
-                        if len(words) > CAPTION_MAX_WORDS:
-                            transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
-                        captions[vid] = transcript_text
+            transcripts = data.get("transcripts") or []
+            if transcripts and isinstance(transcripts, list):
+                transcript_text = " ".join(
+                    t.get("text", "") for t in transcripts
+                    if isinstance(t, dict) and t.get("text")
+                )
+                if transcript_text:
+                    words = transcript_text.split()
+                    if len(words) > CAPTION_MAX_WORDS:
+                        transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
+                    captions[vid] = transcript_text
         except Exception as e:
             _log(f"Transcript fetch failed for {vid}: {e}")
 

skills/last30days/scripts/lib/pinterest.py

@@ -11,11 +11,6 @@ import re
 import sys
 from typing import Any, Dict, List, Optional, Set
 
-try:
-    import requests as _requests
-except ImportError:
-    _requests = None
-
 from . import dates, http, log
 
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/pinterest"
@@ -140,31 +135,17 @@ def search_pinterest(
 
     _log(f"Searching Pinterest for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
-    if not _requests:
-        _log("requests library not installed, falling back to urllib")
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"keyword": core_topic})
-            url = f"{SCRAPECREATORS_BASE}/search?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as e:
-            _log(f"ScrapeCreators error (urllib): {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_BASE}/search",
-                params={"keyword": core_topic},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as e:
-            _log(f"ScrapeCreators error: {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/search",
+            params={"keyword": core_topic},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"ScrapeCreators error: {e}")
+        return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Extract items from response - try common SC response shapes
     raw_items = data.get("pins") or data.get("results") or data.get("data") or data.get("items") or []

skills/last30days/scripts/lib/threads.py

@@ -152,35 +152,16 @@ def search_threads(
     _log(f"Searching for '{core_topic}' (depth={depth}, limit={config['results']})")
 
     try:
-        import requests as _requests
-    except ImportError:
-        _requests = None
-
-    if not _requests:
-        _log("requests library not installed, falling back to urllib")
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"keyword": core_topic})
-            url = f"{SCRAPECREATORS_BASE}/search?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as e:
-            _log(f"ScrapeCreators error (urllib): {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_BASE}/search",
-                params={"keyword": core_topic},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as e:
-            _log(f"ScrapeCreators error: {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/search",
+            params={"keyword": core_topic},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"ScrapeCreators error: {e}")
+        return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Extract items from response (try common SC response shapes)
     raw_items = (

skills/last30days/scripts/lib/tiktok.py

@@ -11,11 +11,6 @@ import re
 import sys
 from typing import Any, Dict, List, Optional, Set
 
-try:
-    import requests as _requests
-except ImportError:
-    _requests = None
-
 from . import dates, http, log
 
 SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
@@ -214,30 +209,17 @@ def _hashtag_search(
         List of raw TikTok item dicts (aweme_info format).
     """
     _log(f"Hashtag search: #{hashtag}")
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"hashtag": hashtag})
-            url = f"{SCRAPECREATORS_BASE}/search/hashtag?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as e:
-            _log(f"Hashtag search error (urllib) for #{hashtag}: {e}")
-            return []
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_BASE}/search/hashtag",
-                params={"hashtag": hashtag},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as e:
-            _log(f"Hashtag search error for #{hashtag}: {e}")
-            return []
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/search/hashtag",
+            params={"hashtag": hashtag},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"Hashtag search error for #{hashtag}: {e}")
+        return []
 
     raw_items = data.get("aweme_list") or data.get("data") or []
     _log(f"  -> {len(raw_items)} results for #{hashtag}")
@@ -261,30 +243,17 @@ def _profile_videos(
     """
     _log(f"Profile videos: @{handle}")
     profile_url = "https://api.scrapecreators.com/v3/tiktok/profile/videos"
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"handle": handle, "sort_by": "latest"})
-            url = f"{profile_url}?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as e:
-            _log(f"Profile videos error (urllib) for @{handle}: {e}")
-            return []
-    else:
-        try:
-            resp = _requests.get(
-                profile_url,
-                params={"handle": handle, "sort_by": "latest"},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as e:
-            _log(f"Profile videos error for @{handle}: {e}")
-            return []
+    try:
+        data = http.get(
+            profile_url,
+            params={"handle": handle, "sort_by": "latest"},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"Profile videos error for @{handle}: {e}")
+        return []
 
     raw_items = data.get("aweme_list") or data.get("data") or []
     _log(f"  -> {len(raw_items)} videos from @{handle}")
@@ -318,31 +287,17 @@ def search_tiktok(
 
     _log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
-    if not _requests:
-        _log("requests library not installed, falling back to urllib")
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"query": core_topic, "sort_by": "relevance"})
-            url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as e:
-            _log(f"ScrapeCreators error (urllib): {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_BASE}/search/keyword",
-                params={"query": core_topic, "sort_by": "relevance"},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as e:
-            _log(f"ScrapeCreators error: {e}")
-            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/search/keyword",
+            params={"query": core_topic, "sort_by": "relevance"},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as e:
+        _log(f"ScrapeCreators error: {e}")
+        return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
     # Items are nested under aweme_info
     raw_entries = data.get("search_item_list") or data.get("data") or []
@@ -397,7 +352,7 @@ def fetch_captions(
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     max_captions = config["max_captions"]
 
-    if not video_items or not token or not _requests:
+    if not video_items or not token:
         return {}
 
     top_items = video_items[:max_captions]
@@ -422,24 +377,23 @@ def fetch_captions(
         if not url:
             continue
         try:
-            resp = _requests.get(
+            data = http.get(
                 f"{SCRAPECREATORS_BASE}/video/transcript",
                 params={"url": url},
                 headers=http.scrapecreators_headers(token),
                 timeout=15,
+                retries=1,
             )
-            if resp.status_code == 200:
-                data = resp.json()
-                transcript = data.get("transcript")
+            transcript = data.get("transcript")
+            if transcript:
+                if isinstance(transcript, list):
+                    transcript = " ".join(str(s) for s in transcript)
+                transcript = _clean_webvtt(transcript)
                 if transcript:
-                    if isinstance(transcript, list):
-                        transcript = " ".join(str(s) for s in transcript)
-                    transcript = _clean_webvtt(transcript)
-                    if transcript:
-                        words = transcript.split()
-                        if len(words) > CAPTION_MAX_WORDS:
-                            transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
-                        captions[vid] = transcript
+                    words = transcript.split()
+                    if len(words) > CAPTION_MAX_WORDS:
+                        transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
+                    captions[vid] = transcript
         except Exception as e:
             _log(f"Transcript fetch failed for {vid}: {e}")
 
@@ -620,30 +574,17 @@ def _fetch_post_comments(
         List of comment dicts with author, text, digg_count (likes), date.
         Empty list on any error — comment failures never crash the pipeline.
     """
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"url": post_url, "trim": "true"})
-            url = f"{SCRAPECREATORS_BASE}/video/comments?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as exc:
-            _log(f"Comment fetch error (urllib) for {post_url}: {exc}")
-            return []
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_BASE}/video/comments",
-                params={"url": post_url, "trim": "true"},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as exc:
-            _log(f"Comment fetch error for {post_url}: {exc}")
-            return []
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_BASE}/video/comments",
+            params={"url": post_url, "trim": "true"},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as exc:
+        _log(f"Comment fetch error for {post_url}: {exc}")
+        return []
 
     raw_comments = data.get("comments") or data.get("data") or []
     # Sort by digg_count desc so normalize sees the highest-signal first.

skills/last30days/scripts/lib/youtube_yt.py

@@ -617,11 +617,6 @@ def parse_youtube_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
 
 SCRAPECREATORS_YT_BASE = "https://api.scrapecreators.com/v1/youtube"
 
-try:
-    import requests as _requests
-except ImportError:
-    _requests = None
-
 
 def _total_engagement(item: Dict[str, Any]) -> int:
     """Combined engagement score for ranking which videos to enrich."""
@@ -701,30 +696,17 @@ def _fetch_video_comments(
         List of comment dicts with author, text, likes, date.
     """
     video_url = f"https://www.youtube.com/watch?v={video_id}"
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"url": video_url})
-            url = f"{SCRAPECREATORS_YT_BASE}/video/comments?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as exc:
-            _log(f"Comment fetch error (urllib) for {video_id}: {exc}")
-            return []
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_YT_BASE}/video/comments",
-                params={"url": video_url},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-        except Exception as exc:
-            _log(f"Comment fetch error for {video_id}: {exc}")
-            return []
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_YT_BASE}/video/comments",
+            params={"url": video_url},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=2,
+        )
+    except Exception as exc:
+        _log(f"Comment fetch error for {video_id}: {exc}")
+        return []
 
     raw_comments = data.get("comments", data.get("data", []))
     comments = []
@@ -883,28 +865,14 @@ def _sc_youtube_search(keyword: str, token: str) -> List[Dict[str, Any]]:
     Returns:
         List of raw video dicts from the API.
     """
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"keyword": keyword})
-            url = f"{SCRAPECREATORS_YT_BASE}/search?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-            return data.get("videos", data.get("data", data.get("items", [])))
-        except Exception as exc:
-            _log(f"SC YouTube search error (urllib): {exc}")
-            return []
-
     try:
-        resp = _requests.get(
+        data = http.get(
             f"{SCRAPECREATORS_YT_BASE}/search",
             params={"keyword": keyword},
             headers=http.scrapecreators_headers(token),
             timeout=30,
+            retries=2,
         )
-        resp.raise_for_status()
-        data = resp.json()
         return data.get("videos", data.get("data", data.get("items", [])))
     except Exception as exc:
         _log(f"SC YouTube search error: {exc}")
@@ -922,32 +890,17 @@ def _sc_fetch_transcript(video_id: str, token: str) -> Optional[str]:
         Plaintext transcript string, or None if unavailable.
     """
     video_url = f"https://www.youtube.com/watch?v={video_id}"
-    if not _requests:
-        try:
-            from urllib.parse import urlencode
-            params = urlencode({"url": video_url})
-            url = f"{SCRAPECREATORS_YT_BASE}/video/transcript?{params}"
-            headers = http.scrapecreators_headers(token)
-            headers["User-Agent"] = http.USER_AGENT
-            data = http.get(url, headers=headers, timeout=30, retries=2)
-        except Exception as exc:
-            _log(f"SC transcript error (urllib) for {video_id}: {exc}")
-            return None
-    else:
-        try:
-            resp = _requests.get(
-                f"{SCRAPECREATORS_YT_BASE}/video/transcript",
-                params={"url": video_url},
-                headers=http.scrapecreators_headers(token),
-                timeout=30,
-            )
-            if resp.status_code != 200:
-                _log(f"SC transcript returned {resp.status_code} for {video_id}")
-                return None
-            data = resp.json()
-        except Exception as exc:
-            _log(f"SC transcript error for {video_id}: {exc}")
-            return None
+    try:
+        data = http.get(
+            f"{SCRAPECREATORS_YT_BASE}/video/transcript",
+            params={"url": video_url},
+            headers=http.scrapecreators_headers(token),
+            timeout=30,
+            retries=1,
+        )
+    except Exception as exc:
+        _log(f"SC transcript error for {video_id}: {exc}")
+        return None
 
     transcript = data.get("transcript")
     if not transcript:

skills/last30days/scripts/watchlist.py

@@ -10,16 +10,11 @@ import sys
 import time
 from pathlib import Path
 
-try:
-    import requests
-except ImportError:
-    requests = None
-
 SCRIPT_DIR = Path(__file__).parent.resolve()
 sys.path.insert(0, str(SCRIPT_DIR))
 
 import store
-from lib import schema
+from lib import http, schema
 
 
 # --- Webhook Delivery Functions ---
@@ -58,34 +53,21 @@ def _format_delivery_message(topic: str, counts: dict, mode: str) -> str:
 
 def _send_slack_webhook(url: str, text: str) -> None:
     """POST to Slack incoming webhook."""
-    if not requests:
-        raise RuntimeError("requests library not available for webhook delivery")
-    
-    response = requests.post(
-        url,
-        json={"text": text},
-        headers={"Content-Type": "application/json"},
-        timeout=10,
-    )
-    response.raise_for_status()
+    http.post(url, json_data={"text": text}, timeout=10, retries=1)
 
 
 def _send_generic_webhook(url: str, text: str) -> None:
     """POST JSON payload to generic webhook."""
-    if not requests:
-        raise RuntimeError("requests library not available for webhook delivery")
-    
-    response = requests.post(
+    http.post(
         url,
-        json={
+        json_data={
             "message": text,
             "source": "last30days",
             "timestamp": time.time(),
         },
-        headers={"Content-Type": "application/json"},
         timeout=10,
+        retries=1,
     )
-    response.raise_for_status()
 
 
 # --- Command Handlers ---