From 4bae05e7fafed2d35d4aa2611496073d4b768e6c Mon Sep 17 00:00:00 2001
From: Trevin Chow <trevin@trevinchow.com>
Date: Sun, 17 May 2026 01:29:26 -0700
Subject: [PATCH] fix(reddit): use browser-like headers to fix HTTP 403 from
 urllib

Reddit's public JSON endpoint returns 403 to requests carrying the
generic User-Agent and minimal header set urllib defaults to, while
matching curl requests succeed. Switch to a current-Chrome User-Agent
and add Accept-Language / Accept-Encoding / Connection headers so the
fingerprint matches a normal browser. Reddit now serves gzip when
Accept-Encoding includes it, so decompress the body before JSON parse.
Update the user-agent assertion in tests/test_reddit_public.py to match
the new browser-like string. Closes #199.

Co-authored-by: Franco Carballar <francocarballar@gmail.com>
---
 skills/last30days/scripts/lib/reddit_public.py | 15 +++++++++++++--
 tests/test_reddit_public.py                    |  2 +-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/skills/last30days/scripts/lib/reddit_public.py b/skills/last30days/scripts/lib/reddit_public.py
index a655a11..e8f3dc8 100644
--- a/skills/last30days/scripts/lib/reddit_public.py
+++ b/skills/last30days/scripts/lib/reddit_public.py
@@ -11,6 +11,7 @@ Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
 network timeouts, and missing subreddits.
 """
 
+import gzip
 import json
 import sys
 import time
@@ -21,7 +22,11 @@ from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeou
 from typing import Any, Dict, List, Optional
 
 
-USER_AGENT = "last30days/3.0 (research tool)"
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
 
 # Depth-aware limits for thread counts
 DEPTH_LIMITS = {
@@ -60,6 +65,9 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
     headers = {
         "User-Agent": USER_AGENT,
         "Accept": "application/json",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Accept-Encoding": "gzip, deflate",
+        "Connection": "keep-alive",
     }
     req = urllib.request.Request(url, headers=headers)
 
@@ -71,7 +79,10 @@ def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
                     _log(f"Anti-bot HTML response (Content-Type: {content_type})")
                     return None
 
-                body = resp.read().decode("utf-8")
+                raw = resp.read()
+                if resp.headers.get("Content-Encoding", "").lower() == "gzip":
+                    raw = gzip.decompress(raw)
+                body = raw.decode("utf-8")
                 return json.loads(body)
 
         except urllib.error.HTTPError as e:
diff --git a/tests/test_reddit_public.py b/tests/test_reddit_public.py
index af2005c..ef0d57c 100644
--- a/tests/test_reddit_public.py
+++ b/tests/test_reddit_public.py
@@ -313,7 +313,7 @@ class TestSearchRedditPublicHighLevel:
         reddit_public.search("test")
 
         req = mock_urlopen.call_args[0][0]
-        assert req.get_header("User-agent") == "last30days/3.0 (research tool)"
+        assert "Mozilla/5.0" in req.get_header("User-agent")
 
 
 class TestMissingSubreddit: