fix(reddit): restore free path via keyless RSS + shreddit scrape (.json is dead) (#457)
* test(reddit): add live RSS + shreddit comment fixtures Captured from reddit.com on 2026-05-29 (search.rss listing + the /svc/shreddit/comments partial), trimmed to a representative subset plus two synthetic edge cases (deleted author, negative score) for offline parser tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(http): add keyless get_text helper Browser-UA text fetch for RSS/HTML endpoints; returns None on any HTTP or network failure so tiered callers fall through cleanly. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(reddit): keyless RSS discovery (search.rss + listing feeds) Replaces the now-403 search.json with keyless Atom feeds, normalized to the existing reddit_public post shape. Scores are placeholder zeros, backfilled during shreddit enrichment. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(reddit): keyless shreddit comment scraper Parses <shreddit-comment> elements from /svc/shreddit/comments/r/{sub}/t3_{id} (score/author/created/permalink + thingId-anchored body) into top comments, matching reddit_enrich output. Replaces the dead {thread}.json enrichment. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(reddit): tiered keyless orchestrator Tier 0 one-shot .json (residential bonus) -> Tier 1 RSS discovery -> Tier 2 shreddit enrichment. Returns [] never raises, so the SC backup still engages when every keyless tier is empty. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(reddit): route free path through keyless pipeline (.json is dead) search_reddit_public is now a thin shim over reddit_keyless, so pipeline.py and other callers need no change. Removes the dead .json enrichment helpers; search/_parse_posts remain as the demoted Tier 0 attempt. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(reddit): request sort=top so true top comments land on page 1 Guarantees the highest-scored comments are captured even on large threads, independent of Reddit's default comment sort. Local score re-sort remains. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(reddit): recover post upvote scores via keyless listing partials The shreddit community-more-posts partial server-renders each post's score and comment count (works for normal users, not IP-gated), unlike RSS or the comments endpoint. Use it as a scored discovery source and to backfill scores onto RSS-discovered posts (subreddits derived from results when not provided). Ranking now uses real upvote score. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(reddit): listings backfill scores only on bare queries, not discovery Caught running the full pipeline on a bare topic: deriving subreddits from noisy RSS results and merging their top/hot listings flooded results with high-upvote off-topic posts. Now derived-subreddit listings are used only to backfill scores onto keyword-matched RSS posts; listing cards are merged as discovery only when the caller explicitly provides subreddits (on-topic). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
@@ -0,0 +1,168 @@
|
||||
"""Tests for scripts/lib/reddit_keyless.py — tiered keyless Reddit pipeline."""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from lib import reddit_keyless
|
||||
|
||||
|
||||
def _post(i, date="2026-05-20", rel=0.0):
|
||||
url = f"https://www.reddit.com/r/test/comments/{i:06d}/post_{i}/"
|
||||
return {
|
||||
"id": "", "title": f"Post {i}", "url": url, "score": 0, "num_comments": 0,
|
||||
"subreddit": "test", "created_utc": None, "author": "u", "selftext": "",
|
||||
"date": date, "engagement": {"score": 0, "num_comments": 0, "upvote_ratio": None},
|
||||
"relevance": rel, "why_relevant": "Reddit RSS", "metadata": {},
|
||||
}
|
||||
|
||||
|
||||
def _scored(i, score, ncmt=0):
|
||||
p = _post(i)
|
||||
p["score"] = score
|
||||
p["num_comments"] = ncmt
|
||||
p["engagement"]["score"] = score
|
||||
p["engagement"]["num_comments"] = ncmt
|
||||
p["why_relevant"] = "Reddit listing"
|
||||
p["metadata"] = {"post_id": f"{i:06d}"}
|
||||
return p
|
||||
|
||||
|
||||
class TestDiscoveryTierOrder:
|
||||
"""Tier 0 (.json) is tried first; RSS + scored listings are the keyless path."""
|
||||
|
||||
def test_tier0_success_skips_keyless(self):
|
||||
with mock.patch.object(reddit_keyless, "_tier0_json", return_value=[_post(1)]) as t0, \
|
||||
mock.patch.object(reddit_keyless.reddit_rss, "search_rss") as rss, \
|
||||
mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings") as lst:
|
||||
out = reddit_keyless._discover("topic", "default", None)
|
||||
assert len(out) == 1
|
||||
t0.assert_called_once()
|
||||
rss.assert_not_called()
|
||||
lst.assert_not_called()
|
||||
|
||||
def test_tier0_empty_falls_to_keyless(self):
|
||||
with mock.patch.object(reddit_keyless, "_tier0_json", return_value=[]), \
|
||||
mock.patch.object(reddit_keyless.reddit_rss, "search_rss",
|
||||
return_value=[_post(1), _post(2)]) as rss, \
|
||||
mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings",
|
||||
return_value=[]):
|
||||
out = reddit_keyless._discover("topic", "default", ["test"])
|
||||
assert len(out) == 2
|
||||
rss.assert_called_once()
|
||||
|
||||
def test_listing_scores_backfill_rss_posts(self):
|
||||
# RSS finds post 1 (no score); listing card for post 1 carries the score.
|
||||
rss_post = _post(1)
|
||||
listing_post = _scored(1, score=52692, ncmt=1743)
|
||||
with mock.patch.object(reddit_keyless, "_tier0_json", return_value=[]), \
|
||||
mock.patch.object(reddit_keyless.reddit_rss, "search_rss",
|
||||
return_value=[rss_post]), \
|
||||
mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings",
|
||||
return_value=[listing_post]):
|
||||
out = reddit_keyless._discover("topic", "default", ["test"])
|
||||
# listing post (scored) is kept; RSS dup of same url is dropped
|
||||
assert len(out) == 1
|
||||
assert out[0]["engagement"]["score"] == 52692
|
||||
assert out[0]["num_comments"] == 1743
|
||||
|
||||
def test_scores_flow_to_distinct_rss_posts(self):
|
||||
# Distinct RSS post whose id matches a listing card gets backfilled.
|
||||
rss_post = _post(7) # url .../000007/...
|
||||
listing_post = _scored(7, score=999)
|
||||
listing_post["url"] = "https://www.reddit.com/r/test/comments/zzzzzz/other/"
|
||||
with mock.patch.object(reddit_keyless, "_tier0_json", return_value=[]), \
|
||||
mock.patch.object(reddit_keyless.reddit_rss, "search_rss",
|
||||
return_value=[rss_post]), \
|
||||
mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings",
|
||||
return_value=[listing_post]):
|
||||
out = reddit_keyless._discover("topic", "default", ["test"])
|
||||
backfilled = [p for p in out if p["url"] == rss_post["url"]][0]
|
||||
assert backfilled["engagement"]["score"] == 999
|
||||
|
||||
def test_bare_query_does_not_merge_listing_discovery(self):
|
||||
# No subreddits provided: derived-subreddit listings must NOT be added as
|
||||
# results (avoids flooding with off-topic high-upvote posts) — only used
|
||||
# to backfill scores onto the keyword-matched RSS posts.
|
||||
rss_post = _post(1) # on-topic keyword match
|
||||
offtopic_listing = _scored(99, score=88888) # high score, unrelated sub
|
||||
offtopic_listing["url"] = "https://www.reddit.com/r/random/comments/zzz999/x/"
|
||||
with mock.patch.object(reddit_keyless, "_tier0_json", return_value=[]), \
|
||||
mock.patch.object(reddit_keyless.reddit_rss, "search_rss",
|
||||
return_value=[rss_post]), \
|
||||
mock.patch.object(reddit_keyless, "_top_subreddits", return_value=["random"]), \
|
||||
mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings",
|
||||
return_value=[offtopic_listing]):
|
||||
out = reddit_keyless._discover("topic", "default", None)
|
||||
urls = [p["url"] for p in out]
|
||||
assert rss_post["url"] in urls
|
||||
assert offtopic_listing["url"] not in urls # not merged as discovery
|
||||
|
||||
def test_tier0_never_raises(self):
|
||||
with mock.patch("lib.reddit_public.search", side_effect=Exception("boom")), \
|
||||
mock.patch.object(reddit_keyless.reddit_rss, "search_rss", return_value=[]), \
|
||||
mock.patch.object(reddit_keyless.reddit_listing, "fetch_listings", return_value=[]):
|
||||
assert reddit_keyless._discover("t", "default", None) == []
|
||||
|
||||
|
||||
class TestSearchAndEnrich:
|
||||
"""Full pipeline: discover -> date filter -> rank -> enrich -> reindex."""
|
||||
|
||||
def _patch_enrich_passthrough(self):
|
||||
return mock.patch.object(
|
||||
reddit_keyless.reddit_shreddit, "fetch_comments",
|
||||
return_value={"top_comments": [], "comment_insights": [], "num_comments": None},
|
||||
)
|
||||
|
||||
def test_returns_empty_when_no_discovery(self):
|
||||
with mock.patch.object(reddit_keyless, "_discover", return_value=[]):
|
||||
assert reddit_keyless.search_and_enrich("t", "2026-05-01", "2026-05-31") == []
|
||||
|
||||
def test_date_filter_keeps_in_range_and_unknown(self):
|
||||
posts = [_post(1, date="2026-05-10"), _post(2, date="2020-01-01"),
|
||||
_post(3, date=None)]
|
||||
with mock.patch.object(reddit_keyless, "_discover", return_value=posts), \
|
||||
self._patch_enrich_passthrough():
|
||||
out = reddit_keyless.search_and_enrich("t", "2026-05-01", "2026-05-31")
|
||||
titles = {p["title"] for p in out}
|
||||
assert "Post 1" in titles and "Post 3" in titles
|
||||
assert "Post 2" not in titles
|
||||
|
||||
def test_reindexes_ids(self):
|
||||
posts = [_post(1), _post(2), _post(3)]
|
||||
with mock.patch.object(reddit_keyless, "_discover", return_value=posts), \
|
||||
self._patch_enrich_passthrough():
|
||||
out = reddit_keyless.search_and_enrich("t", "2026-05-01", "2026-05-31")
|
||||
assert [p["id"] for p in out] == ["R1", "R2", "R3"]
|
||||
|
||||
def test_enrichment_attaches_comments(self):
|
||||
posts = [_post(1)]
|
||||
enriched = {
|
||||
"top_comments": [{"score": 9, "date": "2026-05-19", "author": "a",
|
||||
"excerpt": "great", "url": "https://reddit.com/x"}],
|
||||
"comment_insights": ["great point about X"],
|
||||
"num_comments": 14,
|
||||
}
|
||||
with mock.patch.object(reddit_keyless, "_discover", return_value=posts), \
|
||||
mock.patch.object(reddit_keyless.reddit_shreddit, "fetch_comments",
|
||||
return_value=enriched):
|
||||
out = reddit_keyless.search_and_enrich("t", "2026-05-01", "2026-05-31")
|
||||
assert out[0]["top_comments"][0]["score"] == 9
|
||||
assert out[0]["num_comments"] == 14
|
||||
assert out[0]["engagement"]["num_comments"] == 14
|
||||
|
||||
def test_enrichment_failure_keeps_posts(self):
|
||||
posts = [_post(i) for i in range(8)]
|
||||
with mock.patch.object(reddit_keyless, "_discover", return_value=posts), \
|
||||
mock.patch.object(reddit_keyless.reddit_shreddit, "fetch_comments",
|
||||
side_effect=Exception("svc down")):
|
||||
out = reddit_keyless.search_and_enrich("t", "2026-05-01", "2026-05-31")
|
||||
assert len(out) == 8 # all posts retained despite enrichment failure
|
||||
|
||||
def test_only_top_n_enriched_by_depth(self):
|
||||
posts = [_post(i, rel=1.0 - i / 100) for i in range(10)]
|
||||
with mock.patch.object(reddit_keyless, "_discover", return_value=posts), \
|
||||
mock.patch.object(reddit_keyless.reddit_shreddit, "fetch_comments",
|
||||
return_value={"top_comments": [], "comment_insights": [],
|
||||
"num_comments": None}) as fc:
|
||||
reddit_keyless.search_and_enrich("t", "2026-05-01", "2026-05-31", depth="quick")
|
||||
# quick depth enriches only top 3 posts
|
||||
assert fc.call_count == reddit_keyless.ENRICH_LIMITS["quick"]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Tests for scripts/lib/reddit_listing.py — keyless scored listing scrape."""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from lib import reddit_listing as rl
|
||||
|
||||
FIXTURE = Path(__file__).resolve().parent.parent / "fixtures" / "reddit_listing_cards_sample.html"
|
||||
|
||||
|
||||
def _html():
|
||||
return FIXTURE.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class TestParseCards:
|
||||
"""parse_cards reads <shreddit-post> cards into scored post dicts."""
|
||||
|
||||
def test_parses_five_cards(self):
|
||||
posts = rl.parse_cards(_html(), query="netherlands")
|
||||
assert len(posts) == 5
|
||||
|
||||
def test_real_score_and_count(self):
|
||||
posts = rl.parse_cards(_html())
|
||||
top = posts[0]
|
||||
assert top["score"] == 52692 # the real upvote count
|
||||
assert top["engagement"]["score"] == 52692
|
||||
assert top["num_comments"] == 1743
|
||||
assert top["engagement"]["num_comments"] == 1743
|
||||
|
||||
def test_normalized_shape(self):
|
||||
post = rl.parse_cards(_html())[0]
|
||||
required = {"id", "title", "url", "score", "num_comments", "subreddit",
|
||||
"created_utc", "author", "selftext", "date",
|
||||
"engagement", "relevance", "why_relevant", "metadata"}
|
||||
assert required.issubset(set(post.keys()))
|
||||
assert post["why_relevant"] == "Reddit listing"
|
||||
assert post["metadata"]["post_id"] # post id captured for backfill
|
||||
|
||||
def test_fields_populated(self):
|
||||
post = rl.parse_cards(_html())[0]
|
||||
assert post["title"]
|
||||
assert post["author"] == "AdSpecialist6598"
|
||||
assert post["subreddit"] == "technology"
|
||||
assert "/comments/" in post["url"]
|
||||
assert post["date"] and len(post["date"]) == 10
|
||||
|
||||
def test_empty_html_returns_empty(self):
|
||||
assert rl.parse_cards("") == []
|
||||
assert rl.parse_cards("<div>no cards</div>") == []
|
||||
|
||||
|
||||
class TestListingUrl:
|
||||
def test_top_includes_timeframe(self):
|
||||
u = rl._listing_url("technology", "top")
|
||||
assert "community-more-posts/top/" in u and "name=technology" in u and "t=month" in u
|
||||
|
||||
def test_hot_no_timeframe(self):
|
||||
u = rl._listing_url("r/technology", "hot")
|
||||
assert "community-more-posts/hot/" in u and "name=technology" in u and "t=" not in u
|
||||
assert ".json" not in u
|
||||
|
||||
|
||||
class TestFetchListings:
|
||||
def test_dedupes_across_sorts(self):
|
||||
with mock.patch.object(rl.http, "get_text", return_value=_html()):
|
||||
posts = rl.fetch_listings(["technology"], depth="default")
|
||||
urls = [p["url"] for p in posts]
|
||||
assert len(urls) == len(set(urls)) # top + hot return same cards -> deduped
|
||||
|
||||
def test_no_subreddits_returns_empty(self):
|
||||
assert rl.fetch_listings([], depth="default") == []
|
||||
|
||||
def test_all_fetches_fail_returns_empty(self):
|
||||
with mock.patch.object(rl.http, "get_text", return_value=None):
|
||||
assert rl.fetch_listings(["technology"]) == []
|
||||
|
||||
|
||||
class TestScoreIndex:
|
||||
def test_builds_post_id_to_score_map(self):
|
||||
with mock.patch.object(rl.http, "get_text", return_value=_html()):
|
||||
idx = rl.score_index(["technology"], depth="quick")
|
||||
assert idx # non-empty
|
||||
first = next(iter(idx.values()))
|
||||
assert set(first.keys()) == {"score", "num_comments"}
|
||||
assert any(v["score"] == 52692 for v in idx.values())
|
||||
+16
-75
@@ -326,83 +326,24 @@ class TestMissingSubreddit:
|
||||
assert results == []
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for comment enrichment (Unit 2)
|
||||
# search_reddit_public is now a thin shim over the keyless pipeline.
|
||||
# Full discovery + enrichment behavior is covered in test_reddit_keyless.py.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnrichmentIntegration:
|
||||
"""search_reddit_public enriches top posts with comments."""
|
||||
class TestSearchRedditPublicDelegatesToKeyless:
|
||||
"""search_reddit_public delegates to reddit_keyless.search_and_enrich."""
|
||||
|
||||
@mock.patch("lib.reddit_public._enrich_post")
|
||||
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||
def test_search_enriches_top_5_by_default(self, mock_urlopen, mock_enrich):
|
||||
listing = _make_reddit_listing([
|
||||
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
|
||||
"score": 100 - i, "created_utc": 1711670400}
|
||||
for i in range(10)
|
||||
])
|
||||
mock_urlopen.return_value = _mock_urlopen_ok(listing)
|
||||
mock_enrich.side_effect = lambda item, timeout=10: item # pass-through
|
||||
def test_delegates_with_all_args(self):
|
||||
with mock.patch("lib.reddit_keyless.search_and_enrich") as mock_keyless:
|
||||
mock_keyless.return_value = [{"id": "R1", "title": "x"}]
|
||||
results = reddit_public.search_reddit_public(
|
||||
"test", "2024-03-01", "2024-03-31",
|
||||
depth="quick", subreddits=["ClaudeAI"],
|
||||
)
|
||||
|
||||
results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31")
|
||||
|
||||
assert len(results) == 10
|
||||
# Default depth enriches top 5
|
||||
assert mock_enrich.call_count == 5
|
||||
|
||||
@mock.patch("lib.reddit_public._enrich_post")
|
||||
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||
def test_enrichment_timeout_keeps_posts(self, mock_urlopen, mock_enrich):
|
||||
listing = _make_reddit_listing([
|
||||
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
|
||||
"score": 100 - i, "created_utc": 1711670400}
|
||||
for i in range(10)
|
||||
])
|
||||
mock_urlopen.return_value = _mock_urlopen_ok(listing)
|
||||
|
||||
# Some enrichments raise, some succeed
|
||||
call_count = {"n": 0}
|
||||
def _side_effect(item, timeout=10):
|
||||
call_count["n"] += 1
|
||||
if call_count["n"] % 2 == 0:
|
||||
raise TimeoutError("enrichment timed out")
|
||||
return item
|
||||
mock_enrich.side_effect = _side_effect
|
||||
|
||||
results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31")
|
||||
|
||||
# All 10 posts should still be returned
|
||||
assert len(results) == 10
|
||||
|
||||
@mock.patch("lib.reddit_public._enrich_post")
|
||||
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||
def test_all_enrichment_fails_all_posts_returned(self, mock_urlopen, mock_enrich):
|
||||
listing = _make_reddit_listing([
|
||||
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
|
||||
"score": 100 - i, "created_utc": 1711670400}
|
||||
for i in range(10)
|
||||
])
|
||||
mock_urlopen.return_value = _mock_urlopen_ok(listing)
|
||||
mock_enrich.side_effect = Exception("total failure")
|
||||
|
||||
results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31")
|
||||
|
||||
# All posts returned despite enrichment failure
|
||||
assert len(results) == 10
|
||||
|
||||
@mock.patch("lib.reddit_public._enrich_post")
|
||||
@mock.patch("lib.reddit_public.urllib.request.urlopen")
|
||||
def test_quick_depth_enriches_top_3(self, mock_urlopen, mock_enrich):
|
||||
listing = _make_reddit_listing([
|
||||
{"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/",
|
||||
"score": 100 - i, "created_utc": 1711670400}
|
||||
for i in range(10)
|
||||
])
|
||||
mock_urlopen.return_value = _mock_urlopen_ok(listing)
|
||||
mock_enrich.side_effect = lambda item, timeout=10: item
|
||||
|
||||
results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31", depth="quick")
|
||||
|
||||
assert len(results) == 10
|
||||
# Quick depth enriches only top 3
|
||||
assert mock_enrich.call_count == 3
|
||||
assert results == [{"id": "R1", "title": "x"}]
|
||||
mock_keyless.assert_called_once_with(
|
||||
"test", "2024-03-01", "2024-03-31",
|
||||
depth="quick", subreddits=["ClaudeAI"],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Tests for scripts/lib/reddit_rss.py — keyless Reddit RSS discovery."""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from lib import reddit_rss
|
||||
|
||||
FIXTURE = Path(__file__).resolve().parent.parent / "fixtures" / "reddit_search_rss_sample.xml"
|
||||
|
||||
|
||||
def _feed_text():
|
||||
return FIXTURE.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class TestParseFeed:
|
||||
"""_parse_feed turns Atom entries into normalized post dicts."""
|
||||
|
||||
def test_parses_entries(self):
|
||||
posts = reddit_rss._parse_feed(_feed_text(), query="lifelock")
|
||||
assert len(posts) == 5
|
||||
for p in posts:
|
||||
assert p["title"]
|
||||
assert "/comments/" in p["url"]
|
||||
assert p["url"].startswith("https://www.reddit.com/")
|
||||
|
||||
def test_normalized_shape_matches_scrapecreators(self):
|
||||
post = reddit_rss._parse_feed(_feed_text(), query="x")[0]
|
||||
required = {"id", "title", "url", "score", "num_comments", "subreddit",
|
||||
"created_utc", "author", "selftext", "date",
|
||||
"engagement", "relevance", "why_relevant", "metadata"}
|
||||
assert required.issubset(set(post.keys()))
|
||||
assert set(post["engagement"].keys()) == {"score", "num_comments", "upvote_ratio"}
|
||||
assert post["why_relevant"] == "Reddit RSS"
|
||||
|
||||
def test_score_is_placeholder_zero(self):
|
||||
# RSS carries no engagement score; it is backfilled during enrichment.
|
||||
for p in reddit_rss._parse_feed(_feed_text(), query="x"):
|
||||
assert p["score"] == 0
|
||||
assert p["engagement"]["score"] == 0
|
||||
|
||||
def test_subreddit_derivation(self):
|
||||
post = reddit_rss._parse_feed(_feed_text(), query="x")[0]
|
||||
assert post["subreddit"] == "Rakuten"
|
||||
|
||||
def test_date_parsed_to_iso(self):
|
||||
post = reddit_rss._parse_feed(_feed_text(), query="x")[0]
|
||||
assert post["date"] and len(post["date"]) == 10 # YYYY-MM-DD
|
||||
assert isinstance(post["created_utc"], float)
|
||||
|
||||
def test_author_strips_u_prefix(self):
|
||||
authors = [p["author"] for p in reddit_rss._parse_feed(_feed_text(), query="x")]
|
||||
assert all(not a.startswith("/u/") and not a.startswith("u/") for a in authors)
|
||||
|
||||
def test_empty_and_malformed_feed_never_raises(self):
|
||||
assert reddit_rss._parse_feed("", query="x") == []
|
||||
assert reddit_rss._parse_feed("<not xml", query="x") == []
|
||||
assert reddit_rss._parse_feed("<feed></feed>", query="x") == []
|
||||
|
||||
def test_entry_without_comments_link_skipped(self):
|
||||
feed = (
|
||||
'<feed xmlns="http://www.w3.org/2005/Atom"><entry>'
|
||||
'<title>Subreddit itself</title>'
|
||||
'<link href="https://www.reddit.com/r/test/" />'
|
||||
'<updated>2026-05-20T00:00:00+00:00</updated></entry></feed>'
|
||||
)
|
||||
assert reddit_rss._parse_feed(feed, query="x") == []
|
||||
|
||||
|
||||
class TestSearchRss:
|
||||
"""search_rss fans out, dedupes, assigns IDs, and honors depth limits."""
|
||||
|
||||
def test_dedupe_and_ids(self):
|
||||
# Same feed returned for every URL -> deduped to 5 unique posts.
|
||||
with mock.patch.object(reddit_rss.http, "get_text", return_value=_feed_text()):
|
||||
posts = reddit_rss.search_rss("lifelock", depth="default",
|
||||
subreddits=["Rakuten", "ConsumerAdvice"])
|
||||
urls = [p["url"] for p in posts]
|
||||
assert len(urls) == len(set(urls)) # no duplicates
|
||||
assert [p["id"] for p in posts] == [f"R{i+1}" for i in range(len(posts))]
|
||||
|
||||
def test_depth_limit_quick(self):
|
||||
with mock.patch.object(reddit_rss.http, "get_text", return_value=_feed_text()):
|
||||
posts = reddit_rss.search_rss("lifelock", depth="quick")
|
||||
assert len(posts) <= reddit_rss.DEPTH_LIMITS["quick"]
|
||||
|
||||
def test_all_feeds_fail_returns_empty(self):
|
||||
with mock.patch.object(reddit_rss.http, "get_text", return_value=None):
|
||||
posts = reddit_rss.search_rss("lifelock", subreddits=["Rakuten"])
|
||||
assert posts == []
|
||||
|
||||
def test_builds_keyless_rss_urls(self):
|
||||
urls = reddit_rss._build_urls("life lock", "default", ["Rakuten"])
|
||||
assert any("search.rss?q=life+lock" in u and "/r/" not in u.split("?")[0] for u in urls)
|
||||
assert any("/r/Rakuten/search.rss" in u and "restrict_sr=on" in u for u in urls)
|
||||
assert any("/r/Rakuten/top.rss" in u for u in urls)
|
||||
assert all(".json" not in u for u in urls) # never the dead endpoint
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Tests for scripts/lib/reddit_shreddit.py — keyless shreddit comment scrape."""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from lib import reddit_shreddit as rs
|
||||
|
||||
FIXTURE = Path(__file__).resolve().parent.parent / "fixtures" / "reddit_shreddit_comments_sample.html"
|
||||
|
||||
|
||||
def _html():
|
||||
return FIXTURE.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class TestExtractPostRef:
|
||||
def test_extracts_sub_and_id(self):
|
||||
ref = rs.extract_post_ref("https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/")
|
||||
assert ref == ("Rakuten", "1taeiw0")
|
||||
|
||||
def test_non_thread_url_returns_none(self):
|
||||
assert rs.extract_post_ref("https://www.reddit.com/r/Rakuten/") is None
|
||||
assert rs.extract_post_ref("") is None
|
||||
|
||||
def test_svc_url_shape(self):
|
||||
# sort=top guarantees the highest-scored comments land on page 1.
|
||||
assert rs._svc_url("Rakuten", "1taeiw0") == (
|
||||
"https://www.reddit.com/svc/shreddit/comments/r/Rakuten/t3_1taeiw0?sort=top"
|
||||
)
|
||||
|
||||
|
||||
class TestParseComments:
|
||||
"""parse_comments reads <shreddit-comment> elements into scored dicts."""
|
||||
|
||||
def test_happy_path(self):
|
||||
comments = rs.parse_comments(_html())
|
||||
assert len(comments) >= 1
|
||||
for c in comments:
|
||||
assert isinstance(c["score"], int)
|
||||
assert c["author"] and c["author"] not in ("[deleted]", "[removed]")
|
||||
assert c["body"]
|
||||
|
||||
def test_sorted_by_score_desc(self):
|
||||
scores = [c["score"] for c in rs.parse_comments(_html())]
|
||||
assert scores == sorted(scores, reverse=True)
|
||||
|
||||
def test_deleted_and_removed_filtered(self):
|
||||
authors = [c["author"] for c in rs.parse_comments(_html())]
|
||||
assert "[deleted]" not in authors and "[removed]" not in authors
|
||||
|
||||
def test_negative_score_retained(self):
|
||||
scores = [c["score"] for c in rs.parse_comments(_html())]
|
||||
assert -7 in scores # synthetic downvoted-but-real comment
|
||||
|
||||
def test_limit_honored(self):
|
||||
assert len(rs.parse_comments(_html(), limit=2)) == 2
|
||||
|
||||
def test_body_text_extracted(self):
|
||||
bodies = [c["body"] for c in rs.parse_comments(_html())]
|
||||
assert any("$750" in b or "pending" in b for b in bodies)
|
||||
|
||||
def test_comment_url_built(self):
|
||||
for c in rs.parse_comments(_html()):
|
||||
if c["url"]:
|
||||
assert c["url"].startswith("https://reddit.com/r/")
|
||||
|
||||
def test_empty_html_returns_empty(self):
|
||||
assert rs.parse_comments("") == []
|
||||
assert rs.parse_comments("<html>no comments here</html>") == []
|
||||
|
||||
|
||||
class TestTotalComments:
|
||||
def test_reads_total(self):
|
||||
assert rs._total_comments(_html()) == 14
|
||||
|
||||
def test_missing_returns_none(self):
|
||||
assert rs._total_comments("<html></html>") is None
|
||||
|
||||
|
||||
class TestFetchComments:
|
||||
"""fetch_comments wires URL -> svc fetch -> parse, never raising."""
|
||||
|
||||
def test_happy_path(self):
|
||||
url = "https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/"
|
||||
with mock.patch.object(rs.http, "get_text", return_value=_html()) as m:
|
||||
out = rs.fetch_comments(url)
|
||||
# svc endpoint, not .json
|
||||
assert "/svc/shreddit/comments/" in m.call_args[0][0]
|
||||
assert ".json" not in m.call_args[0][0]
|
||||
assert out["num_comments"] == 14
|
||||
assert len(out["top_comments"]) >= 1
|
||||
first = out["top_comments"][0]
|
||||
assert {"score", "date", "author", "excerpt", "url"} <= set(first.keys())
|
||||
assert isinstance(out["comment_insights"], list)
|
||||
|
||||
def test_bad_url_returns_empty(self):
|
||||
out = rs.fetch_comments("https://www.reddit.com/r/Rakuten/")
|
||||
assert out["top_comments"] == [] and out["num_comments"] is None
|
||||
|
||||
def test_fetch_failure_returns_empty(self):
|
||||
url = "https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/"
|
||||
with mock.patch.object(rs.http, "get_text", return_value=None):
|
||||
out = rs.fetch_comments(url)
|
||||
assert out["top_comments"] == [] and out["num_comments"] is None
|
||||
Reference in New Issue
Block a user