From 0979f506db0a143d01055f820540cdd242c58439 Mon Sep 17 00:00:00 2001 From: P Date: Mon, 9 Mar 2026 16:36:52 -0400 Subject: [PATCH] test: add unit tests for untested modules Add pytest infrastructure (pyproject.toml, conftest.py) and unit tests for modules that previously had zero test coverage: - test_schema_roundtrip.py: to_dict() serialization for all data classes - test_reddit_enrich.py: URL parsing, thread data parsing, comment filtering - test_reddit_sc.py: ScrapeCreators Reddit search (query expansion, subreddit discovery) - test_instagram_sc.py: Instagram relevance scoring, tokenization, depth config Includes fixtures/reddit_thread_sample.json for reddit_enrich tests. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 3 + tests/conftest.py | 33 ++++++++ tests/test_instagram_sc.py | 74 ++++++++++++++++++ tests/test_reddit_enrich.py | 110 +++++++++++++++++++++++++++ tests/test_reddit_sc.py | 135 +++++++++++++++++++++++++++++++++ tests/test_schema_roundtrip.py | 131 ++++++++++++++++++++++++++++++++ 6 files changed, 486 insertions(+) create mode 100644 pyproject.toml create mode 100644 tests/conftest.py create mode 100644 tests/test_instagram_sc.py create mode 100644 tests/test_reddit_enrich.py create mode 100644 tests/test_reddit_sc.py create mode 100644 tests/test_schema_roundtrip.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5364782 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["scripts"] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2a75c30 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,33 @@ +"""Shared pytest fixtures for last30days tests.""" + +import json +from pathlib import Path + +import pytest + + +@pytest.fixture +def project_root(): + """Path to the repository root.""" + return Path(__file__).parent.parent + + +@pytest.fixture +def fixtures_dir(project_root): + """Path to the fixtures directory.""" + return project_root / "fixtures" + + +@pytest.fixture +def tmp_config_dir(tmp_path): + """Temporary directory for config file tests. Auto-cleaned by pytest.""" + return tmp_path + + +@pytest.fixture +def load_fixture(fixtures_dir): + """Load a JSON fixture file by name.""" + def _load(name): + with open(fixtures_dir / name) as f: + return json.load(f) + return _load diff --git a/tests/test_instagram_sc.py b/tests/test_instagram_sc.py new file mode 100644 index 0000000..1b455f9 --- /dev/null +++ b/tests/test_instagram_sc.py @@ -0,0 +1,74 @@ +"""Tests for instagram.py — ScrapeCreators Instagram search module.""" + +from lib import instagram + + +class TestTokenize: + """Tests for _tokenize().""" + + def test_strips_stopwords(self): + tokens = instagram._tokenize("how to use the AI tools") + assert "how" not in tokens + assert "the" not in tokens + assert "to" not in tokens + + def test_expands_synonyms(self): + tokens = instagram._tokenize("ai tools") + assert "artificial" in tokens or "intelligence" in tokens + + def test_removes_single_char(self): + tokens = instagram._tokenize("a b c python") + assert "a" not in tokens + assert "b" not in tokens + assert "python" in tokens + + def test_lowercases(self): + tokens = instagram._tokenize("Python REACT") + assert "python" in tokens + assert "react" in tokens + + def test_strips_punctuation(self): + tokens = instagram._tokenize("hello, world!") + assert "hello" in tokens + assert "world" in tokens + + +class TestComputeRelevance: + """Tests for _compute_relevance().""" + + def test_exact_match_high(self): + rel = instagram._compute_relevance("claude code", "Claude Code tricks and tips") + assert rel >= 0.8 + + def test_partial_match_lower(self): + rel = instagram._compute_relevance("claude code tips", "Best AI tools for coding") + assert rel < 0.5 + + def test_hashtag_boost(self): + base = instagram._compute_relevance("claude code", "random video about stuff") + boosted = instagram._compute_relevance("claude code", "random video about stuff", ["claudecode", "ai"]) + assert boosted > base + + def test_floor_at_01(self): + rel = instagram._compute_relevance("quantum physics", "cat dancing video") + assert rel >= 0.1 + + def test_empty_query_returns_default(self): + rel = instagram._compute_relevance("", "Some video title") + assert rel == 0.5 + + +class TestInstagramDepthConfig: + """Tests for DEPTH_CONFIG.""" + + def test_all_depths_exist(self): + for depth in ("quick", "default", "deep"): + assert depth in instagram.DEPTH_CONFIG + + def test_required_keys(self): + for depth, config in instagram.DEPTH_CONFIG.items(): + assert "results_per_page" in config + assert "max_captions" in config + + def test_deep_has_more_results(self): + assert instagram.DEPTH_CONFIG["deep"]["results_per_page"] > instagram.DEPTH_CONFIG["quick"]["results_per_page"] diff --git a/tests/test_reddit_enrich.py b/tests/test_reddit_enrich.py new file mode 100644 index 0000000..72c1887 --- /dev/null +++ b/tests/test_reddit_enrich.py @@ -0,0 +1,110 @@ +"""Tests for reddit_enrich.py — comment enrichment and parsing.""" + +from lib import reddit_enrich + + +class TestExtractRedditPath: + """Tests for extract_reddit_path().""" + + def test_valid_url(self): + url = "https://www.reddit.com/r/ClaudeAI/comments/abc123/post_title/" + path = reddit_enrich.extract_reddit_path(url) + assert path == "/r/ClaudeAI/comments/abc123/post_title/" + + def test_non_reddit_url(self): + assert reddit_enrich.extract_reddit_path("https://example.com/foo") is None + + def test_empty_string(self): + assert reddit_enrich.extract_reddit_path("") is None + + def test_old_reddit(self): + url = "https://old.reddit.com/r/test/comments/xyz/" + assert reddit_enrich.extract_reddit_path(url) is not None + + +class TestParseThreadData: + """Tests for parse_thread_data() using fixture.""" + + def test_parses_submission(self, load_fixture): + data = load_fixture("reddit_thread_sample.json") + result = reddit_enrich.parse_thread_data(data) + assert result["submission"] is not None + assert result["submission"]["score"] == 847 + assert result["submission"]["num_comments"] == 156 + + def test_parses_comments(self, load_fixture): + data = load_fixture("reddit_thread_sample.json") + result = reddit_enrich.parse_thread_data(data) + assert len(result["comments"]) == 8 + assert result["comments"][0]["author"] == "skill_expert" + + def test_empty_input(self): + result = reddit_enrich.parse_thread_data([]) + assert result["submission"] is None + assert result["comments"] == [] + + def test_malformed_input(self): + result = reddit_enrich.parse_thread_data("not a list") + assert result["submission"] is None + + def test_none_input(self): + result = reddit_enrich.parse_thread_data(None) + assert result["submission"] is None + + +class TestGetTopComments: + """Tests for get_top_comments().""" + + def test_sorted_by_score(self): + comments = [ + {"score": 10, "author": "a"}, + {"score": 100, "author": "b"}, + {"score": 50, "author": "c"}, + ] + top = reddit_enrich.get_top_comments(comments, limit=3) + assert top[0]["score"] == 100 + assert top[1]["score"] == 50 + + def test_filters_deleted(self): + comments = [ + {"score": 100, "author": "[deleted]"}, + {"score": 50, "author": "[removed]"}, + {"score": 10, "author": "real_user"}, + ] + top = reddit_enrich.get_top_comments(comments) + assert len(top) == 1 + assert top[0]["author"] == "real_user" + + def test_respects_limit(self): + comments = [{"score": i, "author": f"u{i}"} for i in range(20)] + top = reddit_enrich.get_top_comments(comments, limit=5) + assert len(top) == 5 + + def test_empty_list(self): + assert reddit_enrich.get_top_comments([]) == [] + + +class TestExtractCommentInsights: + """Tests for extract_comment_insights().""" + + def test_filters_short_comments(self): + comments = [ + {"body": "yes"}, + {"body": "A" * 50 + " this is a substantive comment about the topic."}, + ] + insights = reddit_enrich.extract_comment_insights(comments) + assert len(insights) == 1 + + def test_filters_low_value_patterns(self): + comments = [ + {"body": "This."}, + {"body": "lol that's hilarious"}, + {"body": "A" * 50 + " Here's a real insight about how to approach this problem."}, + ] + insights = reddit_enrich.extract_comment_insights(comments) + assert len(insights) == 1 + + def test_respects_limit(self): + comments = [{"body": f"Comment number {i} " + "x" * 50} for i in range(20)] + insights = reddit_enrich.extract_comment_insights(comments, limit=3) + assert len(insights) <= 3 diff --git a/tests/test_reddit_sc.py b/tests/test_reddit_sc.py new file mode 100644 index 0000000..52b2129 --- /dev/null +++ b/tests/test_reddit_sc.py @@ -0,0 +1,135 @@ +"""Tests for reddit.py — ScrapeCreators Reddit search module.""" + +from lib import reddit + + +class TestExtractCoreSubject: + """Tests for _extract_core_subject().""" + + def test_strips_what_are_prefix(self): + assert reddit._extract_core_subject("what are the best AI tools") == "ai tools" + + def test_strips_how_to_prefix(self): + assert reddit._extract_core_subject("how to use cursor IDE") == "cursor ide" + + def test_strips_noise_words(self): + result = reddit._extract_core_subject("latest trending updates") + assert result == "latest trending updates" + + def test_preserves_product_name(self): + assert reddit._extract_core_subject("cursor IDE") == "cursor ide" + + def test_strips_trailing_punctuation(self): + result = reddit._extract_core_subject("what is Claude?") + assert not result.endswith("?") + + def test_empty_string(self): + result = reddit._extract_core_subject("") + assert result == "" + + def test_strips_what_do_people_think(self): + result = reddit._extract_core_subject("what do people think about React Server Components") + assert result == "react server components" + + +class TestExpandRedditQueries: + """Tests for expand_reddit_queries().""" + + def test_quick_returns_one_query(self): + queries = reddit.expand_reddit_queries("cursor IDE", "quick") + assert len(queries) >= 1 + + def test_default_includes_review_variant(self): + queries = reddit.expand_reddit_queries("cursor IDE", "default") + assert any("worth it" in q or "review" in q for q in queries) + + def test_deep_includes_issues_variant(self): + queries = reddit.expand_reddit_queries("cursor IDE", "deep") + assert any("issues" in q or "problems" in q for q in queries) + + def test_deep_has_more_queries_than_quick(self): + quick = reddit.expand_reddit_queries("cursor IDE", "quick") + deep = reddit.expand_reddit_queries("cursor IDE", "deep") + assert len(deep) > len(quick) + + +class TestDiscoverSubreddits: + """Tests for discover_subreddits().""" + + def test_ranks_by_frequency(self): + results = [ + {"subreddit": "programming", "score": 10}, + {"subreddit": "programming", "score": 20}, + {"subreddit": "python", "score": 5}, + ] + subs = reddit.discover_subreddits(results, max_subs=5) + assert subs[0] == "programming" + + def test_utility_sub_penalty(self): + results = [ + {"subreddit": "tipofmytongue", "score": 100}, + {"subreddit": "tipofmytongue", "score": 100}, + {"subreddit": "python", "score": 10}, + ] + subs = reddit.discover_subreddits(results, topic="python", max_subs=5) + assert subs[0] == "python" + + def test_topic_name_bonus(self): + results = [ + {"subreddit": "reactjs", "score": 10}, + {"subreddit": "webdev", "score": 10}, + ] + subs = reddit.discover_subreddits(results, topic="react hooks", max_subs=5) + assert subs[0] == "reactjs" + + def test_engagement_bonus(self): + results = [ + {"subreddit": "AIsub", "ups": 500}, + {"subreddit": "OtherSub", "ups": 5}, + ] + subs = reddit.discover_subreddits(results, max_subs=5) + assert subs[0] == "AIsub" + + def test_max_subs_limit(self): + results = [{"subreddit": f"sub{i}"} for i in range(20)] + subs = reddit.discover_subreddits(results, max_subs=3) + assert len(subs) <= 3 + + def test_empty_results(self): + assert reddit.discover_subreddits([]) == [] + + def test_missing_subreddit_field(self): + results = [{"title": "no sub field"}] + assert reddit.discover_subreddits(results) == [] + + +class TestParseDate: + """Tests for _parse_date().""" + + def test_valid_timestamp(self): + assert reddit._parse_date(1705363200) == "2024-01-16" + + def test_string_timestamp(self): + assert reddit._parse_date("1705363200") == "2024-01-16" + + def test_none_returns_none(self): + assert reddit._parse_date(None) is None + + def test_zero_returns_none(self): + assert reddit._parse_date(0) is None + + +class TestDepthConfig: + """Tests for DEPTH_CONFIG structure.""" + + def test_all_depths_exist(self): + for depth in ("quick", "default", "deep"): + assert depth in reddit.DEPTH_CONFIG + + def test_required_keys(self): + required = {"global_searches", "subreddit_searches", "comment_enrichments", "timeframe"} + for depth, config in reddit.DEPTH_CONFIG.items(): + assert required.issubset(config.keys()), f"Missing keys in {depth}: {required - config.keys()}" + + def test_deep_has_more_searches(self): + assert reddit.DEPTH_CONFIG["deep"]["global_searches"] > reddit.DEPTH_CONFIG["quick"]["global_searches"] diff --git a/tests/test_schema_roundtrip.py b/tests/test_schema_roundtrip.py new file mode 100644 index 0000000..4362899 --- /dev/null +++ b/tests/test_schema_roundtrip.py @@ -0,0 +1,131 @@ +"""Tests for schema.py — data class serialization roundtrips.""" + +from lib import schema + + +class TestEngagement: + """Tests for Engagement.to_dict().""" + + def test_sparse_fields(self): + eng = schema.Engagement(score=100, num_comments=50) + d = eng.to_dict() + assert d == {"score": 100, "num_comments": 50} + assert "likes" not in d + + def test_all_none_returns_none(self): + eng = schema.Engagement() + assert eng.to_dict() is None + + def test_all_fields(self): + eng = schema.Engagement( + score=1, num_comments=2, upvote_ratio=0.9, + likes=3, reposts=4, replies=5, quotes=6, + views=7, shares=8, volume=9.0, liquidity=10.0, + ) + d = eng.to_dict() + assert len(d) == 11 + + +class TestComment: + def test_basic(self): + c = schema.Comment(score=50, date="2026-03-01", author="user", excerpt="text", url="http://x") + d = c.to_dict() + assert d["score"] == 50 + assert d["author"] == "user" + assert len(d) == 5 + + +class TestRedditItem: + def test_roundtrip(self): + item = schema.RedditItem( + id="R1", title="Test", url="http://reddit.com/r/test", + subreddit="test", date="2026-03-01", + engagement=schema.Engagement(score=100), + ) + d = item.to_dict() + assert d["id"] == "R1" + assert d["subreddit"] == "test" + assert d["engagement"] == {"score": 100} + assert "cross_refs" not in d + + def test_cross_refs_included_when_present(self): + item = schema.RedditItem( + id="R1", title="T", url="u", subreddit="s", + cross_refs=["X1", "HN2"], + ) + d = item.to_dict() + assert d["cross_refs"] == ["X1", "HN2"] + + +class TestXItem: + def test_roundtrip(self): + item = schema.XItem( + id="X1", text="tweet", url="http://x.com/1", + author_handle="user", date="2026-03-01", + ) + d = item.to_dict() + assert d["id"] == "X1" + assert d["author_handle"] == "user" + assert "cross_refs" not in d + + +class TestYouTubeItem: + def test_roundtrip(self): + item = schema.YouTubeItem( + id="YT1", title="Video", url="http://youtube.com/1", + channel_name="chan", + ) + d = item.to_dict() + assert d["channel_name"] == "chan" + assert d["date_confidence"] == "high" + + +class TestTikTokItem: + def test_roundtrip(self): + item = schema.TikTokItem( + id="TK1", text="caption", url="http://tiktok.com/1", + author_name="creator", hashtags=["ai", "code"], + ) + d = item.to_dict() + assert d["hashtags"] == ["ai", "code"] + assert d["author_name"] == "creator" + + +class TestInstagramItem: + def test_roundtrip(self): + item = schema.InstagramItem( + id="IG1", text="caption", url="http://instagram.com/reel/1", + author_name="creator", + ) + d = item.to_dict() + assert d["id"] == "IG1" + + +class TestWebSearchItem: + def test_roundtrip(self): + item = schema.WebSearchItem( + id="W1", title="Article", url="http://example.com", + source_domain="example.com", snippet="text", + ) + d = item.to_dict() + assert d["source_domain"] == "example.com" + + +class TestHackerNewsItem: + def test_roundtrip(self): + item = schema.HackerNewsItem( + id="HN1", title="Show HN", url="http://example.com", + hn_url="http://news.ycombinator.com/item?id=1", author="pg", + ) + d = item.to_dict() + assert d["hn_url"].startswith("http://news.ycombinator.com") + + +class TestPolymarketItem: + def test_roundtrip(self): + item = schema.PolymarketItem( + id="PM1", title="Election", question="Who wins?", + url="http://polymarket.com/1", + ) + d = item.to_dict() + assert d["question"] == "Who wins?"