diff --git a/tests/test_instagram_sc.py b/tests/test_instagram_sc.py new file mode 100644 index 0000000..0076c80 --- /dev/null +++ b/tests/test_instagram_sc.py @@ -0,0 +1,88 @@ +"""Tests for instagram.py — ScrapeCreators Instagram search module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import instagram + + +class TestTokenize(unittest.TestCase): + """Tests for _tokenize().""" + + def test_strips_stopwords(self): + tokens = instagram._tokenize("how to use the AI tools") + self.assertNotIn("how", tokens) + self.assertNotIn("the", tokens) + self.assertNotIn("to", tokens) + + def test_expands_synonyms(self): + tokens = instagram._tokenize("ai tools") + self.assertTrue("artificial" in tokens or "intelligence" in tokens) + + def test_removes_single_char(self): + tokens = instagram._tokenize("a b c python") + self.assertNotIn("a", tokens) + self.assertNotIn("b", tokens) + self.assertIn("python", tokens) + + def test_lowercases(self): + tokens = instagram._tokenize("Python REACT") + self.assertIn("python", tokens) + self.assertIn("react", tokens) + + def test_strips_punctuation(self): + tokens = instagram._tokenize("hello, world!") + self.assertIn("hello", tokens) + self.assertIn("world", tokens) + + +class TestComputeRelevance(unittest.TestCase): + """Tests for _compute_relevance().""" + + def test_exact_match_high(self): + rel = instagram._compute_relevance("claude code", "Claude Code tricks and tips") + self.assertGreaterEqual(rel, 0.8) + + def test_partial_match_lower(self): + rel = instagram._compute_relevance("claude code tips", "Best AI tools for coding") + self.assertLess(rel, 0.5) + + def test_hashtag_boost(self): + base = instagram._compute_relevance("claude code", "random video about stuff") + boosted = instagram._compute_relevance("claude code", "random video about stuff", ["claudecode", "ai"]) + self.assertGreater(boosted, base) + + def test_floor_at_01(self): + rel = instagram._compute_relevance("quantum physics", "cat dancing video") + self.assertGreaterEqual(rel, 0.1) + + def test_empty_query_returns_default(self): + rel = instagram._compute_relevance("", "Some video title") + self.assertEqual(rel, 0.5) + + +class TestInstagramDepthConfig(unittest.TestCase): + """Tests for DEPTH_CONFIG.""" + + def test_all_depths_exist(self): + for depth in ("quick", "default", "deep"): + self.assertIn(depth, instagram.DEPTH_CONFIG) + + def test_required_keys(self): + for depth, config in instagram.DEPTH_CONFIG.items(): + self.assertIn("results_per_page", config) + self.assertIn("max_captions", config) + + def test_deep_has_more_results(self): + self.assertGreater( + instagram.DEPTH_CONFIG["deep"]["results_per_page"], + instagram.DEPTH_CONFIG["quick"]["results_per_page"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reddit_enrich.py b/tests/test_reddit_enrich.py new file mode 100644 index 0000000..c093f82 --- /dev/null +++ b/tests/test_reddit_enrich.py @@ -0,0 +1,129 @@ +"""Tests for reddit_enrich.py — comment enrichment and parsing.""" + +import json +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import reddit_enrich + +FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" + + +def _load_fixture(name): + with open(FIXTURES_DIR / name) as f: + return json.load(f) + + +class TestExtractRedditPath(unittest.TestCase): + """Tests for extract_reddit_path().""" + + def test_valid_url(self): + url = "https://www.reddit.com/r/ClaudeAI/comments/abc123/post_title/" + path = reddit_enrich.extract_reddit_path(url) + self.assertEqual(path, "/r/ClaudeAI/comments/abc123/post_title/") + + def test_non_reddit_url(self): + self.assertIsNone(reddit_enrich.extract_reddit_path("https://example.com/foo")) + + def test_empty_string(self): + self.assertIsNone(reddit_enrich.extract_reddit_path("")) + + def test_old_reddit(self): + url = "https://old.reddit.com/r/test/comments/xyz/" + self.assertIsNotNone(reddit_enrich.extract_reddit_path(url)) + + +class TestParseThreadData(unittest.TestCase): + """Tests for parse_thread_data() using fixture.""" + + def test_parses_submission(self): + data = _load_fixture("reddit_thread_sample.json") + result = reddit_enrich.parse_thread_data(data) + self.assertIsNotNone(result["submission"]) + self.assertEqual(result["submission"]["score"], 847) + self.assertEqual(result["submission"]["num_comments"], 156) + + def test_parses_comments(self): + data = _load_fixture("reddit_thread_sample.json") + result = reddit_enrich.parse_thread_data(data) + self.assertEqual(len(result["comments"]), 8) + self.assertEqual(result["comments"][0]["author"], "skill_expert") + + def test_empty_input(self): + result = reddit_enrich.parse_thread_data([]) + self.assertIsNone(result["submission"]) + self.assertEqual(result["comments"], []) + + def test_malformed_input(self): + result = reddit_enrich.parse_thread_data("not a list") + self.assertIsNone(result["submission"]) + + def test_none_input(self): + result = reddit_enrich.parse_thread_data(None) + self.assertIsNone(result["submission"]) + + +class TestGetTopComments(unittest.TestCase): + """Tests for get_top_comments().""" + + def test_sorted_by_score(self): + comments = [ + {"score": 10, "author": "a"}, + {"score": 100, "author": "b"}, + {"score": 50, "author": "c"}, + ] + top = reddit_enrich.get_top_comments(comments, limit=3) + self.assertEqual(top[0]["score"], 100) + self.assertEqual(top[1]["score"], 50) + + def test_filters_deleted(self): + comments = [ + {"score": 100, "author": "[deleted]"}, + {"score": 50, "author": "[removed]"}, + {"score": 10, "author": "real_user"}, + ] + top = reddit_enrich.get_top_comments(comments) + self.assertEqual(len(top), 1) + self.assertEqual(top[0]["author"], "real_user") + + def test_respects_limit(self): + comments = [{"score": i, "author": f"u{i}"} for i in range(20)] + top = reddit_enrich.get_top_comments(comments, limit=5) + self.assertEqual(len(top), 5) + + def test_empty_list(self): + self.assertEqual(reddit_enrich.get_top_comments([]), []) + + +class TestExtractCommentInsights(unittest.TestCase): + """Tests for extract_comment_insights().""" + + def test_filters_short_comments(self): + comments = [ + {"body": "yes"}, + {"body": "A" * 50 + " this is a substantive comment about the topic."}, + ] + insights = reddit_enrich.extract_comment_insights(comments) + self.assertEqual(len(insights), 1) + + def test_filters_low_value_patterns(self): + comments = [ + {"body": "This."}, + {"body": "lol that's hilarious"}, + {"body": "A" * 50 + " Here's a real insight about how to approach this problem."}, + ] + insights = reddit_enrich.extract_comment_insights(comments) + self.assertEqual(len(insights), 1) + + def test_respects_limit(self): + comments = [{"body": f"Comment number {i} " + "x" * 50} for i in range(20)] + insights = reddit_enrich.extract_comment_insights(comments, limit=3) + self.assertLessEqual(len(insights), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reddit_sc.py b/tests/test_reddit_sc.py new file mode 100644 index 0000000..d621de8 --- /dev/null +++ b/tests/test_reddit_sc.py @@ -0,0 +1,150 @@ +"""Tests for reddit.py — ScrapeCreators Reddit search module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import reddit + + +class TestExtractCoreSubject(unittest.TestCase): + """Tests for _extract_core_subject().""" + + def test_strips_what_are_prefix(self): + self.assertEqual(reddit._extract_core_subject("what are the best AI tools"), "ai tools") + + def test_strips_how_to_prefix(self): + self.assertEqual(reddit._extract_core_subject("how to use cursor IDE"), "cursor ide") + + def test_strips_noise_words(self): + result = reddit._extract_core_subject("latest trending updates") + self.assertEqual(result, "latest trending updates") + + def test_preserves_product_name(self): + self.assertEqual(reddit._extract_core_subject("cursor IDE"), "cursor ide") + + def test_strips_trailing_punctuation(self): + result = reddit._extract_core_subject("what is Claude?") + self.assertFalse(result.endswith("?")) + + def test_empty_string(self): + result = reddit._extract_core_subject("") + self.assertEqual(result, "") + + def test_strips_what_do_people_think(self): + result = reddit._extract_core_subject("what do people think about React Server Components") + self.assertEqual(result, "react server components") + + +class TestExpandRedditQueries(unittest.TestCase): + """Tests for expand_reddit_queries().""" + + def test_quick_returns_one_query(self): + queries = reddit.expand_reddit_queries("cursor IDE", "quick") + self.assertGreaterEqual(len(queries), 1) + + def test_default_includes_review_variant(self): + queries = reddit.expand_reddit_queries("cursor IDE", "default") + self.assertTrue(any("worth it" in q or "review" in q for q in queries)) + + def test_deep_includes_issues_variant(self): + queries = reddit.expand_reddit_queries("cursor IDE", "deep") + self.assertTrue(any("issues" in q or "problems" in q for q in queries)) + + def test_deep_has_more_queries_than_quick(self): + quick = reddit.expand_reddit_queries("cursor IDE", "quick") + deep = reddit.expand_reddit_queries("cursor IDE", "deep") + self.assertGreater(len(deep), len(quick)) + + +class TestDiscoverSubreddits(unittest.TestCase): + """Tests for discover_subreddits().""" + + def test_ranks_by_frequency(self): + results = [ + {"subreddit": "programming", "score": 10}, + {"subreddit": "programming", "score": 20}, + {"subreddit": "python", "score": 5}, + ] + subs = reddit.discover_subreddits(results, max_subs=5) + self.assertEqual(subs[0], "programming") + + def test_utility_sub_penalty(self): + results = [ + {"subreddit": "tipofmytongue", "score": 100}, + {"subreddit": "tipofmytongue", "score": 100}, + {"subreddit": "python", "score": 10}, + ] + subs = reddit.discover_subreddits(results, topic="python", max_subs=5) + self.assertEqual(subs[0], "python") + + def test_topic_name_bonus(self): + results = [ + {"subreddit": "reactjs", "score": 10}, + {"subreddit": "webdev", "score": 10}, + ] + subs = reddit.discover_subreddits(results, topic="react hooks", max_subs=5) + self.assertEqual(subs[0], "reactjs") + + def test_engagement_bonus(self): + results = [ + {"subreddit": "AIsub", "ups": 500}, + {"subreddit": "OtherSub", "ups": 5}, + ] + subs = reddit.discover_subreddits(results, max_subs=5) + self.assertEqual(subs[0], "AIsub") + + def test_max_subs_limit(self): + results = [{"subreddit": f"sub{i}"} for i in range(20)] + subs = reddit.discover_subreddits(results, max_subs=3) + self.assertLessEqual(len(subs), 3) + + def test_empty_results(self): + self.assertEqual(reddit.discover_subreddits([]), []) + + def test_missing_subreddit_field(self): + results = [{"title": "no sub field"}] + self.assertEqual(reddit.discover_subreddits(results), []) + + +class TestParseDate(unittest.TestCase): + """Tests for _parse_date().""" + + def test_valid_timestamp(self): + self.assertEqual(reddit._parse_date(1705363200), "2024-01-16") + + def test_string_timestamp(self): + self.assertEqual(reddit._parse_date("1705363200"), "2024-01-16") + + def test_none_returns_none(self): + self.assertIsNone(reddit._parse_date(None)) + + def test_zero_returns_none(self): + self.assertIsNone(reddit._parse_date(0)) + + +class TestDepthConfig(unittest.TestCase): + """Tests for DEPTH_CONFIG structure.""" + + def test_all_depths_exist(self): + for depth in ("quick", "default", "deep"): + self.assertIn(depth, reddit.DEPTH_CONFIG) + + def test_required_keys(self): + required = {"global_searches", "subreddit_searches", "comment_enrichments", "timeframe"} + for depth, config in reddit.DEPTH_CONFIG.items(): + self.assertTrue(required.issubset(config.keys()), + f"Missing keys in {depth}: {required - config.keys()}") + + def test_deep_has_more_searches(self): + self.assertGreater( + reddit.DEPTH_CONFIG["deep"]["global_searches"], + reddit.DEPTH_CONFIG["quick"]["global_searches"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_schema_roundtrip.py b/tests/test_schema_roundtrip.py new file mode 100644 index 0000000..25f0b50 --- /dev/null +++ b/tests/test_schema_roundtrip.py @@ -0,0 +1,142 @@ +"""Tests for schema.py — data class serialization roundtrips.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import schema + + +class TestEngagement(unittest.TestCase): + """Tests for Engagement.to_dict().""" + + def test_sparse_fields(self): + eng = schema.Engagement(score=100, num_comments=50) + d = eng.to_dict() + self.assertEqual(d, {"score": 100, "num_comments": 50}) + self.assertNotIn("likes", d) + + def test_all_none_returns_none(self): + eng = schema.Engagement() + self.assertIsNone(eng.to_dict()) + + def test_all_fields(self): + eng = schema.Engagement( + score=1, num_comments=2, upvote_ratio=0.9, + likes=3, reposts=4, replies=5, quotes=6, + views=7, shares=8, volume=9.0, liquidity=10.0, + ) + d = eng.to_dict() + self.assertEqual(len(d), 11) + + +class TestComment(unittest.TestCase): + def test_basic(self): + c = schema.Comment(score=50, date="2026-03-01", author="user", excerpt="text", url="http://x") + d = c.to_dict() + self.assertEqual(d["score"], 50) + self.assertEqual(d["author"], "user") + self.assertEqual(len(d), 5) + + +class TestRedditItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.RedditItem( + id="R1", title="Test", url="http://reddit.com/r/test", + subreddit="test", date="2026-03-01", + engagement=schema.Engagement(score=100), + ) + d = item.to_dict() + self.assertEqual(d["id"], "R1") + self.assertEqual(d["subreddit"], "test") + self.assertEqual(d["engagement"], {"score": 100}) + self.assertNotIn("cross_refs", d) + + def test_cross_refs_included_when_present(self): + item = schema.RedditItem( + id="R1", title="T", url="u", subreddit="s", + cross_refs=["X1", "HN2"], + ) + d = item.to_dict() + self.assertEqual(d["cross_refs"], ["X1", "HN2"]) + + +class TestXItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.XItem( + id="X1", text="tweet", url="http://x.com/1", + author_handle="user", date="2026-03-01", + ) + d = item.to_dict() + self.assertEqual(d["id"], "X1") + self.assertEqual(d["author_handle"], "user") + self.assertNotIn("cross_refs", d) + + +class TestYouTubeItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.YouTubeItem( + id="YT1", title="Video", url="http://youtube.com/1", + channel_name="chan", + ) + d = item.to_dict() + self.assertEqual(d["channel_name"], "chan") + self.assertEqual(d["date_confidence"], "high") + + +class TestTikTokItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.TikTokItem( + id="TK1", text="caption", url="http://tiktok.com/1", + author_name="creator", hashtags=["ai", "code"], + ) + d = item.to_dict() + self.assertEqual(d["hashtags"], ["ai", "code"]) + self.assertEqual(d["author_name"], "creator") + + +class TestInstagramItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.InstagramItem( + id="IG1", text="caption", url="http://instagram.com/reel/1", + author_name="creator", + ) + d = item.to_dict() + self.assertEqual(d["id"], "IG1") + + +class TestWebSearchItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.WebSearchItem( + id="W1", title="Article", url="http://example.com", + source_domain="example.com", snippet="text", + ) + d = item.to_dict() + self.assertEqual(d["source_domain"], "example.com") + + +class TestHackerNewsItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.HackerNewsItem( + id="HN1", title="Show HN", url="http://example.com", + hn_url="http://news.ycombinator.com/item?id=1", author="pg", + ) + d = item.to_dict() + self.assertTrue(d["hn_url"].startswith("http://news.ycombinator.com")) + + +class TestPolymarketItem(unittest.TestCase): + def test_roundtrip(self): + item = schema.PolymarketItem( + id="PM1", title="Election", question="Who wins?", + url="http://polymarket.com/1", + ) + d = item.to_dict() + self.assertEqual(d["question"], "Who wins?") + + +if __name__ == "__main__": + unittest.main()