Merge pull request #56 from phjlljp/feat/unit-tests-untested-modules
test: add unit tests for untested modules
This commit is contained in:
Matt Van Horn
@@ -0,0 +1,88 @@
|
||||
"""Tests for instagram.py — ScrapeCreators Instagram search module."""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
# Add lib to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||
|
||||
from lib import instagram
|
||||
|
||||
|
||||
class TestTokenize(unittest.TestCase):
|
||||
"""Tests for _tokenize()."""
|
||||
|
||||
def test_strips_stopwords(self):
|
||||
tokens = instagram._tokenize("how to use the AI tools")
|
||||
self.assertNotIn("how", tokens)
|
||||
self.assertNotIn("the", tokens)
|
||||
self.assertNotIn("to", tokens)
|
||||
|
||||
def test_expands_synonyms(self):
|
||||
tokens = instagram._tokenize("ai tools")
|
||||
self.assertTrue("artificial" in tokens or "intelligence" in tokens)
|
||||
|
||||
def test_removes_single_char(self):
|
||||
tokens = instagram._tokenize("a b c python")
|
||||
self.assertNotIn("a", tokens)
|
||||
self.assertNotIn("b", tokens)
|
||||
self.assertIn("python", tokens)
|
||||
|
||||
def test_lowercases(self):
|
||||
tokens = instagram._tokenize("Python REACT")
|
||||
self.assertIn("python", tokens)
|
||||
self.assertIn("react", tokens)
|
||||
|
||||
def test_strips_punctuation(self):
|
||||
tokens = instagram._tokenize("hello, world!")
|
||||
self.assertIn("hello", tokens)
|
||||
self.assertIn("world", tokens)
|
||||
|
||||
|
||||
class TestComputeRelevance(unittest.TestCase):
|
||||
"""Tests for _compute_relevance()."""
|
||||
|
||||
def test_exact_match_high(self):
|
||||
rel = instagram._compute_relevance("claude code", "Claude Code tricks and tips")
|
||||
self.assertGreaterEqual(rel, 0.8)
|
||||
|
||||
def test_partial_match_lower(self):
|
||||
rel = instagram._compute_relevance("claude code tips", "Best AI tools for coding")
|
||||
self.assertLess(rel, 0.5)
|
||||
|
||||
def test_hashtag_boost(self):
|
||||
base = instagram._compute_relevance("claude code", "random video about stuff")
|
||||
boosted = instagram._compute_relevance("claude code", "random video about stuff", ["claudecode", "ai"])
|
||||
self.assertGreater(boosted, base)
|
||||
|
||||
def test_floor_at_01(self):
|
||||
rel = instagram._compute_relevance("quantum physics", "cat dancing video")
|
||||
self.assertGreaterEqual(rel, 0.1)
|
||||
|
||||
def test_empty_query_returns_default(self):
|
||||
rel = instagram._compute_relevance("", "Some video title")
|
||||
self.assertEqual(rel, 0.5)
|
||||
|
||||
|
||||
class TestInstagramDepthConfig(unittest.TestCase):
|
||||
"""Tests for DEPTH_CONFIG."""
|
||||
|
||||
def test_all_depths_exist(self):
|
||||
for depth in ("quick", "default", "deep"):
|
||||
self.assertIn(depth, instagram.DEPTH_CONFIG)
|
||||
|
||||
def test_required_keys(self):
|
||||
for depth, config in instagram.DEPTH_CONFIG.items():
|
||||
self.assertIn("results_per_page", config)
|
||||
self.assertIn("max_captions", config)
|
||||
|
||||
def test_deep_has_more_results(self):
|
||||
self.assertGreater(
|
||||
instagram.DEPTH_CONFIG["deep"]["results_per_page"],
|
||||
instagram.DEPTH_CONFIG["quick"]["results_per_page"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Tests for reddit_enrich.py — comment enrichment and parsing."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
# Add lib to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||
|
||||
from lib import reddit_enrich
|
||||
|
||||
FIXTURES_DIR = Path(__file__).parent.parent / "fixtures"
|
||||
|
||||
|
||||
def _load_fixture(name):
|
||||
with open(FIXTURES_DIR / name) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
class TestExtractRedditPath(unittest.TestCase):
|
||||
"""Tests for extract_reddit_path()."""
|
||||
|
||||
def test_valid_url(self):
|
||||
url = "https://www.reddit.com/r/ClaudeAI/comments/abc123/post_title/"
|
||||
path = reddit_enrich.extract_reddit_path(url)
|
||||
self.assertEqual(path, "/r/ClaudeAI/comments/abc123/post_title/")
|
||||
|
||||
def test_non_reddit_url(self):
|
||||
self.assertIsNone(reddit_enrich.extract_reddit_path("https://example.com/foo"))
|
||||
|
||||
def test_empty_string(self):
|
||||
self.assertIsNone(reddit_enrich.extract_reddit_path(""))
|
||||
|
||||
def test_old_reddit(self):
|
||||
url = "https://old.reddit.com/r/test/comments/xyz/"
|
||||
self.assertIsNotNone(reddit_enrich.extract_reddit_path(url))
|
||||
|
||||
|
||||
class TestParseThreadData(unittest.TestCase):
|
||||
"""Tests for parse_thread_data() using fixture."""
|
||||
|
||||
def test_parses_submission(self):
|
||||
data = _load_fixture("reddit_thread_sample.json")
|
||||
result = reddit_enrich.parse_thread_data(data)
|
||||
self.assertIsNotNone(result["submission"])
|
||||
self.assertEqual(result["submission"]["score"], 847)
|
||||
self.assertEqual(result["submission"]["num_comments"], 156)
|
||||
|
||||
def test_parses_comments(self):
|
||||
data = _load_fixture("reddit_thread_sample.json")
|
||||
result = reddit_enrich.parse_thread_data(data)
|
||||
self.assertEqual(len(result["comments"]), 8)
|
||||
self.assertEqual(result["comments"][0]["author"], "skill_expert")
|
||||
|
||||
def test_empty_input(self):
|
||||
result = reddit_enrich.parse_thread_data([])
|
||||
self.assertIsNone(result["submission"])
|
||||
self.assertEqual(result["comments"], [])
|
||||
|
||||
def test_malformed_input(self):
|
||||
result = reddit_enrich.parse_thread_data("not a list")
|
||||
self.assertIsNone(result["submission"])
|
||||
|
||||
def test_none_input(self):
|
||||
result = reddit_enrich.parse_thread_data(None)
|
||||
self.assertIsNone(result["submission"])
|
||||
|
||||
|
||||
class TestGetTopComments(unittest.TestCase):
|
||||
"""Tests for get_top_comments()."""
|
||||
|
||||
def test_sorted_by_score(self):
|
||||
comments = [
|
||||
{"score": 10, "author": "a"},
|
||||
{"score": 100, "author": "b"},
|
||||
{"score": 50, "author": "c"},
|
||||
]
|
||||
top = reddit_enrich.get_top_comments(comments, limit=3)
|
||||
self.assertEqual(top[0]["score"], 100)
|
||||
self.assertEqual(top[1]["score"], 50)
|
||||
|
||||
def test_filters_deleted(self):
|
||||
comments = [
|
||||
{"score": 100, "author": "[deleted]"},
|
||||
{"score": 50, "author": "[removed]"},
|
||||
{"score": 10, "author": "real_user"},
|
||||
]
|
||||
top = reddit_enrich.get_top_comments(comments)
|
||||
self.assertEqual(len(top), 1)
|
||||
self.assertEqual(top[0]["author"], "real_user")
|
||||
|
||||
def test_respects_limit(self):
|
||||
comments = [{"score": i, "author": f"u{i}"} for i in range(20)]
|
||||
top = reddit_enrich.get_top_comments(comments, limit=5)
|
||||
self.assertEqual(len(top), 5)
|
||||
|
||||
def test_empty_list(self):
|
||||
self.assertEqual(reddit_enrich.get_top_comments([]), [])
|
||||
|
||||
|
||||
class TestExtractCommentInsights(unittest.TestCase):
|
||||
"""Tests for extract_comment_insights()."""
|
||||
|
||||
def test_filters_short_comments(self):
|
||||
comments = [
|
||||
{"body": "yes"},
|
||||
{"body": "A" * 50 + " this is a substantive comment about the topic."},
|
||||
]
|
||||
insights = reddit_enrich.extract_comment_insights(comments)
|
||||
self.assertEqual(len(insights), 1)
|
||||
|
||||
def test_filters_low_value_patterns(self):
|
||||
comments = [
|
||||
{"body": "This."},
|
||||
{"body": "lol that's hilarious"},
|
||||
{"body": "A" * 50 + " Here's a real insight about how to approach this problem."},
|
||||
]
|
||||
insights = reddit_enrich.extract_comment_insights(comments)
|
||||
self.assertEqual(len(insights), 1)
|
||||
|
||||
def test_respects_limit(self):
|
||||
comments = [{"body": f"Comment number {i} " + "x" * 50} for i in range(20)]
|
||||
insights = reddit_enrich.extract_comment_insights(comments, limit=3)
|
||||
self.assertLessEqual(len(insights), 3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Tests for reddit.py — ScrapeCreators Reddit search module."""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
# Add lib to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||
|
||||
from lib import reddit
|
||||
|
||||
|
||||
class TestExtractCoreSubject(unittest.TestCase):
|
||||
"""Tests for _extract_core_subject()."""
|
||||
|
||||
def test_strips_what_are_prefix(self):
|
||||
self.assertEqual(reddit._extract_core_subject("what are the best AI tools"), "ai tools")
|
||||
|
||||
def test_strips_how_to_prefix(self):
|
||||
self.assertEqual(reddit._extract_core_subject("how to use cursor IDE"), "cursor ide")
|
||||
|
||||
def test_strips_noise_words(self):
|
||||
result = reddit._extract_core_subject("latest trending updates")
|
||||
self.assertEqual(result, "latest trending updates")
|
||||
|
||||
def test_preserves_product_name(self):
|
||||
self.assertEqual(reddit._extract_core_subject("cursor IDE"), "cursor ide")
|
||||
|
||||
def test_strips_trailing_punctuation(self):
|
||||
result = reddit._extract_core_subject("what is Claude?")
|
||||
self.assertFalse(result.endswith("?"))
|
||||
|
||||
def test_empty_string(self):
|
||||
result = reddit._extract_core_subject("")
|
||||
self.assertEqual(result, "")
|
||||
|
||||
def test_strips_what_do_people_think(self):
|
||||
result = reddit._extract_core_subject("what do people think about React Server Components")
|
||||
self.assertEqual(result, "react server components")
|
||||
|
||||
|
||||
class TestExpandRedditQueries(unittest.TestCase):
|
||||
"""Tests for expand_reddit_queries()."""
|
||||
|
||||
def test_quick_returns_one_query(self):
|
||||
queries = reddit.expand_reddit_queries("cursor IDE", "quick")
|
||||
self.assertGreaterEqual(len(queries), 1)
|
||||
|
||||
def test_default_includes_review_variant(self):
|
||||
queries = reddit.expand_reddit_queries("cursor IDE", "default")
|
||||
self.assertTrue(any("worth it" in q or "review" in q for q in queries))
|
||||
|
||||
def test_deep_includes_issues_variant(self):
|
||||
queries = reddit.expand_reddit_queries("cursor IDE", "deep")
|
||||
self.assertTrue(any("issues" in q or "problems" in q for q in queries))
|
||||
|
||||
def test_deep_has_more_queries_than_quick(self):
|
||||
quick = reddit.expand_reddit_queries("cursor IDE", "quick")
|
||||
deep = reddit.expand_reddit_queries("cursor IDE", "deep")
|
||||
self.assertGreater(len(deep), len(quick))
|
||||
|
||||
|
||||
class TestDiscoverSubreddits(unittest.TestCase):
|
||||
"""Tests for discover_subreddits()."""
|
||||
|
||||
def test_ranks_by_frequency(self):
|
||||
results = [
|
||||
{"subreddit": "programming", "score": 10},
|
||||
{"subreddit": "programming", "score": 20},
|
||||
{"subreddit": "python", "score": 5},
|
||||
]
|
||||
subs = reddit.discover_subreddits(results, max_subs=5)
|
||||
self.assertEqual(subs[0], "programming")
|
||||
|
||||
def test_utility_sub_penalty(self):
|
||||
results = [
|
||||
{"subreddit": "tipofmytongue", "score": 100},
|
||||
{"subreddit": "tipofmytongue", "score": 100},
|
||||
{"subreddit": "python", "score": 10},
|
||||
]
|
||||
subs = reddit.discover_subreddits(results, topic="python", max_subs=5)
|
||||
self.assertEqual(subs[0], "python")
|
||||
|
||||
def test_topic_name_bonus(self):
|
||||
results = [
|
||||
{"subreddit": "reactjs", "score": 10},
|
||||
{"subreddit": "webdev", "score": 10},
|
||||
]
|
||||
subs = reddit.discover_subreddits(results, topic="react hooks", max_subs=5)
|
||||
self.assertEqual(subs[0], "reactjs")
|
||||
|
||||
def test_engagement_bonus(self):
|
||||
results = [
|
||||
{"subreddit": "AIsub", "ups": 500},
|
||||
{"subreddit": "OtherSub", "ups": 5},
|
||||
]
|
||||
subs = reddit.discover_subreddits(results, max_subs=5)
|
||||
self.assertEqual(subs[0], "AIsub")
|
||||
|
||||
def test_max_subs_limit(self):
|
||||
results = [{"subreddit": f"sub{i}"} for i in range(20)]
|
||||
subs = reddit.discover_subreddits(results, max_subs=3)
|
||||
self.assertLessEqual(len(subs), 3)
|
||||
|
||||
def test_empty_results(self):
|
||||
self.assertEqual(reddit.discover_subreddits([]), [])
|
||||
|
||||
def test_missing_subreddit_field(self):
|
||||
results = [{"title": "no sub field"}]
|
||||
self.assertEqual(reddit.discover_subreddits(results), [])
|
||||
|
||||
|
||||
class TestParseDate(unittest.TestCase):
|
||||
"""Tests for _parse_date()."""
|
||||
|
||||
def test_valid_timestamp(self):
|
||||
self.assertEqual(reddit._parse_date(1705363200), "2024-01-16")
|
||||
|
||||
def test_string_timestamp(self):
|
||||
self.assertEqual(reddit._parse_date("1705363200"), "2024-01-16")
|
||||
|
||||
def test_none_returns_none(self):
|
||||
self.assertIsNone(reddit._parse_date(None))
|
||||
|
||||
def test_zero_returns_none(self):
|
||||
self.assertIsNone(reddit._parse_date(0))
|
||||
|
||||
|
||||
class TestDepthConfig(unittest.TestCase):
|
||||
"""Tests for DEPTH_CONFIG structure."""
|
||||
|
||||
def test_all_depths_exist(self):
|
||||
for depth in ("quick", "default", "deep"):
|
||||
self.assertIn(depth, reddit.DEPTH_CONFIG)
|
||||
|
||||
def test_required_keys(self):
|
||||
required = {"global_searches", "subreddit_searches", "comment_enrichments", "timeframe"}
|
||||
for depth, config in reddit.DEPTH_CONFIG.items():
|
||||
self.assertTrue(required.issubset(config.keys()),
|
||||
f"Missing keys in {depth}: {required - config.keys()}")
|
||||
|
||||
def test_deep_has_more_searches(self):
|
||||
self.assertGreater(
|
||||
reddit.DEPTH_CONFIG["deep"]["global_searches"],
|
||||
reddit.DEPTH_CONFIG["quick"]["global_searches"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Tests for schema.py — data class serialization roundtrips."""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
# Add lib to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||
|
||||
from lib import schema
|
||||
|
||||
|
||||
class TestEngagement(unittest.TestCase):
|
||||
"""Tests for Engagement.to_dict()."""
|
||||
|
||||
def test_sparse_fields(self):
|
||||
eng = schema.Engagement(score=100, num_comments=50)
|
||||
d = eng.to_dict()
|
||||
self.assertEqual(d, {"score": 100, "num_comments": 50})
|
||||
self.assertNotIn("likes", d)
|
||||
|
||||
def test_all_none_returns_none(self):
|
||||
eng = schema.Engagement()
|
||||
self.assertIsNone(eng.to_dict())
|
||||
|
||||
def test_all_fields(self):
|
||||
eng = schema.Engagement(
|
||||
score=1, num_comments=2, upvote_ratio=0.9,
|
||||
likes=3, reposts=4, replies=5, quotes=6,
|
||||
views=7, shares=8, volume=9.0, liquidity=10.0,
|
||||
)
|
||||
d = eng.to_dict()
|
||||
self.assertEqual(len(d), 11)
|
||||
|
||||
|
||||
class TestComment(unittest.TestCase):
|
||||
def test_basic(self):
|
||||
c = schema.Comment(score=50, date="2026-03-01", author="user", excerpt="text", url="http://x")
|
||||
d = c.to_dict()
|
||||
self.assertEqual(d["score"], 50)
|
||||
self.assertEqual(d["author"], "user")
|
||||
self.assertEqual(len(d), 5)
|
||||
|
||||
|
||||
class TestRedditItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.RedditItem(
|
||||
id="R1", title="Test", url="http://reddit.com/r/test",
|
||||
subreddit="test", date="2026-03-01",
|
||||
engagement=schema.Engagement(score=100),
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["id"], "R1")
|
||||
self.assertEqual(d["subreddit"], "test")
|
||||
self.assertEqual(d["engagement"], {"score": 100})
|
||||
self.assertNotIn("cross_refs", d)
|
||||
|
||||
def test_cross_refs_included_when_present(self):
|
||||
item = schema.RedditItem(
|
||||
id="R1", title="T", url="u", subreddit="s",
|
||||
cross_refs=["X1", "HN2"],
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["cross_refs"], ["X1", "HN2"])
|
||||
|
||||
|
||||
class TestXItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.XItem(
|
||||
id="X1", text="tweet", url="http://x.com/1",
|
||||
author_handle="user", date="2026-03-01",
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["id"], "X1")
|
||||
self.assertEqual(d["author_handle"], "user")
|
||||
self.assertNotIn("cross_refs", d)
|
||||
|
||||
|
||||
class TestYouTubeItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.YouTubeItem(
|
||||
id="YT1", title="Video", url="http://youtube.com/1",
|
||||
channel_name="chan",
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["channel_name"], "chan")
|
||||
self.assertEqual(d["date_confidence"], "high")
|
||||
|
||||
|
||||
class TestTikTokItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.TikTokItem(
|
||||
id="TK1", text="caption", url="http://tiktok.com/1",
|
||||
author_name="creator", hashtags=["ai", "code"],
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["hashtags"], ["ai", "code"])
|
||||
self.assertEqual(d["author_name"], "creator")
|
||||
|
||||
|
||||
class TestInstagramItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.InstagramItem(
|
||||
id="IG1", text="caption", url="http://instagram.com/reel/1",
|
||||
author_name="creator",
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["id"], "IG1")
|
||||
|
||||
|
||||
class TestWebSearchItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.WebSearchItem(
|
||||
id="W1", title="Article", url="http://example.com",
|
||||
source_domain="example.com", snippet="text",
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["source_domain"], "example.com")
|
||||
|
||||
|
||||
class TestHackerNewsItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.HackerNewsItem(
|
||||
id="HN1", title="Show HN", url="http://example.com",
|
||||
hn_url="http://news.ycombinator.com/item?id=1", author="pg",
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertTrue(d["hn_url"].startswith("http://news.ycombinator.com"))
|
||||
|
||||
|
||||
class TestPolymarketItem(unittest.TestCase):
|
||||
def test_roundtrip(self):
|
||||
item = schema.PolymarketItem(
|
||||
id="PM1", title="Election", question="Who wins?",
|
||||
url="http://polymarket.com/1",
|
||||
)
|
||||
d = item.to_dict()
|
||||
self.assertEqual(d["question"], "Who wins?")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user