feat: v2.8 — Instagram Reels source + TikTok ScrapeCreators migration
Add Instagram Reels as the 8th research source via ScrapeCreators API. One API key (SCRAPECREATORS_API_KEY) now covers both TikTok and Instagram. - Add scripts/lib/instagram.py: keyword search, transcript extraction, relevance scoring, engagement metrics (views, likes, comments) - Add InstagramItem to schema, normalization, scoring, dedup, rendering - Add Instagram to orchestrator pipeline, watchlist, and UI spinners - Update SKILL.md: stats template, citation priority, item format, URL-to-name extraction rules, anti-Sources instruction - Update README and CHANGELOG for v2.8 - Fix: Instagram/TikTok not running in --search= web-only path - Fix: web stats line showing full URLs instead of domain names - Replace APIFY_API_TOKEN with SCRAPECREATORS_API_KEY throughout Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+13
-1
@@ -46,7 +46,7 @@ def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
|
||||
|
||||
|
||||
AnyItem = Union[schema.RedditItem, schema.XItem, schema.YouTubeItem, schema.TikTokItem,
|
||||
schema.HackerNewsItem, schema.PolymarketItem, schema.WebSearchItem]
|
||||
schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem, schema.WebSearchItem]
|
||||
|
||||
|
||||
def get_item_text(item: AnyItem) -> str:
|
||||
@@ -59,6 +59,8 @@ def get_item_text(item: AnyItem) -> str:
|
||||
return f"{item.title} {item.channel_name}"
|
||||
elif isinstance(item, schema.TikTokItem):
|
||||
return f"{item.text} {item.author_name}"
|
||||
elif isinstance(item, schema.InstagramItem):
|
||||
return f"{item.text} {item.author_name}"
|
||||
elif isinstance(item, schema.PolymarketItem):
|
||||
return f"{item.title} {item.question}"
|
||||
elif isinstance(item, schema.WebSearchItem):
|
||||
@@ -78,6 +80,8 @@ def _get_cross_source_text(item: AnyItem) -> str:
|
||||
return item.text[:100]
|
||||
if isinstance(item, schema.TikTokItem):
|
||||
return item.text[:100]
|
||||
if isinstance(item, schema.InstagramItem):
|
||||
return item.text[:100]
|
||||
if isinstance(item, schema.HackerNewsItem):
|
||||
title = item.title
|
||||
if title.startswith("Show HN:"):
|
||||
@@ -206,6 +210,14 @@ def dedupe_tiktok(
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_instagram(
|
||||
items: List[schema.InstagramItem],
|
||||
threshold: float = 0.7,
|
||||
) -> List[schema.InstagramItem]:
|
||||
"""Dedupe Instagram items."""
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_hackernews(
|
||||
items: List[schema.HackerNewsItem],
|
||||
threshold: float = 0.7,
|
||||
|
||||
Reference in New Issue
Block a user