feat: v2.8 — Instagram Reels source + TikTok ScrapeCreators migration

Add Instagram Reels as the 8th research source via ScrapeCreators API.
One API key (SCRAPECREATORS_API_KEY) now covers both TikTok and Instagram.

- Add scripts/lib/instagram.py: keyword search, transcript extraction,
  relevance scoring, engagement metrics (views, likes, comments)
- Add InstagramItem to schema, normalization, scoring, dedup, rendering
- Add Instagram to orchestrator pipeline, watchlist, and UI spinners
- Update SKILL.md: stats template, citation priority, item format,
  URL-to-name extraction rules, anti-Sources instruction
- Update README and CHANGELOG for v2.8
- Fix: Instagram/TikTok not running in --search= web-only path
- Fix: web stats line showing full URLs instead of domain names
- Replace APIFY_API_TOKEN with SCRAPECREATORS_API_KEY throughout

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-04 06:57:52 -08:00
parent 740dcc5789
commit db75f9e341
17 changed files with 1609 additions and 67 deletions
+47 -1
View File
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
from . import dates, schema
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.HackerNewsItem, schema.PolymarketItem)
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem)
def filter_by_date_range(
@@ -247,6 +247,52 @@ def normalize_tiktok_items(
return normalized
def normalize_instagram_items(
items: List[Dict[str, Any]],
from_date: str,
to_date: str,
) -> List[schema.InstagramItem]:
"""Normalize raw Instagram items to schema.
Args:
items: Raw Instagram items from ScrapeCreators
from_date: Start of date range
to_date: End of date range
Returns:
List of InstagramItem objects
"""
normalized = []
for i, item in enumerate(items):
# Parse engagement
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
views=eng_raw.get("views"),
likes=eng_raw.get("likes"),
num_comments=eng_raw.get("comments"),
)
# Instagram dates are reliable (exact timestamps from ScrapeCreators)
date_str = item.get("date")
normalized.append(schema.InstagramItem(
id=f"IG{i+1}",
text=item.get("text", ""),
url=item.get("url", ""),
author_name=item.get("author_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
caption_snippet=item.get("caption_snippet", ""),
hashtags=item.get("hashtags", []),
relevance=item.get("relevance", 0.7),
why_relevant=item.get("why_relevant", ""),
))
return normalized
def normalize_hackernews_items(
items: List[Dict[str, Any]],
from_date: str,