feat: v2.8 — Instagram Reels source + TikTok ScrapeCreators migration
Add Instagram Reels as the 8th research source via ScrapeCreators API. One API key (SCRAPECREATORS_API_KEY) now covers both TikTok and Instagram. - Add scripts/lib/instagram.py: keyword search, transcript extraction, relevance scoring, engagement metrics (views, likes, comments) - Add InstagramItem to schema, normalization, scoring, dedup, rendering - Add Instagram to orchestrator pipeline, watchlist, and UI spinners - Update SKILL.md: stats template, citation priority, item format, URL-to-name extraction rules, anti-Sources instruction - Update README and CHANGELOG for v2.8 - Fix: Instagram/TikTok not running in --search= web-only path - Fix: web stats line showing full URLs instead of domain names - Replace APIFY_API_TOKEN with SCRAPECREATORS_API_KEY throughout Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.HackerNewsItem, schema.PolymarketItem)
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem)
|
||||
|
||||
|
||||
def filter_by_date_range(
|
||||
@@ -247,6 +247,52 @@ def normalize_tiktok_items(
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_instagram_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.InstagramItem]:
|
||||
"""Normalize raw Instagram items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw Instagram items from ScrapeCreators
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of InstagramItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for i, item in enumerate(items):
|
||||
# Parse engagement
|
||||
eng_raw = item.get("engagement") or {}
|
||||
engagement = schema.Engagement(
|
||||
views=eng_raw.get("views"),
|
||||
likes=eng_raw.get("likes"),
|
||||
num_comments=eng_raw.get("comments"),
|
||||
)
|
||||
|
||||
# Instagram dates are reliable (exact timestamps from ScrapeCreators)
|
||||
date_str = item.get("date")
|
||||
|
||||
normalized.append(schema.InstagramItem(
|
||||
id=f"IG{i+1}",
|
||||
text=item.get("text", ""),
|
||||
url=item.get("url", ""),
|
||||
author_name=item.get("author_name", ""),
|
||||
date=date_str,
|
||||
date_confidence="high",
|
||||
engagement=engagement,
|
||||
caption_snippet=item.get("caption_snippet", ""),
|
||||
hashtags=item.get("hashtags", []),
|
||||
relevance=item.get("relevance", 0.7),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_hackernews_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
|
||||
Reference in New Issue
Block a user