feat: v2.8 — Instagram Reels source + TikTok ScrapeCreators migration
Add Instagram Reels as the 8th research source via ScrapeCreators API. One API key (SCRAPECREATORS_API_KEY) now covers both TikTok and Instagram. - Add scripts/lib/instagram.py: keyword search, transcript extraction, relevance scoring, engagement metrics (views, likes, comments) - Add InstagramItem to schema, normalization, scoring, dedup, rendering - Add Instagram to orchestrator pipeline, watchlist, and UI spinners - Update SKILL.md: stats template, citation priority, item format, URL-to-name extraction rules, anti-Sources instruction - Update README and CHANGELOG for v2.8 - Fix: Instagram/TikTok not running in --search= web-only path - Fix: web stats line showing full URLs instead of domain names - Replace APIFY_API_TOKEN with SCRAPECREATORS_API_KEY throughout Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
+128
-13
@@ -38,13 +38,13 @@ _child_pids: set = set()
|
||||
_child_pids_lock = threading.Lock()
|
||||
|
||||
TIMEOUT_PROFILES = {
|
||||
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
|
||||
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
|
||||
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
||||
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
|
||||
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
|
||||
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
||||
}
|
||||
|
||||
# Valid source names for the --search flag
|
||||
VALID_SEARCH_SOURCES = {"reddit", "x", "hn", "youtube", "tiktok", "polymarket", "web"}
|
||||
VALID_SEARCH_SOURCES = {"reddit", "x", "hn", "youtube", "tiktok", "instagram", "polymarket", "web"}
|
||||
|
||||
|
||||
def parse_search_flag(search_str: str) -> set:
|
||||
@@ -146,6 +146,7 @@ from lib import (
|
||||
score,
|
||||
ui,
|
||||
tiktok,
|
||||
instagram,
|
||||
websearch,
|
||||
xai_x,
|
||||
youtube_yt,
|
||||
@@ -373,6 +374,35 @@ def _search_tiktok(
|
||||
return tiktok_items, tiktok_error
|
||||
|
||||
|
||||
def _search_instagram(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str,
|
||||
token: str,
|
||||
) -> tuple:
|
||||
"""Search Instagram via ScrapeCreators (runs in thread).
|
||||
|
||||
Returns:
|
||||
Tuple of (instagram_items, instagram_error)
|
||||
"""
|
||||
instagram_error = None
|
||||
|
||||
try:
|
||||
response = instagram.search_and_enrich(
|
||||
topic, from_date, to_date, depth=depth, token=token,
|
||||
)
|
||||
except Exception as e:
|
||||
return [], f"{type(e).__name__}: {e}"
|
||||
|
||||
instagram_items = instagram.parse_instagram_response(response)
|
||||
|
||||
if response.get("error"):
|
||||
instagram_error = response["error"]
|
||||
|
||||
return instagram_items, instagram_error
|
||||
|
||||
|
||||
def _search_hackernews(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
@@ -664,6 +694,7 @@ def run_research(
|
||||
x_source: str = "xai",
|
||||
run_youtube: bool = False,
|
||||
run_tiktok: bool = False,
|
||||
run_instagram: bool = False,
|
||||
timeouts: dict = None,
|
||||
resolved_handle: str = None,
|
||||
do_hackernews: bool = True,
|
||||
@@ -673,9 +704,11 @@ def run_research(
|
||||
"""Run the research pipeline.
|
||||
|
||||
Returns:
|
||||
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, web_items, web_needed,
|
||||
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items,
|
||||
hackernews_items, polymarket_items, web_items, web_needed,
|
||||
raw_openai, raw_xai, raw_reddit_enriched,
|
||||
reddit_error, x_error, youtube_error, tiktok_error, web_error)
|
||||
reddit_error, x_error, youtube_error, tiktok_error, instagram_error,
|
||||
hackernews_error, polymarket_error, web_error)
|
||||
|
||||
Note: web_needed is True when web search should be performed by the assistant
|
||||
(i.e., no native web search API keys are configured). When native web search
|
||||
@@ -689,6 +722,7 @@ def run_research(
|
||||
x_items = []
|
||||
youtube_items = []
|
||||
tiktok_items = []
|
||||
instagram_items = []
|
||||
hackernews_items = []
|
||||
polymarket_items = []
|
||||
web_items = []
|
||||
@@ -699,6 +733,7 @@ def run_research(
|
||||
x_error = None
|
||||
youtube_error = None
|
||||
tiktok_error = None
|
||||
instagram_error = None
|
||||
hackernews_error = None
|
||||
polymarket_error = None
|
||||
web_error = None
|
||||
@@ -729,7 +764,7 @@ def run_research(
|
||||
if progress:
|
||||
progress.start_web_only()
|
||||
progress.end_web_only()
|
||||
# Still run YouTube in web-only mode if yt-dlp is available
|
||||
# Still run YouTube/TikTok/Instagram in web-only mode if available
|
||||
if run_youtube:
|
||||
if progress:
|
||||
progress.start_youtube()
|
||||
@@ -743,7 +778,34 @@ def run_research(
|
||||
progress.show_error(f"YouTube error: {e}")
|
||||
if progress:
|
||||
progress.end_youtube(len(youtube_items))
|
||||
return reddit_items, x_items, youtube_items, tiktok_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, hackernews_error, polymarket_error, web_error
|
||||
if run_tiktok:
|
||||
if progress:
|
||||
progress.start_tiktok()
|
||||
try:
|
||||
tiktok_items, tiktok_error = _search_tiktok(topic, from_date, to_date, depth, env.get_tiktok_token(config))
|
||||
if tiktok_error and progress:
|
||||
progress.show_error(f"TikTok error: {tiktok_error}")
|
||||
except Exception as e:
|
||||
tiktok_error = f"{type(e).__name__}: {e}"
|
||||
if progress:
|
||||
progress.show_error(f"TikTok error: {e}")
|
||||
if progress:
|
||||
progress.end_tiktok(len(tiktok_items))
|
||||
if run_instagram:
|
||||
if progress:
|
||||
progress.start_instagram()
|
||||
try:
|
||||
ig_timeout = timeouts.get("instagram_future", future_timeout)
|
||||
instagram_items, instagram_error = _search_instagram(topic, from_date, to_date, depth, env.get_instagram_token(config))
|
||||
if instagram_error and progress:
|
||||
progress.show_error(f"Instagram error: {instagram_error}")
|
||||
except Exception as e:
|
||||
instagram_error = f"{type(e).__name__}: {e}"
|
||||
if progress:
|
||||
progress.show_error(f"Instagram error: {e}")
|
||||
if progress:
|
||||
progress.end_instagram(len(instagram_items))
|
||||
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error
|
||||
|
||||
# Determine which searches to run
|
||||
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
|
||||
@@ -756,10 +818,11 @@ def run_research(
|
||||
x_future = None
|
||||
youtube_future = None
|
||||
tiktok_future = None
|
||||
instagram_future = None
|
||||
hackernews_future = None
|
||||
polymarket_future = None
|
||||
web_future = None
|
||||
max_workers = 2 + (1 if run_youtube else 0) + (1 if run_tiktok else 0) + (1 if do_hackernews else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0)
|
||||
max_workers = 2 + (1 if run_youtube else 0) + (1 if run_tiktok else 0) + (1 if run_instagram else 0) + (1 if do_hackernews else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
# Submit searches
|
||||
@@ -794,6 +857,14 @@ def run_research(
|
||||
env.get_tiktok_token(config),
|
||||
)
|
||||
|
||||
if run_instagram:
|
||||
if progress:
|
||||
progress.start_instagram()
|
||||
instagram_future = executor.submit(
|
||||
_search_instagram, topic, from_date, to_date, depth,
|
||||
env.get_instagram_token(config),
|
||||
)
|
||||
|
||||
if do_hackernews:
|
||||
if progress:
|
||||
progress.start_hackernews()
|
||||
@@ -883,6 +954,23 @@ def run_research(
|
||||
if progress:
|
||||
progress.end_tiktok(len(tiktok_items))
|
||||
|
||||
if instagram_future:
|
||||
ig_timeout = timeouts.get("instagram_future", future_timeout)
|
||||
try:
|
||||
instagram_items, instagram_error = instagram_future.result(timeout=ig_timeout)
|
||||
if instagram_error and progress:
|
||||
progress.show_error(f"Instagram error: {instagram_error}")
|
||||
except TimeoutError:
|
||||
instagram_error = f"Instagram search timed out after {ig_timeout}s"
|
||||
if progress:
|
||||
progress.show_error(instagram_error)
|
||||
except Exception as e:
|
||||
instagram_error = f"{type(e).__name__}: {e}"
|
||||
if progress:
|
||||
progress.show_error(f"Instagram error: {e}")
|
||||
if progress:
|
||||
progress.end_instagram(len(instagram_items))
|
||||
|
||||
if hackernews_future:
|
||||
hn_timeout = timeouts.get("hackernews_future", future_timeout)
|
||||
try:
|
||||
@@ -1025,7 +1113,7 @@ def run_research(
|
||||
if sup_x:
|
||||
x_items.extend(sup_x)
|
||||
|
||||
return reddit_items, x_items, youtube_items, tiktok_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, hackernews_error, polymarket_error, web_error
|
||||
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error
|
||||
|
||||
|
||||
def main():
|
||||
@@ -1163,6 +1251,9 @@ def main():
|
||||
# Auto-detect ScrapeCreators/Apify for TikTok
|
||||
has_tiktok = env.is_tiktok_available(config)
|
||||
|
||||
# Auto-detect ScrapeCreators for Instagram
|
||||
has_instagram = env.is_instagram_available(config)
|
||||
|
||||
# --diagnose: show source availability and exit
|
||||
if args.diagnose:
|
||||
web_source = env.get_web_search_source(config)
|
||||
@@ -1175,6 +1266,7 @@ def main():
|
||||
"bird_username": x_source_status.get("bird_username"),
|
||||
"youtube": has_ytdlp,
|
||||
"tiktok": has_tiktok,
|
||||
"instagram": has_instagram,
|
||||
"hackernews": True,
|
||||
"polymarket": True,
|
||||
"web_search_backend": web_source,
|
||||
@@ -1290,6 +1382,7 @@ def main():
|
||||
search_do_polymarket = True
|
||||
search_run_youtube = has_ytdlp
|
||||
search_run_tiktok = has_tiktok
|
||||
search_run_instagram = has_instagram
|
||||
if args.search:
|
||||
search_sources = parse_search_flag(args.search)
|
||||
has_reddit = "reddit" in search_sources
|
||||
@@ -1298,6 +1391,7 @@ def main():
|
||||
search_do_polymarket = "polymarket" in search_sources
|
||||
search_run_youtube = "youtube" in search_sources and has_ytdlp
|
||||
search_run_tiktok = "tiktok" in search_sources and has_tiktok
|
||||
search_run_instagram = "instagram" in search_sources and has_instagram
|
||||
include_search_web = "web" in search_sources
|
||||
# Map to existing sources string
|
||||
if has_reddit and has_x:
|
||||
@@ -1311,7 +1405,7 @@ def main():
|
||||
sources = "web" # hn/polymarket only; no Reddit/X
|
||||
|
||||
# Run research
|
||||
reddit_items, x_items, youtube_items, tiktok_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, hackernews_error, polymarket_error, web_error = run_research(
|
||||
reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error = run_research(
|
||||
args.topic,
|
||||
sources,
|
||||
config,
|
||||
@@ -1324,6 +1418,7 @@ def main():
|
||||
x_source=x_source or "xai",
|
||||
run_youtube=search_run_youtube,
|
||||
run_tiktok=search_run_tiktok,
|
||||
run_instagram=search_run_instagram,
|
||||
timeouts=timeouts,
|
||||
resolved_handle=args.x_handle,
|
||||
do_hackernews=search_do_hackernews,
|
||||
@@ -1339,6 +1434,7 @@ def main():
|
||||
normalized_x = normalize.normalize_x_items(x_items, from_date, to_date)
|
||||
normalized_youtube = normalize.normalize_youtube_items(youtube_items, from_date, to_date) if youtube_items else []
|
||||
normalized_tiktok = normalize.normalize_tiktok_items(tiktok_items, from_date, to_date) if tiktok_items else []
|
||||
normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else []
|
||||
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
|
||||
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
|
||||
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
|
||||
@@ -1353,6 +1449,8 @@ def main():
|
||||
filtered_youtube = normalized_youtube
|
||||
# TikTok: hard date filter (tiktok.py already pre-filters, but safety net)
|
||||
filtered_tiktok = normalize.filter_by_date_range(normalized_tiktok, from_date, to_date) if normalized_tiktok else []
|
||||
# Instagram: hard date filter (instagram.py already pre-filters, but safety net)
|
||||
filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else []
|
||||
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
|
||||
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
|
||||
filtered_pm = normalized_pm
|
||||
@@ -1363,6 +1461,7 @@ def main():
|
||||
scored_x = score.score_x_items(filtered_x)
|
||||
scored_youtube = score.score_youtube_items(filtered_youtube) if filtered_youtube else []
|
||||
scored_tiktok = score.score_tiktok_items(filtered_tiktok) if filtered_tiktok else []
|
||||
scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else []
|
||||
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
|
||||
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
|
||||
scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
|
||||
@@ -1372,6 +1471,7 @@ def main():
|
||||
sorted_x = score.sort_items(scored_x)
|
||||
sorted_youtube = score.sort_items(scored_youtube) if scored_youtube else []
|
||||
sorted_tiktok = score.sort_items(scored_tiktok) if scored_tiktok else []
|
||||
sorted_ig = score.sort_items(scored_ig) if scored_ig else []
|
||||
sorted_hn = score.sort_items(scored_hn) if scored_hn else []
|
||||
sorted_pm = score.sort_items(scored_pm) if scored_pm else []
|
||||
sorted_web = score.sort_items(scored_web) if scored_web else []
|
||||
@@ -1381,6 +1481,7 @@ def main():
|
||||
deduped_x = dedupe.dedupe_x(sorted_x)
|
||||
deduped_youtube = dedupe.dedupe_youtube(sorted_youtube) if sorted_youtube else []
|
||||
deduped_tiktok = dedupe.dedupe_tiktok(sorted_tiktok) if sorted_tiktok else []
|
||||
deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else []
|
||||
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
|
||||
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
|
||||
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
|
||||
@@ -1394,7 +1495,7 @@ def main():
|
||||
|
||||
# Cross-source linking: annotate items that discuss the same story
|
||||
dedupe.cross_source_link(
|
||||
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_hn, deduped_pm, deduped_web,
|
||||
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_pm, deduped_web,
|
||||
)
|
||||
|
||||
progress.end_processing()
|
||||
@@ -1412,6 +1513,7 @@ def main():
|
||||
report.x = deduped_x
|
||||
report.youtube = deduped_youtube
|
||||
report.tiktok = deduped_tiktok
|
||||
report.instagram = deduped_ig
|
||||
report.hackernews = deduped_hn
|
||||
report.polymarket = deduped_pm
|
||||
report.web = deduped_web
|
||||
@@ -1419,6 +1521,7 @@ def main():
|
||||
report.x_error = x_error
|
||||
report.youtube_error = youtube_error
|
||||
report.tiktok_error = tiktok_error
|
||||
report.instagram_error = instagram_error
|
||||
report.hackernews_error = hackernews_error
|
||||
report.polymarket_error = polymarket_error
|
||||
report.web_error = web_error
|
||||
@@ -1434,7 +1537,7 @@ def main():
|
||||
if sources == "web":
|
||||
progress.show_web_only_complete()
|
||||
else:
|
||||
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn), len(deduped_pm), len(deduped_tiktok))
|
||||
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn), len(deduped_pm), len(deduped_tiktok), len(deduped_ig))
|
||||
|
||||
# Build source info for status footer
|
||||
source_info = {}
|
||||
@@ -1451,6 +1554,8 @@ def main():
|
||||
source_info["youtube_skip_reason"] = "0 results (query may be too specific)"
|
||||
if not has_tiktok:
|
||||
source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)"
|
||||
if not has_instagram:
|
||||
source_info["instagram_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)"
|
||||
if not web_source:
|
||||
source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)"
|
||||
|
||||
@@ -1516,6 +1621,16 @@ def main():
|
||||
"engagement_score": item.engagement.volume if item.engagement and item.engagement.volume else 0,
|
||||
"relevance_score": item.relevance,
|
||||
})
|
||||
for item in deduped_ig:
|
||||
findings.append({
|
||||
"source": "instagram",
|
||||
"url": item.url,
|
||||
"title": item.text[:100],
|
||||
"author": item.author_name,
|
||||
"content": item.caption_snippet[:500] if item.caption_snippet else item.text,
|
||||
"engagement_score": item.engagement.views if item.engagement and item.engagement.views else 0,
|
||||
"relevance_score": item.relevance,
|
||||
})
|
||||
for item in deduped_web:
|
||||
findings.append({
|
||||
"source": "web",
|
||||
|
||||
+13
-1
@@ -46,7 +46,7 @@ def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
|
||||
|
||||
|
||||
AnyItem = Union[schema.RedditItem, schema.XItem, schema.YouTubeItem, schema.TikTokItem,
|
||||
schema.HackerNewsItem, schema.PolymarketItem, schema.WebSearchItem]
|
||||
schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem, schema.WebSearchItem]
|
||||
|
||||
|
||||
def get_item_text(item: AnyItem) -> str:
|
||||
@@ -59,6 +59,8 @@ def get_item_text(item: AnyItem) -> str:
|
||||
return f"{item.title} {item.channel_name}"
|
||||
elif isinstance(item, schema.TikTokItem):
|
||||
return f"{item.text} {item.author_name}"
|
||||
elif isinstance(item, schema.InstagramItem):
|
||||
return f"{item.text} {item.author_name}"
|
||||
elif isinstance(item, schema.PolymarketItem):
|
||||
return f"{item.title} {item.question}"
|
||||
elif isinstance(item, schema.WebSearchItem):
|
||||
@@ -78,6 +80,8 @@ def _get_cross_source_text(item: AnyItem) -> str:
|
||||
return item.text[:100]
|
||||
if isinstance(item, schema.TikTokItem):
|
||||
return item.text[:100]
|
||||
if isinstance(item, schema.InstagramItem):
|
||||
return item.text[:100]
|
||||
if isinstance(item, schema.HackerNewsItem):
|
||||
title = item.title
|
||||
if title.startswith("Show HN:"):
|
||||
@@ -206,6 +210,14 @@ def dedupe_tiktok(
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_instagram(
|
||||
items: List[schema.InstagramItem],
|
||||
threshold: float = 0.7,
|
||||
) -> List[schema.InstagramItem]:
|
||||
"""Dedupe Instagram items."""
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_hackernews(
|
||||
items: List[schema.HackerNewsItem],
|
||||
threshold: float = 0.7,
|
||||
|
||||
@@ -421,6 +421,20 @@ def get_tiktok_token(config: Dict[str, Any]) -> str:
|
||||
return config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN') or ''
|
||||
|
||||
|
||||
def is_instagram_available(config: Dict[str, Any]) -> bool:
|
||||
"""Check if Instagram source is available (ScrapeCreators).
|
||||
|
||||
Returns True if SCRAPECREATORS_API_KEY is set.
|
||||
Instagram uses the same key as TikTok.
|
||||
"""
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY'))
|
||||
|
||||
|
||||
def get_instagram_token(config: Dict[str, Any]) -> str:
|
||||
"""Get Instagram API token (same ScrapeCreators key as TikTok)."""
|
||||
return config.get('SCRAPECREATORS_API_KEY') or ''
|
||||
|
||||
|
||||
# Backward compat alias
|
||||
is_apify_available = is_tiktok_available
|
||||
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
"""Instagram Reels search via ScrapeCreators API for /last30days.
|
||||
|
||||
Uses ScrapeCreators REST API to search Instagram Reels by keyword, extract
|
||||
engagement metrics (views, likes, comments), and fetch video transcripts.
|
||||
|
||||
Requires SCRAPECREATORS_API_KEY in config. 100 free credits, then PAYG.
|
||||
API docs: https://scrapecreators.com/docs
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
|
||||
|
||||
# Depth configurations: how many results to fetch / captions to extract
|
||||
DEPTH_CONFIG = {
|
||||
"quick": {"results_per_page": 10, "max_captions": 3},
|
||||
"default": {"results_per_page": 20, "max_captions": 5},
|
||||
"deep": {"results_per_page": 40, "max_captions": 8},
|
||||
}
|
||||
|
||||
# Max words to keep from each caption
|
||||
CAPTION_MAX_WORDS = 500
|
||||
|
||||
# Stopwords for relevance computation (shared with tiktok.py pattern)
|
||||
STOPWORDS = frozenset({
|
||||
'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
|
||||
'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
|
||||
'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
|
||||
'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
|
||||
'all', 'just', 'get', 'has', 'have', 'was', 'will',
|
||||
})
|
||||
|
||||
# Synonym groups for relevance scoring
|
||||
SYNONYMS = {
|
||||
'hip': {'rap', 'hiphop'},
|
||||
'hop': {'rap', 'hiphop'},
|
||||
'rap': {'hip', 'hop', 'hiphop'},
|
||||
'hiphop': {'rap', 'hip', 'hop'},
|
||||
'js': {'javascript'},
|
||||
'javascript': {'js'},
|
||||
'ts': {'typescript'},
|
||||
'typescript': {'ts'},
|
||||
'ai': {'artificial', 'intelligence'},
|
||||
'ml': {'machine', 'learning'},
|
||||
'react': {'reactjs'},
|
||||
'reactjs': {'react'},
|
||||
}
|
||||
|
||||
|
||||
def _tokenize(text: str) -> Set[str]:
|
||||
"""Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
|
||||
words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
|
||||
tokens = {w for w in words if w not in STOPWORDS and len(w) > 1}
|
||||
expanded = set(tokens)
|
||||
for t in tokens:
|
||||
if t in SYNONYMS:
|
||||
expanded.update(SYNONYMS[t])
|
||||
return expanded
|
||||
|
||||
|
||||
def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> float:
|
||||
"""Compute relevance as ratio of query tokens found in text + hashtags.
|
||||
|
||||
Uses ratio overlap (intersection / query_length). Hashtags provide
|
||||
an Instagram-specific relevance boost. Floors at 0.1.
|
||||
"""
|
||||
q_tokens = _tokenize(query)
|
||||
|
||||
# Combine text and hashtags for matching
|
||||
combined = text
|
||||
if hashtags:
|
||||
combined = f"{text} {' '.join(hashtags)}"
|
||||
t_tokens = _tokenize(combined)
|
||||
|
||||
# Split concatenated hashtags (e.g., "claudecode" -> "claude", "code")
|
||||
if hashtags:
|
||||
for tag in hashtags:
|
||||
tag_lower = tag.lower()
|
||||
for qt in q_tokens:
|
||||
if qt in tag_lower and qt != tag_lower:
|
||||
t_tokens.add(qt)
|
||||
|
||||
if not q_tokens:
|
||||
return 0.5 # Neutral fallback
|
||||
|
||||
overlap = len(q_tokens & t_tokens)
|
||||
ratio = overlap / len(q_tokens)
|
||||
return max(0.1, min(1.0, ratio))
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from verbose query for Instagram search.
|
||||
|
||||
Strips meta/research words to keep only the core product/concept name.
|
||||
"""
|
||||
text = topic.lower().strip()
|
||||
|
||||
# Strip multi-word prefixes
|
||||
prefixes = [
|
||||
'what are the best', 'what is the best', 'what are the latest',
|
||||
'what are people saying about', 'what do people think about',
|
||||
'how do i use', 'how to use', 'how to',
|
||||
'what are', 'what is', 'tips for', 'best practices for',
|
||||
]
|
||||
for p in prefixes:
|
||||
if text.startswith(p + ' '):
|
||||
text = text[len(p):].strip()
|
||||
|
||||
# Strip individual noise words
|
||||
noise = {
|
||||
'best', 'top', 'good', 'great', 'awesome', 'killer',
|
||||
'latest', 'new', 'news', 'update', 'updates',
|
||||
'trending', 'hottest', 'popular', 'viral',
|
||||
'practices', 'features',
|
||||
'recommendations', 'advice',
|
||||
'prompt', 'prompts', 'prompting',
|
||||
'methods', 'strategies', 'approaches',
|
||||
}
|
||||
words = text.split()
|
||||
filtered = [w for w in words if w not in noise]
|
||||
|
||||
result = ' '.join(filtered) if filtered else text
|
||||
return result.rstrip('?!.')
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
"""Log to stderr (only in interactive terminals; spinner handles non-TTY)."""
|
||||
if sys.stderr.isatty():
|
||||
sys.stderr.write(f"[Instagram] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def _sc_headers(token: str) -> Dict[str, str]:
|
||||
"""Build ScrapeCreators request headers."""
|
||||
return {
|
||||
"x-api-key": token,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
||||
"""Parse date from ScrapeCreators Instagram item to YYYY-MM-DD.
|
||||
|
||||
Handles taken_at as ISO string (e.g. "2026-02-26T16:00:00.000Z")
|
||||
or unix timestamp.
|
||||
"""
|
||||
ts = item.get("taken_at")
|
||||
if not ts:
|
||||
return None
|
||||
|
||||
# Try ISO string first (ScrapeCreators reels/search returns this)
|
||||
if isinstance(ts, str):
|
||||
try:
|
||||
# Handle "2026-02-26T16:00:00.000Z" format
|
||||
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
# Try just the date portion
|
||||
if len(ts) >= 10:
|
||||
return ts[:10]
|
||||
|
||||
# Fall back to unix timestamp
|
||||
try:
|
||||
dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError, OSError):
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_hashtags(caption_text: str) -> List[str]:
|
||||
"""Extract hashtags from Instagram caption text."""
|
||||
if not caption_text:
|
||||
return []
|
||||
return re.findall(r'#(\w+)', caption_text)
|
||||
|
||||
|
||||
def search_instagram(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
token: str = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Search Instagram Reels via ScrapeCreators API.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
token: ScrapeCreators API key
|
||||
|
||||
Returns:
|
||||
Dict with 'items' list and optional 'error'.
|
||||
"""
|
||||
if not token:
|
||||
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
|
||||
|
||||
if not _requests:
|
||||
return {"items": [], "error": "requests library not installed"}
|
||||
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
core_topic = _extract_core_subject(topic)
|
||||
|
||||
_log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
|
||||
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v1/instagram/reels/search",
|
||||
params={"query": core_topic},
|
||||
headers=_sc_headers(token),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
_log(f"ScrapeCreators error: {e}")
|
||||
return {"items": [], "error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Items are in the 'reels' array (ScrapeCreators v1 response)
|
||||
raw_items = data.get("reels") or data.get("items") or data.get("data") or []
|
||||
|
||||
# Limit to configured count
|
||||
raw_items = raw_items[:config["results_per_page"]]
|
||||
|
||||
# Parse items
|
||||
items = []
|
||||
for raw in raw_items:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
|
||||
# Extract reel ID and shortcode
|
||||
reel_pk = str(raw.get("id", raw.get("pk", "")))
|
||||
shortcode = raw.get("shortcode", raw.get("code", ""))
|
||||
|
||||
# Caption text — can be a string or dict depending on endpoint
|
||||
caption_obj = raw.get("caption", "")
|
||||
if isinstance(caption_obj, dict):
|
||||
text = caption_obj.get("text", "")
|
||||
elif isinstance(caption_obj, str):
|
||||
text = caption_obj
|
||||
else:
|
||||
text = raw.get("desc", raw.get("text", ""))
|
||||
|
||||
# Engagement metrics
|
||||
play_count = raw.get("video_play_count") or raw.get("video_view_count") or raw.get("play_count") or 0
|
||||
like_count = raw.get("like_count") or 0
|
||||
comment_count = raw.get("comment_count") or 0
|
||||
|
||||
# Author info — 'owner' in reels/search, 'user' in user/reels
|
||||
owner = raw.get("owner") or raw.get("user") or {}
|
||||
author_name = owner.get("username", "")
|
||||
|
||||
# Duration
|
||||
duration = raw.get("video_duration")
|
||||
|
||||
# Date
|
||||
date_str = _parse_date(raw)
|
||||
|
||||
# Hashtags from caption text
|
||||
hashtags = _extract_hashtags(text)
|
||||
|
||||
# Compute relevance with hashtag boost
|
||||
relevance = _compute_relevance(core_topic, text, hashtags)
|
||||
|
||||
# Build URL — prefer API-provided url, fallback to shortcode
|
||||
url = raw.get("url", "")
|
||||
if not url and shortcode:
|
||||
url = f"https://www.instagram.com/reel/{shortcode}"
|
||||
|
||||
items.append({
|
||||
"video_id": reel_pk,
|
||||
"text": text,
|
||||
"url": url,
|
||||
"author_name": author_name,
|
||||
"date": date_str,
|
||||
"engagement": {
|
||||
"views": play_count,
|
||||
"likes": like_count,
|
||||
"comments": comment_count,
|
||||
},
|
||||
"hashtags": hashtags,
|
||||
"duration": duration,
|
||||
"relevance": relevance,
|
||||
"why_relevant": f"Instagram: {text[:60]}" if text else f"Instagram: {core_topic}",
|
||||
"caption_snippet": "", # populated by fetch_captions
|
||||
})
|
||||
|
||||
# Hard date filter
|
||||
in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
|
||||
out_of_range = len(items) - len(in_range)
|
||||
if in_range:
|
||||
items = in_range
|
||||
if out_of_range:
|
||||
_log(f"Filtered {out_of_range} reels outside date range")
|
||||
else:
|
||||
_log(f"No reels within date range, keeping all {len(items)}")
|
||||
|
||||
# Sort by views descending
|
||||
items.sort(key=lambda x: x["engagement"]["views"], reverse=True)
|
||||
|
||||
_log(f"Found {len(items)} Instagram reels")
|
||||
return {"items": items}
|
||||
|
||||
|
||||
def fetch_captions(
|
||||
video_items: List[Dict[str, Any]],
|
||||
token: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, str]:
|
||||
"""Fetch transcripts for top N Instagram reels via ScrapeCreators.
|
||||
|
||||
Strategy:
|
||||
1. Use the 'text' field (caption) as baseline
|
||||
2. For top N, call /v2/instagram/media/transcript for spoken-word captions
|
||||
|
||||
Args:
|
||||
video_items: Items from search_instagram()
|
||||
token: ScrapeCreators API key
|
||||
depth: Depth level for caption limit
|
||||
|
||||
Returns:
|
||||
Dict mapping video_id -> caption text (truncated to 500 words)
|
||||
"""
|
||||
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
max_captions = config["max_captions"]
|
||||
|
||||
if not video_items or not token or not _requests:
|
||||
return {}
|
||||
|
||||
top_items = video_items[:max_captions]
|
||||
_log(f"Enriching captions for {len(top_items)} reels")
|
||||
|
||||
captions = {}
|
||||
|
||||
# First pass: use text field as caption (always available, free)
|
||||
for item in top_items:
|
||||
vid = item["video_id"]
|
||||
text = item.get("text", "")
|
||||
if text:
|
||||
words = text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = text
|
||||
|
||||
# Second pass: try to get spoken-word transcripts (1 credit each)
|
||||
for item in top_items:
|
||||
vid = item["video_id"]
|
||||
url = item.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
resp = _requests.get(
|
||||
f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
|
||||
params={"url": url},
|
||||
headers=_sc_headers(token),
|
||||
timeout=15,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
transcripts = data.get("transcripts") or []
|
||||
if transcripts and isinstance(transcripts, list):
|
||||
# Combine all transcript segments
|
||||
transcript_text = " ".join(
|
||||
t.get("text", "") for t in transcripts
|
||||
if isinstance(t, dict) and t.get("text")
|
||||
)
|
||||
if transcript_text:
|
||||
words = transcript_text.split()
|
||||
if len(words) > CAPTION_MAX_WORDS:
|
||||
transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
|
||||
captions[vid] = transcript_text
|
||||
except Exception as e:
|
||||
_log(f"Transcript fetch failed for {vid}: {e}")
|
||||
|
||||
got = sum(1 for v in captions.values() if v)
|
||||
_log(f"Got captions for {got}/{len(top_items)} reels")
|
||||
return captions
|
||||
|
||||
|
||||
def search_and_enrich(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
token: str = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Full Instagram search: find reels, then fetch captions for top results.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
token: ScrapeCreators API key
|
||||
|
||||
Returns:
|
||||
Dict with 'items' list. Each item has a 'caption_snippet' field.
|
||||
"""
|
||||
# Step 1: Search
|
||||
search_result = search_instagram(topic, from_date, to_date, depth, token)
|
||||
items = search_result.get("items", [])
|
||||
|
||||
if not items:
|
||||
return search_result
|
||||
|
||||
# Step 2: Fetch captions for top N
|
||||
captions = fetch_captions(items, token, depth)
|
||||
|
||||
# Step 3: Attach captions to items
|
||||
for item in items:
|
||||
vid = item["video_id"]
|
||||
caption = captions.get(vid)
|
||||
if caption:
|
||||
item["caption_snippet"] = caption
|
||||
|
||||
return {"items": items, "error": search_result.get("error")}
|
||||
|
||||
|
||||
def parse_instagram_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Parse Instagram search response to normalized format.
|
||||
|
||||
Returns:
|
||||
List of item dicts ready for normalization.
|
||||
"""
|
||||
return response.get("items", [])
|
||||
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.HackerNewsItem, schema.PolymarketItem)
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem)
|
||||
|
||||
|
||||
def filter_by_date_range(
|
||||
@@ -247,6 +247,52 @@ def normalize_tiktok_items(
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_instagram_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.InstagramItem]:
|
||||
"""Normalize raw Instagram items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw Instagram items from ScrapeCreators
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of InstagramItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for i, item in enumerate(items):
|
||||
# Parse engagement
|
||||
eng_raw = item.get("engagement") or {}
|
||||
engagement = schema.Engagement(
|
||||
views=eng_raw.get("views"),
|
||||
likes=eng_raw.get("likes"),
|
||||
num_comments=eng_raw.get("comments"),
|
||||
)
|
||||
|
||||
# Instagram dates are reliable (exact timestamps from ScrapeCreators)
|
||||
date_str = item.get("date")
|
||||
|
||||
normalized.append(schema.InstagramItem(
|
||||
id=f"IG{i+1}",
|
||||
text=item.get("text", ""),
|
||||
url=item.get("url", ""),
|
||||
author_name=item.get("author_name", ""),
|
||||
date=date_str,
|
||||
date_confidence="high",
|
||||
engagement=engagement,
|
||||
caption_snippet=item.get("caption_snippet", ""),
|
||||
hashtags=item.get("hashtags", []),
|
||||
relevance=item.get("relevance", 0.7),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_hackernews_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
|
||||
+74
-2
@@ -26,6 +26,8 @@ def _xref_tag(item) -> str:
|
||||
source_names.add('YouTube')
|
||||
elif ref_id.startswith('TK'):
|
||||
source_names.add('TikTok')
|
||||
elif ref_id.startswith('IG'):
|
||||
source_names.add('Instagram')
|
||||
elif ref_id.startswith('HN'):
|
||||
source_names.add('HN')
|
||||
elif ref_id.startswith('PM'):
|
||||
@@ -60,9 +62,10 @@ def _assess_data_freshness(report: schema.Report) -> dict:
|
||||
pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from)
|
||||
|
||||
tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from)
|
||||
ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from)
|
||||
|
||||
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent + tiktok_recent
|
||||
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket) + len(report.tiktok)
|
||||
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent + tiktok_recent + ig_recent
|
||||
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket) + len(report.tiktok) + len(report.instagram)
|
||||
|
||||
return {
|
||||
"reddit_recent": reddit_recent,
|
||||
@@ -284,6 +287,42 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
# Instagram items
|
||||
if report.instagram_error:
|
||||
lines.append("### Instagram Reels")
|
||||
lines.append("")
|
||||
lines.append(f"**ERROR:** {report.instagram_error}")
|
||||
lines.append("")
|
||||
elif report.instagram:
|
||||
lines.append("### Instagram Reels")
|
||||
lines.append("")
|
||||
for item in report.instagram[:limit]:
|
||||
eng_str = ""
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
parts = []
|
||||
if eng.views is not None:
|
||||
parts.append(f"{eng.views:,} views")
|
||||
if eng.likes is not None:
|
||||
parts.append(f"{eng.likes:,} likes")
|
||||
if parts:
|
||||
eng_str = f" [{', '.join(parts)}]"
|
||||
|
||||
date_str = f" ({item.date})" if item.date else ""
|
||||
|
||||
lines.append(f"**{item.id}** (score:{item.score}) @{item.author_name}{date_str}{eng_str}{_xref_tag(item)}")
|
||||
lines.append(f" {item.text[:200]}")
|
||||
lines.append(f" {item.url}")
|
||||
if item.caption_snippet and item.caption_snippet != item.text[:len(item.caption_snippet)]:
|
||||
snippet = item.caption_snippet[:200]
|
||||
if len(item.caption_snippet) > 200:
|
||||
snippet += "..."
|
||||
lines.append(f" Caption: {snippet}")
|
||||
if item.hashtags:
|
||||
lines.append(f" Tags: {' '.join('#' + h for h in item.hashtags[:8])}")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
# Hacker News items
|
||||
if report.hackernews_error:
|
||||
lines.append("### Hacker News Stories")
|
||||
@@ -455,6 +494,14 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
|
||||
lines.append(f" ✅ TikTok: {len(report.tiktok)} videos ({with_captions} with captions)")
|
||||
# Hide when zero results
|
||||
|
||||
# Instagram
|
||||
if report.instagram_error:
|
||||
lines.append(f" ❌ Instagram: error — {report.instagram_error}")
|
||||
elif report.instagram:
|
||||
with_captions = sum(1 for v in report.instagram if getattr(v, 'caption_snippet', None))
|
||||
lines.append(f" ✅ Instagram: {len(report.instagram)} reels ({with_captions} with captions)")
|
||||
# Hide when zero results
|
||||
|
||||
# Hacker News
|
||||
if report.hackernews_error:
|
||||
lines.append(f" ❌ HN: error - {report.hackernews_error}")
|
||||
@@ -508,6 +555,8 @@ def render_context_snippet(report: schema.Report) -> str:
|
||||
all_items.append((item.score, "X", item.text[:50] + "...", item.url))
|
||||
for item in report.tiktok[:5]:
|
||||
all_items.append((item.score, "TikTok", item.text[:50] + "...", item.url))
|
||||
for item in report.instagram[:5]:
|
||||
all_items.append((item.score, "Instagram", item.text[:50] + "...", item.url))
|
||||
for item in report.hackernews[:5]:
|
||||
all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url))
|
||||
for item in report.polymarket[:5]:
|
||||
@@ -624,6 +673,29 @@ def render_full_report(report: schema.Report) -> str:
|
||||
lines.append(f"> {item.text[:300]}")
|
||||
lines.append("")
|
||||
|
||||
# Instagram section
|
||||
if report.instagram:
|
||||
lines.append("## Instagram Reels")
|
||||
lines.append("")
|
||||
for item in report.instagram:
|
||||
lines.append(f"### {item.id}: @{item.author_name}")
|
||||
lines.append("")
|
||||
lines.append(f"- **URL:** {item.url}")
|
||||
lines.append(f"- **Date:** {item.date or 'Unknown'}")
|
||||
lines.append(f"- **Score:** {item.score}/100")
|
||||
lines.append(f"- **Relevance:** {item.why_relevant}")
|
||||
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
lines.append(f"- **Engagement:** {eng.views or '?'} views, {eng.likes or '?'} likes, {eng.num_comments or '?'} comments")
|
||||
|
||||
if item.hashtags:
|
||||
lines.append(f"- **Hashtags:** {' '.join('#' + h for h in item.hashtags[:10])}")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"> {item.text[:300]}")
|
||||
lines.append("")
|
||||
|
||||
# HN section
|
||||
if report.hackernews:
|
||||
lines.append("## Hacker News Stories")
|
||||
|
||||
@@ -275,6 +275,45 @@ class TikTokItem:
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class InstagramItem:
|
||||
"""Normalized Instagram item."""
|
||||
id: str # "IG1", "IG2", ...
|
||||
text: str # caption text
|
||||
url: str # https://www.instagram.com/reel/{code}
|
||||
author_name: str # Instagram handle
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "high" # ScrapeCreators provides exact timestamps
|
||||
engagement: Optional[Engagement] = None # views, likes, num_comments
|
||||
caption_snippet: str = "" # spoken-word caption (if available), else text
|
||||
hashtags: List[str] = field(default_factory=list)
|
||||
relevance: float = 0.7
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
cross_refs: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {
|
||||
'id': self.id,
|
||||
'text': self.text,
|
||||
'url': self.url,
|
||||
'author_name': self.author_name,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||
'caption_snippet': self.caption_snippet,
|
||||
'hashtags': self.hashtags,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
if self.cross_refs:
|
||||
d['cross_refs'] = self.cross_refs
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class HackerNewsItem:
|
||||
"""Normalized Hacker News item."""
|
||||
@@ -374,6 +413,7 @@ class Report:
|
||||
web: List[WebSearchItem] = field(default_factory=list)
|
||||
youtube: List[YouTubeItem] = field(default_factory=list)
|
||||
tiktok: List[TikTokItem] = field(default_factory=list)
|
||||
instagram: List[InstagramItem] = field(default_factory=list)
|
||||
hackernews: List[HackerNewsItem] = field(default_factory=list)
|
||||
polymarket: List[PolymarketItem] = field(default_factory=list)
|
||||
best_practices: List[str] = field(default_factory=list)
|
||||
@@ -385,6 +425,7 @@ class Report:
|
||||
web_error: Optional[str] = None
|
||||
youtube_error: Optional[str] = None
|
||||
tiktok_error: Optional[str] = None
|
||||
instagram_error: Optional[str] = None
|
||||
hackernews_error: Optional[str] = None
|
||||
polymarket_error: Optional[str] = None
|
||||
# Handle resolution
|
||||
@@ -409,6 +450,7 @@ class Report:
|
||||
'web': [w.to_dict() for w in self.web],
|
||||
'youtube': [y.to_dict() for y in self.youtube],
|
||||
'tiktok': [t.to_dict() for t in self.tiktok],
|
||||
'instagram': [ig.to_dict() for ig in self.instagram],
|
||||
'hackernews': [h.to_dict() for h in self.hackernews],
|
||||
'polymarket': [p.to_dict() for p in self.polymarket],
|
||||
'best_practices': self.best_practices,
|
||||
@@ -427,6 +469,8 @@ class Report:
|
||||
d['youtube_error'] = self.youtube_error
|
||||
if self.tiktok_error:
|
||||
d['tiktok_error'] = self.tiktok_error
|
||||
if self.instagram_error:
|
||||
d['instagram_error'] = self.instagram_error
|
||||
if self.hackernews_error:
|
||||
d['hackernews_error'] = self.hackernews_error
|
||||
if self.polymarket_error:
|
||||
@@ -558,6 +602,30 @@ class Report:
|
||||
cross_refs=t.get('cross_refs', []),
|
||||
))
|
||||
|
||||
# Reconstruct Instagram items
|
||||
ig_items = []
|
||||
for ig in data.get('instagram', []):
|
||||
eng = None
|
||||
if ig.get('engagement'):
|
||||
eng = Engagement(**ig['engagement'])
|
||||
subs = SubScores(**ig.get('subs', {})) if ig.get('subs') else SubScores()
|
||||
ig_items.append(InstagramItem(
|
||||
id=ig['id'],
|
||||
text=ig.get('text', ''),
|
||||
url=ig['url'],
|
||||
author_name=ig.get('author_name', ''),
|
||||
date=ig.get('date'),
|
||||
date_confidence=ig.get('date_confidence', 'high'),
|
||||
engagement=eng,
|
||||
caption_snippet=ig.get('caption_snippet', ''),
|
||||
hashtags=ig.get('hashtags', []),
|
||||
relevance=ig.get('relevance', 0.7),
|
||||
why_relevant=ig.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=ig.get('score', 0),
|
||||
cross_refs=ig.get('cross_refs', []),
|
||||
))
|
||||
|
||||
# Reconstruct HackerNews items
|
||||
hn_items = []
|
||||
for h in data.get('hackernews', []):
|
||||
@@ -623,6 +691,7 @@ class Report:
|
||||
web=web_items,
|
||||
youtube=youtube_items,
|
||||
tiktok=tiktok_items,
|
||||
instagram=ig_items,
|
||||
hackernews=hn_items,
|
||||
polymarket=pm_items,
|
||||
best_practices=data.get('best_practices', []),
|
||||
@@ -633,6 +702,7 @@ class Report:
|
||||
web_error=data.get('web_error'),
|
||||
youtube_error=data.get('youtube_error'),
|
||||
tiktok_error=data.get('tiktok_error'),
|
||||
instagram_error=data.get('instagram_error'),
|
||||
hackernews_error=data.get('hackernews_error'),
|
||||
polymarket_error=data.get('polymarket_error'),
|
||||
resolved_x_handle=data.get('resolved_x_handle'),
|
||||
|
||||
+65
-4
@@ -339,6 +339,65 @@ def score_tiktok_items(items: List[schema.TikTokItem]) -> List[schema.TikTokItem
|
||||
return items
|
||||
|
||||
|
||||
def compute_instagram_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
"""Compute raw engagement score for Instagram item.
|
||||
|
||||
Formula: 0.50*log1p(views) + 0.30*log1p(likes) + 0.20*log1p(comments)
|
||||
Views dominate on Instagram Reels — they're the primary discovery signal.
|
||||
"""
|
||||
if engagement is None:
|
||||
return None
|
||||
|
||||
if engagement.views is None and engagement.likes is None:
|
||||
return None
|
||||
|
||||
views = log1p_safe(engagement.views)
|
||||
likes = log1p_safe(engagement.likes)
|
||||
comments = log1p_safe(engagement.num_comments)
|
||||
|
||||
return 0.50 * views + 0.30 * likes + 0.20 * comments
|
||||
|
||||
|
||||
def score_instagram_items(items: List[schema.InstagramItem]) -> List[schema.InstagramItem]:
|
||||
"""Compute scores for Instagram items.
|
||||
|
||||
Uses same weight structure as TikTok (relevance + recency + engagement).
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
|
||||
eng_raw = [compute_instagram_engagement_raw(item.engagement) for item in items]
|
||||
eng_normalized = normalize_to_100(eng_raw)
|
||||
|
||||
for i, item in enumerate(items):
|
||||
rel_score = int(item.relevance * 100)
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
if eng_normalized[i] is not None:
|
||||
eng_score = int(eng_normalized[i])
|
||||
else:
|
||||
eng_score = DEFAULT_ENGAGEMENT
|
||||
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=eng_score,
|
||||
)
|
||||
|
||||
overall = (
|
||||
WEIGHT_RELEVANCE * rel_score +
|
||||
WEIGHT_RECENCY * rec_score +
|
||||
WEIGHT_ENGAGEMENT * eng_score
|
||||
)
|
||||
|
||||
if eng_raw[i] is None:
|
||||
overall -= UNKNOWN_ENGAGEMENT_PENALTY
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def compute_hackernews_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
"""Compute raw engagement score for Hacker News item.
|
||||
|
||||
@@ -512,7 +571,7 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS
|
||||
return items
|
||||
|
||||
|
||||
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.HackerNewsItem, schema.PolymarketItem]]) -> List:
|
||||
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem]]) -> List:
|
||||
"""Sort items by score (descending), then date, then source priority.
|
||||
|
||||
Args:
|
||||
@@ -538,12 +597,14 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
|
||||
source_priority = 2
|
||||
elif isinstance(item, schema.TikTokItem):
|
||||
source_priority = 3
|
||||
elif isinstance(item, schema.HackerNewsItem):
|
||||
elif isinstance(item, schema.InstagramItem):
|
||||
source_priority = 4
|
||||
elif isinstance(item, schema.PolymarketItem):
|
||||
elif isinstance(item, schema.HackerNewsItem):
|
||||
source_priority = 5
|
||||
else: # WebSearchItem
|
||||
elif isinstance(item, schema.PolymarketItem):
|
||||
source_priority = 6
|
||||
else: # WebSearchItem
|
||||
source_priority = 7
|
||||
|
||||
# Quaternary: title/text for stability
|
||||
text = getattr(item, "title", "") or getattr(item, "text", "")
|
||||
|
||||
+20
-1
@@ -77,6 +77,12 @@ TIKTOK_MESSAGES = [
|
||||
"Scanning TikTok for relevant content...",
|
||||
]
|
||||
|
||||
INSTAGRAM_MESSAGES = [
|
||||
"Searching Instagram Reels...",
|
||||
"Finding what's trending on Instagram...",
|
||||
"Scanning Instagram for relevant reels...",
|
||||
]
|
||||
|
||||
HN_MESSAGES = [
|
||||
"Searching Hacker News...",
|
||||
"Scanning HN front page stories...",
|
||||
@@ -286,6 +292,15 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.PURPLE}TikTok{Colors.RESET} Found {count} videos")
|
||||
|
||||
def start_instagram(self):
|
||||
msg = random.choice(INSTAGRAM_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.PURPLE}Instagram{Colors.RESET} {msg}", Colors.PURPLE)
|
||||
self.spinner.start()
|
||||
|
||||
def end_instagram(self, count: int):
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.PURPLE}Instagram{Colors.RESET} Found {count} reels")
|
||||
|
||||
def start_hackernews(self):
|
||||
msg = random.choice(HN_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.YELLOW}HN{Colors.RESET} {msg}", Colors.YELLOW, quiet=True)
|
||||
@@ -313,7 +328,7 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop()
|
||||
|
||||
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0, pm_count: int = 0, tiktok_count: int = 0):
|
||||
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0, pm_count: int = 0, tiktok_count: int = 0, ig_count: int = 0):
|
||||
elapsed = time.time() - self.start_time
|
||||
if IS_TTY:
|
||||
sys.stderr.write(f"\n{Colors.GREEN}{Colors.BOLD}✓ Research complete{Colors.RESET} ")
|
||||
@@ -324,6 +339,8 @@ class ProgressDisplay:
|
||||
sys.stderr.write(f" {Colors.RED}YouTube:{Colors.RESET} {youtube_count} videos")
|
||||
if tiktok_count:
|
||||
sys.stderr.write(f" {Colors.PURPLE}TikTok:{Colors.RESET} {tiktok_count} videos")
|
||||
if ig_count:
|
||||
sys.stderr.write(f" {Colors.PURPLE}Instagram:{Colors.RESET} {ig_count} reels")
|
||||
if hn_count:
|
||||
sys.stderr.write(f" {Colors.YELLOW}HN:{Colors.RESET} {hn_count} stories")
|
||||
if pm_count:
|
||||
@@ -335,6 +352,8 @@ class ProgressDisplay:
|
||||
parts.append(f"YouTube: {youtube_count} videos")
|
||||
if tiktok_count:
|
||||
parts.append(f"TikTok: {tiktok_count} videos")
|
||||
if ig_count:
|
||||
parts.append(f"Instagram: {ig_count} reels")
|
||||
if hn_count:
|
||||
parts.append(f"HN: {hn_count} stories")
|
||||
if pm_count:
|
||||
|
||||
@@ -211,6 +211,16 @@ def _run_topic(topic: dict) -> dict:
|
||||
"engagement_score": (item.get("engagement") or {}).get("views", 0),
|
||||
"relevance_score": item.get("relevance", 0),
|
||||
})
|
||||
for item in data.get("instagram", []):
|
||||
findings.append({
|
||||
"source": "instagram",
|
||||
"url": item.get("url", ""),
|
||||
"title": (item.get("caption_snippet", "") or "")[:120],
|
||||
"author": item.get("author_name", ""),
|
||||
"content": item.get("caption_snippet", ""),
|
||||
"engagement_score": (item.get("engagement") or {}).get("views", 0),
|
||||
"relevance_score": item.get("relevance", 0),
|
||||
})
|
||||
|
||||
# Store with dedup
|
||||
counts = store.store_findings(run_id, topic_id, findings)
|
||||
|
||||
Reference in New Issue
Block a user