sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v2.8 — Instagram Reels source + TikTok ScrapeCreators migration

Browse Source 
Add Instagram Reels as the 8th research source via ScrapeCreators API.
One API key (SCRAPECREATORS_API_KEY) now covers both TikTok and Instagram.

- Add scripts/lib/instagram.py: keyword search, transcript extraction,
  relevance scoring, engagement metrics (views, likes, comments)
- Add InstagramItem to schema, normalization, scoring, dedup, rendering
- Add Instagram to orchestrator pipeline, watchlist, and UI spinners
- Update SKILL.md: stats template, citation priority, item format,
  URL-to-name extraction rules, anti-Sources instruction
- Update README and CHANGELOG for v2.8
- Fix: Instagram/TikTok not running in --search= web-only path
- Fix: web stats line showing full URLs instead of domain names
- Replace APIFY_API_TOKEN with SCRAPECREATORS_API_KEY throughout

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-04 06:57:52 -08:00
parent 740dcc5789
commit db75f9e341
17 changed files with 1609 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lib/render.py
+74 -2
View File
@@ -26,6 +26,8 @@ def _xref_tag(item) -> str:
source_names.add('YouTube')
elif ref_id.startswith('TK'):
source_names.add('TikTok')
elif ref_id.startswith('IG'):
source_names.add('Instagram')
elif ref_id.startswith('HN'):
source_names.add('HN')
elif ref_id.startswith('PM'):
@@ -60,9 +62,10 @@ def _assess_data_freshness(report: schema.Report) -> dict:
pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from)
tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from)
ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from)
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent + tiktok_recent
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket) + len(report.tiktok)
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent + tiktok_recent + ig_recent
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket) + len(report.tiktok) + len(report.instagram)
return {
"reddit_recent": reddit_recent,
@@ -284,6 +287,42 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
lines.append(f" *{item.why_relevant}*")
lines.append("")
# Instagram items
if report.instagram_error:
lines.append("### Instagram Reels")
lines.append("")
lines.append(f"**ERROR:** {report.instagram_error}")
lines.append("")
elif report.instagram:
lines.append("### Instagram Reels")
lines.append("")
for item in report.instagram[:limit]:
eng_str = ""
if item.engagement:
eng = item.engagement
parts = []
if eng.views is not None:
parts.append(f"{eng.views:,} views")
if eng.likes is not None:
parts.append(f"{eng.likes:,} likes")
if parts:
eng_str = f" [{', '.join(parts)}]"
date_str = f" ({item.date})" if item.date else ""
lines.append(f"**{item.id}** (score:{item.score}) @{item.author_name}{date_str}{eng_str}{_xref_tag(item)}")
lines.append(f" {item.text[:200]}")
lines.append(f" {item.url}")
if item.caption_snippet and item.caption_snippet != item.text[:len(item.caption_snippet)]:
snippet = item.caption_snippet[:200]
if len(item.caption_snippet) > 200:
snippet += "..."
lines.append(f" Caption: {snippet}")
if item.hashtags:
lines.append(f" Tags: {' '.join('#' + h for h in item.hashtags[:8])}")
lines.append(f" *{item.why_relevant}*")
lines.append("")
# Hacker News items
if report.hackernews_error:
lines.append("### Hacker News Stories")
@@ -455,6 +494,14 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
lines.append(f" ✅ TikTok: {len(report.tiktok)} videos ({with_captions} with captions)")
# Hide when zero results
# Instagram
if report.instagram_error:
lines.append(f" ❌ Instagram: error — {report.instagram_error}")
elif report.instagram:
with_captions = sum(1 for v in report.instagram if getattr(v, 'caption_snippet', None))
lines.append(f" ✅ Instagram: {len(report.instagram)} reels ({with_captions} with captions)")
# Hide when zero results
# Hacker News
if report.hackernews_error:
lines.append(f" ❌ HN: error - {report.hackernews_error}")
@@ -508,6 +555,8 @@ def render_context_snippet(report: schema.Report) -> str:
all_items.append((item.score, "X", item.text[:50] + "...", item.url))
for item in report.tiktok[:5]:
all_items.append((item.score, "TikTok", item.text[:50] + "...", item.url))
for item in report.instagram[:5]:
all_items.append((item.score, "Instagram", item.text[:50] + "...", item.url))
for item in report.hackernews[:5]:
all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url))
for item in report.polymarket[:5]:
@@ -624,6 +673,29 @@ def render_full_report(report: schema.Report) -> str:
lines.append(f"> {item.text[:300]}")
lines.append("")
# Instagram section
if report.instagram:
lines.append("## Instagram Reels")
lines.append("")
for item in report.instagram:
lines.append(f"### {item.id}: @{item.author_name}")
lines.append("")
lines.append(f"- **URL:** {item.url}")
lines.append(f"- **Date:** {item.date or 'Unknown'}")
lines.append(f"- **Score:** {item.score}/100")
lines.append(f"- **Relevance:** {item.why_relevant}")
if item.engagement:
eng = item.engagement
lines.append(f"- **Engagement:** {eng.views or '?'} views, {eng.likes or '?'} likes, {eng.num_comments or '?'} comments")
if item.hashtags:
lines.append(f"- **Hashtags:** {' '.join('#' + h for h in item.hashtags[:10])}")
lines.append("")
lines.append(f"> {item.text[:300]}")
lines.append("")
# HN section
if report.hackernews:
lines.append("## Hacker News Stories")