sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v2.8 — Instagram Reels source + TikTok ScrapeCreators migration

Browse Source 
Add Instagram Reels as the 8th research source via ScrapeCreators API.
One API key (SCRAPECREATORS_API_KEY) now covers both TikTok and Instagram.

- Add scripts/lib/instagram.py: keyword search, transcript extraction,
  relevance scoring, engagement metrics (views, likes, comments)
- Add InstagramItem to schema, normalization, scoring, dedup, rendering
- Add Instagram to orchestrator pipeline, watchlist, and UI spinners
- Update SKILL.md: stats template, citation priority, item format,
  URL-to-name extraction rules, anti-Sources instruction
- Update README and CHANGELOG for v2.8
- Fix: Instagram/TikTok not running in --search= web-only path
- Fix: web stats line showing full URLs instead of domain names
- Replace APIFY_API_TOKEN with SCRAPECREATORS_API_KEY throughout

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-04 06:57:52 -08:00
parent 740dcc5789
commit db75f9e341
17 changed files with 1609 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/last30days.py
+128 -13
View File
@@ -38,13 +38,13 @@ _child_pids: set = set()
_child_pids_lock = threading.Lock()
TIMEOUT_PROFILES = {
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
}
# Valid source names for the --search flag
VALID_SEARCH_SOURCES = {"reddit", "x", "hn", "youtube", "tiktok", "polymarket", "web"}
VALID_SEARCH_SOURCES = {"reddit", "x", "hn", "youtube", "tiktok", "instagram", "polymarket", "web"}
def parse_search_flag(search_str: str) -> set:
@@ -146,6 +146,7 @@ from lib import (
score,
ui,
tiktok,
instagram,
websearch,
xai_x,
youtube_yt,
@@ -373,6 +374,35 @@ def _search_tiktok(
return tiktok_items, tiktok_error
def _search_instagram(
topic: str,
from_date: str,
to_date: str,
depth: str,
token: str,
) -> tuple:
"""Search Instagram via ScrapeCreators (runs in thread).
Returns:
Tuple of (instagram_items, instagram_error)
"""
instagram_error = None
try:
response = instagram.search_and_enrich(
topic, from_date, to_date, depth=depth, token=token,
)
except Exception as e:
return [], f"{type(e).__name__}: {e}"
instagram_items = instagram.parse_instagram_response(response)
if response.get("error"):
instagram_error = response["error"]
return instagram_items, instagram_error
def _search_hackernews(
topic: str,
from_date: str,
@@ -664,6 +694,7 @@ def run_research(
x_source: str = "xai",
run_youtube: bool = False,
run_tiktok: bool = False,
run_instagram: bool = False,
timeouts: dict = None,
resolved_handle: str = None,
do_hackernews: bool = True,
@@ -673,9 +704,11 @@ def run_research(
"""Run the research pipeline.
Returns:
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, web_items, web_needed,
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items,
hackernews_items, polymarket_items, web_items, web_needed,
raw_openai, raw_xai, raw_reddit_enriched,
reddit_error, x_error, youtube_error, tiktok_error, web_error)
reddit_error, x_error, youtube_error, tiktok_error, instagram_error,
hackernews_error, polymarket_error, web_error)
Note: web_needed is True when web search should be performed by the assistant
(i.e., no native web search API keys are configured). When native web search
@@ -689,6 +722,7 @@ def run_research(
x_items = []
youtube_items = []
tiktok_items = []
instagram_items = []
hackernews_items = []
polymarket_items = []
web_items = []
@@ -699,6 +733,7 @@ def run_research(
x_error = None
youtube_error = None
tiktok_error = None
instagram_error = None
hackernews_error = None
polymarket_error = None
web_error = None
@@ -729,7 +764,7 @@ def run_research(
if progress:
progress.start_web_only()
progress.end_web_only()
# Still run YouTube in web-only mode if yt-dlp is available
# Still run YouTube/TikTok/Instagram in web-only mode if available
if run_youtube:
if progress:
progress.start_youtube()
@@ -743,7 +778,34 @@ def run_research(
progress.show_error(f"YouTube error: {e}")
if progress:
progress.end_youtube(len(youtube_items))
return reddit_items, x_items, youtube_items, tiktok_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, hackernews_error, polymarket_error, web_error
if run_tiktok:
if progress:
progress.start_tiktok()
try:
tiktok_items, tiktok_error = _search_tiktok(topic, from_date, to_date, depth, env.get_tiktok_token(config))
if tiktok_error and progress:
progress.show_error(f"TikTok error: {tiktok_error}")
except Exception as e:
tiktok_error = f"{type(e).__name__}: {e}"
if progress:
progress.show_error(f"TikTok error: {e}")
if progress:
progress.end_tiktok(len(tiktok_items))
if run_instagram:
if progress:
progress.start_instagram()
try:
ig_timeout = timeouts.get("instagram_future", future_timeout)
instagram_items, instagram_error = _search_instagram(topic, from_date, to_date, depth, env.get_instagram_token(config))
if instagram_error and progress:
progress.show_error(f"Instagram error: {instagram_error}")
except Exception as e:
instagram_error = f"{type(e).__name__}: {e}"
if progress:
progress.show_error(f"Instagram error: {e}")
if progress:
progress.end_instagram(len(instagram_items))
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error
# Determine which searches to run
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
@@ -756,10 +818,11 @@ def run_research(
x_future = None
youtube_future = None
tiktok_future = None
instagram_future = None
hackernews_future = None
polymarket_future = None
web_future = None
max_workers = 2 + (1 if run_youtube else 0) + (1 if run_tiktok else 0) + (1 if do_hackernews else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0)
max_workers = 2 + (1 if run_youtube else 0) + (1 if run_tiktok else 0) + (1 if run_instagram else 0) + (1 if do_hackernews else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit searches
@@ -794,6 +857,14 @@ def run_research(
env.get_tiktok_token(config),
)
if run_instagram:
if progress:
progress.start_instagram()
instagram_future = executor.submit(
_search_instagram, topic, from_date, to_date, depth,
env.get_instagram_token(config),
)
if do_hackernews:
if progress:
progress.start_hackernews()
@@ -883,6 +954,23 @@ def run_research(
if progress:
progress.end_tiktok(len(tiktok_items))
if instagram_future:
ig_timeout = timeouts.get("instagram_future", future_timeout)
try:
instagram_items, instagram_error = instagram_future.result(timeout=ig_timeout)
if instagram_error and progress:
progress.show_error(f"Instagram error: {instagram_error}")
except TimeoutError:
instagram_error = f"Instagram search timed out after {ig_timeout}s"
if progress:
progress.show_error(instagram_error)
except Exception as e:
instagram_error = f"{type(e).__name__}: {e}"
if progress:
progress.show_error(f"Instagram error: {e}")
if progress:
progress.end_instagram(len(instagram_items))
if hackernews_future:
hn_timeout = timeouts.get("hackernews_future", future_timeout)
try:
@@ -1025,7 +1113,7 @@ def run_research(
if sup_x:
x_items.extend(sup_x)
return reddit_items, x_items, youtube_items, tiktok_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, hackernews_error, polymarket_error, web_error
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error
def main():
@@ -1163,6 +1251,9 @@ def main():
# Auto-detect ScrapeCreators/Apify for TikTok
has_tiktok = env.is_tiktok_available(config)
# Auto-detect ScrapeCreators for Instagram
has_instagram = env.is_instagram_available(config)
# --diagnose: show source availability and exit
if args.diagnose:
web_source = env.get_web_search_source(config)
@@ -1175,6 +1266,7 @@ def main():
"bird_username": x_source_status.get("bird_username"),
"youtube": has_ytdlp,
"tiktok": has_tiktok,
"instagram": has_instagram,
"hackernews": True,
"polymarket": True,
"web_search_backend": web_source,
@@ -1290,6 +1382,7 @@ def main():
search_do_polymarket = True
search_run_youtube = has_ytdlp
search_run_tiktok = has_tiktok
search_run_instagram = has_instagram
if args.search:
search_sources = parse_search_flag(args.search)
has_reddit = "reddit" in search_sources
@@ -1298,6 +1391,7 @@ def main():
search_do_polymarket = "polymarket" in search_sources
search_run_youtube = "youtube" in search_sources and has_ytdlp
search_run_tiktok = "tiktok" in search_sources and has_tiktok
search_run_instagram = "instagram" in search_sources and has_instagram
include_search_web = "web" in search_sources
# Map to existing sources string
if has_reddit and has_x:
@@ -1311,7 +1405,7 @@ def main():
sources = "web" # hn/polymarket only; no Reddit/X
# Run research
reddit_items, x_items, youtube_items, tiktok_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, hackernews_error, polymarket_error, web_error = run_research(
reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error = run_research(
args.topic,
sources,
config,
@@ -1324,6 +1418,7 @@ def main():
x_source=x_source or "xai",
run_youtube=search_run_youtube,
run_tiktok=search_run_tiktok,
run_instagram=search_run_instagram,
timeouts=timeouts,
resolved_handle=args.x_handle,
do_hackernews=search_do_hackernews,
@@ -1339,6 +1434,7 @@ def main():
normalized_x = normalize.normalize_x_items(x_items, from_date, to_date)
normalized_youtube = normalize.normalize_youtube_items(youtube_items, from_date, to_date) if youtube_items else []
normalized_tiktok = normalize.normalize_tiktok_items(tiktok_items, from_date, to_date) if tiktok_items else []
normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else []
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
@@ -1353,6 +1449,8 @@ def main():
filtered_youtube = normalized_youtube
# TikTok: hard date filter (tiktok.py already pre-filters, but safety net)
filtered_tiktok = normalize.filter_by_date_range(normalized_tiktok, from_date, to_date) if normalized_tiktok else []
# Instagram: hard date filter (instagram.py already pre-filters, but safety net)
filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else []
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
filtered_pm = normalized_pm
@@ -1363,6 +1461,7 @@ def main():
scored_x = score.score_x_items(filtered_x)
scored_youtube = score.score_youtube_items(filtered_youtube) if filtered_youtube else []
scored_tiktok = score.score_tiktok_items(filtered_tiktok) if filtered_tiktok else []
scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else []
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
@@ -1372,6 +1471,7 @@ def main():
sorted_x = score.sort_items(scored_x)
sorted_youtube = score.sort_items(scored_youtube) if scored_youtube else []
sorted_tiktok = score.sort_items(scored_tiktok) if scored_tiktok else []
sorted_ig = score.sort_items(scored_ig) if scored_ig else []
sorted_hn = score.sort_items(scored_hn) if scored_hn else []
sorted_pm = score.sort_items(scored_pm) if scored_pm else []
sorted_web = score.sort_items(scored_web) if scored_web else []
@@ -1381,6 +1481,7 @@ def main():
deduped_x = dedupe.dedupe_x(sorted_x)
deduped_youtube = dedupe.dedupe_youtube(sorted_youtube) if sorted_youtube else []
deduped_tiktok = dedupe.dedupe_tiktok(sorted_tiktok) if sorted_tiktok else []
deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else []
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
@@ -1394,7 +1495,7 @@ def main():
# Cross-source linking: annotate items that discuss the same story
dedupe.cross_source_link(
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_hn, deduped_pm, deduped_web,
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_pm, deduped_web,
)
progress.end_processing()
@@ -1412,6 +1513,7 @@ def main():
report.x = deduped_x
report.youtube = deduped_youtube
report.tiktok = deduped_tiktok
report.instagram = deduped_ig
report.hackernews = deduped_hn
report.polymarket = deduped_pm
report.web = deduped_web
@@ -1419,6 +1521,7 @@ def main():
report.x_error = x_error
report.youtube_error = youtube_error
report.tiktok_error = tiktok_error
report.instagram_error = instagram_error
report.hackernews_error = hackernews_error
report.polymarket_error = polymarket_error
report.web_error = web_error
@@ -1434,7 +1537,7 @@ def main():
if sources == "web":
progress.show_web_only_complete()
else:
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn), len(deduped_pm), len(deduped_tiktok))
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn), len(deduped_pm), len(deduped_tiktok), len(deduped_ig))
# Build source info for status footer
source_info = {}
@@ -1451,6 +1554,8 @@ def main():
source_info["youtube_skip_reason"] = "0 results (query may be too specific)"
if not has_tiktok:
source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)"
if not has_instagram:
source_info["instagram_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)"
if not web_source:
source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)"
@@ -1516,6 +1621,16 @@ def main():
"engagement_score": item.engagement.volume if item.engagement and item.engagement.volume else 0,
"relevance_score": item.relevance,
})
for item in deduped_ig:
findings.append({
"source": "instagram",
"url": item.url,
"title": item.text[:100],
"author": item.author_name,
"content": item.caption_snippet[:500] if item.caption_snippet else item.text,
"engagement_score": item.engagement.views if item.engagement and item.engagement.views else 0,
"relevance_score": item.relevance,
})
for item in deduped_web:
findings.append({
"source": "web",