sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources

Browse Source 
v3 rewrites the search engine from the ground up:

- Intelligent pre-research: resolves X handles, GitHub repos, subreddits,
  TikTok hashtags, and YouTube channels before searching
- GitHub person-mode: PR velocity, top repos by stars, release notes
- GitHub project-mode: live star counts, README, releases, top issues
- ELI5 mode: plain language synthesis, no jargon
- 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket,
  GitHub, Threads, Pinterest, Perplexity, Bluesky, Web
- Free Reddit comments via public JSON (no API key needed)
- Fun judge v2: humor scoring baked into narrative
- Cookie consent before browser scanning
- 10,000 free ScrapeCreators calls
- 1,012 tests

Thank you to the community contributors whose issues and PRs shaped v3:
@uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116),
@thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78),
@zl190 (#115), @hnshah (#84, #85, #86)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-04-08 10:52:23 -07:00
parent 61904b31e3
commit 0a9ff16dfc
397 changed files with 21427 additions and 53106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lib/instagram.py
+219 -76
View File
@@ -9,7 +9,7 @@ API docs: https://scrapecreators.com/docs
import re
import sys
from datetime import datetime, timezone
from datetime import datetime
from typing import Any, Dict, List, Optional, Set
try:
@@ -17,7 +17,7 @@ try:
except ImportError:
_requests = None
from . import http
from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
@@ -49,11 +49,67 @@ def _extract_core_subject(topic: str) -> str:
return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
def _infer_query_intent(topic: str) -> str:
"""Tiny local intent classifier for Instagram query expansion."""
text = topic.lower().strip()
if re.search(r"\b(vs|versus|compare|difference between)\b", text):
return "comparison"
if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install)\b", text):
return "how_to"
if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
return "opinion"
if re.search(r"\b(pricing|feature|features|best .* for)\b", text):
return "product"
return "breaking_news"
def expand_instagram_queries(topic: str, depth: str) -> List[str]:
"""Generate multiple Instagram search queries from a topic.
Mirrors reddit.py's expand_reddit_queries() pattern:
1. Extract core subject (strip noise words)
2. Include original topic if different from core
3. Add intent-specific OR-joined content-type variants
4. Cap by depth: 1 for quick, 2 for default, 3 for deep
Returns 1-3 query strings depending on depth.
"""
core = _extract_core_subject(topic)
queries = [core]
# Include cleaned original topic as variant if different from core
original_clean = topic.strip().rstrip('?!.')
if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
queries.append(original_clean)
qtype = _infer_query_intent(topic)
# Intent-specific Instagram content-type variants
if qtype == "breaking_news":
queries.append(f"{core} reaction OR edit")
elif qtype == "opinion":
queries.append(f"{core} reaction OR edit")
elif qtype == "product":
queries.append(f"{core} review OR haul")
elif qtype == "comparison":
queries.append(f"{core} vs OR compared")
elif qtype == "how_to":
queries.append(f"{core} tutorial OR hack")
else:
queries.append(f"{core} reaction OR edit")
# Deep depth: add viral content variant
if depth == "deep":
queries.append(f"{core} viral OR trending OR reel")
# Cap by depth budget
caps = {"quick": 1, "default": 2, "deep": 3}
cap = caps.get(depth, 2)
return queries[:cap]
def _log(msg: str):
"""Log to stderr (only in interactive terminals; spinner handles non-TTY)."""
if sys.stderr.isatty():
sys.stderr.write(f"[Instagram] {msg}\n")
sys.stderr.flush()
log.source_log("Instagram", msg)
def _sc_headers(token: str) -> Dict[str, str]:
@@ -88,9 +144,8 @@ def _parse_date(item: Dict[str, Any]) -> Optional[str]:
# Fall back to unix timestamp
try:
dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError, OSError):
return dates.timestamp_to_date(int(ts))
except (ValueError, TypeError):
pass
return None
@@ -103,6 +158,122 @@ def _extract_hashtags(caption_text: str) -> List[str]:
return re.findall(r'#(\w+)', caption_text)
def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]:
"""Parse raw Instagram items into normalized dicts."""
items = []
for raw in raw_items:
if not isinstance(raw, dict):
continue
# Extract reel ID and shortcode
reel_pk = str(raw.get("id", raw.get("pk", "")))
shortcode = raw.get("shortcode", raw.get("code", ""))
# Caption text -- can be a string or dict depending on endpoint
caption_obj = raw.get("caption", "")
if isinstance(caption_obj, dict):
text = caption_obj.get("text", "")
elif isinstance(caption_obj, str):
text = caption_obj
else:
text = raw.get("desc", raw.get("text", ""))
# Engagement metrics
play_count = raw.get("video_play_count") or raw.get("video_view_count") or raw.get("play_count") or 0
like_count = raw.get("like_count") or 0
comment_count = raw.get("comment_count") or 0
# Author info -- 'owner' in reels/search, 'user' in user/reels
owner_raw = raw.get("owner") or raw.get("user")
if isinstance(owner_raw, dict):
author_name = owner_raw.get("username", "")
elif isinstance(owner_raw, str):
author_name = owner_raw
else:
author_name = ""
# Duration
duration = raw.get("video_duration")
# Date
date_str = _parse_date(raw)
# Hashtags from caption text
hashtags = _extract_hashtags(text)
# Compute relevance with hashtag boost
relevance = _compute_relevance(core_topic, text, hashtags)
# Build URL -- prefer API-provided url, fallback to shortcode
url = raw.get("url", "")
if not url and shortcode:
url = f"https://www.instagram.com/reel/{shortcode}"
items.append({
"video_id": reel_pk,
"text": text,
"url": url,
"author_name": author_name,
"date": date_str,
"engagement": {
"views": play_count,
"likes": like_count,
"comments": comment_count,
},
"hashtags": hashtags,
"duration": duration,
"relevance": relevance,
"why_relevant": f"Instagram: {text[:60]}" if text else f"Instagram: {core_topic}",
"caption_snippet": "", # populated by fetch_captions
})
return items
def _user_reels(
handle: str,
token: str,
) -> List[Dict[str, Any]]:
"""Fetch an Instagram user's recent reels via ScrapeCreators.
Args:
handle: Instagram username (without @)
token: ScrapeCreators API key
Returns:
List of raw Instagram reel dicts.
"""
_log(f"User reels: @{handle}")
reels_url = f"{SCRAPECREATORS_BASE}/v1/instagram/user/reels"
if not _requests:
try:
from urllib.parse import urlencode
params = urlencode({"handle": handle})
url = f"{reels_url}?{params}"
headers = _sc_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"User reels error (urllib) for @{handle}: {e}")
return []
else:
try:
resp = _requests.get(
reels_url,
params={"handle": handle},
headers=_sc_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"User reels error for @{handle}: {e}")
return []
raw_items = data.get("items") or data.get("reels") or data.get("data") or []
_log(f" -> {len(raw_items)} reels from @{handle}")
return raw_items
def search_instagram(
topic: str,
from_date: str,
@@ -163,67 +334,7 @@ def search_instagram(
raw_items = raw_items[:config["results_per_page"]]
# Parse items
items = []
for raw in raw_items:
if not isinstance(raw, dict):
continue
# Extract reel ID and shortcode
reel_pk = str(raw.get("id", raw.get("pk", "")))
shortcode = raw.get("shortcode", raw.get("code", ""))
# Caption text — can be a string or dict depending on endpoint
caption_obj = raw.get("caption", "")
if isinstance(caption_obj, dict):
text = caption_obj.get("text", "")
elif isinstance(caption_obj, str):
text = caption_obj
else:
text = raw.get("desc", raw.get("text", ""))
# Engagement metrics
play_count = raw.get("video_play_count") or raw.get("video_view_count") or raw.get("play_count") or 0
like_count = raw.get("like_count") or 0
comment_count = raw.get("comment_count") or 0
# Author info — 'owner' in reels/search, 'user' in user/reels
owner = raw.get("owner") or raw.get("user") or {}
author_name = owner.get("username", "")
# Duration
duration = raw.get("video_duration")
# Date
date_str = _parse_date(raw)
# Hashtags from caption text
hashtags = _extract_hashtags(text)
# Compute relevance with hashtag boost
relevance = _compute_relevance(core_topic, text, hashtags)
# Build URL — prefer API-provided url, fallback to shortcode
url = raw.get("url", "")
if not url and shortcode:
url = f"https://www.instagram.com/reel/{shortcode}"
items.append({
"video_id": reel_pk,
"text": text,
"url": url,
"author_name": author_name,
"date": date_str,
"engagement": {
"views": play_count,
"likes": like_count,
"comments": comment_count,
},
"hashtags": hashtags,
"duration": duration,
"relevance": relevance,
"why_relevant": f"Instagram: {text[:60]}" if text else f"Instagram: {core_topic}",
"caption_snippet": "", # populated by fetch_captions
})
items = _parse_items(raw_items, core_topic)
# Hard date filter
in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
@@ -323,25 +434,57 @@ def search_and_enrich(
to_date: str,
depth: str = "default",
token: str = None,
ig_creators: List[str] | None = None,
) -> Dict[str, Any]:
"""Full Instagram search: find reels, then fetch captions for top results.
Uses expand_instagram_queries() to generate multiple search queries,
runs ScrapeCreators for each, and merges/deduplicates results by video ID.
Args:
topic: Search topic
topic: Search topic (raw topic, not planner's narrowed query)
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
ig_creators: Optional list of Instagram creator handles to fetch reels from
Returns:
Dict with 'items' list. Each item has a 'caption_snippet' field.
"""
# Step 1: Search
search_result = search_instagram(topic, from_date, to_date, depth, token)
items = search_result.get("items", [])
core_topic = _extract_core_subject(topic)
seen_ids: Set[str] = set()
items: List[Dict[str, Any]] = []
last_error = None
# Step 0: Creator reels (high-signal, runs first)
if ig_creators and token:
for creator in ig_creators:
raw_items = _user_reels(creator, token)
parsed = _parse_items(raw_items, core_topic)
for item in parsed:
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Step 1: Multi-query keyword search — run ScrapeCreators for each expanded query
queries = expand_instagram_queries(topic, depth)
for q in queries:
search_result = search_instagram(q, from_date, to_date, depth, token)
if search_result.get("error"):
last_error = search_result["error"]
for item in search_result.get("items", []):
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Sort merged results by views descending
items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True)
if not items:
return search_result
return {"items": [], "error": last_error}
# Step 2: Fetch captions for top N
captions = fetch_captions(items, token, depth)
@@ -353,7 +496,7 @@ def search_and_enrich(
if caption:
item["caption_snippet"] = caption
return {"items": items, "error": search_result.get("error")}
return {"items": items, "error": last_error}
def parse_instagram_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: