Agent Eyes v1.0.0 — search + read the entire internet
Based on x-reader by @runes_leo (MIT License). Extended with: - Reddit support (posts + comments, proxy support) - GitHub support (repos, issues, PRs) - Web search via Exa semantic search - Reddit search (bypasses IP blocks via Exa) - GitHub search (repos by stars) - Renamed package: x_reader → agent_eyes - New MCP tools: search, search_reddit, search_github - Agent-first positioning and documentation
This commit is contained in:
Panniantong
@@ -0,0 +1,46 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Bilibili video fetcher — uses official web API."""
|
||||
|
||||
import re
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
API_URL = "https://api.bilibili.com/x/web-interface/view"
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
}
|
||||
|
||||
|
||||
async def fetch_bilibili(url_or_bv: str) -> Dict[str, Any]:
|
||||
"""Fetch Bilibili video metadata via official API."""
|
||||
logger.info(f"Fetching Bilibili: {url_or_bv}")
|
||||
|
||||
bv_id = url_or_bv
|
||||
if "bilibili.com" in url_or_bv or "b23.tv" in url_or_bv:
|
||||
match = re.search(r'BV\w+', url_or_bv)
|
||||
if match:
|
||||
bv_id = match.group()
|
||||
else:
|
||||
raise ValueError(f"Cannot extract BV ID from: {url_or_bv}")
|
||||
|
||||
resp = requests.get(API_URL, params={"bvid": bv_id}, headers=HEADERS, timeout=10)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if data.get("code") != 0:
|
||||
raise ValueError(f"Bilibili API error: {data.get('message')}")
|
||||
|
||||
video = data["data"]
|
||||
return {
|
||||
"title": video.get("title", ""),
|
||||
"description": video.get("desc", ""),
|
||||
"author": video.get("owner", {}).get("name", ""),
|
||||
"url": f"https://www.bilibili.com/video/{bv_id}",
|
||||
"cover": video.get("pic", ""),
|
||||
"bvid": bv_id,
|
||||
"duration": video.get("duration", 0),
|
||||
"view_count": video.get("stat", {}).get("view", 0),
|
||||
"platform": "bilibili",
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Playwright browser fetcher — headless Chromium fallback for anti-scraping sites.
|
||||
|
||||
Used when Jina Reader fails (403/451/timeout). Supports persistent login
|
||||
sessions via Playwright's storage_state for platforms requiring authentication.
|
||||
|
||||
Install: pip install "x-reader[browser]" && playwright install chromium
|
||||
"""
|
||||
|
||||
from loguru import logger
|
||||
from pathlib import Path
|
||||
|
||||
SESSION_DIR = Path.home() / ".x-reader" / "sessions"
|
||||
TIMEOUT_MS = 30_000
|
||||
|
||||
|
||||
async def fetch_via_browser(url: str, storage_state: str = None) -> dict:
|
||||
"""
|
||||
Fetch a URL using headless Chromium via Playwright.
|
||||
|
||||
Args:
|
||||
url: Target URL to fetch.
|
||||
storage_state: Path to a Playwright storage state JSON file (cookies/localStorage).
|
||||
If provided, the browser context will load this session.
|
||||
|
||||
Returns:
|
||||
dict with keys: title, content, url, author
|
||||
"""
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except ImportError:
|
||||
raise RuntimeError(
|
||||
"Playwright is not installed. Run:\n"
|
||||
' pip install "x-reader[browser]"\n'
|
||||
" playwright install chromium"
|
||||
)
|
||||
|
||||
logger.info(f"Browser fetch: {url}")
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
|
||||
context_kwargs = {}
|
||||
if storage_state and Path(storage_state).exists():
|
||||
context_kwargs["storage_state"] = storage_state
|
||||
logger.info(f"Using session: {storage_state}")
|
||||
|
||||
context = await browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
**context_kwargs,
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=TIMEOUT_MS)
|
||||
# Extra wait for JS-heavy pages
|
||||
await page.wait_for_timeout(2000)
|
||||
|
||||
title = await page.title()
|
||||
# Extract main text content, stripping scripts/styles
|
||||
content = await page.evaluate("""() => {
|
||||
const el = document.querySelector('article')
|
||||
|| document.querySelector('main')
|
||||
|| document.querySelector('.content')
|
||||
|| document.body;
|
||||
return el ? el.innerText : '';
|
||||
}""")
|
||||
|
||||
result = {
|
||||
"title": (title or "").strip()[:200],
|
||||
"content": (content or "").strip(),
|
||||
"url": url,
|
||||
"author": "",
|
||||
}
|
||||
logger.info(f"Browser fetch OK: {title[:60]}")
|
||||
return result
|
||||
|
||||
finally:
|
||||
await context.close()
|
||||
await browser.close()
|
||||
|
||||
|
||||
def get_session_path(platform: str) -> str:
|
||||
"""Get the session file path for a platform."""
|
||||
return str(SESSION_DIR / f"{platform}.json")
|
||||
@@ -0,0 +1,190 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GitHub fetcher — extracts repo info, issues, PRs, and README content.
|
||||
|
||||
Uses GitHub public API (no token needed for public repos).
|
||||
For higher rate limits, set GITHUB_TOKEN env var.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import base64
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
API_BASE = "https://api.github.com"
|
||||
|
||||
|
||||
def _get_headers() -> Dict[str, str]:
|
||||
"""Get request headers, optionally with auth token."""
|
||||
headers = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "AgentEyes/1.0",
|
||||
}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def _parse_github_url(url: str) -> Dict[str, str]:
|
||||
"""Parse GitHub URL into components."""
|
||||
# Match: github.com/owner/repo[/type/number]
|
||||
match = re.search(
|
||||
r'github\.com/([^/]+)/([^/]+?)(?:\.git)?(?:/(issues|pull|tree|blob)/(.+))?/?$',
|
||||
url
|
||||
)
|
||||
if not match:
|
||||
raise ValueError(f"Cannot parse GitHub URL: {url}")
|
||||
|
||||
return {
|
||||
"owner": match.group(1),
|
||||
"repo": match.group(2),
|
||||
"type": match.group(3), # issues, pull, tree, blob, or None
|
||||
"ref": match.group(4), # issue number, branch, file path, or None
|
||||
}
|
||||
|
||||
|
||||
async def fetch_github(url: str) -> Dict[str, Any]:
|
||||
"""Fetch content from a GitHub URL."""
|
||||
logger.info(f"Fetching GitHub: {url}")
|
||||
|
||||
parsed = _parse_github_url(url)
|
||||
owner = parsed["owner"]
|
||||
repo = parsed["repo"]
|
||||
content_type = parsed["type"]
|
||||
ref = parsed["ref"]
|
||||
|
||||
headers = _get_headers()
|
||||
|
||||
if content_type == "issues" and ref:
|
||||
return await _fetch_issue(owner, repo, ref, headers)
|
||||
elif content_type == "pull" and ref:
|
||||
return await _fetch_pull(owner, repo, ref, headers)
|
||||
else:
|
||||
return await _fetch_repo(owner, repo, headers)
|
||||
|
||||
|
||||
async def _fetch_repo(owner: str, repo: str, headers: Dict) -> Dict[str, Any]:
|
||||
"""Fetch repo info + README."""
|
||||
# Get repo info
|
||||
repo_resp = requests.get(f"{API_BASE}/repos/{owner}/{repo}", headers=headers, timeout=10)
|
||||
repo_resp.raise_for_status()
|
||||
repo_data = repo_resp.json()
|
||||
|
||||
# Get README
|
||||
readme_content = ""
|
||||
try:
|
||||
readme_resp = requests.get(
|
||||
f"{API_BASE}/repos/{owner}/{repo}/readme",
|
||||
headers=headers, timeout=10,
|
||||
)
|
||||
if readme_resp.status_code == 200:
|
||||
readme_data = readme_resp.json()
|
||||
readme_content = base64.b64decode(readme_data.get("content", "")).decode("utf-8")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not fetch README: {e}")
|
||||
|
||||
return {
|
||||
"title": f"{owner}/{repo}",
|
||||
"content": readme_content or repo_data.get("description", ""),
|
||||
"description": repo_data.get("description", ""),
|
||||
"author": owner,
|
||||
"url": repo_data.get("html_url", ""),
|
||||
"stars": repo_data.get("stargazers_count", 0),
|
||||
"forks": repo_data.get("forks_count", 0),
|
||||
"language": repo_data.get("language", ""),
|
||||
"topics": repo_data.get("topics", []),
|
||||
"license": (repo_data.get("license") or {}).get("spdx_id", ""),
|
||||
"platform": "github",
|
||||
}
|
||||
|
||||
|
||||
async def _fetch_issue(owner: str, repo: str, number: str, headers: Dict) -> Dict[str, Any]:
|
||||
"""Fetch a GitHub issue with comments."""
|
||||
issue_num = re.match(r'(\d+)', number).group(1)
|
||||
|
||||
# Get issue
|
||||
resp = requests.get(
|
||||
f"{API_BASE}/repos/{owner}/{repo}/issues/{issue_num}",
|
||||
headers=headers, timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
issue = resp.json()
|
||||
|
||||
# Get comments
|
||||
comments_text = ""
|
||||
if issue.get("comments", 0) > 0:
|
||||
c_resp = requests.get(
|
||||
f"{API_BASE}/repos/{owner}/{repo}/issues/{issue_num}/comments",
|
||||
headers=headers, params={"per_page": 20}, timeout=10,
|
||||
)
|
||||
if c_resp.status_code == 200:
|
||||
comments = c_resp.json()
|
||||
parts = ["\n---\n## Comments\n"]
|
||||
for c in comments:
|
||||
parts.append(f"**@{c.get('user', {}).get('login', '?')}**:\n{c.get('body', '')}\n")
|
||||
comments_text = "\n".join(parts)
|
||||
|
||||
return {
|
||||
"title": f"[{owner}/{repo}#{issue_num}] {issue.get('title', '')}",
|
||||
"content": (issue.get("body", "") or "") + comments_text,
|
||||
"author": issue.get("user", {}).get("login", ""),
|
||||
"url": issue.get("html_url", ""),
|
||||
"state": issue.get("state", ""),
|
||||
"labels": [l.get("name", "") for l in issue.get("labels", [])],
|
||||
"platform": "github",
|
||||
}
|
||||
|
||||
|
||||
async def _fetch_pull(owner: str, repo: str, number: str, headers: Dict) -> Dict[str, Any]:
|
||||
"""Fetch a GitHub pull request."""
|
||||
pr_num = re.match(r'(\d+)', number).group(1)
|
||||
|
||||
resp = requests.get(
|
||||
f"{API_BASE}/repos/{owner}/{repo}/pulls/{pr_num}",
|
||||
headers=headers, timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
pr = resp.json()
|
||||
|
||||
return {
|
||||
"title": f"[{owner}/{repo}#{pr_num}] {pr.get('title', '')}",
|
||||
"content": pr.get("body", "") or "",
|
||||
"author": pr.get("user", {}).get("login", ""),
|
||||
"url": pr.get("html_url", ""),
|
||||
"state": pr.get("state", ""),
|
||||
"merged": pr.get("merged", False),
|
||||
"additions": pr.get("additions", 0),
|
||||
"deletions": pr.get("deletions", 0),
|
||||
"changed_files": pr.get("changed_files", 0),
|
||||
"platform": "github",
|
||||
}
|
||||
|
||||
|
||||
async def search_github(query: str, limit: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Search GitHub repositories."""
|
||||
logger.info(f"Searching GitHub: {query}")
|
||||
|
||||
resp = requests.get(
|
||||
f"{API_BASE}/search/repositories",
|
||||
headers=_get_headers(),
|
||||
params={"q": query, "sort": "stars", "per_page": limit},
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for item in data.get("items", []):
|
||||
results.append({
|
||||
"title": item.get("full_name", ""),
|
||||
"description": item.get("description", ""),
|
||||
"url": item.get("html_url", ""),
|
||||
"stars": item.get("stargazers_count", 0),
|
||||
"language": item.get("language", ""),
|
||||
"updated_at": item.get("updated_at", ""),
|
||||
})
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,63 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Jina Reader — universal fallback for content extraction.
|
||||
|
||||
Uses https://r.jina.ai/{url} to extract markdown from any web page.
|
||||
Free, no API key required, handles JS rendering and anti-scraping.
|
||||
"""
|
||||
|
||||
import requests
|
||||
from loguru import logger
|
||||
|
||||
|
||||
JINA_BASE = "https://r.jina.ai"
|
||||
TIMEOUT = 30
|
||||
|
||||
HEADERS = {
|
||||
"Accept": "text/markdown",
|
||||
"User-Agent": "x-reader/0.1",
|
||||
}
|
||||
|
||||
|
||||
def fetch_via_jina(url: str) -> dict:
|
||||
"""
|
||||
Fetch any URL via Jina Reader and return structured data.
|
||||
|
||||
Returns:
|
||||
dict with keys: title, content, url, author (best-effort)
|
||||
"""
|
||||
jina_url = f"{JINA_BASE}/{url}"
|
||||
logger.info(f"Jina fetch: {url}")
|
||||
|
||||
try:
|
||||
resp = requests.get(jina_url, headers=HEADERS, timeout=TIMEOUT)
|
||||
resp.raise_for_status()
|
||||
text = resp.text
|
||||
|
||||
# Jina returns markdown; first line is usually the title
|
||||
lines = text.strip().split("\n")
|
||||
title = ""
|
||||
content_lines = []
|
||||
|
||||
for line in lines:
|
||||
if not title and line.strip():
|
||||
# First non-empty line as title, strip markdown heading
|
||||
title = line.lstrip("#").strip()
|
||||
else:
|
||||
content_lines.append(line)
|
||||
|
||||
content = "\n".join(content_lines).strip()
|
||||
|
||||
return {
|
||||
"title": title[:200],
|
||||
"content": content,
|
||||
"url": url,
|
||||
"author": "",
|
||||
}
|
||||
|
||||
except requests.Timeout:
|
||||
logger.error(f"Jina timeout: {url}")
|
||||
raise
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Jina fetch failed: {url} — {e}")
|
||||
raise
|
||||
@@ -0,0 +1,132 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Reddit fetcher — extracts posts and comments via JSON API.
|
||||
|
||||
Supports optional proxy via REDDIT_PROXY env var (many IPs are blocked by Reddit).
|
||||
Example: REDDIT_PROXY=http://user:pass@host:port
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
|
||||
def _get_proxies() -> Optional[Dict[str, str]]:
|
||||
"""Get proxy config from env."""
|
||||
proxy = os.environ.get("REDDIT_PROXY")
|
||||
if proxy:
|
||||
return {"http": proxy, "https": proxy}
|
||||
return None
|
||||
|
||||
|
||||
def _extract_post(post_data: Dict) -> Dict[str, Any]:
|
||||
"""Extract post info from Reddit JSON."""
|
||||
data = post_data.get("data", {})
|
||||
return {
|
||||
"title": data.get("title", ""),
|
||||
"author": data.get("author", "[deleted]"),
|
||||
"selftext": data.get("selftext", ""),
|
||||
"score": data.get("score", 0),
|
||||
"num_comments": data.get("num_comments", 0),
|
||||
"url": f"https://www.reddit.com{data.get('permalink', '')}",
|
||||
"created_utc": data.get("created_utc", 0),
|
||||
"subreddit": data.get("subreddit", ""),
|
||||
}
|
||||
|
||||
|
||||
def _extract_comments(comments_data: Dict, limit: int = 20) -> List[Dict[str, str]]:
|
||||
"""Extract top-level comments."""
|
||||
comments = []
|
||||
children = comments_data.get("data", {}).get("children", [])
|
||||
for child in children[:limit]:
|
||||
if child.get("kind") != "t1":
|
||||
continue
|
||||
data = child.get("data", {})
|
||||
comments.append({
|
||||
"author": data.get("author", "[deleted]"),
|
||||
"body": data.get("body", ""),
|
||||
"score": data.get("score", 0),
|
||||
})
|
||||
return comments
|
||||
|
||||
|
||||
async def fetch_reddit(url: str) -> Dict[str, Any]:
|
||||
"""Fetch Reddit post + comments via JSON API."""
|
||||
logger.info(f"Fetching Reddit: {url}")
|
||||
|
||||
# Normalize URL and append .json
|
||||
clean_url = re.sub(r'\?.*$', '', url.rstrip('/'))
|
||||
json_url = f"{clean_url}.json"
|
||||
|
||||
resp = requests.get(
|
||||
json_url,
|
||||
headers=HEADERS,
|
||||
proxies=_get_proxies(),
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Reddit returns [post_listing, comments_listing]
|
||||
if not isinstance(data, list) or len(data) < 2:
|
||||
raise ValueError(f"Unexpected Reddit response format")
|
||||
|
||||
post_listing = data[0].get("data", {}).get("children", [])
|
||||
if not post_listing:
|
||||
raise ValueError("No post found")
|
||||
|
||||
post = _extract_post(post_listing[0])
|
||||
comments = _extract_comments(data[1])
|
||||
|
||||
# Build readable content
|
||||
content_parts = [post["selftext"]] if post["selftext"] else []
|
||||
if comments:
|
||||
content_parts.append("\n---\n## Top Comments\n")
|
||||
for c in comments:
|
||||
content_parts.append(f"**u/{c['author']}** ({c['score']} pts):\n{c['body']}\n")
|
||||
|
||||
return {
|
||||
"title": post["title"],
|
||||
"content": "\n".join(content_parts),
|
||||
"author": f"u/{post['author']}",
|
||||
"url": post["url"],
|
||||
"subreddit": post["subreddit"],
|
||||
"score": post["score"],
|
||||
"num_comments": post["num_comments"],
|
||||
"platform": "reddit",
|
||||
}
|
||||
|
||||
|
||||
async def search_reddit(query: str, subreddit: str = None, limit: int = 10) -> List[Dict[str, Any]]:
|
||||
"""Search Reddit posts."""
|
||||
logger.info(f"Searching Reddit: {query} (sub={subreddit})")
|
||||
|
||||
if subreddit:
|
||||
search_url = f"https://www.reddit.com/r/{subreddit}/search.json"
|
||||
params = {"q": query, "restrict_sr": "on", "limit": limit, "sort": "relevance"}
|
||||
else:
|
||||
search_url = "https://www.reddit.com/search.json"
|
||||
params = {"q": query, "limit": limit, "sort": "relevance"}
|
||||
|
||||
resp = requests.get(
|
||||
search_url,
|
||||
headers=HEADERS,
|
||||
params=params,
|
||||
proxies=_get_proxies(),
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for child in data.get("data", {}).get("children", []):
|
||||
results.append(_extract_post(child))
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,47 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""RSS feed fetcher — uses feedparser."""
|
||||
|
||||
import feedparser
|
||||
from loguru import logger
|
||||
from typing import Dict, Any, List
|
||||
|
||||
|
||||
async def fetch_rss(url: str, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch and parse an RSS/Atom feed.
|
||||
|
||||
Args:
|
||||
url: RSS feed URL
|
||||
limit: Max number of entries to return
|
||||
|
||||
Returns:
|
||||
List of article dicts with: title, summary, url, source, published
|
||||
"""
|
||||
logger.info(f"Fetching RSS: {url}")
|
||||
|
||||
feed = feedparser.parse(url)
|
||||
|
||||
if feed.bozo and not feed.entries:
|
||||
raise ValueError(f"Failed to parse RSS feed: {feed.bozo_exception}")
|
||||
|
||||
source_name = feed.feed.get("title", url)
|
||||
articles = []
|
||||
|
||||
for entry in feed.entries[:limit]:
|
||||
summary = ""
|
||||
if hasattr(entry, "summary"):
|
||||
summary = entry.summary
|
||||
elif hasattr(entry, "content"):
|
||||
summary = entry.content[0].get("value", "")
|
||||
|
||||
articles.append({
|
||||
"title": entry.get("title", ""),
|
||||
"summary": summary,
|
||||
"url": entry.get("link", ""),
|
||||
"source": source_name,
|
||||
"published": entry.get("published", ""),
|
||||
"platform": "rss",
|
||||
})
|
||||
|
||||
logger.info(f"RSS: {len(articles)} articles from {source_name}")
|
||||
return articles
|
||||
@@ -0,0 +1,94 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Search fetcher — semantic web search via Exa API.
|
||||
|
||||
Requires EXA_API_KEY env var. Get a free key at https://exa.ai
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
EXA_API_URL = "https://api.exa.ai/search"
|
||||
|
||||
|
||||
async def search_web(
|
||||
query: str,
|
||||
num_results: int = 5,
|
||||
search_type: str = "auto",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search the web using Exa semantic search.
|
||||
|
||||
Args:
|
||||
query: Search query (supports site: prefix, e.g. "site:reddit.com AI agent")
|
||||
num_results: Number of results to return (default 5, max 10)
|
||||
search_type: "auto" (default) or "neural" or "keyword"
|
||||
|
||||
Returns:
|
||||
List of search results with title, url, snippet
|
||||
"""
|
||||
api_key = os.environ.get("EXA_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
"EXA_API_KEY not set. Get a free key at https://exa.ai\n"
|
||||
"Then: export EXA_API_KEY=your_key_here"
|
||||
)
|
||||
|
||||
logger.info(f"Exa search: {query} (n={num_results})")
|
||||
|
||||
resp = requests.post(
|
||||
EXA_API_URL,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": api_key,
|
||||
},
|
||||
json={
|
||||
"query": query,
|
||||
"numResults": min(num_results, 10),
|
||||
"type": search_type,
|
||||
"contents": {
|
||||
"text": {"maxCharacters": 500},
|
||||
},
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for item in data.get("results", []):
|
||||
results.append({
|
||||
"title": item.get("title", ""),
|
||||
"url": item.get("url", ""),
|
||||
"snippet": item.get("text", ""),
|
||||
"published_date": item.get("publishedDate", ""),
|
||||
"score": item.get("score", 0),
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def search_reddit_via_exa(
|
||||
query: str,
|
||||
subreddit: Optional[str] = None,
|
||||
num_results: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Reddit content via Exa (bypasses Reddit IP blocks).
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
subreddit: Optional subreddit to limit search (e.g. "LocalLLaMA")
|
||||
num_results: Number of results
|
||||
|
||||
Returns:
|
||||
List of Reddit posts found
|
||||
"""
|
||||
if subreddit:
|
||||
full_query = f"site:reddit.com/r/{subreddit} {query}"
|
||||
else:
|
||||
full_query = f"site:reddit.com {query}"
|
||||
|
||||
return await search_web(full_query, num_results=num_results)
|
||||
@@ -0,0 +1,71 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Telegram channel fetcher — uses Telethon.
|
||||
|
||||
Requires: pip install x-reader[telegram]
|
||||
Requires: TG_API_ID + TG_API_HASH in .env
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from loguru import logger
|
||||
from typing import Dict, Any, List
|
||||
|
||||
|
||||
async def fetch_telegram(
|
||||
channel: str,
|
||||
limit: int = 20,
|
||||
hours: int = 24,
|
||||
session_path: str = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch recent messages from a Telegram channel.
|
||||
|
||||
Args:
|
||||
channel: Channel username (e.g. 'predictionmkt')
|
||||
limit: Max messages per channel
|
||||
hours: Only fetch messages from the last N hours
|
||||
session_path: Path to Telethon session file
|
||||
|
||||
Returns:
|
||||
List of message dicts
|
||||
"""
|
||||
try:
|
||||
from telethon import TelegramClient
|
||||
from telethon.tl.types import Message
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"Telethon is required for Telegram fetching. "
|
||||
"Install with: pip install x-reader[telegram]"
|
||||
)
|
||||
|
||||
api_id = os.getenv("TG_API_ID", "")
|
||||
api_hash = os.getenv("TG_API_HASH", "")
|
||||
|
||||
if not api_id or not api_hash:
|
||||
raise ValueError("TG_API_ID and TG_API_HASH must be set in .env")
|
||||
|
||||
session = session_path or os.getenv("TG_SESSION_PATH", "./tg_session")
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||
|
||||
messages = []
|
||||
async with TelegramClient(session, int(api_id), api_hash) as client:
|
||||
logger.info(f"Fetching TG channel: {channel}")
|
||||
entity = await client.get_entity(channel)
|
||||
|
||||
async for msg in client.iter_messages(entity, limit=limit):
|
||||
if not isinstance(msg, Message) or not msg.text:
|
||||
continue
|
||||
if msg.date < cutoff:
|
||||
break
|
||||
|
||||
messages.append({
|
||||
"text": msg.text,
|
||||
"views": msg.views or 0,
|
||||
"date": msg.date.isoformat(),
|
||||
"url": f"https://t.me/{channel}/{msg.id}",
|
||||
"platform": "telegram",
|
||||
})
|
||||
|
||||
logger.info(f"TG {channel}: {len(messages)} messages")
|
||||
return messages
|
||||
@@ -0,0 +1,217 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
X/Twitter fetcher — three-tier fallback:
|
||||
|
||||
1. X oEmbed API (fast, reliable for individual tweets, no login needed)
|
||||
2. Jina Reader (handles non-tweet X pages like profiles)
|
||||
3. Playwright + saved session (handles login-required content)
|
||||
|
||||
Install browser tier: pip install "x-reader[browser]" && playwright install chromium
|
||||
Save X session: x-reader login twitter
|
||||
"""
|
||||
|
||||
import re
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Dict, Any
|
||||
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
|
||||
|
||||
OEMBED_URL = "https://publish.twitter.com/oembed"
|
||||
|
||||
|
||||
def _extract_author(url: str) -> str:
|
||||
"""Extract @username from tweet URL."""
|
||||
match = re.search(r'x\.com/(\w+)/status', url)
|
||||
return f"@{match.group(1)}" if match else ""
|
||||
|
||||
|
||||
def _is_tweet_url(url: str) -> bool:
|
||||
"""Check if this is a direct tweet/status URL (vs profile or other X page)."""
|
||||
return bool(re.search(r'x\.com/\w+/status/\d+', url))
|
||||
|
||||
|
||||
def _fetch_via_oembed(url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch tweet text via X's oEmbed API.
|
||||
Free, reliable, no auth needed. Works for public tweets.
|
||||
Note: oEmbed requires twitter.com URLs (not x.com).
|
||||
"""
|
||||
# oEmbed API requires twitter.com format
|
||||
oembed_query_url = url.replace("x.com", "twitter.com")
|
||||
resp = requests.get(
|
||||
OEMBED_URL,
|
||||
params={"url": oembed_query_url, "omit_script": "true"},
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Strip HTML tags from the embedded HTML to get clean text
|
||||
html = data.get("html", "")
|
||||
text = re.sub(r'<[^>]+>', ' ', html)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
return {
|
||||
"text": text,
|
||||
"author": data.get("author_name", ""),
|
||||
"author_url": data.get("author_url", ""),
|
||||
"title": text[:100] if text else "",
|
||||
}
|
||||
|
||||
|
||||
async def _fetch_via_playwright(url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch tweet via Playwright with X-specific DOM selectors.
|
||||
Uses saved login session if available (~/.x-reader/sessions/twitter.json).
|
||||
"""
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except ImportError:
|
||||
raise RuntimeError(
|
||||
"Playwright not installed. Run:\n"
|
||||
' pip install "x-reader[browser]"\n'
|
||||
" playwright install chromium"
|
||||
)
|
||||
|
||||
from agent_eyes.fetchers.browser import get_session_path
|
||||
from pathlib import Path
|
||||
|
||||
session_path = get_session_path("twitter")
|
||||
has_session = Path(session_path).exists()
|
||||
if has_session:
|
||||
logger.info(f"Using saved X session: {session_path}")
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
|
||||
context_kwargs = {}
|
||||
if has_session:
|
||||
context_kwargs["storage_state"] = session_path
|
||||
|
||||
context = await browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
**context_kwargs,
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30_000)
|
||||
|
||||
# Wait for tweet text to render (X is a SPA, needs JS execution)
|
||||
try:
|
||||
await page.wait_for_selector(
|
||||
'[data-testid="tweetText"]', timeout=10_000
|
||||
)
|
||||
except Exception:
|
||||
pass # May not appear if login required
|
||||
|
||||
# Extract tweet content with X-specific selectors
|
||||
tweet_text = await page.evaluate("""() => {
|
||||
// Priority 1: tweet text element
|
||||
const tweetEl = document.querySelector('[data-testid="tweetText"]');
|
||||
if (tweetEl) return tweetEl.innerText;
|
||||
|
||||
// Priority 2: article element (thread view)
|
||||
const article = document.querySelector('article');
|
||||
if (article) return article.innerText;
|
||||
|
||||
// Priority 3: main content area
|
||||
const main = document.querySelector('main');
|
||||
if (main) return main.innerText;
|
||||
|
||||
return '';
|
||||
}""")
|
||||
|
||||
title = await page.title()
|
||||
|
||||
return {
|
||||
"text": (tweet_text or "").strip(),
|
||||
"title": (title or "").strip()[:200],
|
||||
}
|
||||
finally:
|
||||
await context.close()
|
||||
await browser.close()
|
||||
|
||||
|
||||
async def fetch_twitter(url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch a tweet or X post with three-tier fallback.
|
||||
|
||||
Args:
|
||||
url: Tweet URL (x.com or twitter.com)
|
||||
|
||||
Returns:
|
||||
Dict with: text, author, url, title, platform
|
||||
"""
|
||||
url = url.replace("twitter.com", "x.com")
|
||||
author = _extract_author(url)
|
||||
|
||||
# Tier 1: oEmbed API (best for individual tweets)
|
||||
if _is_tweet_url(url):
|
||||
try:
|
||||
logger.info(f"[Twitter] Tier 1 — oEmbed: {url}")
|
||||
data = _fetch_via_oembed(url)
|
||||
if data.get("text") and len(data["text"].strip()) > 20:
|
||||
return {
|
||||
"text": data["text"],
|
||||
"author": author or data.get("author", ""),
|
||||
"url": url,
|
||||
"title": data.get("title", ""),
|
||||
"platform": "twitter",
|
||||
}
|
||||
logger.warning("[Twitter] oEmbed returned thin content")
|
||||
except Exception as e:
|
||||
logger.warning(f"[Twitter] oEmbed failed ({e})")
|
||||
|
||||
# Tier 2: Jina Reader (handles profiles, threads, non-tweet pages)
|
||||
try:
|
||||
logger.info(f"[Twitter] Tier 2 — Jina: {url}")
|
||||
data = fetch_via_jina(url)
|
||||
content = data.get("content", "")
|
||||
title = data.get("title", "")
|
||||
jina_ok = (
|
||||
content
|
||||
and len(content.strip()) > 100
|
||||
and "not yet fully loaded" not in content.lower()
|
||||
and title.lower() not in ("x", "title: x", "")
|
||||
)
|
||||
if jina_ok:
|
||||
return {
|
||||
"text": content,
|
||||
"author": author,
|
||||
"url": url,
|
||||
"title": title,
|
||||
"platform": "twitter",
|
||||
}
|
||||
logger.warning("[Twitter] Jina returned unusable content")
|
||||
except Exception as e:
|
||||
logger.warning(f"[Twitter] Jina failed ({e})")
|
||||
|
||||
# Tier 3: Playwright + session with X-specific extraction
|
||||
try:
|
||||
logger.info(f"[Twitter] Tier 3 — Playwright: {url}")
|
||||
data = await _fetch_via_playwright(url)
|
||||
content = data.get("text", "")
|
||||
if content and len(content.strip()) > 20:
|
||||
return {
|
||||
"text": content,
|
||||
"author": author,
|
||||
"url": url,
|
||||
"title": data.get("title", ""),
|
||||
"platform": "twitter",
|
||||
}
|
||||
logger.warning("[Twitter] Playwright returned empty content")
|
||||
except RuntimeError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[Twitter] All methods failed: {e}")
|
||||
|
||||
raise RuntimeError(
|
||||
f"❌ All Twitter fetch methods failed for: {url}\n"
|
||||
f" Try: x-reader login twitter (to save session for browser fallback)\n"
|
||||
f" Then retry: x-reader {url}"
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
WeChat article fetcher — two-tier fallback:
|
||||
|
||||
1. Jina Reader (fast, no deps)
|
||||
2. Playwright headless (no login needed for public articles)
|
||||
"""
|
||||
|
||||
from loguru import logger
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
async def fetch_wechat(url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch a WeChat public account article with fallback.
|
||||
|
||||
Args:
|
||||
url: mp.weixin.qq.com article URL
|
||||
|
||||
Returns:
|
||||
Dict with: title, content, author, url, platform
|
||||
"""
|
||||
# Tier 1: Jina Reader
|
||||
try:
|
||||
logger.info(f"[WeChat] Tier 1 — Jina: {url}")
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
|
||||
data = fetch_via_jina(url)
|
||||
if data.get("content"):
|
||||
return {
|
||||
"title": data["title"],
|
||||
"content": data["content"],
|
||||
"author": data.get("author", ""),
|
||||
"url": url,
|
||||
"platform": "wechat",
|
||||
}
|
||||
logger.warning("[WeChat] Jina returned empty content, falling back to browser")
|
||||
except Exception as e:
|
||||
logger.warning(f"[WeChat] Jina failed ({e}), falling back to browser")
|
||||
|
||||
# Tier 2: Playwright headless (no session needed)
|
||||
try:
|
||||
logger.info(f"[WeChat] Tier 2 — Playwright headless: {url}")
|
||||
from agent_eyes.fetchers.browser import fetch_via_browser
|
||||
|
||||
data = await fetch_via_browser(url)
|
||||
return {
|
||||
"title": data["title"],
|
||||
"content": data["content"],
|
||||
"author": data.get("author", ""),
|
||||
"url": url,
|
||||
"platform": "wechat",
|
||||
}
|
||||
except RuntimeError:
|
||||
# Playwright not installed — re-raise with original Jina error context
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[WeChat] Browser fetch also failed: {e}")
|
||||
raise RuntimeError(
|
||||
f"❌ All WeChat fetch methods failed.\n"
|
||||
f" Last error: {e}"
|
||||
)
|
||||
@@ -0,0 +1,78 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Xiaohongshu (RED) note fetcher — three-tier fallback:
|
||||
|
||||
1. Jina Reader (fast, no deps)
|
||||
2. Playwright + saved session (handles 451/403)
|
||||
3. Error with login instructions
|
||||
|
||||
Install browser tier: pip install "x-reader[browser]" && playwright install chromium
|
||||
"""
|
||||
|
||||
from loguru import logger
|
||||
from typing import Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
|
||||
|
||||
async def fetch_xhs(url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch a Xiaohongshu note with three-tier fallback.
|
||||
|
||||
Args:
|
||||
url: xiaohongshu.com or xhslink.com URL
|
||||
|
||||
Returns:
|
||||
Dict with: title, content, author, url, platform
|
||||
"""
|
||||
# Tier 1: Jina Reader
|
||||
try:
|
||||
logger.info(f"[XHS] Tier 1 — Jina: {url}")
|
||||
data = fetch_via_jina(url)
|
||||
if data.get("content"):
|
||||
return {
|
||||
"title": data["title"],
|
||||
"content": data["content"],
|
||||
"author": data.get("author", ""),
|
||||
"url": url,
|
||||
"platform": "xhs",
|
||||
}
|
||||
logger.warning("[XHS] Jina returned empty content, falling back to browser")
|
||||
except Exception as e:
|
||||
logger.warning(f"[XHS] Jina failed ({e}), falling back to browser")
|
||||
|
||||
# Tier 2: Playwright with session
|
||||
from agent_eyes.fetchers.browser import get_session_path, SESSION_DIR
|
||||
|
||||
session_path = get_session_path("xhs")
|
||||
if not Path(session_path).exists():
|
||||
# Tier 3: No session — guide user
|
||||
raise RuntimeError(
|
||||
f"❌ XHS blocked Jina and no saved session found.\n"
|
||||
f" Run: x-reader login xhs\n"
|
||||
f" Then retry this URL."
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"[XHS] Tier 2 — Playwright with session: {url}")
|
||||
from agent_eyes.fetchers.browser import fetch_via_browser
|
||||
|
||||
data = await fetch_via_browser(url, storage_state=session_path)
|
||||
return {
|
||||
"title": data["title"],
|
||||
"content": data["content"],
|
||||
"author": data.get("author", ""),
|
||||
"url": url,
|
||||
"platform": "xhs",
|
||||
}
|
||||
except RuntimeError:
|
||||
# Playwright not installed
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[XHS] Browser fetch also failed: {e}")
|
||||
raise RuntimeError(
|
||||
f"❌ All XHS fetch methods failed.\n"
|
||||
f" Last error: {e}\n"
|
||||
f" Try: x-reader login xhs (to refresh session)"
|
||||
)
|
||||
@@ -0,0 +1,211 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
YouTube video fetcher — three-tier content extraction:
|
||||
|
||||
1. yt-dlp auto-subtitles (fastest, best quality for subtitled videos)
|
||||
2. yt-dlp audio download → Groq Whisper API transcription (for non-subtitled videos)
|
||||
3. Jina Reader fallback (page description only)
|
||||
|
||||
Requires: yt-dlp installed (brew install yt-dlp / pip install yt-dlp)
|
||||
Optional: GROQ_API_KEY env var for Whisper transcription
|
||||
"""
|
||||
|
||||
import re
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from loguru import logger
|
||||
from typing import Dict, Any
|
||||
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
|
||||
|
||||
def _extract_video_id(url: str) -> str:
|
||||
"""Extract video ID from YouTube URL."""
|
||||
match = re.search(r'(?:v=|youtu\.be/)([a-zA-Z0-9_-]{11})', url)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def _get_subtitles_via_ytdlp(url: str, lang: str = "en") -> str:
|
||||
"""
|
||||
Download auto-generated subtitles using yt-dlp.
|
||||
Returns subtitle text, or empty string if unavailable.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_path = os.path.join(tmpdir, "sub")
|
||||
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"--write-auto-sub",
|
||||
"--write-sub",
|
||||
"--sub-lang", lang,
|
||||
"--sub-format", "srt",
|
||||
"--skip-download",
|
||||
"-o", output_path,
|
||||
url,
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, text=True, timeout=60)
|
||||
except FileNotFoundError:
|
||||
logger.warning("yt-dlp not found. Install with: brew install yt-dlp")
|
||||
return ""
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("yt-dlp subtitle download timed out")
|
||||
return ""
|
||||
|
||||
for ext in [f".{lang}.srt", f".{lang}.vtt"]:
|
||||
sub_file = output_path + ext
|
||||
if os.path.exists(sub_file):
|
||||
return _parse_srt(sub_file)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_srt(filepath: str) -> str:
|
||||
"""Parse SRT file into clean text (strip timestamps and sequence numbers)."""
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
text_lines = []
|
||||
seen = set()
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or line.isdigit() or '-->' in line:
|
||||
continue
|
||||
if line.startswith('[') and line.endswith(']'):
|
||||
continue
|
||||
if line not in seen:
|
||||
seen.add(line)
|
||||
text_lines.append(line)
|
||||
|
||||
return " ".join(text_lines)
|
||||
|
||||
|
||||
def _transcribe_via_whisper(url: str) -> str:
|
||||
"""
|
||||
Download audio with yt-dlp and transcribe via Groq Whisper API.
|
||||
|
||||
Requires: GROQ_API_KEY env var + yt-dlp + ffmpeg installed.
|
||||
Groq Whisper limit: 25MB audio file.
|
||||
Returns transcript text, or empty string if unavailable.
|
||||
"""
|
||||
api_key = os.getenv("GROQ_API_KEY")
|
||||
if not api_key:
|
||||
logger.info("GROQ_API_KEY not set, skipping Whisper transcription")
|
||||
return ""
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_template = os.path.join(tmpdir, "audio.%(ext)s")
|
||||
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"-x",
|
||||
"--audio-format", "m4a",
|
||||
"--audio-quality", "5",
|
||||
"-o", output_template,
|
||||
"--no-playlist",
|
||||
url,
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||
except FileNotFoundError:
|
||||
logger.warning("yt-dlp not found for audio download")
|
||||
return ""
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("yt-dlp audio download timed out")
|
||||
return ""
|
||||
|
||||
# Find the downloaded audio file
|
||||
audio_path = os.path.join(tmpdir, "audio.m4a")
|
||||
if not os.path.exists(audio_path):
|
||||
for f in os.listdir(tmpdir):
|
||||
if f.startswith("audio."):
|
||||
audio_path = os.path.join(tmpdir, f)
|
||||
break
|
||||
else:
|
||||
logger.warning("No audio file downloaded")
|
||||
return ""
|
||||
|
||||
file_size = os.path.getsize(audio_path)
|
||||
if file_size > 25 * 1024 * 1024:
|
||||
logger.warning(f"Audio file too large ({file_size // 1024 // 1024}MB > 25MB limit)")
|
||||
return ""
|
||||
|
||||
logger.info(f"Transcribing {file_size // 1024}KB audio via Groq Whisper...")
|
||||
|
||||
import requests
|
||||
try:
|
||||
with open(audio_path, "rb") as f:
|
||||
response = requests.post(
|
||||
"https://api.groq.com/openai/v1/audio/transcriptions",
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
files={"file": (os.path.basename(audio_path), f, "audio/mp4")},
|
||||
data={"model": "whisper-large-v3", "response_format": "text"},
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
transcript = response.text.strip()
|
||||
logger.info(f"Whisper transcript: {len(transcript)} chars")
|
||||
return transcript
|
||||
else:
|
||||
logger.warning(f"Groq Whisper API error: {response.status_code} {response.text[:200]}")
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.warning(f"Whisper transcription failed: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
async def fetch_youtube(url: str, sub_lang: str = "en") -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch YouTube video content with three-tier extraction.
|
||||
|
||||
Strategy:
|
||||
1. yt-dlp auto-subtitles (full transcript, fastest)
|
||||
2. yt-dlp audio + Groq Whisper API (for non-subtitled videos)
|
||||
3. Jina Reader fallback (page description only)
|
||||
|
||||
Args:
|
||||
url: YouTube video URL
|
||||
sub_lang: Subtitle language code (default: "en")
|
||||
|
||||
Returns:
|
||||
Dict with: title, description, author, url, video_id, has_transcript, platform
|
||||
"""
|
||||
logger.info(f"Fetching YouTube: {url}")
|
||||
video_id = _extract_video_id(url)
|
||||
|
||||
# Step 1: Get metadata via Jina (fast, always works)
|
||||
jina_data = fetch_via_jina(url)
|
||||
title = jina_data["title"]
|
||||
|
||||
# Step 2: Try yt-dlp auto-subtitles
|
||||
logger.info(f"Extracting subtitles ({sub_lang})...")
|
||||
transcript = _get_subtitles_via_ytdlp(url, lang=sub_lang)
|
||||
|
||||
# Step 3: No subtitles? Try Whisper transcription
|
||||
if not transcript:
|
||||
logger.info("No subtitles available, trying Whisper transcription...")
|
||||
transcript = _transcribe_via_whisper(url)
|
||||
|
||||
if transcript:
|
||||
logger.info(f"Got transcript: {len(transcript)} chars")
|
||||
content = transcript
|
||||
has_transcript = True
|
||||
else:
|
||||
logger.info("No transcript available, using page description")
|
||||
content = jina_data["content"]
|
||||
has_transcript = False
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"description": content,
|
||||
"author": jina_data.get("author", ""),
|
||||
"url": url,
|
||||
"video_id": video_id,
|
||||
"has_transcript": has_transcript,
|
||||
"platform": "youtube",
|
||||
}
|
||||
Reference in New Issue
Block a user