v1.0.0 — Agent Eyes: search + read the entire internet
Major restructure from x-reader fork to independent project: Architecture: - readers/ — content extraction from 10+ platforms (based on x-reader, MIT) - search/ — semantic search via Exa, GitHub API, birdx (NEW) - config.py — configuration management (~/.agent-eyes/config.yaml) (NEW) - doctor.py — environment health checker (NEW) - core.py — AgentEyes unified entry point (NEW) - cli.py — full CLI: read, search, setup, doctor (NEW) - integrations/mcp_server.py — 8 MCP tools (NEW) - guides/ — 6 Agent-readable setup guides (NEW) - integrations/skill/ — OpenClaw Skill package (NEW) Platforms (zero config): - Web pages, GitHub, Bilibili, YouTube, RSS, single tweets Platforms (one free API key): - Web search, Reddit search, Twitter search (via Exa) Platforms (optional setup): - Reddit full reader, Twitter advanced, WeChat, XiaoHongShu Tests: 34/34 passing Credits: Built on x-reader by @runes_leo (MIT License)
This commit is contained in:
Panniantong
@@ -0,0 +1,16 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Search module init."""
|
||||
|
||||
from agent_eyes.search.exa import search_web
|
||||
from agent_eyes.search.reddit import search_reddit
|
||||
from agent_eyes.search.github import search_github, search_github_issues
|
||||
from agent_eyes.search.twitter import search_twitter, get_user_tweets
|
||||
|
||||
__all__ = [
|
||||
"search_web",
|
||||
"search_reddit",
|
||||
"search_github",
|
||||
"search_github_issues",
|
||||
"search_twitter",
|
||||
"get_user_tweets",
|
||||
]
|
||||
@@ -0,0 +1,79 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Exa semantic web search.
|
||||
|
||||
Get a free API key at https://exa.ai (1000 searches/month free).
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
EXA_API_URL = "https://api.exa.ai/search"
|
||||
|
||||
|
||||
def _get_api_key(config=None) -> str:
|
||||
"""Get Exa API key from config or env."""
|
||||
if config:
|
||||
key = config.get("exa_api_key")
|
||||
if key:
|
||||
return key
|
||||
key = os.environ.get("EXA_API_KEY")
|
||||
if key:
|
||||
return key
|
||||
raise ValueError(
|
||||
"Exa API key not configured.\n"
|
||||
"Get a free key at https://exa.ai (1000 searches/month free)\n"
|
||||
"Then run: agent-eyes setup"
|
||||
)
|
||||
|
||||
|
||||
async def search_web(
|
||||
query: str,
|
||||
num_results: int = 5,
|
||||
search_type: str = "auto",
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Semantic web search via Exa.
|
||||
|
||||
Args:
|
||||
query: Search query (supports site: prefix)
|
||||
num_results: Number of results (default 5, max 10)
|
||||
search_type: "auto" / "neural" / "keyword"
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {title, url, snippet, published_date, score}
|
||||
"""
|
||||
api_key = _get_api_key(config)
|
||||
logger.info(f"Exa search: {query} (n={num_results})")
|
||||
|
||||
resp = requests.post(
|
||||
EXA_API_URL,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": api_key,
|
||||
},
|
||||
json={
|
||||
"query": query,
|
||||
"numResults": min(num_results, 10),
|
||||
"type": search_type,
|
||||
"contents": {"text": {"maxCharacters": 500}},
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for item in data.get("results", []):
|
||||
results.append({
|
||||
"title": item.get("title", ""),
|
||||
"url": item.get("url", ""),
|
||||
"snippet": item.get("text", ""),
|
||||
"published_date": item.get("publishedDate", ""),
|
||||
"score": item.get("score", 0),
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,118 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GitHub search via public API (no key required)."""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
GITHUB_API = "https://api.github.com"
|
||||
|
||||
|
||||
def _get_headers(config=None) -> dict:
|
||||
"""Get GitHub API headers with optional auth token."""
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
token = None
|
||||
if config:
|
||||
token = config.get("github_token")
|
||||
if not token:
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
async def search_github(
|
||||
query: str,
|
||||
language: Optional[str] = None,
|
||||
sort: str = "stars",
|
||||
limit: int = 5,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search GitHub repositories.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
language: Filter by language (e.g. "python")
|
||||
sort: Sort by "stars" / "forks" / "updated"
|
||||
limit: Number of results (max 30)
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {name, url, description, stars, language, updated}
|
||||
"""
|
||||
q = query
|
||||
if language:
|
||||
q += f" language:{language}"
|
||||
|
||||
logger.info(f"GitHub search: {q} (n={limit})")
|
||||
|
||||
resp = requests.get(
|
||||
f"{GITHUB_API}/search/repositories",
|
||||
headers=_get_headers(config),
|
||||
params={"q": q, "sort": sort, "per_page": min(limit, 30)},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for repo in data.get("items", []):
|
||||
results.append({
|
||||
"name": repo.get("full_name", ""),
|
||||
"url": repo.get("html_url", ""),
|
||||
"description": repo.get("description", ""),
|
||||
"stars": repo.get("stargazers_count", 0),
|
||||
"forks": repo.get("forks_count", 0),
|
||||
"language": repo.get("language", ""),
|
||||
"updated": repo.get("updated_at", ""),
|
||||
"topics": repo.get("topics", []),
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
async def search_github_issues(
|
||||
query: str,
|
||||
repo: Optional[str] = None,
|
||||
state: str = "open",
|
||||
limit: int = 5,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search GitHub issues and discussions.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
repo: Filter by repo (e.g. "owner/repo")
|
||||
state: "open" / "closed"
|
||||
limit: Number of results
|
||||
config: Optional Config instance
|
||||
"""
|
||||
q = query
|
||||
if repo:
|
||||
q += f" repo:{repo}"
|
||||
q += f" state:{state}"
|
||||
|
||||
resp = requests.get(
|
||||
f"{GITHUB_API}/search/issues",
|
||||
headers=_get_headers(config),
|
||||
params={"q": q, "sort": "reactions", "per_page": min(limit, 30)},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for issue in data.get("items", []):
|
||||
results.append({
|
||||
"title": issue.get("title", ""),
|
||||
"url": issue.get("html_url", ""),
|
||||
"body": (issue.get("body", "") or "")[:500],
|
||||
"state": issue.get("state", ""),
|
||||
"comments": issue.get("comments", 0),
|
||||
"reactions": issue.get("reactions", {}).get("total_count", 0),
|
||||
"created": issue.get("created_at", ""),
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,30 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Reddit search via Exa (bypasses Reddit IP blocks)."""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from agent_eyes.search.exa import search_web
|
||||
|
||||
|
||||
async def search_reddit(
|
||||
query: str,
|
||||
subreddit: Optional[str] = None,
|
||||
num_results: int = 10,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Reddit content via Exa semantic search.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
subreddit: Optional subreddit (e.g. "LocalLLaMA")
|
||||
num_results: Number of results
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {title, url, snippet, published_date, score}
|
||||
"""
|
||||
if subreddit:
|
||||
full_query = f"site:reddit.com/r/{subreddit} {query}"
|
||||
else:
|
||||
full_query = f"site:reddit.com {query}"
|
||||
return await search_web(full_query, num_results=num_results, config=config)
|
||||
@@ -0,0 +1,112 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Twitter search — uses birdx if available, falls back to Exa."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from loguru import logger
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
async def search_twitter(
|
||||
query: str,
|
||||
limit: int = 10,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Twitter/X content.
|
||||
|
||||
Strategy:
|
||||
1. If birdx is installed → use it (full search, timeline, threads)
|
||||
2. Otherwise → use Exa with site:x.com (basic search)
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
limit: Number of results
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {author, text, url, likes, retweets, date}
|
||||
"""
|
||||
if shutil.which("birdx"):
|
||||
return await _search_birdx(query, limit)
|
||||
else:
|
||||
return await _search_exa(query, limit, config)
|
||||
|
||||
|
||||
async def _search_birdx(query: str, limit: int) -> List[Dict[str, Any]]:
|
||||
"""Search Twitter via birdx CLI."""
|
||||
logger.info(f"birdx search: {query} (n={limit})")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["birdx", "search", query, "-n", str(limit), "--json"],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
# birdx might not support --json, try plain output
|
||||
result = subprocess.run(
|
||||
["birdx", "search", query, "-n", str(limit)],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
return _parse_birdx_text(result.stdout)
|
||||
|
||||
data = json.loads(result.stdout)
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
return data.get("tweets", data.get("results", []))
|
||||
except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError) as e:
|
||||
logger.error(f"birdx search failed: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _parse_birdx_text(text: str) -> List[Dict[str, Any]]:
|
||||
"""Parse birdx plain text output into structured data."""
|
||||
results = []
|
||||
current = {}
|
||||
for line in text.strip().split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
if current:
|
||||
results.append(current)
|
||||
current = {}
|
||||
continue
|
||||
if line.startswith("@"):
|
||||
current["author"] = line.split()[0] if line else ""
|
||||
elif line.startswith("http"):
|
||||
current["url"] = line
|
||||
else:
|
||||
current["text"] = current.get("text", "") + " " + line
|
||||
if current:
|
||||
results.append(current)
|
||||
return results
|
||||
|
||||
|
||||
async def _search_exa(query: str, limit: int, config=None) -> List[Dict[str, Any]]:
|
||||
"""Search Twitter via Exa (site:x.com)."""
|
||||
from agent_eyes.search.exa import search_web
|
||||
return await search_web(
|
||||
f"site:x.com {query}",
|
||||
num_results=limit,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
async def get_user_tweets(
|
||||
username: str,
|
||||
limit: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get recent tweets from a user (requires birdx)."""
|
||||
if not shutil.which("birdx"):
|
||||
raise RuntimeError(
|
||||
"birdx not installed. Install: pip install birdx\n"
|
||||
"Then configure cookies: agent-eyes setup"
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["birdx", "user-tweets", f"@{username.lstrip('@')}", "-n", str(limit)],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
return _parse_birdx_text(result.stdout)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("birdx timed out")
|
||||
return []
|
||||
Reference in New Issue
Block a user