v1.0.0 — Agent Eyes: search + read the entire internet
Major restructure from x-reader fork to independent project: Architecture: - readers/ — content extraction from 10+ platforms (based on x-reader, MIT) - search/ — semantic search via Exa, GitHub API, birdx (NEW) - config.py — configuration management (~/.agent-eyes/config.yaml) (NEW) - doctor.py — environment health checker (NEW) - core.py — AgentEyes unified entry point (NEW) - cli.py — full CLI: read, search, setup, doctor (NEW) - integrations/mcp_server.py — 8 MCP tools (NEW) - guides/ — 6 Agent-readable setup guides (NEW) - integrations/skill/ — OpenClaw Skill package (NEW) Platforms (zero config): - Web pages, GitHub, Bilibili, YouTube, RSS, single tweets Platforms (one free API key): - Web search, Reddit search, Twitter search (via Exa) Platforms (optional setup): - Reddit full reader, Twitter advanced, WeChat, XiaoHongShu Tests: 34/34 passing Credits: Built on x-reader by @runes_leo (MIT License)
This commit is contained in:
Panniantong
@@ -1,3 +1,9 @@
|
||||
"""x-reader: Universal content reader for 7+ platforms."""
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Agent Eyes — Give your AI Agent eyes to see the entire internet."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__version__ = "1.0.0"
|
||||
__author__ = "Neo Reid"
|
||||
|
||||
from agent_eyes.core import AgentEyes
|
||||
|
||||
__all__ = ["AgentEyes"]
|
||||
|
||||
+225
-116
@@ -1,138 +1,247 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
x-reader CLI — fetch content from any platform.
|
||||
Agent Eyes CLI — command-line interface.
|
||||
|
||||
Usage:
|
||||
x-reader <url> # Fetch a single URL
|
||||
x-reader <url1> <url2> ... # Fetch multiple URLs
|
||||
x-reader list # Show inbox contents
|
||||
x-reader clear # Clear inbox
|
||||
agent-eyes read <url>
|
||||
agent-eyes search <query>
|
||||
agent-eyes search-reddit <query> [--sub <subreddit>]
|
||||
agent-eyes search-github <query> [--lang <language>]
|
||||
agent-eyes search-twitter <query>
|
||||
agent-eyes setup
|
||||
agent-eyes doctor
|
||||
agent-eyes version
|
||||
"""
|
||||
|
||||
import sys
|
||||
import asyncio
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
from agent_eyes.reader import UniversalReader
|
||||
from agent_eyes.schema import UnifiedInbox, SourceType
|
||||
|
||||
|
||||
def get_inbox_path() -> str:
|
||||
import os
|
||||
return os.getenv("INBOX_FILE", "unified_inbox.json")
|
||||
|
||||
|
||||
def cmd_fetch(urls: list[str]):
|
||||
"""Fetch one or more URLs."""
|
||||
inbox = UnifiedInbox(get_inbox_path())
|
||||
reader = UniversalReader(inbox=inbox)
|
||||
|
||||
async def run():
|
||||
if len(urls) == 1:
|
||||
item = await reader.read(urls[0])
|
||||
print(f"✅ [{item.source_type.value}] {item.title[:60]}")
|
||||
print(f" {item.url}")
|
||||
print(f" {item.content[:200]}...")
|
||||
else:
|
||||
items = await reader.read_batch(urls)
|
||||
for item in items:
|
||||
print(f"✅ [{item.source_type.value}] {item.title[:60]}")
|
||||
print(f"\n📦 Fetched {len(items)}/{len(urls)} URLs")
|
||||
|
||||
try:
|
||||
asyncio.run(run())
|
||||
except KeyboardInterrupt:
|
||||
print("\n⏹ Cancelled")
|
||||
except Exception as e:
|
||||
print(f"❌ {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_list():
|
||||
"""Show inbox contents."""
|
||||
inbox = UnifiedInbox(get_inbox_path())
|
||||
if not inbox.items:
|
||||
print("📦 Inbox is empty")
|
||||
return
|
||||
|
||||
print(f"📦 Inbox: {len(inbox.items)} items\n")
|
||||
|
||||
emoji_map = {
|
||||
SourceType.TELEGRAM: "📢", SourceType.RSS: "📰",
|
||||
SourceType.BILIBILI: "🎬", SourceType.XIAOHONGSHU: "📕",
|
||||
SourceType.TWITTER: "🐦", SourceType.WECHAT: "💬",
|
||||
SourceType.YOUTUBE: "▶️", SourceType.MANUAL: "✏️",
|
||||
}
|
||||
|
||||
for i, item in enumerate(inbox.items[-20:], 1):
|
||||
emoji = emoji_map.get(item.source_type, "📄")
|
||||
print(f" {i:2d}. {emoji} [{item.source_type.value:8s}] {item.title[:50]}")
|
||||
|
||||
|
||||
def cmd_clear():
|
||||
"""Clear inbox."""
|
||||
path = Path(get_inbox_path())
|
||||
if path.exists():
|
||||
confirm = input("Clear inbox? (y/N) ")
|
||||
if confirm.lower() == 'y':
|
||||
path.write_text("[]")
|
||||
print("✅ Inbox cleared")
|
||||
else:
|
||||
print("📦 Inbox is already empty")
|
||||
|
||||
|
||||
def cmd_login(platform: str):
|
||||
"""Open browser for manual login to a platform."""
|
||||
from agent_eyes.login import login
|
||||
login(platform)
|
||||
from agent_eyes import __version__
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("""
|
||||
📖 x-reader — Universal content reader
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="agent-eyes",
|
||||
description="👁️ Give your AI Agent eyes to see the entire internet",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", help="Available commands")
|
||||
|
||||
Usage:
|
||||
x-reader <url> Fetch content from any URL
|
||||
x-reader <url1> <url2> Fetch multiple URLs
|
||||
x-reader login <platform> Login to a platform (saves session for browser fallback)
|
||||
x-reader list Show inbox contents
|
||||
x-reader clear Clear inbox
|
||||
# ── read ──
|
||||
p_read = sub.add_parser("read", help="Read content from a URL")
|
||||
p_read.add_argument("url", help="URL to read")
|
||||
p_read.add_argument("--json", dest="as_json", action="store_true", help="Output as JSON")
|
||||
|
||||
Supported platforms:
|
||||
WeChat, Telegram, X/Twitter, YouTube,
|
||||
Bilibili, Xiaohongshu, RSS, and any web page
|
||||
# ── search ──
|
||||
p_search = sub.add_parser("search", help="Search the web (Exa)")
|
||||
p_search.add_argument("query", nargs="+", help="Search query")
|
||||
p_search.add_argument("-n", "--num", type=int, default=5, help="Number of results")
|
||||
|
||||
Examples:
|
||||
x-reader https://mp.weixin.qq.com/s/abc123
|
||||
x-reader https://x.com/elonmusk/status/123456
|
||||
x-reader https://www.xiaohongshu.com/explore/abc123
|
||||
x-reader login xhs
|
||||
""")
|
||||
return
|
||||
# ── search-reddit ──
|
||||
p_sr = sub.add_parser("search-reddit", help="Search Reddit")
|
||||
p_sr.add_argument("query", nargs="+", help="Search query")
|
||||
p_sr.add_argument("--sub", help="Subreddit filter")
|
||||
p_sr.add_argument("-n", "--num", type=int, default=10, help="Number of results")
|
||||
|
||||
cmd = sys.argv[1].lower()
|
||||
# ── search-github ──
|
||||
p_sg = sub.add_parser("search-github", help="Search GitHub")
|
||||
p_sg.add_argument("query", nargs="+", help="Search query")
|
||||
p_sg.add_argument("--lang", help="Language filter")
|
||||
p_sg.add_argument("-n", "--num", type=int, default=5, help="Number of results")
|
||||
|
||||
if cmd == "login":
|
||||
if len(sys.argv) < 3:
|
||||
print("❌ Usage: x-reader login <platform>")
|
||||
print(" Supported: xhs, wechat")
|
||||
sys.exit(1)
|
||||
cmd_login(sys.argv[2])
|
||||
elif cmd == "list":
|
||||
cmd_list()
|
||||
elif cmd == "clear":
|
||||
cmd_clear()
|
||||
elif cmd.startswith("http") or cmd.startswith("www.") or "." in cmd:
|
||||
urls = [arg for arg in sys.argv[1:] if arg.startswith(("http", "www.")) or "." in arg]
|
||||
cmd_fetch(urls)
|
||||
# ── search-twitter ──
|
||||
p_st = sub.add_parser("search-twitter", help="Search Twitter")
|
||||
p_st.add_argument("query", nargs="+", help="Search query")
|
||||
p_st.add_argument("-n", "--num", type=int, default=10, help="Number of results")
|
||||
|
||||
# ── setup ──
|
||||
sub.add_parser("setup", help="Interactive configuration wizard")
|
||||
|
||||
# ── doctor ──
|
||||
sub.add_parser("doctor", help="Check platform availability")
|
||||
|
||||
# ── version ──
|
||||
sub.add_parser("version", help="Show version")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.command:
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
if args.command == "version":
|
||||
print(f"Agent Eyes v{__version__}")
|
||||
sys.exit(0)
|
||||
|
||||
if args.command == "doctor":
|
||||
_cmd_doctor()
|
||||
elif args.command == "setup":
|
||||
_cmd_setup()
|
||||
elif args.command == "read":
|
||||
asyncio.run(_cmd_read(args))
|
||||
elif args.command.startswith("search"):
|
||||
asyncio.run(_cmd_search(args))
|
||||
|
||||
|
||||
# ── Command handlers ────────────────────────────────
|
||||
|
||||
|
||||
def _cmd_doctor():
|
||||
from agent_eyes.config import Config
|
||||
from agent_eyes.doctor import check_all, format_report
|
||||
config = Config()
|
||||
results = check_all(config)
|
||||
print(format_report(results))
|
||||
|
||||
|
||||
def _cmd_setup():
|
||||
from agent_eyes.config import Config
|
||||
|
||||
config = Config()
|
||||
print()
|
||||
print("👁️ Agent Eyes Setup")
|
||||
print("=" * 40)
|
||||
print()
|
||||
|
||||
# Step 1: Exa
|
||||
print("【推荐】全网搜索 — Exa Search API")
|
||||
print(" 免费 1000 次/月，注册地址: https://exa.ai")
|
||||
current = config.get("exa_api_key")
|
||||
if current:
|
||||
print(f" 当前状态: ✅ 已配置 ({current[:8]}...)")
|
||||
change = input(" 要更换吗？[y/N]: ").strip().lower()
|
||||
if change != "y":
|
||||
print()
|
||||
else:
|
||||
key = input(" EXA_API_KEY: ").strip()
|
||||
if key:
|
||||
config.set("exa_api_key", key)
|
||||
print(" ✅ 已更新！")
|
||||
print()
|
||||
else:
|
||||
print(f"❌ Unknown command: {cmd}")
|
||||
print(" Run 'x-reader' with no args for help")
|
||||
print(" 当前状态: ⬜ 未配置")
|
||||
key = input(" EXA_API_KEY (回车跳过): ").strip()
|
||||
if key:
|
||||
config.set("exa_api_key", key)
|
||||
print(" ✅ 全网搜索 + Reddit搜索 + Twitter搜索 已开启！")
|
||||
else:
|
||||
print(" ℹ️ 跳过。稍后可运行 agent-eyes setup 配置")
|
||||
print()
|
||||
|
||||
# Step 2: GitHub token
|
||||
print("【可选】GitHub Token — 提高 API 限额")
|
||||
print(" 无 token: 60 次/小时 | 有 token: 5000 次/小时")
|
||||
print(" 获取: https://github.com/settings/tokens (无需任何权限)")
|
||||
current = config.get("github_token")
|
||||
if current:
|
||||
print(f" 当前状态: ✅ 已配置")
|
||||
else:
|
||||
key = input(" GITHUB_TOKEN (回车跳过): ").strip()
|
||||
if key:
|
||||
config.set("github_token", key)
|
||||
print(" ✅ GitHub API 已提升至 5000 次/小时！")
|
||||
else:
|
||||
print(" ℹ️ 跳过。公开 API 也能用")
|
||||
print()
|
||||
|
||||
# Step 3: Reddit proxy
|
||||
print("【可选】Reddit 代理 — 完整阅读 Reddit 帖子+评论")
|
||||
print(" Reddit 封锁很多 IP，需要 ISP 代理才能直接访问")
|
||||
print(" 格式: http://用户名:密码@IP:端口")
|
||||
current = config.get("reddit_proxy")
|
||||
if current:
|
||||
print(f" 当前状态: ✅ 已配置")
|
||||
else:
|
||||
proxy = input(" REDDIT_PROXY (回车跳过): ").strip()
|
||||
if proxy:
|
||||
config.set("reddit_proxy", proxy)
|
||||
print(" ✅ Reddit 完整阅读已开启！")
|
||||
else:
|
||||
print(" ℹ️ 跳过。仍可通过搜索获取 Reddit 内容")
|
||||
print()
|
||||
|
||||
# Step 4: Groq (Whisper)
|
||||
print("【可选】Groq API — 视频无字幕时的语音转文字")
|
||||
print(" 免费额度，注册: https://console.groq.com")
|
||||
current = config.get("groq_api_key")
|
||||
if current:
|
||||
print(f" 当前状态: ✅ 已配置")
|
||||
else:
|
||||
key = input(" GROQ_API_KEY (回车跳过): ").strip()
|
||||
if key:
|
||||
config.set("groq_api_key", key)
|
||||
print(" ✅ 语音转文字已开启！")
|
||||
else:
|
||||
print(" ℹ️ 跳过")
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print("=" * 40)
|
||||
print(f"✅ 配置已保存到 {config.config_path}")
|
||||
print("运行 agent-eyes doctor 查看完整状态")
|
||||
print()
|
||||
|
||||
|
||||
async def _cmd_read(args):
|
||||
from agent_eyes.core import AgentEyes
|
||||
eyes = AgentEyes()
|
||||
try:
|
||||
result = await eyes.read(args.url)
|
||||
if args.as_json:
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"\n📖 {result.get('title', 'Untitled')}")
|
||||
print(f"🔗 {result.get('url', '')}")
|
||||
if result.get("author"):
|
||||
print(f"👤 {result['author']}")
|
||||
print(f"\n{result.get('content', '')}")
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
async def _cmd_search(args):
|
||||
from agent_eyes.core import AgentEyes
|
||||
eyes = AgentEyes()
|
||||
query = " ".join(args.query)
|
||||
num = args.num
|
||||
|
||||
try:
|
||||
if args.command == "search":
|
||||
results = await eyes.search(query, num_results=num)
|
||||
elif args.command == "search-reddit":
|
||||
results = await eyes.search_reddit(query, subreddit=getattr(args, "sub", None), limit=num)
|
||||
elif args.command == "search-github":
|
||||
results = await eyes.search_github(query, language=getattr(args, "lang", None), limit=num)
|
||||
elif args.command == "search-twitter":
|
||||
results = await eyes.search_twitter(query, limit=num)
|
||||
else:
|
||||
print(f"Unknown command: {args.command}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if not results:
|
||||
print("No results found.")
|
||||
return
|
||||
|
||||
for i, r in enumerate(results, 1):
|
||||
title = r.get("title") or r.get("name") or r.get("text", "")[:60]
|
||||
url = r.get("url", "")
|
||||
snippet = r.get("snippet") or r.get("description") or r.get("text", "")
|
||||
print(f"\n{i}. {title}")
|
||||
print(f" 🔗 {url}")
|
||||
if snippet:
|
||||
print(f" {snippet[:200]}")
|
||||
# Extra info for GitHub
|
||||
if "stars" in r:
|
||||
print(f" ⭐ {r['stars']} 🍴 {r.get('forks', 0)} 📝 {r.get('language', '')}")
|
||||
|
||||
except ValueError as e:
|
||||
print(f"⚠️ {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Configuration management for Agent Eyes.
|
||||
|
||||
Stores settings in ~/.agent-eyes/config.yaml.
|
||||
Auto-creates directory on first use.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class Config:
|
||||
"""Manages Agent Eyes configuration."""
|
||||
|
||||
CONFIG_DIR = Path.home() / ".agent-eyes"
|
||||
CONFIG_FILE = CONFIG_DIR / "config.yaml"
|
||||
|
||||
# Feature → required config keys
|
||||
FEATURE_REQUIREMENTS = {
|
||||
"exa_search": ["exa_api_key"],
|
||||
"reddit_proxy": ["reddit_proxy"],
|
||||
"twitter_birdx": ["twitter_auth_token", "twitter_ct0"],
|
||||
"groq_whisper": ["groq_api_key"],
|
||||
"github_token": ["github_token"],
|
||||
}
|
||||
|
||||
def __init__(self, config_path: Optional[Path] = None):
|
||||
self.config_path = Path(config_path) if config_path else self.CONFIG_FILE
|
||||
self.config_dir = self.config_path.parent
|
||||
self.data: dict = {}
|
||||
self._ensure_dir()
|
||||
self.load()
|
||||
|
||||
def _ensure_dir(self):
|
||||
"""Create config directory if it doesn't exist."""
|
||||
self.config_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def load(self):
|
||||
"""Load config from YAML file."""
|
||||
if self.config_path.exists():
|
||||
with open(self.config_path, "r") as f:
|
||||
self.data = yaml.safe_load(f) or {}
|
||||
else:
|
||||
self.data = {}
|
||||
|
||||
def save(self):
|
||||
"""Save config to YAML file."""
|
||||
self._ensure_dir()
|
||||
with open(self.config_path, "w") as f:
|
||||
yaml.dump(self.data, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""Get a config value. Also checks environment variables (uppercase)."""
|
||||
# Config file first
|
||||
if key in self.data:
|
||||
return self.data[key]
|
||||
# Then env var (uppercase)
|
||||
env_val = os.environ.get(key.upper())
|
||||
if env_val:
|
||||
return env_val
|
||||
return default
|
||||
|
||||
def set(self, key: str, value: Any):
|
||||
"""Set a config value and save."""
|
||||
self.data[key] = value
|
||||
self.save()
|
||||
|
||||
def delete(self, key: str):
|
||||
"""Delete a config key and save."""
|
||||
self.data.pop(key, None)
|
||||
self.save()
|
||||
|
||||
def is_configured(self, feature: str) -> bool:
|
||||
"""Check if a feature has all required config."""
|
||||
required = self.FEATURE_REQUIREMENTS.get(feature, [])
|
||||
return all(self.get(k) for k in required)
|
||||
|
||||
def get_configured_features(self) -> dict:
|
||||
"""Return status of all optional features."""
|
||||
return {
|
||||
feature: self.is_configured(feature)
|
||||
for feature in self.FEATURE_REQUIREMENTS
|
||||
}
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Return config as dict (masks sensitive values)."""
|
||||
masked = {}
|
||||
for k, v in self.data.items():
|
||||
if any(s in k.lower() for s in ("key", "token", "password", "proxy")):
|
||||
masked[k] = f"{str(v)[:8]}..." if v else None
|
||||
else:
|
||||
masked[k] = v
|
||||
return masked
|
||||
@@ -0,0 +1,133 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
AgentEyes — the unified entry point.
|
||||
|
||||
Usage:
|
||||
from agent_eyes import AgentEyes
|
||||
|
||||
eyes = AgentEyes()
|
||||
content = await eyes.read("https://github.com/openai/gpt-4")
|
||||
results = await eyes.search("AI agent framework")
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent_eyes.config import Config
|
||||
from agent_eyes.reader import UniversalReader
|
||||
|
||||
|
||||
class AgentEyes:
|
||||
"""Give your AI Agent eyes to see the entire internet."""
|
||||
|
||||
def __init__(self, config: Optional[Config] = None):
|
||||
self.config = config or Config()
|
||||
self.reader = UniversalReader()
|
||||
|
||||
# ── Reading ─────────────────────────────────────────
|
||||
|
||||
async def read(self, url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Read content from any URL. Auto-detects platform.
|
||||
|
||||
Supported: Web, GitHub, Reddit, Twitter, YouTube,
|
||||
Bilibili, WeChat, XiaoHongShu, RSS, Telegram, etc.
|
||||
|
||||
Returns:
|
||||
Dict with title, content, url, author, etc.
|
||||
"""
|
||||
content = await self.reader.read(url)
|
||||
return content.to_dict()
|
||||
|
||||
async def read_batch(self, urls: List[str]) -> List[Dict[str, Any]]:
|
||||
"""Read multiple URLs concurrently."""
|
||||
contents = await self.reader.read_batch(urls)
|
||||
return [c.to_dict() for c in contents]
|
||||
|
||||
def detect_platform(self, url: str) -> str:
|
||||
"""Detect what platform a URL belongs to."""
|
||||
return self.reader._detect_platform(url)
|
||||
|
||||
# ── Searching ───────────────────────────────────────
|
||||
|
||||
async def search(self, query: str, num_results: int = 5) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Semantic web search via Exa. Requires Exa API key.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
num_results: Number of results (max 10)
|
||||
"""
|
||||
from agent_eyes.search.exa import search_web
|
||||
return await search_web(query, num_results=num_results, config=self.config)
|
||||
|
||||
async def search_reddit(
|
||||
self,
|
||||
query: str,
|
||||
subreddit: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Reddit via Exa (bypasses IP blocks).
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
subreddit: Optional subreddit filter
|
||||
limit: Number of results
|
||||
"""
|
||||
from agent_eyes.search.reddit import search_reddit
|
||||
return await search_reddit(query, subreddit=subreddit, num_results=limit, config=self.config)
|
||||
|
||||
async def search_github(
|
||||
self,
|
||||
query: str,
|
||||
language: Optional[str] = None,
|
||||
limit: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search GitHub repositories.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
language: Filter by language
|
||||
limit: Number of results
|
||||
"""
|
||||
from agent_eyes.search.github import search_github
|
||||
return await search_github(query, language=language, limit=limit, config=self.config)
|
||||
|
||||
async def search_twitter(
|
||||
self,
|
||||
query: str,
|
||||
limit: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Twitter. Uses birdx if available, else Exa.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
limit: Number of results
|
||||
"""
|
||||
from agent_eyes.search.twitter import search_twitter
|
||||
return await search_twitter(query, limit=limit, config=self.config)
|
||||
|
||||
# ── Health ──────────────────────────────────────────
|
||||
|
||||
def doctor(self) -> Dict[str, dict]:
|
||||
"""Check all platform/feature availability."""
|
||||
from agent_eyes.doctor import check_all
|
||||
return check_all(self.config)
|
||||
|
||||
def doctor_report(self) -> str:
|
||||
"""Get formatted health report."""
|
||||
from agent_eyes.doctor import check_all, format_report
|
||||
return format_report(check_all(self.config))
|
||||
|
||||
# ── Sync wrappers ───────────────────────────────────
|
||||
|
||||
def read_sync(self, url: str) -> Dict[str, Any]:
|
||||
"""Synchronous version of read()."""
|
||||
return asyncio.run(self.read(url))
|
||||
|
||||
def search_sync(self, query: str, num_results: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Synchronous version of search()."""
|
||||
return asyncio.run(self.search(query, num_results))
|
||||
@@ -0,0 +1,226 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Environment health checker for Agent Eyes.
|
||||
|
||||
Checks all platforms, tools, and API keys.
|
||||
Outputs a rich-formatted status report.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Dict
|
||||
|
||||
from agent_eyes.config import Config
|
||||
|
||||
|
||||
STATUS_OK = "ok"
|
||||
STATUS_WARN = "warn"
|
||||
STATUS_OFF = "off"
|
||||
STATUS_ERROR = "error"
|
||||
|
||||
|
||||
def _check_command(cmd: str) -> bool:
|
||||
"""Check if a command exists on PATH."""
|
||||
return shutil.which(cmd) is not None
|
||||
|
||||
|
||||
def _check_python_import(module: str) -> bool:
|
||||
"""Check if a Python module is importable."""
|
||||
try:
|
||||
__import__(module)
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def check_all(config: Config) -> Dict[str, dict]:
|
||||
"""Check all features and return status dict."""
|
||||
results = {}
|
||||
|
||||
# === Zero-config (always available) ===
|
||||
results["web"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Web Pages",
|
||||
"message": "Jina Reader (built-in)",
|
||||
"tier": 0,
|
||||
}
|
||||
results["github_read"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "GitHub",
|
||||
"message": "Public API (built-in)",
|
||||
"tier": 0,
|
||||
}
|
||||
results["bilibili"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Bilibili",
|
||||
"message": "Public API (built-in)",
|
||||
"tier": 0,
|
||||
}
|
||||
results["rss"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "RSS",
|
||||
"message": "feedparser (built-in)",
|
||||
"tier": 0,
|
||||
}
|
||||
results["tweet_read"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Tweet (single)",
|
||||
"message": "Jina Reader (built-in)",
|
||||
"tier": 0,
|
||||
}
|
||||
|
||||
# YouTube — needs yt-dlp
|
||||
if _check_command("yt-dlp"):
|
||||
results["youtube"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "YouTube",
|
||||
"message": "yt-dlp found",
|
||||
"tier": 0,
|
||||
}
|
||||
else:
|
||||
results["youtube"] = {
|
||||
"status": STATUS_WARN,
|
||||
"name": "YouTube",
|
||||
"message": "Install yt-dlp: pip install yt-dlp",
|
||||
"tier": 0,
|
||||
}
|
||||
|
||||
# === Needs API key (Tier 1) ===
|
||||
exa_key = config.get("exa_api_key")
|
||||
if exa_key:
|
||||
results["search_web"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Web Search",
|
||||
"message": "Exa API configured",
|
||||
"tier": 1,
|
||||
}
|
||||
results["search_reddit"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Reddit Search",
|
||||
"message": "Via Exa (site:reddit.com)",
|
||||
"tier": 1,
|
||||
}
|
||||
results["search_twitter"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Twitter Search",
|
||||
"message": "Via Exa (site:x.com)",
|
||||
"tier": 1,
|
||||
}
|
||||
else:
|
||||
for key in ("search_web", "search_reddit", "search_twitter"):
|
||||
results[key] = {
|
||||
"status": STATUS_OFF,
|
||||
"name": {"search_web": "Web Search", "search_reddit": "Reddit Search",
|
||||
"search_twitter": "Twitter Search"}[key],
|
||||
"message": "Need Exa API key (free). Run: agent-eyes setup",
|
||||
"tier": 1,
|
||||
}
|
||||
|
||||
# GitHub search (always works, token optional for higher limits)
|
||||
github_token = config.get("github_token")
|
||||
results["search_github"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "GitHub Search",
|
||||
"message": f"API ({'authenticated' if github_token else 'public, 60 req/hr'})",
|
||||
"tier": 0,
|
||||
}
|
||||
|
||||
# === Optional tools (Tier 2) ===
|
||||
|
||||
# Twitter advanced (birdx)
|
||||
if _check_command("birdx"):
|
||||
results["twitter_advanced"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Twitter Advanced",
|
||||
"message": "birdx found",
|
||||
"tier": 2,
|
||||
}
|
||||
else:
|
||||
results["twitter_advanced"] = {
|
||||
"status": STATUS_OFF,
|
||||
"name": "Twitter Advanced",
|
||||
"message": "Install birdx for timeline/deep search",
|
||||
"tier": 2,
|
||||
}
|
||||
|
||||
# Reddit full reader
|
||||
reddit_proxy = config.get("reddit_proxy")
|
||||
if reddit_proxy:
|
||||
results["reddit_full"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "Reddit Reader",
|
||||
"message": "Proxy configured",
|
||||
"tier": 2,
|
||||
}
|
||||
else:
|
||||
results["reddit_full"] = {
|
||||
"status": STATUS_OFF,
|
||||
"name": "Reddit Reader",
|
||||
"message": "Need proxy for full post reading",
|
||||
"tier": 2,
|
||||
}
|
||||
|
||||
# WeChat / XHS (playwright)
|
||||
if _check_python_import("playwright"):
|
||||
results["wechat"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "WeChat",
|
||||
"message": "Playwright available",
|
||||
"tier": 2,
|
||||
}
|
||||
results["xhs"] = {
|
||||
"status": STATUS_OK,
|
||||
"name": "XiaoHongShu",
|
||||
"message": "Playwright available",
|
||||
"tier": 2,
|
||||
}
|
||||
else:
|
||||
for key in ("wechat", "xhs"):
|
||||
results[key] = {
|
||||
"status": STATUS_OFF,
|
||||
"name": "WeChat" if key == "wechat" else "XiaoHongShu",
|
||||
"message": "pip install agent-eyes[browser]",
|
||||
"tier": 2,
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def format_report(results: Dict[str, dict]) -> str:
|
||||
"""Format results as a readable text report."""
|
||||
lines = []
|
||||
lines.append("👁️ Agent Eyes Status")
|
||||
lines.append("=" * 40)
|
||||
|
||||
# Count stats
|
||||
ok_count = sum(1 for r in results.values() if r["status"] == STATUS_OK)
|
||||
total = len(results)
|
||||
|
||||
# Group by tier
|
||||
lines.append("")
|
||||
lines.append("✅ Ready (no setup needed):")
|
||||
for key, r in results.items():
|
||||
if r["tier"] == 0 and r["status"] == STATUS_OK:
|
||||
lines.append(f" ✅ {r['name']}")
|
||||
elif r["tier"] == 0 and r["status"] == STATUS_WARN:
|
||||
lines.append(f" ⚠️ {r['name']} — {r['message']}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("🔍 Search (need free Exa API key):")
|
||||
for key, r in results.items():
|
||||
if r["tier"] == 1:
|
||||
icon = "✅" if r["status"] == STATUS_OK else "⬜"
|
||||
lines.append(f" {icon} {r['name']}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("🔧 Optional (advanced setup):")
|
||||
for key, r in results.items():
|
||||
if r["tier"] == 2:
|
||||
icon = "✅" if r["status"] == STATUS_OK else "⬜"
|
||||
lines.append(f" {icon} {r['name']} — {r['message']}")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"Status: {ok_count}/{total} platforms active")
|
||||
if ok_count < total:
|
||||
lines.append("Run `agent-eyes setup` to unlock more!")
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -1,94 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Search fetcher — semantic web search via Exa API.
|
||||
|
||||
Requires EXA_API_KEY env var. Get a free key at https://exa.ai
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
|
||||
EXA_API_URL = "https://api.exa.ai/search"
|
||||
|
||||
|
||||
async def search_web(
|
||||
query: str,
|
||||
num_results: int = 5,
|
||||
search_type: str = "auto",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search the web using Exa semantic search.
|
||||
|
||||
Args:
|
||||
query: Search query (supports site: prefix, e.g. "site:reddit.com AI agent")
|
||||
num_results: Number of results to return (default 5, max 10)
|
||||
search_type: "auto" (default) or "neural" or "keyword"
|
||||
|
||||
Returns:
|
||||
List of search results with title, url, snippet
|
||||
"""
|
||||
api_key = os.environ.get("EXA_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
"EXA_API_KEY not set. Get a free key at https://exa.ai\n"
|
||||
"Then: export EXA_API_KEY=your_key_here"
|
||||
)
|
||||
|
||||
logger.info(f"Exa search: {query} (n={num_results})")
|
||||
|
||||
resp = requests.post(
|
||||
EXA_API_URL,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": api_key,
|
||||
},
|
||||
json={
|
||||
"query": query,
|
||||
"numResults": min(num_results, 10),
|
||||
"type": search_type,
|
||||
"contents": {
|
||||
"text": {"maxCharacters": 500},
|
||||
},
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for item in data.get("results", []):
|
||||
results.append({
|
||||
"title": item.get("title", ""),
|
||||
"url": item.get("url", ""),
|
||||
"snippet": item.get("text", ""),
|
||||
"published_date": item.get("publishedDate", ""),
|
||||
"score": item.get("score", 0),
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def search_reddit_via_exa(
|
||||
query: str,
|
||||
subreddit: Optional[str] = None,
|
||||
num_results: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Reddit content via Exa (bypasses Reddit IP blocks).
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
subreddit: Optional subreddit to limit search (e.g. "LocalLLaMA")
|
||||
num_results: Number of results
|
||||
|
||||
Returns:
|
||||
List of Reddit posts found
|
||||
"""
|
||||
if subreddit:
|
||||
full_query = f"site:reddit.com/r/{subreddit} {query}"
|
||||
else:
|
||||
full_query = f"site:reddit.com {query}"
|
||||
|
||||
return await search_web(full_query, num_results=num_results)
|
||||
@@ -0,0 +1,54 @@
|
||||
# Exa Search 配置指南
|
||||
|
||||
## 功能说明
|
||||
Exa 是一个语义搜索引擎，免费提供 1000 次/月搜索。配置后解锁：
|
||||
- 全网搜索
|
||||
- Reddit 搜索（通过 site:reddit.com）
|
||||
- Twitter 搜索（通过 site:x.com）
|
||||
|
||||
## Agent 可自动完成的步骤
|
||||
|
||||
1. 检查是否已配置：
|
||||
```bash
|
||||
agent-eyes doctor | grep "Web Search"
|
||||
```
|
||||
|
||||
2. 如果用户提供了 key，写入配置：
|
||||
```bash
|
||||
# 方法1: 通过 setup 命令
|
||||
agent-eyes setup
|
||||
|
||||
# 方法2: 直接写入配置文件
|
||||
python3 -c "
|
||||
from agent_eyes.config import Config
|
||||
c = Config()
|
||||
c.set('exa_api_key', '用户提供的KEY')
|
||||
print('✅ Exa API key saved')
|
||||
"
|
||||
```
|
||||
|
||||
3. 测试是否有效：
|
||||
```bash
|
||||
agent-eyes search "test query" -n 1
|
||||
```
|
||||
|
||||
## 需要用户手动做的步骤
|
||||
|
||||
请告诉用户：
|
||||
|
||||
> 我需要一个 Exa API Key 来开启搜索功能。完全免费，每月 1000 次搜索。
|
||||
>
|
||||
> 步骤：
|
||||
> 1. 打开 https://exa.ai
|
||||
> 2. 点击 "Get API Key" 或 "Sign Up"
|
||||
> 3. 用 Google 账号或邮箱注册
|
||||
> 4. 注册后在 Dashboard 页面复制你的 API Key
|
||||
> 5. 把 Key 发给我
|
||||
>
|
||||
> 大概 30 秒就能搞定。
|
||||
|
||||
## Agent 收到 key 后的操作
|
||||
|
||||
1. 写入配置：`config.set("exa_api_key", key)`
|
||||
2. 测试：运行一次搜索确认可用
|
||||
3. 反馈："✅ 全网搜索已开启！现在我可以帮你搜索全网、Reddit 和 Twitter 了。"
|
||||
@@ -0,0 +1,47 @@
|
||||
# Groq Whisper 配置指南
|
||||
|
||||
## 功能说明
|
||||
当 YouTube/Bilibili 视频没有字幕时，用 Groq 的 Whisper API 进行语音转文字。Groq 提供免费额度。
|
||||
|
||||
## Agent 可自动完成的步骤
|
||||
|
||||
1. 检查是否已配置：
|
||||
```bash
|
||||
agent-eyes doctor | grep -i "groq\|whisper"
|
||||
```
|
||||
|
||||
2. 如果用户提供了 key，写入配置：
|
||||
```python
|
||||
from agent_eyes.config import Config
|
||||
c = Config()
|
||||
c.set("groq_api_key", "用户提供的KEY")
|
||||
```
|
||||
|
||||
3. 测试（可选）：
|
||||
```bash
|
||||
curl -s https://api.groq.com/openai/v1/models \
|
||||
-H "Authorization: Bearer 用户提供的KEY" \
|
||||
-o /dev/null -w "%{http_code}"
|
||||
```
|
||||
返回 200 = 可用
|
||||
|
||||
## 需要用户手动做的步骤
|
||||
|
||||
请告诉用户：
|
||||
|
||||
> 视频语音转文字需要一个 Groq API Key（免费）。
|
||||
>
|
||||
> 步骤：
|
||||
> 1. 打开 https://console.groq.com
|
||||
> 2. 用 Google 账号或邮箱注册
|
||||
> 3. 点击左侧 "API Keys"
|
||||
> 4. 点击 "Create API Key"
|
||||
> 5. 复制生成的 Key，发给我
|
||||
>
|
||||
> Groq 提供免费额度，日常使用完全够用。
|
||||
|
||||
## Agent 收到 key 后的操作
|
||||
|
||||
1. 写入配置：`config.set("groq_api_key", key)`
|
||||
2. 测试 API 可用性
|
||||
3. 反馈："✅ 语音转文字已开启！现在遇到没有字幕的视频，我也能帮你提取内容了。"
|
||||
@@ -0,0 +1,60 @@
|
||||
# Reddit 代理配置指南
|
||||
|
||||
## 功能说明
|
||||
Reddit 封锁了大量服务器 IP，直接访问会返回 403。需要 ISP 代理（Residential/ISP Proxy）才能读取完整的 Reddit 帖子和评论。
|
||||
|
||||
**注意**：即使没有代理，也可以通过 Exa 搜索找到 Reddit 内容（只需要配 Exa API key）。代理只是用于读取完整帖子+评论。
|
||||
|
||||
## Agent 可自动完成的步骤
|
||||
|
||||
1. 检查当前状态：
|
||||
```bash
|
||||
agent-eyes doctor | grep "Reddit"
|
||||
```
|
||||
|
||||
2. 如果用户提供了代理，测试连通性：
|
||||
```bash
|
||||
curl -s --proxy "用户提供的代理" \
|
||||
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" \
|
||||
"https://www.reddit.com/r/test.json?limit=1" \
|
||||
-o /dev/null -w "%{http_code}"
|
||||
```
|
||||
返回 200 = 可用，403 = 代理被封，其他 = 配置错误
|
||||
|
||||
3. 写入配置：
|
||||
```python
|
||||
from agent_eyes.config import Config
|
||||
c = Config()
|
||||
c.set("reddit_proxy", "http://用户名:密码@IP:端口")
|
||||
```
|
||||
|
||||
## 需要用户手动做的步骤
|
||||
|
||||
请告诉用户：
|
||||
|
||||
> 要完整阅读 Reddit 帖子和评论，需要一个 ISP 代理（约 $3-10/月）。
|
||||
>
|
||||
> 推荐代理服务商（选一个就行）：
|
||||
> 1. **Smartproxy** (https://smartproxy.com) — ISP 代理，按流量计费
|
||||
> 2. **Bright Data** (https://brightdata.com) — 大厂，ISP 代理
|
||||
> 3. **IPRoyal** (https://iproyal.com) — 便宜，适合入门
|
||||
> 4. **ProxyEmpire** (https://proxyempire.io) — 有 Reddit 专用代理
|
||||
>
|
||||
> 购买时选择：
|
||||
> - 类型：**ISP Proxy**（不要选 Datacenter，会被封）
|
||||
> - 地区：**美国**
|
||||
> - 协议：**HTTP**
|
||||
>
|
||||
> 购买后你会得到一个代理地址，格式类似：
|
||||
> `http://用户名:密码@IP地址:端口号`
|
||||
>
|
||||
> 把这个地址发给我就行。
|
||||
>
|
||||
> ⚠️ 如果不想花钱，可以跳过。我仍然可以通过搜索引擎找到 Reddit 上的内容，只是不能读完整的帖子和评论。
|
||||
|
||||
## Agent 收到代理后的操作
|
||||
|
||||
1. 测试代理：用 curl 测试 reddit.com 是否返回 200
|
||||
2. 如果成功，写入配置：`config.set("reddit_proxy", proxy_url)`
|
||||
3. 反馈："✅ Reddit 完整阅读已开启！现在我可以读取 Reddit 帖子和所有评论了。"
|
||||
4. 如果失败，告诉用户："❌ 这个代理无法访问 Reddit，请检查代理是否有效，或换一个试试。"
|
||||
@@ -0,0 +1,69 @@
|
||||
# Twitter 高级功能配置指南（birdx）
|
||||
|
||||
## 功能说明
|
||||
基础 Twitter 功能（搜索+读单条推文）无需配置，开箱即用。
|
||||
|
||||
高级功能需要 birdx：
|
||||
- 查看用户时间线
|
||||
- 深度搜索（更精确、更多结果）
|
||||
- 读取完整线程（thread）
|
||||
- 查看关注列表推文
|
||||
|
||||
birdx 是免费开源工具，但需要你的 Twitter 账号 cookie。
|
||||
|
||||
## Agent 可自动完成的步骤
|
||||
|
||||
1. 检查 birdx 是否安装：
|
||||
```bash
|
||||
which birdx && echo "installed" || echo "not installed"
|
||||
```
|
||||
|
||||
2. 安装 birdx：
|
||||
```bash
|
||||
pip install birdx
|
||||
```
|
||||
|
||||
3. 检查是否已配置 cookie：
|
||||
```bash
|
||||
birdx whoami 2>&1
|
||||
```
|
||||
|
||||
4. 如果用户提供了 cookie，配置 birdx：
|
||||
```bash
|
||||
# birdx 的 cookie 配置文件位置
|
||||
# 通常在 ~/.birdx/cookies.json 或通过环境变量
|
||||
export TWITTER_AUTH_TOKEN="用户提供的auth_token"
|
||||
export TWITTER_CT0="用户提供的ct0"
|
||||
```
|
||||
|
||||
5. 测试：
|
||||
```bash
|
||||
birdx search "test" -n 1
|
||||
```
|
||||
|
||||
## 需要用户手动做的步骤
|
||||
|
||||
请告诉用户：
|
||||
|
||||
> Twitter 高级功能需要你的 Twitter 账号 cookie（完全免费）。
|
||||
>
|
||||
> 步骤：
|
||||
> 1. 用 Chrome 打开 https://x.com 并确保你已登录
|
||||
> 2. 按 **F12** 打开开发者工具（Mac 按 Cmd+Option+I）
|
||||
> 3. 点击顶部的 **Application**（应用）标签
|
||||
> 4. 左侧找到 **Cookies** → **https://x.com**
|
||||
> 5. 在列表中找到以下两个值，双击复制：
|
||||
> - **auth_token** — 一串字母数字
|
||||
> - **ct0** — 一串字母数字
|
||||
> 6. 把这两个值发给我
|
||||
>
|
||||
> ⚠️ 这些 cookie 让我能以你的身份读取推文（只读）。我不会发推、点赞或做任何操作。
|
||||
> ⚠️ cookie 大约 1-3 个月会过期，届时需要重新导出。
|
||||
|
||||
## Agent 收到 cookie 后的操作
|
||||
|
||||
1. 安装 birdx（如果没装）：`pip install birdx`
|
||||
2. 配置 cookie：写入 birdx 配置
|
||||
3. 测试：`birdx whoami` 确认身份
|
||||
4. 反馈："✅ Twitter 高级功能已开启！你的账号是 @xxx。现在我可以查看时间线、读取线程了。"
|
||||
5. 如果失败："❌ Cookie 无效或已过期，请重新导出。"
|
||||
@@ -0,0 +1,40 @@
|
||||
# 微信公众号配置指南
|
||||
|
||||
## 功能说明
|
||||
读取微信公众号文章。需要 Playwright 来处理微信的反爬机制。
|
||||
|
||||
## Agent 可自动完成的步骤
|
||||
|
||||
1. 检查 Playwright 是否安装：
|
||||
```bash
|
||||
python3 -c "import playwright; print('installed')" 2>&1
|
||||
```
|
||||
|
||||
2. 安装 Playwright + 浏览器：
|
||||
```bash
|
||||
pip install playwright
|
||||
playwright install chromium
|
||||
```
|
||||
|
||||
3. 安装完成后测试：
|
||||
```bash
|
||||
agent-eyes read "https://mp.weixin.qq.com/s/一个测试链接"
|
||||
```
|
||||
|
||||
## 需要用户手动做的步骤
|
||||
|
||||
请告诉用户：
|
||||
|
||||
> 微信公众号的配置很简单，只需要安装一个浏览器组件（约 150MB）。
|
||||
>
|
||||
> 我来帮你安装，你不需要做任何事情。安装过程大约 1-2 分钟。
|
||||
>
|
||||
> 安装好之后就可以直接读取微信公众号文章了，不需要登录。
|
||||
|
||||
## Agent 操作流程
|
||||
|
||||
1. 安装 Playwright：`pip install playwright`
|
||||
2. 安装 Chromium：`playwright install chromium`
|
||||
3. 测试：读一篇微信文章
|
||||
4. 反馈："✅ 微信公众号已配置！发给我任何公众号文章链接，我都能读取。"
|
||||
5. 如果安装失败（空间不足等）："❌ 浏览器组件安装失败。可能是磁盘空间不足（需要约 150MB）。"
|
||||
@@ -0,0 +1,42 @@
|
||||
# 小红书配置指南
|
||||
|
||||
## 功能说明
|
||||
读取小红书笔记内容。需要 Playwright（浏览器自动化）和一次性登录。
|
||||
|
||||
## Agent 可自动完成的步骤
|
||||
|
||||
1. 检查 Playwright 是否安装：
|
||||
```bash
|
||||
python3 -c "import playwright; print('installed')" 2>&1
|
||||
```
|
||||
|
||||
2. 安装 Playwright + 浏览器：
|
||||
```bash
|
||||
pip install playwright
|
||||
playwright install chromium
|
||||
```
|
||||
|
||||
3. 检查是否已有登录态：
|
||||
```bash
|
||||
# 检查 cookie 文件是否存在
|
||||
ls ~/.agent-eyes/xhs_cookies.json 2>/dev/null
|
||||
```
|
||||
|
||||
## 需要用户手动做的步骤
|
||||
|
||||
请告诉用户：
|
||||
|
||||
> 小红书需要登录一次（之后会记住你的登录状态）。
|
||||
>
|
||||
> 我现在会打开一个浏览器窗口，显示小红书登录页面。你需要：
|
||||
> 1. 用手机小红书 App 扫描屏幕上的二维码
|
||||
> 2. 在手机上确认登录
|
||||
> 3. 看到首页后告诉我"登录好了"
|
||||
>
|
||||
> 之后就不需要再登录了（除非 cookie 过期，大约 1-3 个月）。
|
||||
|
||||
## Agent 收到确认后的操作
|
||||
|
||||
1. 保存浏览器 cookie 到 `~/.agent-eyes/xhs_cookies.json`
|
||||
2. 测试：读取一条小红书笔记
|
||||
3. 反馈："✅ 小红书已配置！现在我可以读取小红书笔记了。"
|
||||
@@ -0,0 +1 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
@@ -0,0 +1,198 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Agent Eyes MCP Server — expose all capabilities as MCP tools.
|
||||
|
||||
Run: python -m agent_eyes.integrations.mcp_server
|
||||
Or: agent-eyes serve (after pip install)
|
||||
|
||||
10 tools for any MCP-compatible AI Agent.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
|
||||
from agent_eyes.config import Config
|
||||
from agent_eyes.core import AgentEyes
|
||||
|
||||
try:
|
||||
from mcp.server import Server
|
||||
from mcp.server.stdio import stdio_server
|
||||
from mcp.types import Tool, TextContent
|
||||
HAS_MCP = True
|
||||
except ImportError:
|
||||
HAS_MCP = False
|
||||
|
||||
|
||||
def create_server():
|
||||
"""Create and configure the MCP server."""
|
||||
if not HAS_MCP:
|
||||
print("MCP not installed. Install: pip install agent-eyes[mcp]", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
server = Server("agent-eyes")
|
||||
config = Config()
|
||||
eyes = AgentEyes(config)
|
||||
|
||||
@server.list_tools()
|
||||
async def list_tools():
|
||||
return [
|
||||
Tool(
|
||||
name="read_url",
|
||||
description="Read content from any URL. Supports: web pages, GitHub, Reddit, Twitter, YouTube, Bilibili, WeChat, XiaoHongShu, RSS, Telegram.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {"type": "string", "description": "URL to read"},
|
||||
},
|
||||
"required": ["url"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="read_batch",
|
||||
description="Read multiple URLs concurrently.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"urls": {"type": "array", "items": {"type": "string"}, "description": "List of URLs"},
|
||||
},
|
||||
"required": ["urls"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="detect_platform",
|
||||
description="Detect what platform a URL belongs to (github, reddit, twitter, youtube, etc).",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {"type": "string", "description": "URL to detect"},
|
||||
},
|
||||
"required": ["url"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="search",
|
||||
description="Semantic web search using Exa. Find any information on the internet.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query"},
|
||||
"num_results": {"type": "integer", "description": "Number of results (1-10)", "default": 5},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="search_reddit",
|
||||
description="Search Reddit posts and discussions. Works even when Reddit blocks your IP.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query"},
|
||||
"subreddit": {"type": "string", "description": "Optional subreddit filter (e.g. 'LocalLLaMA')"},
|
||||
"limit": {"type": "integer", "description": "Number of results", "default": 10},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="search_github",
|
||||
description="Search GitHub repositories by topic, keyword, or technology.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query"},
|
||||
"language": {"type": "string", "description": "Filter by language (e.g. 'python')"},
|
||||
"limit": {"type": "integer", "description": "Number of results", "default": 5},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="search_twitter",
|
||||
description="Search Twitter/X posts. Uses birdx if available, otherwise Exa.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query"},
|
||||
"limit": {"type": "integer", "description": "Number of results", "default": 10},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="get_status",
|
||||
description="Get Agent Eyes status: which platforms are active, which need configuration.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
@server.call_tool()
|
||||
async def call_tool(name: str, arguments: dict):
|
||||
try:
|
||||
if name == "read_url":
|
||||
result = await eyes.read(arguments["url"])
|
||||
return [TextContent(type="text", text=json.dumps(result, ensure_ascii=False, indent=2))]
|
||||
|
||||
elif name == "read_batch":
|
||||
results = await eyes.read_batch(arguments["urls"])
|
||||
return [TextContent(type="text", text=json.dumps(results, ensure_ascii=False, indent=2))]
|
||||
|
||||
elif name == "detect_platform":
|
||||
platform = eyes.detect_platform(arguments["url"])
|
||||
return [TextContent(type="text", text=f"Platform: {platform}")]
|
||||
|
||||
elif name == "search":
|
||||
results = await eyes.search(
|
||||
arguments["query"],
|
||||
num_results=arguments.get("num_results", 5),
|
||||
)
|
||||
return [TextContent(type="text", text=json.dumps(results, ensure_ascii=False, indent=2))]
|
||||
|
||||
elif name == "search_reddit":
|
||||
results = await eyes.search_reddit(
|
||||
arguments["query"],
|
||||
subreddit=arguments.get("subreddit"),
|
||||
limit=arguments.get("limit", 10),
|
||||
)
|
||||
return [TextContent(type="text", text=json.dumps(results, ensure_ascii=False, indent=2))]
|
||||
|
||||
elif name == "search_github":
|
||||
results = await eyes.search_github(
|
||||
arguments["query"],
|
||||
language=arguments.get("language"),
|
||||
limit=arguments.get("limit", 5),
|
||||
)
|
||||
return [TextContent(type="text", text=json.dumps(results, ensure_ascii=False, indent=2))]
|
||||
|
||||
elif name == "search_twitter":
|
||||
results = await eyes.search_twitter(
|
||||
arguments["query"],
|
||||
limit=arguments.get("limit", 10),
|
||||
)
|
||||
return [TextContent(type="text", text=json.dumps(results, ensure_ascii=False, indent=2))]
|
||||
|
||||
elif name == "get_status":
|
||||
report = eyes.doctor_report()
|
||||
return [TextContent(type="text", text=report)]
|
||||
|
||||
else:
|
||||
return [TextContent(type="text", text=f"Unknown tool: {name}")]
|
||||
|
||||
except Exception as e:
|
||||
return [TextContent(type="text", text=f"Error: {str(e)}")]
|
||||
|
||||
return server
|
||||
|
||||
|
||||
async def main():
|
||||
server = create_server()
|
||||
async with stdio_server() as (read_stream, write_stream):
|
||||
await server.run(read_stream, write_stream, server.create_initialization_options())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
name: agent-eyes
|
||||
description: "Give your AI Agent eyes to see the entire internet. Search + read 10+ platforms with one install."
|
||||
install: pip install git+https://github.com/Panniantong/agent-eyes.git
|
||||
---
|
||||
|
||||
# Agent Eyes — 给 Agent 装上眼睛 👁️
|
||||
|
||||
安装后你就能搜索和阅读全网内容。
|
||||
|
||||
## 安装
|
||||
|
||||
```bash
|
||||
pip install git+https://github.com/Panniantong/agent-eyes.git
|
||||
```
|
||||
|
||||
## 装完直接能用的（零配置）
|
||||
|
||||
- `agent-eyes read <URL>` — 读取任意网页、GitHub、B站、YouTube、RSS
|
||||
- `agent-eyes search-github "query"` — 搜索 GitHub 仓库
|
||||
- `agent-eyes doctor` — 查看所有平台状态
|
||||
|
||||
## 解锁更多功能
|
||||
|
||||
运行 `agent-eyes setup` 进行交互式配置，或者读取以下指南：
|
||||
|
||||
| 功能 | 指南文件 |
|
||||
|------|---------|
|
||||
| 全网搜索 + Reddit搜索 + Twitter搜索 | `agent_eyes/guides/setup-exa.md` |
|
||||
| Reddit 完整阅读 | `agent_eyes/guides/setup-reddit.md` |
|
||||
| Twitter 高级（时间线/线程） | `agent_eyes/guides/setup-twitter.md` |
|
||||
| 小红书 | `agent_eyes/guides/setup-xiaohongshu.md` |
|
||||
| 微信公众号 | `agent_eyes/guides/setup-wechat.md` |
|
||||
| 视频语音转文字 | `agent_eyes/guides/setup-groq.md` |
|
||||
|
||||
每个指南里有详细的步骤说明，你（Agent）可以照着做，只有需要人类操作的部分（登录、复制 key）才需要问用户。
|
||||
|
||||
## MCP Server
|
||||
|
||||
如果你的 Agent 平台支持 MCP：
|
||||
|
||||
```bash
|
||||
pip install agent-eyes[mcp]
|
||||
python -m agent_eyes.integrations.mcp_server
|
||||
```
|
||||
|
||||
提供 8 个工具：read_url, read_batch, detect_platform, search, search_reddit, search_github, search_twitter, get_status
|
||||
|
||||
## Python API
|
||||
|
||||
```python
|
||||
from agent_eyes import AgentEyes
|
||||
import asyncio
|
||||
|
||||
eyes = AgentEyes()
|
||||
|
||||
# 读取
|
||||
result = asyncio.run(eyes.read("https://github.com/openai/gpt-4"))
|
||||
|
||||
# 搜索
|
||||
results = asyncio.run(eyes.search("AI agent framework"))
|
||||
|
||||
# 健康检查
|
||||
print(eyes.doctor_report())
|
||||
```
|
||||
@@ -1,91 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Login manager — opens a visible browser for manual login, saves session.
|
||||
|
||||
Usage:
|
||||
x-reader login xhs # Login to Xiaohongshu
|
||||
x-reader login wechat # Login to WeChat (if needed)
|
||||
|
||||
Sessions are saved as Playwright storage_state JSON files.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
|
||||
SESSION_DIR = Path.home() / ".x-reader" / "sessions"
|
||||
|
||||
PLATFORM_URLS = {
|
||||
"xhs": "https://www.xiaohongshu.com/explore",
|
||||
"xiaohongshu": "https://www.xiaohongshu.com/explore",
|
||||
"wechat": "https://mp.weixin.qq.com",
|
||||
"twitter": "https://x.com/login",
|
||||
"x": "https://x.com/login",
|
||||
}
|
||||
|
||||
|
||||
def login(platform: str) -> None:
|
||||
"""
|
||||
Open a visible browser for the user to log in manually.
|
||||
After login, saves cookies/localStorage to a session file.
|
||||
|
||||
Args:
|
||||
platform: Platform key (e.g. 'xhs', 'wechat')
|
||||
"""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
print(
|
||||
"❌ Playwright is not installed. Run:\n"
|
||||
' pip install "x-reader[browser]"\n'
|
||||
" playwright install chromium"
|
||||
)
|
||||
return
|
||||
|
||||
platform = platform.lower()
|
||||
login_url = PLATFORM_URLS.get(platform)
|
||||
if not login_url:
|
||||
supported = ", ".join(sorted(PLATFORM_URLS.keys()))
|
||||
print(f"❌ Unknown platform: {platform}")
|
||||
print(f" Supported: {supported}")
|
||||
return
|
||||
|
||||
SESSION_DIR.mkdir(parents=True, exist_ok=True)
|
||||
session_path = SESSION_DIR / f"{platform}.json"
|
||||
# Normalize alias to canonical name
|
||||
if platform in ("xhs", "xiaohongshu"):
|
||||
canonical = "xhs"
|
||||
elif platform in ("twitter", "x"):
|
||||
canonical = "twitter"
|
||||
else:
|
||||
canonical = platform
|
||||
session_path = SESSION_DIR / f"{canonical}.json"
|
||||
|
||||
print(f"🌐 Opening {platform} login page: {login_url}")
|
||||
print(" Please log in manually in the browser window.")
|
||||
print(" When done, close the browser or press Ctrl+C.\n")
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=False)
|
||||
context = browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
)
|
||||
page = context.new_page()
|
||||
page.goto(login_url)
|
||||
|
||||
try:
|
||||
# Wait for user to log in — blocks until browser is closed
|
||||
page.wait_for_event("close", timeout=300_000) # 5 min max
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
except Exception:
|
||||
pass # Browser closed by user
|
||||
|
||||
# Save session regardless of how we got here
|
||||
context.storage_state(path=str(session_path))
|
||||
logger.info(f"Session saved: {session_path}")
|
||||
print(f"\n✅ Session saved to {session_path}")
|
||||
|
||||
context.close()
|
||||
browser.close()
|
||||
+10
-10
@@ -15,7 +15,7 @@ from agent_eyes.schema import (
|
||||
from_bilibili, from_twitter, from_wechat,
|
||||
from_xiaohongshu, from_youtube, from_rss, from_telegram,
|
||||
)
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
from agent_eyes.readers.jina import fetch_via_jina
|
||||
|
||||
|
||||
class UniversalReader:
|
||||
@@ -91,39 +91,39 @@ class UniversalReader:
|
||||
"""Dispatch to platform-specific fetcher."""
|
||||
|
||||
if platform == "bilibili":
|
||||
from agent_eyes.fetchers.bilibili import fetch_bilibili
|
||||
from agent_eyes.readers.bilibili import fetch_bilibili
|
||||
data = await fetch_bilibili(url)
|
||||
return from_bilibili(data)
|
||||
|
||||
if platform == "twitter":
|
||||
from agent_eyes.fetchers.twitter import fetch_twitter
|
||||
from agent_eyes.readers.twitter import fetch_twitter
|
||||
data = await fetch_twitter(url)
|
||||
return from_twitter(data)
|
||||
|
||||
if platform == "wechat":
|
||||
from agent_eyes.fetchers.wechat import fetch_wechat
|
||||
from agent_eyes.readers.wechat import fetch_wechat
|
||||
data = await fetch_wechat(url)
|
||||
return from_wechat(data)
|
||||
|
||||
if platform == "xhs":
|
||||
from agent_eyes.fetchers.xhs import fetch_xhs
|
||||
from agent_eyes.readers.xhs import fetch_xhs
|
||||
data = await fetch_xhs(url)
|
||||
return from_xiaohongshu(data)
|
||||
|
||||
if platform == "youtube":
|
||||
from agent_eyes.fetchers.youtube import fetch_youtube
|
||||
from agent_eyes.readers.youtube import fetch_youtube
|
||||
data = await fetch_youtube(url)
|
||||
return from_youtube(data)
|
||||
|
||||
if platform == "rss":
|
||||
from agent_eyes.fetchers.rss import fetch_rss
|
||||
from agent_eyes.readers.rss import fetch_rss
|
||||
articles = await fetch_rss(url, limit=1)
|
||||
if articles:
|
||||
return from_rss(articles[0])
|
||||
raise ValueError(f"No articles found in RSS feed: {url}")
|
||||
|
||||
if platform == "reddit":
|
||||
from agent_eyes.fetchers.reddit import fetch_reddit
|
||||
from agent_eyes.readers.reddit import fetch_reddit
|
||||
data = await fetch_reddit(url)
|
||||
return UnifiedContent(
|
||||
source_type=SourceType.REDDIT,
|
||||
@@ -137,7 +137,7 @@ class UniversalReader:
|
||||
)
|
||||
|
||||
if platform == "github":
|
||||
from agent_eyes.fetchers.github import fetch_github
|
||||
from agent_eyes.readers.github import fetch_github
|
||||
data = await fetch_github(url)
|
||||
return UnifiedContent(
|
||||
source_type=SourceType.GITHUB,
|
||||
@@ -151,7 +151,7 @@ class UniversalReader:
|
||||
)
|
||||
|
||||
if platform == "telegram":
|
||||
from agent_eyes.fetchers.telegram import fetch_telegram
|
||||
from agent_eyes.readers.telegram import fetch_telegram
|
||||
# Extract channel username from t.me URL
|
||||
path = urlparse(url).path.strip("/").split("/")[0]
|
||||
channel = path if path else url
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Search module init."""
|
||||
|
||||
from agent_eyes.search.exa import search_web
|
||||
from agent_eyes.search.reddit import search_reddit
|
||||
from agent_eyes.search.github import search_github, search_github_issues
|
||||
from agent_eyes.search.twitter import search_twitter, get_user_tweets
|
||||
|
||||
__all__ = [
|
||||
"search_web",
|
||||
"search_reddit",
|
||||
"search_github",
|
||||
"search_github_issues",
|
||||
"search_twitter",
|
||||
"get_user_tweets",
|
||||
]
|
||||
@@ -0,0 +1,79 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Exa semantic web search.
|
||||
|
||||
Get a free API key at https://exa.ai (1000 searches/month free).
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
EXA_API_URL = "https://api.exa.ai/search"
|
||||
|
||||
|
||||
def _get_api_key(config=None) -> str:
|
||||
"""Get Exa API key from config or env."""
|
||||
if config:
|
||||
key = config.get("exa_api_key")
|
||||
if key:
|
||||
return key
|
||||
key = os.environ.get("EXA_API_KEY")
|
||||
if key:
|
||||
return key
|
||||
raise ValueError(
|
||||
"Exa API key not configured.\n"
|
||||
"Get a free key at https://exa.ai (1000 searches/month free)\n"
|
||||
"Then run: agent-eyes setup"
|
||||
)
|
||||
|
||||
|
||||
async def search_web(
|
||||
query: str,
|
||||
num_results: int = 5,
|
||||
search_type: str = "auto",
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Semantic web search via Exa.
|
||||
|
||||
Args:
|
||||
query: Search query (supports site: prefix)
|
||||
num_results: Number of results (default 5, max 10)
|
||||
search_type: "auto" / "neural" / "keyword"
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {title, url, snippet, published_date, score}
|
||||
"""
|
||||
api_key = _get_api_key(config)
|
||||
logger.info(f"Exa search: {query} (n={num_results})")
|
||||
|
||||
resp = requests.post(
|
||||
EXA_API_URL,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": api_key,
|
||||
},
|
||||
json={
|
||||
"query": query,
|
||||
"numResults": min(num_results, 10),
|
||||
"type": search_type,
|
||||
"contents": {"text": {"maxCharacters": 500}},
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for item in data.get("results", []):
|
||||
results.append({
|
||||
"title": item.get("title", ""),
|
||||
"url": item.get("url", ""),
|
||||
"snippet": item.get("text", ""),
|
||||
"published_date": item.get("publishedDate", ""),
|
||||
"score": item.get("score", 0),
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,118 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GitHub search via public API (no key required)."""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from loguru import logger
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
GITHUB_API = "https://api.github.com"
|
||||
|
||||
|
||||
def _get_headers(config=None) -> dict:
|
||||
"""Get GitHub API headers with optional auth token."""
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
token = None
|
||||
if config:
|
||||
token = config.get("github_token")
|
||||
if not token:
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
async def search_github(
|
||||
query: str,
|
||||
language: Optional[str] = None,
|
||||
sort: str = "stars",
|
||||
limit: int = 5,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search GitHub repositories.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
language: Filter by language (e.g. "python")
|
||||
sort: Sort by "stars" / "forks" / "updated"
|
||||
limit: Number of results (max 30)
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {name, url, description, stars, language, updated}
|
||||
"""
|
||||
q = query
|
||||
if language:
|
||||
q += f" language:{language}"
|
||||
|
||||
logger.info(f"GitHub search: {q} (n={limit})")
|
||||
|
||||
resp = requests.get(
|
||||
f"{GITHUB_API}/search/repositories",
|
||||
headers=_get_headers(config),
|
||||
params={"q": q, "sort": sort, "per_page": min(limit, 30)},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for repo in data.get("items", []):
|
||||
results.append({
|
||||
"name": repo.get("full_name", ""),
|
||||
"url": repo.get("html_url", ""),
|
||||
"description": repo.get("description", ""),
|
||||
"stars": repo.get("stargazers_count", 0),
|
||||
"forks": repo.get("forks_count", 0),
|
||||
"language": repo.get("language", ""),
|
||||
"updated": repo.get("updated_at", ""),
|
||||
"topics": repo.get("topics", []),
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
async def search_github_issues(
|
||||
query: str,
|
||||
repo: Optional[str] = None,
|
||||
state: str = "open",
|
||||
limit: int = 5,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search GitHub issues and discussions.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
repo: Filter by repo (e.g. "owner/repo")
|
||||
state: "open" / "closed"
|
||||
limit: Number of results
|
||||
config: Optional Config instance
|
||||
"""
|
||||
q = query
|
||||
if repo:
|
||||
q += f" repo:{repo}"
|
||||
q += f" state:{state}"
|
||||
|
||||
resp = requests.get(
|
||||
f"{GITHUB_API}/search/issues",
|
||||
headers=_get_headers(config),
|
||||
params={"q": q, "sort": "reactions", "per_page": min(limit, 30)},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
results = []
|
||||
for issue in data.get("items", []):
|
||||
results.append({
|
||||
"title": issue.get("title", ""),
|
||||
"url": issue.get("html_url", ""),
|
||||
"body": (issue.get("body", "") or "")[:500],
|
||||
"state": issue.get("state", ""),
|
||||
"comments": issue.get("comments", 0),
|
||||
"reactions": issue.get("reactions", {}).get("total_count", 0),
|
||||
"created": issue.get("created_at", ""),
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,30 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Reddit search via Exa (bypasses Reddit IP blocks)."""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from agent_eyes.search.exa import search_web
|
||||
|
||||
|
||||
async def search_reddit(
|
||||
query: str,
|
||||
subreddit: Optional[str] = None,
|
||||
num_results: int = 10,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Reddit content via Exa semantic search.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
subreddit: Optional subreddit (e.g. "LocalLLaMA")
|
||||
num_results: Number of results
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {title, url, snippet, published_date, score}
|
||||
"""
|
||||
if subreddit:
|
||||
full_query = f"site:reddit.com/r/{subreddit} {query}"
|
||||
else:
|
||||
full_query = f"site:reddit.com {query}"
|
||||
return await search_web(full_query, num_results=num_results, config=config)
|
||||
@@ -0,0 +1,112 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Twitter search — uses birdx if available, falls back to Exa."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from loguru import logger
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
async def search_twitter(
|
||||
query: str,
|
||||
limit: int = 10,
|
||||
config=None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Twitter/X content.
|
||||
|
||||
Strategy:
|
||||
1. If birdx is installed → use it (full search, timeline, threads)
|
||||
2. Otherwise → use Exa with site:x.com (basic search)
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
limit: Number of results
|
||||
config: Optional Config instance
|
||||
|
||||
Returns:
|
||||
List of {author, text, url, likes, retweets, date}
|
||||
"""
|
||||
if shutil.which("birdx"):
|
||||
return await _search_birdx(query, limit)
|
||||
else:
|
||||
return await _search_exa(query, limit, config)
|
||||
|
||||
|
||||
async def _search_birdx(query: str, limit: int) -> List[Dict[str, Any]]:
|
||||
"""Search Twitter via birdx CLI."""
|
||||
logger.info(f"birdx search: {query} (n={limit})")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["birdx", "search", query, "-n", str(limit), "--json"],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
# birdx might not support --json, try plain output
|
||||
result = subprocess.run(
|
||||
["birdx", "search", query, "-n", str(limit)],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
return _parse_birdx_text(result.stdout)
|
||||
|
||||
data = json.loads(result.stdout)
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
return data.get("tweets", data.get("results", []))
|
||||
except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError) as e:
|
||||
logger.error(f"birdx search failed: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _parse_birdx_text(text: str) -> List[Dict[str, Any]]:
|
||||
"""Parse birdx plain text output into structured data."""
|
||||
results = []
|
||||
current = {}
|
||||
for line in text.strip().split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
if current:
|
||||
results.append(current)
|
||||
current = {}
|
||||
continue
|
||||
if line.startswith("@"):
|
||||
current["author"] = line.split()[0] if line else ""
|
||||
elif line.startswith("http"):
|
||||
current["url"] = line
|
||||
else:
|
||||
current["text"] = current.get("text", "") + " " + line
|
||||
if current:
|
||||
results.append(current)
|
||||
return results
|
||||
|
||||
|
||||
async def _search_exa(query: str, limit: int, config=None) -> List[Dict[str, Any]]:
|
||||
"""Search Twitter via Exa (site:x.com)."""
|
||||
from agent_eyes.search.exa import search_web
|
||||
return await search_web(
|
||||
f"site:x.com {query}",
|
||||
num_results=limit,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
async def get_user_tweets(
|
||||
username: str,
|
||||
limit: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get recent tweets from a user (requires birdx)."""
|
||||
if not shutil.which("birdx"):
|
||||
raise RuntimeError(
|
||||
"birdx not installed. Install: pip install birdx\n"
|
||||
"Then configure cookies: agent-eyes setup"
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["birdx", "user-tweets", f"@{username.lstrip('@')}", "-n", str(limit)],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
return _parse_birdx_text(result.stdout)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("birdx timed out")
|
||||
return []
|
||||
Reference in New Issue
Block a user