rename: Agent Eyes → Agent Reach
全局重命名： - 包名: agent_eyes → agent_reach - CLI: agent-eyes → agent-reach - 类名: AgentEyes → AgentReach - 显示名: Agent Eyes → Agent Reach - GitHub: Panniantong/agent-eyes → Panniantong/Agent-Reach 所有 36 个测试通过，CLI/doctor/read/search 全部正常。
This commit is contained in:
Panniantong
@@ -0,0 +1,69 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Channel registry — routes URLs to the right channel.
|
||||
|
||||
This is the core of Agent Reach' pluggable architecture.
|
||||
Add a new channel: just create a file and register it here.
|
||||
Swap a backend: just change the implementation inside the channel file.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
from .base import Channel, ReadResult, SearchResult
|
||||
|
||||
# Import all channels
|
||||
from .web import WebChannel
|
||||
from .github import GitHubChannel
|
||||
from .twitter import TwitterChannel
|
||||
from .youtube import YouTubeChannel
|
||||
from .reddit import RedditChannel
|
||||
from .rss import RSSChannel
|
||||
from .bilibili import BilibiliChannel
|
||||
from .exa_search import ExaSearchChannel
|
||||
from .xiaohongshu import XiaoHongShuChannel
|
||||
|
||||
|
||||
# Channel registry — order matters (first match wins, web is last as fallback)
|
||||
ALL_CHANNELS: List[Channel] = [
|
||||
GitHubChannel(),
|
||||
TwitterChannel(),
|
||||
YouTubeChannel(),
|
||||
RedditChannel(),
|
||||
BilibiliChannel(),
|
||||
XiaoHongShuChannel(),
|
||||
RSSChannel(),
|
||||
ExaSearchChannel(),
|
||||
WebChannel(), # Fallback — handles any URL
|
||||
]
|
||||
|
||||
# Search-capable channels
|
||||
SEARCH_CHANNELS: Dict[str, Channel] = {
|
||||
ch.name: ch for ch in ALL_CHANNELS if ch.can_search()
|
||||
}
|
||||
|
||||
|
||||
def get_channel_for_url(url: str) -> Channel:
|
||||
"""Find the right channel for a URL."""
|
||||
for channel in ALL_CHANNELS:
|
||||
if channel.can_handle(url):
|
||||
return channel
|
||||
return WebChannel() # Should never reach here, but just in case
|
||||
|
||||
|
||||
def get_channel(name: str) -> Optional[Channel]:
|
||||
"""Get a channel by name."""
|
||||
for ch in ALL_CHANNELS:
|
||||
if ch.name == name:
|
||||
return ch
|
||||
return None
|
||||
|
||||
|
||||
def get_all_channels() -> List[Channel]:
|
||||
"""Get all registered channels."""
|
||||
return ALL_CHANNELS
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Channel", "ReadResult", "SearchResult",
|
||||
"ALL_CHANNELS", "SEARCH_CHANNELS",
|
||||
"get_channel_for_url", "get_channel", "get_all_channels",
|
||||
]
|
||||
@@ -0,0 +1,140 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Channel base class — the universal interface for all platforms.
|
||||
|
||||
Every channel (YouTube, Twitter, GitHub, etc.) implements this interface.
|
||||
The backend tool can be swapped anytime without changing anything else.
|
||||
|
||||
Example:
|
||||
class YouTubeChannel(Channel):
|
||||
name = "youtube"
|
||||
backends = ["yt-dlp"] # current backend, can be swapped
|
||||
|
||||
async def read(self, url, config):
|
||||
# Just call yt-dlp, return standardized dict
|
||||
...
|
||||
"""
|
||||
|
||||
import shutil
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReadResult:
|
||||
"""Standardized read result. Every channel returns this."""
|
||||
title: str
|
||||
content: str
|
||||
url: str
|
||||
author: str = ""
|
||||
date: str = ""
|
||||
platform: str = ""
|
||||
extra: dict = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.extra = self.extra or {}
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = {
|
||||
"title": self.title,
|
||||
"content": self.content,
|
||||
"url": self.url,
|
||||
"platform": self.platform,
|
||||
}
|
||||
if self.author:
|
||||
d["author"] = self.author
|
||||
if self.date:
|
||||
d["date"] = self.date
|
||||
if self.extra:
|
||||
d["extra"] = self.extra
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Standardized search result."""
|
||||
title: str
|
||||
url: str
|
||||
snippet: str = ""
|
||||
author: str = ""
|
||||
date: str = ""
|
||||
score: float = 0
|
||||
extra: dict = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.extra = self.extra or {}
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = {
|
||||
"title": self.title,
|
||||
"url": self.url,
|
||||
"snippet": self.snippet,
|
||||
}
|
||||
if self.author:
|
||||
d["author"] = self.author
|
||||
if self.date:
|
||||
d["date"] = self.date
|
||||
if self.extra:
|
||||
d["extra"] = self.extra
|
||||
return d
|
||||
|
||||
|
||||
class Channel(ABC):
|
||||
"""
|
||||
Base class for all channels.
|
||||
|
||||
Subclasses just need to implement:
|
||||
- read(url, config) → ReadResult
|
||||
- can_handle(url) → bool
|
||||
- check(config) → (status, message)
|
||||
|
||||
Optionally:
|
||||
- search(query, config, **kwargs) → list[SearchResult]
|
||||
"""
|
||||
|
||||
name: str = "" # e.g. "youtube"
|
||||
description: str = "" # e.g. "YouTube video transcripts"
|
||||
backends: List[str] = [] # e.g. ["yt-dlp"] — what external tool is used
|
||||
requires_config: List[str] = [] # e.g. ["reddit_proxy"]
|
||||
requires_tools: List[str] = [] # e.g. ["yt-dlp"]
|
||||
tier: int = 0 # 0=zero-config, 1=needs free key, 2=needs setup
|
||||
|
||||
@abstractmethod
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
"""Read content from a URL. Must return ReadResult."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def can_handle(self, url: str) -> bool:
|
||||
"""Check if this channel can handle this URL."""
|
||||
...
|
||||
|
||||
def check(self, config=None) -> Tuple[str, str]:
|
||||
"""
|
||||
Check if this channel is available.
|
||||
Returns (status, message) where status is 'ok'/'warn'/'off'/'error'.
|
||||
"""
|
||||
# Check required tools
|
||||
for tool in self.requires_tools:
|
||||
if not shutil.which(tool):
|
||||
return "off", f"需要安装：pip install {tool}"
|
||||
|
||||
# Check required config
|
||||
for key in self.requires_config:
|
||||
if config and not config.get(key):
|
||||
return "off", f"需要配置 {key}，运行 agent-reach setup"
|
||||
|
||||
return "ok", f"{'、'.join(self.backends) if self.backends else '内置'}"
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
"""Search this platform. Override if supported."""
|
||||
raise NotImplementedError(f"{self.name} does not support search")
|
||||
|
||||
def can_search(self) -> bool:
|
||||
"""Whether this channel supports search."""
|
||||
try:
|
||||
# Check if search is overridden
|
||||
return type(self).search is not Channel.search
|
||||
except:
|
||||
return False
|
||||
@@ -0,0 +1,121 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Bilibili — via public API (free, no config needed).
|
||||
|
||||
Backend: Bilibili public API
|
||||
Swap to: any Bilibili access method
|
||||
"""
|
||||
|
||||
import requests
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class BilibiliChannel(Channel):
|
||||
name = "bilibili"
|
||||
description = "B站视频信息和字幕"
|
||||
backends = ["Bilibili API"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "bilibili.com" in domain or "b23.tv" in domain
|
||||
|
||||
def check(self, config=None):
|
||||
proxy = config.get("bilibili_proxy") if config else None
|
||||
if proxy:
|
||||
return "ok", "已配置代理，完整可用"
|
||||
# Detect if we're on a server (same logic as cli._detect_environment)
|
||||
import os
|
||||
indicators = [
|
||||
os.path.exists("/var/run/docker.sock"),
|
||||
os.path.exists("/etc/cloud"),
|
||||
"SSH_CONNECTION" in os.environ,
|
||||
"container" in os.environ.get("container", ""),
|
||||
]
|
||||
is_server = any(indicators)
|
||||
if is_server:
|
||||
return "warn", "服务器 IP 可能被封，配置代理即可解决：agent-reach configure proxy URL"
|
||||
return "ok", "本地直连可用"
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
# Proxy support (Bilibili blocks server IPs)
|
||||
proxy = config.get("bilibili_proxy") if config else None
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
|
||||
# Extract BV id from URL
|
||||
path = urlparse(url).path
|
||||
bv_id = ""
|
||||
for part in path.split("/"):
|
||||
if part.startswith("BV"):
|
||||
bv_id = part
|
||||
break
|
||||
|
||||
if not bv_id:
|
||||
# Fallback to Jina Reader
|
||||
from agent_reach.channels.web import WebChannel
|
||||
return await WebChannel().read(url, config)
|
||||
|
||||
# Get video info
|
||||
resp = requests.get(
|
||||
"https://api.bilibili.com/x/web-interface/view",
|
||||
params={"bvid": bv_id},
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
proxies=proxies,
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
api_data = resp.json()
|
||||
|
||||
# Check for API errors (IP blocked, video not found, etc.)
|
||||
if api_data.get("code") != 0:
|
||||
msg = api_data.get("message", "Unknown error")
|
||||
# Bilibili returns -404 when server IP is blocked
|
||||
if api_data.get("code") in (-404, -403, -412):
|
||||
return ReadResult(
|
||||
title=f"Bilibili: {bv_id}",
|
||||
content=f"⚠️ Bilibili blocked this request ({msg}). "
|
||||
f"This usually means the server IP is blocked. "
|
||||
f"Try: agent-reach configure proxy http://user:pass@ip:port",
|
||||
url=url,
|
||||
platform="bilibili",
|
||||
)
|
||||
return ReadResult(
|
||||
title=f"Bilibili: {bv_id}",
|
||||
content=f"Bilibili API error: {msg} (code: {api_data.get('code')})",
|
||||
url=url,
|
||||
platform="bilibili",
|
||||
)
|
||||
|
||||
data = api_data.get("data", {})
|
||||
|
||||
title = data.get("title", "")
|
||||
desc = data.get("desc", "")
|
||||
author = data.get("owner", {}).get("name", "")
|
||||
|
||||
# Try to get subtitles
|
||||
subtitle_text = ""
|
||||
subtitle_list = data.get("subtitle", {}).get("list", [])
|
||||
if subtitle_list:
|
||||
sub_url = subtitle_list[0].get("subtitle_url", "")
|
||||
if sub_url:
|
||||
if sub_url.startswith("//"):
|
||||
sub_url = "https:" + sub_url
|
||||
sr = requests.get(sub_url, timeout=10)
|
||||
if sr.ok:
|
||||
sub_data = sr.json()
|
||||
lines = [item.get("content", "") for item in sub_data.get("body", [])]
|
||||
subtitle_text = "\n".join(lines)
|
||||
|
||||
content = desc
|
||||
if subtitle_text:
|
||||
content += f"\n\n## Transcript\n{subtitle_text}"
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=content,
|
||||
url=url,
|
||||
author=author,
|
||||
platform="bilibili",
|
||||
extra={"view": data.get("stat", {}).get("view", 0),
|
||||
"like": data.get("stat", {}).get("like", 0)},
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Exa semantic search — the search backbone for Agent Reach.
|
||||
|
||||
Backend: Exa API (https://exa.ai) — free 1000 searches/month
|
||||
Swap to: Tavily, SerpAPI, or any search API
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from .base import Channel, SearchResult
|
||||
from typing import List
|
||||
|
||||
|
||||
class ExaSearchChannel(Channel):
|
||||
name = "exa_search"
|
||||
description = "全网语义搜索（同时支持 Reddit/Twitter 搜索）"
|
||||
backends = ["Exa API"]
|
||||
requires_config = ["exa_api_key"]
|
||||
tier = 1
|
||||
|
||||
API_URL = "https://api.exa.ai/search"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
return False # Search-only channel, doesn't read URLs
|
||||
|
||||
async def read(self, url: str, config=None) -> None:
|
||||
raise NotImplementedError("Exa is a search engine, not a reader")
|
||||
|
||||
def _get_key(self, config=None) -> str:
|
||||
if config:
|
||||
key = config.get("exa_api_key")
|
||||
if key:
|
||||
return key
|
||||
key = os.environ.get("EXA_API_KEY")
|
||||
if key:
|
||||
return key
|
||||
raise ValueError(
|
||||
"Exa API key not configured.\n"
|
||||
"Get a free key at https://exa.ai (1000 searches/month free)\n"
|
||||
"Then run: agent-reach setup"
|
||||
)
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
api_key = self._get_key(config)
|
||||
limit = kwargs.get("limit", 5)
|
||||
|
||||
resp = requests.post(
|
||||
self.API_URL,
|
||||
headers={"Content-Type": "application/json", "x-api-key": api_key},
|
||||
json={
|
||||
"query": query,
|
||||
"numResults": min(limit, 10),
|
||||
"type": "auto",
|
||||
"contents": {"text": {"maxCharacters": 500}},
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
results = []
|
||||
for item in resp.json().get("results", []):
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
url=item.get("url", ""),
|
||||
snippet=item.get("text", ""),
|
||||
date=item.get("publishedDate", ""),
|
||||
score=item.get("score", 0),
|
||||
))
|
||||
return results
|
||||
@@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GitHub — via GitHub REST API (free, no config needed).
|
||||
|
||||
Backend: GitHub API v3
|
||||
Swap to: gh CLI, or any GitHub API wrapper
|
||||
"""
|
||||
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult, SearchResult
|
||||
from typing import List
|
||||
|
||||
|
||||
class GitHubChannel(Channel):
|
||||
name = "github"
|
||||
description = "GitHub 仓库和代码"
|
||||
backends = ["GitHub API"]
|
||||
tier = 0
|
||||
|
||||
API = "https://api.github.com"
|
||||
|
||||
def _headers(self, config=None):
|
||||
h = {"Accept": "application/vnd.github+json"}
|
||||
token = config.get("github_token") if config else None
|
||||
if token:
|
||||
h["Authorization"] = f"Bearer {token}"
|
||||
return h
|
||||
|
||||
def check(self, config=None):
|
||||
import shutil
|
||||
token = config.get("github_token") if config else None
|
||||
has_gh = shutil.which("gh")
|
||||
if token or has_gh:
|
||||
return "ok", "完整可用（读取、搜索、Fork、Issue、PR 等）"
|
||||
return "ok", "公开仓库可读可搜。配置 gh CLI 或 github_token 可解锁 Fork、Issue、PR 等操作"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "github.com" in domain
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
path = urlparse(url).path.strip("/").split("/")
|
||||
|
||||
if len(path) < 2:
|
||||
raise ValueError(f"Invalid GitHub URL: {url}")
|
||||
|
||||
owner, repo = path[0], path[1]
|
||||
headers = self._headers(config)
|
||||
|
||||
# Issues/PRs
|
||||
if len(path) >= 4 and path[2] in ("issues", "pull"):
|
||||
num = path[3]
|
||||
resp = requests.get(f"{self.API}/repos/{owner}/{repo}/issues/{num}", headers=headers, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Get comments
|
||||
comments_text = ""
|
||||
if data.get("comments", 0) > 0:
|
||||
cr = requests.get(f"{self.API}/repos/{owner}/{repo}/issues/{num}/comments",
|
||||
headers=headers, params={"per_page": 20}, timeout=15)
|
||||
if cr.ok:
|
||||
for c in cr.json():
|
||||
comments_text += f"\n\n---\n**{c.get('user', {}).get('login', '')}** ({c.get('created_at', '')}):\n{c.get('body', '')}"
|
||||
|
||||
return ReadResult(
|
||||
title=data.get("title", ""),
|
||||
content=(data.get("body", "") or "") + comments_text,
|
||||
url=url,
|
||||
author=data.get("user", {}).get("login", ""),
|
||||
date=data.get("created_at", ""),
|
||||
platform="github",
|
||||
extra={"state": data.get("state"), "comments": data.get("comments", 0),
|
||||
"reactions": data.get("reactions", {}).get("total_count", 0)},
|
||||
)
|
||||
|
||||
# Repo
|
||||
resp = requests.get(f"{self.API}/repos/{owner}/{repo}", headers=headers, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Get README
|
||||
readme_text = ""
|
||||
rr = requests.get(f"{self.API}/repos/{owner}/{repo}/readme", headers=headers, timeout=15)
|
||||
if rr.ok:
|
||||
import base64
|
||||
readme_data = rr.json()
|
||||
if readme_data.get("encoding") == "base64":
|
||||
readme_text = base64.b64decode(readme_data["content"]).decode("utf-8", errors="replace")
|
||||
|
||||
return ReadResult(
|
||||
title=f"{owner}/{repo}",
|
||||
content=readme_text or data.get("description", ""),
|
||||
url=url,
|
||||
author=owner,
|
||||
platform="github",
|
||||
extra={"stars": data.get("stargazers_count", 0), "forks": data.get("forks_count", 0),
|
||||
"language": data.get("language", ""), "description": data.get("description", "")},
|
||||
)
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
language = kwargs.get("language")
|
||||
limit = kwargs.get("limit", 5)
|
||||
|
||||
q = query
|
||||
if language:
|
||||
q += f" language:{language}"
|
||||
|
||||
resp = requests.get(
|
||||
f"{self.API}/search/repositories",
|
||||
headers=self._headers(config),
|
||||
params={"q": q, "sort": "stars", "per_page": min(limit, 30)},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
results = []
|
||||
for repo in resp.json().get("items", []):
|
||||
results.append(SearchResult(
|
||||
title=repo.get("full_name", ""),
|
||||
url=repo.get("html_url", ""),
|
||||
snippet=repo.get("description", ""),
|
||||
date=repo.get("updated_at", ""),
|
||||
extra={"stars": repo.get("stargazers_count", 0),
|
||||
"forks": repo.get("forks_count", 0),
|
||||
"language": repo.get("language", "")},
|
||||
))
|
||||
return results
|
||||
@@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Reddit — via Reddit JSON API + optional proxy.
|
||||
|
||||
Backend: Reddit public JSON API (append .json to any URL)
|
||||
Swap to: any Reddit access method
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class RedditChannel(Channel):
|
||||
name = "reddit"
|
||||
description = "Reddit 帖子和评论"
|
||||
backends = ["Reddit JSON API"]
|
||||
tier = 2
|
||||
|
||||
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "reddit.com" in domain or "redd.it" in domain
|
||||
|
||||
def check(self, config=None):
|
||||
proxy = config.get("reddit_proxy") if config else None
|
||||
has_bot = bool(os.environ.get("REDDIT_CLIENT_ID"))
|
||||
if proxy and has_bot:
|
||||
return "ok", "完整可用（代理 + OAuth Bot）"
|
||||
elif proxy:
|
||||
return "ok", "代理已配置，可读取帖子。配置 REDDIT_CLIENT_ID/SECRET 可解锁高级搜索和发帖"
|
||||
elif has_bot:
|
||||
return "warn", "OAuth Bot 已配置，但服务器直连可能被封。配个代理更稳定：agent-reach configure proxy URL"
|
||||
else:
|
||||
return "off", "搜索用 Exa 免费可用。读帖子需配个代理：agent-reach configure proxy URL"
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
proxy = config.get("reddit_proxy") if config else None
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
|
||||
# Clean URL: remove query params, trailing slash, then add .json
|
||||
parsed = urlparse(url)
|
||||
clean_path = parsed.path.rstrip("/")
|
||||
# Remove trailing .json if already present (avoid double .json)
|
||||
if clean_path.endswith(".json"):
|
||||
clean_path = clean_path[:-5]
|
||||
json_url = f"https://www.reddit.com{clean_path}.json"
|
||||
|
||||
try:
|
||||
resp = requests.get(
|
||||
json_url,
|
||||
headers={"User-Agent": self.USER_AGENT},
|
||||
proxies=proxies,
|
||||
params={"limit": 50},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
status = e.response.status_code if e.response is not None else 0
|
||||
if status in (403, 429):
|
||||
return ReadResult(
|
||||
title="Reddit",
|
||||
content="⚠️ Reddit blocked this request (403 Forbidden). "
|
||||
"Reddit blocks most server IPs.\n"
|
||||
"Fix: agent-reach configure proxy http://user:pass@ip:port\n"
|
||||
"Cheap option: https://www.webshare.io ($1/month)\n\n"
|
||||
"Alternatively, search Reddit via Exa (free, no proxy needed): "
|
||||
"agent-reach search-reddit \"your query\"",
|
||||
url=url,
|
||||
platform="reddit",
|
||||
)
|
||||
raise
|
||||
|
||||
data = resp.json()
|
||||
|
||||
if isinstance(data, list) and len(data) >= 1:
|
||||
# Post page: [post_listing, comments_listing]
|
||||
post = data[0]["data"]["children"][0]["data"]
|
||||
title = post.get("title", "")
|
||||
author = post.get("author", "")
|
||||
selftext = post.get("selftext", "")
|
||||
score = post.get("score", 0)
|
||||
subreddit = post.get("subreddit", "")
|
||||
|
||||
# Extract comments
|
||||
comments_text = ""
|
||||
if len(data) >= 2:
|
||||
comments_text = self._extract_comments(data[1])
|
||||
|
||||
content = selftext
|
||||
if comments_text:
|
||||
content += f"\n\n---\n## Comments\n{comments_text}"
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=content,
|
||||
url=url,
|
||||
author=f"u/{author}",
|
||||
platform="reddit",
|
||||
extra={"subreddit": subreddit, "score": score},
|
||||
)
|
||||
|
||||
raise ValueError(f"Could not parse Reddit response for: {url}")
|
||||
|
||||
def _extract_comments(self, comments_data: dict, depth: int = 0, max_depth: int = 3) -> str:
|
||||
"""Recursively extract comments."""
|
||||
lines = []
|
||||
children = comments_data.get("data", {}).get("children", [])
|
||||
|
||||
for child in children:
|
||||
if child.get("kind") != "t1":
|
||||
continue
|
||||
data = child.get("data", {})
|
||||
author = data.get("author", "[deleted]")
|
||||
body = data.get("body", "")
|
||||
score = data.get("score", 0)
|
||||
indent = " " * depth
|
||||
|
||||
lines.append(f"{indent}**u/{author}** ({score} points):")
|
||||
lines.append(f"{indent}{body}")
|
||||
lines.append("")
|
||||
|
||||
# Recurse into replies
|
||||
if depth < max_depth and data.get("replies") and isinstance(data["replies"], dict):
|
||||
lines.append(self._extract_comments(data["replies"], depth + 1, max_depth))
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,57 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""RSS feeds — via feedparser (free, pip dependency).
|
||||
|
||||
Backend: feedparser (https://github.com/kurtmckee/feedparser)
|
||||
Swap to: any RSS parser
|
||||
"""
|
||||
|
||||
import feedparser
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class RSSChannel(Channel):
|
||||
name = "rss"
|
||||
description = "RSS/Atom 订阅源"
|
||||
backends = ["feedparser"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
lower = url.lower()
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return (lower.endswith(".xml") or "/rss" in lower or "/feed" in lower
|
||||
or "/atom" in lower or "rss" in domain)
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
feed = feedparser.parse(url)
|
||||
|
||||
if feed.bozo and not feed.entries:
|
||||
raise ValueError(f"Failed to parse RSS feed: {url}")
|
||||
|
||||
if not feed.entries:
|
||||
raise ValueError(f"No entries in RSS feed: {url}")
|
||||
|
||||
# Return latest entry
|
||||
entry = feed.entries[0]
|
||||
content = entry.get("summary", "") or entry.get("description", "")
|
||||
|
||||
# If multiple entries, summarize all
|
||||
if len(feed.entries) > 1:
|
||||
lines = [f"# {feed.feed.get('title', 'RSS Feed')}\n"]
|
||||
for i, e in enumerate(feed.entries[:20], 1):
|
||||
title = e.get("title", "Untitled")
|
||||
link = e.get("link", "")
|
||||
summary = e.get("summary", "")[:200]
|
||||
lines.append(f"## {i}. {title}")
|
||||
lines.append(f"🔗 {link}")
|
||||
if summary:
|
||||
lines.append(summary)
|
||||
lines.append("")
|
||||
content = "\n".join(lines)
|
||||
|
||||
return ReadResult(
|
||||
title=feed.feed.get("title", entry.get("title", url)),
|
||||
content=content,
|
||||
url=url,
|
||||
platform="rss",
|
||||
)
|
||||
@@ -0,0 +1,150 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Twitter/X — via birdx CLI (free) or Jina Reader fallback.
|
||||
|
||||
Backend: birdx (https://github.com/runesleo/birdx) for search/timeline
|
||||
Jina Reader for single tweets
|
||||
Swap to: any Twitter access tool
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult, SearchResult
|
||||
from typing import List
|
||||
import requests
|
||||
|
||||
|
||||
class TwitterChannel(Channel):
|
||||
name = "twitter"
|
||||
description = "Twitter/X 推文"
|
||||
backends = ["birdx", "Jina Reader"]
|
||||
tier = 0 # Single tweet reading is zero-config
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "x.com" in domain or "twitter.com" in domain
|
||||
|
||||
def check(self, config=None):
|
||||
# Basic reading always works (Jina fallback)
|
||||
if shutil.which("birdx"):
|
||||
return "ok", "搜索、时间线、发推全部可用"
|
||||
return "ok", "可读取推文。安装 birdx + 配置 Cookie 可解锁搜索和发推"
|
||||
return "ok", "Jina Reader (single tweets only)"
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
# Try birdx first
|
||||
if shutil.which("birdx"):
|
||||
return await self._read_birdx(url)
|
||||
# Fallback: Jina Reader
|
||||
return await self._read_jina(url)
|
||||
|
||||
async def _read_birdx(self, url: str) -> ReadResult:
|
||||
result = subprocess.run(
|
||||
["birdx", "read", url],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return await self._read_jina(url)
|
||||
|
||||
text = result.stdout.strip()
|
||||
# Extract author from first line
|
||||
author = ""
|
||||
lines = text.split("\n")
|
||||
if lines and lines[0].startswith("@"):
|
||||
author = lines[0].split()[0]
|
||||
|
||||
return ReadResult(
|
||||
title=text[:100],
|
||||
content=text,
|
||||
url=url,
|
||||
author=author,
|
||||
platform="twitter",
|
||||
)
|
||||
|
||||
async def _read_jina(self, url: str) -> ReadResult:
|
||||
resp = requests.get(
|
||||
f"https://r.jina.ai/{url}",
|
||||
headers={"Accept": "text/markdown"},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.text
|
||||
title = text[:100] if text else url
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=text,
|
||||
url=url,
|
||||
platform="twitter",
|
||||
)
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
limit = kwargs.get("limit", 10)
|
||||
|
||||
if shutil.which("birdx"):
|
||||
return await self._search_birdx(query, limit)
|
||||
|
||||
# Fallback to Exa
|
||||
return await self._search_exa(query, limit, config)
|
||||
|
||||
async def _search_birdx(self, query: str, limit: int) -> List[SearchResult]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["birdx", "search", query, "-n", str(limit)],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return []
|
||||
|
||||
return self._parse_birdx_output(result.stdout)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return []
|
||||
|
||||
def _parse_birdx_output(self, text: str) -> List[SearchResult]:
|
||||
"""Parse birdx text output into SearchResults."""
|
||||
results = []
|
||||
current = {}
|
||||
text_lines = []
|
||||
|
||||
for line in text.strip().split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("─"):
|
||||
if current:
|
||||
current["text"] = "\n".join(text_lines).strip()
|
||||
results.append(SearchResult(
|
||||
title=current.get("text", "")[:80],
|
||||
url=current.get("url", ""),
|
||||
snippet=current.get("text", ""),
|
||||
author=current.get("author", ""),
|
||||
date=current.get("date", ""),
|
||||
))
|
||||
current = {}
|
||||
text_lines = []
|
||||
continue
|
||||
if line.startswith("@") and line.endswith(":") and "(" in line:
|
||||
current["author"] = line.split()[0]
|
||||
continue
|
||||
if line.startswith("date:"):
|
||||
current["date"] = line[5:].strip()
|
||||
continue
|
||||
if line.startswith("url:"):
|
||||
current["url"] = line[4:].strip()
|
||||
continue
|
||||
if current is not None:
|
||||
text_lines.append(line)
|
||||
|
||||
if current and text_lines:
|
||||
current["text"] = "\n".join(text_lines).strip()
|
||||
results.append(SearchResult(
|
||||
title=current.get("text", "")[:80],
|
||||
url=current.get("url", ""),
|
||||
snippet=current.get("text", ""),
|
||||
author=current.get("author", ""),
|
||||
date=current.get("date", ""),
|
||||
))
|
||||
return results
|
||||
|
||||
async def _search_exa(self, query: str, limit: int, config=None) -> List[SearchResult]:
|
||||
from agent_reach.channels.exa_search import ExaSearchChannel
|
||||
exa = ExaSearchChannel()
|
||||
return await exa.search(f"site:x.com {query}", config=config, limit=limit)
|
||||
@@ -0,0 +1,49 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Web pages — via Jina Reader API (free, no config needed).
|
||||
|
||||
Backend: Jina Reader (https://r.jina.ai)
|
||||
Swap to: Firecrawl, Trafilatura, or any other reader API
|
||||
"""
|
||||
|
||||
import requests
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class WebChannel(Channel):
|
||||
name = "web"
|
||||
description = "网页（任意 URL）"
|
||||
backends = ["Jina Reader API"]
|
||||
tier = 0
|
||||
|
||||
JINA_URL = "https://r.jina.ai/"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
# Fallback — handles any URL not matched by other channels
|
||||
return True
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
resp = requests.get(
|
||||
f"{self.JINA_URL}{url}",
|
||||
headers={"Accept": "text/markdown"},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.text
|
||||
|
||||
# Extract title from first markdown heading
|
||||
title = url
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("# "):
|
||||
title = line[2:].strip()
|
||||
break
|
||||
if line.startswith("Title:"):
|
||||
title = line[6:].strip()
|
||||
break
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=text,
|
||||
url=url,
|
||||
platform="web",
|
||||
)
|
||||
@@ -0,0 +1,120 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""XiaoHongShu (小红书) — via cookie-based API access.
|
||||
|
||||
Backend: XHS web API + cookies
|
||||
Swap to: any XHS access method
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class XiaoHongShuChannel(Channel):
|
||||
name = "xiaohongshu"
|
||||
description = "小红书笔记"
|
||||
backends = ["XHS Web API"]
|
||||
tier = 2
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "xiaohongshu.com" in domain or "xhslink.com" in domain
|
||||
|
||||
def check(self, config=None):
|
||||
cookie = config.get("xhs_cookie") if config else None
|
||||
if cookie:
|
||||
return "ok", "Cookie 已配置，完整可用"
|
||||
return "off", "需要配置 Cookie 才能访问。导入浏览器 Cookie 即可：agent-reach configure --from-browser chrome"
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
cookie = config.get("xhs_cookie") if config else None
|
||||
|
||||
if not cookie:
|
||||
return ReadResult(
|
||||
title="XiaoHongShu",
|
||||
content="⚠️ XiaoHongShu requires cookies to access.\n"
|
||||
"Set up: agent-reach configure xhs-cookie \"YOUR_COOKIE_STRING\"\n"
|
||||
"How to get it: install Cookie-Editor extension → go to xiaohongshu.com → Export → Header String",
|
||||
url=url,
|
||||
platform="xiaohongshu",
|
||||
)
|
||||
|
||||
# Extract note ID from URL
|
||||
note_id = self._extract_note_id(url)
|
||||
if not note_id:
|
||||
from agent_reach.channels.web import WebChannel
|
||||
return await WebChannel().read(url, config)
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||||
"Cookie": cookie,
|
||||
"Referer": "https://www.xiaohongshu.com/",
|
||||
}
|
||||
|
||||
# Fetch note page
|
||||
resp = requests.get(
|
||||
f"https://www.xiaohongshu.com/explore/{note_id}",
|
||||
headers=headers,
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
html = resp.text
|
||||
|
||||
# Extract note data from HTML
|
||||
title, content, author = self._parse_html(html)
|
||||
|
||||
return ReadResult(
|
||||
title=title or f"XHS Note {note_id}",
|
||||
content=content or "Could not extract content. Cookie may be expired.",
|
||||
url=url,
|
||||
author=author,
|
||||
platform="xiaohongshu",
|
||||
)
|
||||
|
||||
def _extract_note_id(self, url: str) -> str:
|
||||
"""Extract note ID from various XHS URL formats."""
|
||||
# https://www.xiaohongshu.com/explore/xxxxx
|
||||
# https://www.xiaohongshu.com/discovery/item/xxxxx
|
||||
# https://xhslink.com/xxxxx
|
||||
path = urlparse(url).path
|
||||
parts = path.strip("/").split("/")
|
||||
if parts:
|
||||
return parts[-1]
|
||||
return ""
|
||||
|
||||
def _parse_html(self, html: str):
|
||||
"""Extract title, content, author from XHS HTML."""
|
||||
title = ""
|
||||
content = ""
|
||||
author = ""
|
||||
|
||||
# Try to find JSON data in page
|
||||
match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.*?})\s*</script>', html, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
# XHS embeds note data in initial state
|
||||
state = json.loads(match.group(1).replace('undefined', 'null'))
|
||||
note_data = state.get("note", {}).get("noteDetailMap", {})
|
||||
if note_data:
|
||||
first_note = list(note_data.values())[0]
|
||||
note = first_note.get("note", {})
|
||||
title = note.get("title", "")
|
||||
content = note.get("desc", "")
|
||||
author = note.get("user", {}).get("nickname", "")
|
||||
except (json.JSONDecodeError, KeyError, IndexError):
|
||||
pass
|
||||
|
||||
# Fallback: extract from meta tags
|
||||
if not title:
|
||||
m = re.search(r'<title>(.*?)</title>', html)
|
||||
if m:
|
||||
title = m.group(1)
|
||||
|
||||
if not content:
|
||||
m = re.search(r'<meta name="description" content="(.*?)"', html)
|
||||
if m:
|
||||
content = m.group(1)
|
||||
|
||||
return title, content, author
|
||||
@@ -0,0 +1,94 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""YouTube — via yt-dlp (free, pip install yt-dlp).
|
||||
|
||||
Backend: yt-dlp (https://github.com/yt-dlp/yt-dlp)
|
||||
Swap to: any YouTube subtitle extractor
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class YouTubeChannel(Channel):
|
||||
name = "youtube"
|
||||
description = "YouTube 视频字幕"
|
||||
backends = ["yt-dlp"]
|
||||
requires_tools = ["yt-dlp"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "youtube.com" in domain or "youtu.be" in domain
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
if not shutil.which("yt-dlp"):
|
||||
raise RuntimeError("yt-dlp not installed. Install: pip install yt-dlp")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Get video info
|
||||
info = self._get_info(url)
|
||||
title = info.get("title", url)
|
||||
author = info.get("uploader", "")
|
||||
|
||||
# Try to get subtitles
|
||||
transcript = self._get_subtitles(url, tmpdir)
|
||||
|
||||
if not transcript:
|
||||
transcript = f"[Video: {title}]\n[No subtitles available. Use Groq Whisper for transcription.]"
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=transcript,
|
||||
url=url,
|
||||
author=author,
|
||||
platform="youtube",
|
||||
extra={
|
||||
"duration": info.get("duration"),
|
||||
"view_count": info.get("view_count"),
|
||||
"upload_date": info.get("upload_date"),
|
||||
},
|
||||
)
|
||||
|
||||
def _get_info(self, url: str) -> dict:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["yt-dlp", "--dump-json", "--no-download", url],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return json.loads(result.stdout)
|
||||
except (subprocess.TimeoutExpired, json.JSONDecodeError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _get_subtitles(self, url: str, tmpdir: str) -> str:
|
||||
"""Extract subtitles using yt-dlp."""
|
||||
try:
|
||||
subprocess.run(
|
||||
["yt-dlp", "--write-auto-sub", "--write-sub",
|
||||
"--sub-lang", "en,zh-Hans,zh",
|
||||
"--skip-download", "--sub-format", "vtt",
|
||||
"-o", f"{tmpdir}/%(id)s.%(ext)s", url],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
|
||||
# Find and read subtitle file
|
||||
for f in Path(tmpdir).glob("*.vtt"):
|
||||
text = f.read_text(errors="replace")
|
||||
# Strip VTT headers and timestamps
|
||||
lines = []
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("WEBVTT") or "-->" in line or line.isdigit():
|
||||
continue
|
||||
if line not in lines[-1:]: # deduplicate
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
return ""
|
||||
Reference in New Issue
Block a user