v2.0.0 — Pure glue architecture: zero copied code, pluggable channels
BREAKING: Complete architectural rewrite.
Before: Copied x-reader's fetcher code into readers/ (1205 lines of borrowed code)
After: Pluggable channel system where each channel is a thin wrapper (~50 lines)
around the best external tool for that platform. Zero copied code.
Architecture:
- channels/base.py — Universal Channel interface (read, search, check)
- channels/web.py — Jina Reader API (swappable)
- channels/github.py — GitHub API (swappable)
- channels/twitter.py — birdx + Jina fallback (swappable)
- channels/youtube.py — yt-dlp (swappable)
- channels/reddit.py — Reddit JSON API + proxy (swappable)
- channels/rss.py — feedparser (swappable)
- channels/bilibili.py — Bilibili API (swappable)
- channels/exa_search.py — Exa semantic search (swappable)
Key design: every backend can be swapped by changing ONE file.
YouTube dies? Change youtube.py. Exa sucks? Swap exa_search.py for Tavily.
Nothing else changes.
Removed: reader.py, schema.py, readers/, search/, utils/ (all x-reader code)
Tests: 36/36 passing
This commit is contained in:
Panniantong
@@ -0,0 +1,67 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Channel registry — routes URLs to the right channel.
|
||||
|
||||
This is the core of Agent Eyes' pluggable architecture.
|
||||
Add a new channel: just create a file and register it here.
|
||||
Swap a backend: just change the implementation inside the channel file.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
from .base import Channel, ReadResult, SearchResult
|
||||
|
||||
# Import all channels
|
||||
from .web import WebChannel
|
||||
from .github import GitHubChannel
|
||||
from .twitter import TwitterChannel
|
||||
from .youtube import YouTubeChannel
|
||||
from .reddit import RedditChannel
|
||||
from .rss import RSSChannel
|
||||
from .bilibili import BilibiliChannel
|
||||
from .exa_search import ExaSearchChannel
|
||||
|
||||
|
||||
# Channel registry — order matters (first match wins, web is last as fallback)
|
||||
ALL_CHANNELS: List[Channel] = [
|
||||
GitHubChannel(),
|
||||
TwitterChannel(),
|
||||
YouTubeChannel(),
|
||||
RedditChannel(),
|
||||
BilibiliChannel(),
|
||||
RSSChannel(),
|
||||
ExaSearchChannel(),
|
||||
WebChannel(), # Fallback — handles any URL
|
||||
]
|
||||
|
||||
# Search-capable channels
|
||||
SEARCH_CHANNELS: Dict[str, Channel] = {
|
||||
ch.name: ch for ch in ALL_CHANNELS if ch.can_search()
|
||||
}
|
||||
|
||||
|
||||
def get_channel_for_url(url: str) -> Channel:
|
||||
"""Find the right channel for a URL."""
|
||||
for channel in ALL_CHANNELS:
|
||||
if channel.can_handle(url):
|
||||
return channel
|
||||
return WebChannel() # Should never reach here, but just in case
|
||||
|
||||
|
||||
def get_channel(name: str) -> Optional[Channel]:
|
||||
"""Get a channel by name."""
|
||||
for ch in ALL_CHANNELS:
|
||||
if ch.name == name:
|
||||
return ch
|
||||
return None
|
||||
|
||||
|
||||
def get_all_channels() -> List[Channel]:
|
||||
"""Get all registered channels."""
|
||||
return ALL_CHANNELS
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Channel", "ReadResult", "SearchResult",
|
||||
"ALL_CHANNELS", "SEARCH_CHANNELS",
|
||||
"get_channel_for_url", "get_channel", "get_all_channels",
|
||||
]
|
||||
@@ -0,0 +1,140 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Channel base class — the universal interface for all platforms.
|
||||
|
||||
Every channel (YouTube, Twitter, GitHub, etc.) implements this interface.
|
||||
The backend tool can be swapped anytime without changing anything else.
|
||||
|
||||
Example:
|
||||
class YouTubeChannel(Channel):
|
||||
name = "youtube"
|
||||
backends = ["yt-dlp"] # current backend, can be swapped
|
||||
|
||||
async def read(self, url, config):
|
||||
# Just call yt-dlp, return standardized dict
|
||||
...
|
||||
"""
|
||||
|
||||
import shutil
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReadResult:
|
||||
"""Standardized read result. Every channel returns this."""
|
||||
title: str
|
||||
content: str
|
||||
url: str
|
||||
author: str = ""
|
||||
date: str = ""
|
||||
platform: str = ""
|
||||
extra: dict = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.extra = self.extra or {}
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = {
|
||||
"title": self.title,
|
||||
"content": self.content,
|
||||
"url": self.url,
|
||||
"platform": self.platform,
|
||||
}
|
||||
if self.author:
|
||||
d["author"] = self.author
|
||||
if self.date:
|
||||
d["date"] = self.date
|
||||
if self.extra:
|
||||
d["extra"] = self.extra
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Standardized search result."""
|
||||
title: str
|
||||
url: str
|
||||
snippet: str = ""
|
||||
author: str = ""
|
||||
date: str = ""
|
||||
score: float = 0
|
||||
extra: dict = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.extra = self.extra or {}
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = {
|
||||
"title": self.title,
|
||||
"url": self.url,
|
||||
"snippet": self.snippet,
|
||||
}
|
||||
if self.author:
|
||||
d["author"] = self.author
|
||||
if self.date:
|
||||
d["date"] = self.date
|
||||
if self.extra:
|
||||
d["extra"] = self.extra
|
||||
return d
|
||||
|
||||
|
||||
class Channel(ABC):
|
||||
"""
|
||||
Base class for all channels.
|
||||
|
||||
Subclasses just need to implement:
|
||||
- read(url, config) → ReadResult
|
||||
- can_handle(url) → bool
|
||||
- check(config) → (status, message)
|
||||
|
||||
Optionally:
|
||||
- search(query, config, **kwargs) → list[SearchResult]
|
||||
"""
|
||||
|
||||
name: str = "" # e.g. "youtube"
|
||||
description: str = "" # e.g. "YouTube video transcripts"
|
||||
backends: List[str] = [] # e.g. ["yt-dlp"] — what external tool is used
|
||||
requires_config: List[str] = [] # e.g. ["reddit_proxy"]
|
||||
requires_tools: List[str] = [] # e.g. ["yt-dlp"]
|
||||
tier: int = 0 # 0=zero-config, 1=needs free key, 2=needs setup
|
||||
|
||||
@abstractmethod
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
"""Read content from a URL. Must return ReadResult."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def can_handle(self, url: str) -> bool:
|
||||
"""Check if this channel can handle this URL."""
|
||||
...
|
||||
|
||||
def check(self, config=None) -> Tuple[str, str]:
|
||||
"""
|
||||
Check if this channel is available.
|
||||
Returns (status, message) where status is 'ok'/'warn'/'off'/'error'.
|
||||
"""
|
||||
# Check required tools
|
||||
for tool in self.requires_tools:
|
||||
if not shutil.which(tool):
|
||||
return "off", f"Install: pip install {tool}"
|
||||
|
||||
# Check required config
|
||||
for key in self.requires_config:
|
||||
if config and not config.get(key):
|
||||
return "off", f"Need config: {key}. Run: agent-eyes setup"
|
||||
|
||||
return "ok", f"{', '.join(self.backends) if self.backends else 'built-in'}"
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
"""Search this platform. Override if supported."""
|
||||
raise NotImplementedError(f"{self.name} does not support search")
|
||||
|
||||
def can_search(self) -> bool:
|
||||
"""Whether this channel supports search."""
|
||||
try:
|
||||
# Check if search is overridden
|
||||
return type(self).search is not Channel.search
|
||||
except:
|
||||
return False
|
||||
@@ -0,0 +1,77 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Bilibili — via public API (free, no config needed).
|
||||
|
||||
Backend: Bilibili public API
|
||||
Swap to: any Bilibili access method
|
||||
"""
|
||||
|
||||
import requests
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class BilibiliChannel(Channel):
|
||||
name = "bilibili"
|
||||
description = "Bilibili video info and subtitles"
|
||||
backends = ["Bilibili API"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "bilibili.com" in domain or "b23.tv" in domain
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
# Extract BV id from URL
|
||||
path = urlparse(url).path
|
||||
bv_id = ""
|
||||
for part in path.split("/"):
|
||||
if part.startswith("BV"):
|
||||
bv_id = part
|
||||
break
|
||||
|
||||
if not bv_id:
|
||||
# Fallback to Jina Reader
|
||||
from agent_eyes.channels.web import WebChannel
|
||||
return await WebChannel().read(url, config)
|
||||
|
||||
# Get video info
|
||||
resp = requests.get(
|
||||
"https://api.bilibili.com/x/web-interface/view",
|
||||
params={"bvid": bv_id},
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json().get("data", {})
|
||||
|
||||
title = data.get("title", "")
|
||||
desc = data.get("desc", "")
|
||||
author = data.get("owner", {}).get("name", "")
|
||||
|
||||
# Try to get subtitles
|
||||
subtitle_text = ""
|
||||
subtitle_list = data.get("subtitle", {}).get("list", [])
|
||||
if subtitle_list:
|
||||
sub_url = subtitle_list[0].get("subtitle_url", "")
|
||||
if sub_url:
|
||||
if sub_url.startswith("//"):
|
||||
sub_url = "https:" + sub_url
|
||||
sr = requests.get(sub_url, timeout=10)
|
||||
if sr.ok:
|
||||
sub_data = sr.json()
|
||||
lines = [item.get("content", "") for item in sub_data.get("body", [])]
|
||||
subtitle_text = "\n".join(lines)
|
||||
|
||||
content = desc
|
||||
if subtitle_text:
|
||||
content += f"\n\n## Transcript\n{subtitle_text}"
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=content,
|
||||
url=url,
|
||||
author=author,
|
||||
platform="bilibili",
|
||||
extra={"view": data.get("stat", {}).get("view", 0),
|
||||
"like": data.get("stat", {}).get("like", 0)},
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Exa semantic search — the search backbone for Agent Eyes.
|
||||
|
||||
Backend: Exa API (https://exa.ai) — free 1000 searches/month
|
||||
Swap to: Tavily, SerpAPI, or any search API
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from .base import Channel, SearchResult
|
||||
from typing import List
|
||||
|
||||
|
||||
class ExaSearchChannel(Channel):
|
||||
name = "exa_search"
|
||||
description = "Semantic web search (powers Reddit/Twitter search too)"
|
||||
backends = ["Exa API"]
|
||||
requires_config = ["exa_api_key"]
|
||||
tier = 1
|
||||
|
||||
API_URL = "https://api.exa.ai/search"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
return False # Search-only channel, doesn't read URLs
|
||||
|
||||
async def read(self, url: str, config=None) -> None:
|
||||
raise NotImplementedError("Exa is a search engine, not a reader")
|
||||
|
||||
def _get_key(self, config=None) -> str:
|
||||
if config:
|
||||
key = config.get("exa_api_key")
|
||||
if key:
|
||||
return key
|
||||
key = os.environ.get("EXA_API_KEY")
|
||||
if key:
|
||||
return key
|
||||
raise ValueError(
|
||||
"Exa API key not configured.\n"
|
||||
"Get a free key at https://exa.ai (1000 searches/month free)\n"
|
||||
"Then run: agent-eyes setup"
|
||||
)
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
api_key = self._get_key(config)
|
||||
limit = kwargs.get("limit", 5)
|
||||
|
||||
resp = requests.post(
|
||||
self.API_URL,
|
||||
headers={"Content-Type": "application/json", "x-api-key": api_key},
|
||||
json={
|
||||
"query": query,
|
||||
"numResults": min(limit, 10),
|
||||
"type": "auto",
|
||||
"contents": {"text": {"maxCharacters": 500}},
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
results = []
|
||||
for item in resp.json().get("results", []):
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
url=item.get("url", ""),
|
||||
snippet=item.get("text", ""),
|
||||
date=item.get("publishedDate", ""),
|
||||
score=item.get("score", 0),
|
||||
))
|
||||
return results
|
||||
@@ -0,0 +1,120 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GitHub — via GitHub REST API (free, no config needed).
|
||||
|
||||
Backend: GitHub API v3
|
||||
Swap to: gh CLI, or any GitHub API wrapper
|
||||
"""
|
||||
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult, SearchResult
|
||||
from typing import List
|
||||
|
||||
|
||||
class GitHubChannel(Channel):
|
||||
name = "github"
|
||||
description = "GitHub repos, issues, PRs, code"
|
||||
backends = ["GitHub API"]
|
||||
tier = 0
|
||||
|
||||
API = "https://api.github.com"
|
||||
|
||||
def _headers(self, config=None):
|
||||
h = {"Accept": "application/vnd.github+json"}
|
||||
token = config.get("github_token") if config else None
|
||||
if token:
|
||||
h["Authorization"] = f"Bearer {token}"
|
||||
return h
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "github.com" in domain
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
path = urlparse(url).path.strip("/").split("/")
|
||||
|
||||
if len(path) < 2:
|
||||
raise ValueError(f"Invalid GitHub URL: {url}")
|
||||
|
||||
owner, repo = path[0], path[1]
|
||||
headers = self._headers(config)
|
||||
|
||||
# Issues/PRs
|
||||
if len(path) >= 4 and path[2] in ("issues", "pull"):
|
||||
num = path[3]
|
||||
resp = requests.get(f"{self.API}/repos/{owner}/{repo}/issues/{num}", headers=headers, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Get comments
|
||||
comments_text = ""
|
||||
if data.get("comments", 0) > 0:
|
||||
cr = requests.get(f"{self.API}/repos/{owner}/{repo}/issues/{num}/comments",
|
||||
headers=headers, params={"per_page": 20}, timeout=15)
|
||||
if cr.ok:
|
||||
for c in cr.json():
|
||||
comments_text += f"\n\n---\n**{c.get('user', {}).get('login', '')}** ({c.get('created_at', '')}):\n{c.get('body', '')}"
|
||||
|
||||
return ReadResult(
|
||||
title=data.get("title", ""),
|
||||
content=(data.get("body", "") or "") + comments_text,
|
||||
url=url,
|
||||
author=data.get("user", {}).get("login", ""),
|
||||
date=data.get("created_at", ""),
|
||||
platform="github",
|
||||
extra={"state": data.get("state"), "comments": data.get("comments", 0),
|
||||
"reactions": data.get("reactions", {}).get("total_count", 0)},
|
||||
)
|
||||
|
||||
# Repo
|
||||
resp = requests.get(f"{self.API}/repos/{owner}/{repo}", headers=headers, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Get README
|
||||
readme_text = ""
|
||||
rr = requests.get(f"{self.API}/repos/{owner}/{repo}/readme", headers=headers, timeout=15)
|
||||
if rr.ok:
|
||||
import base64
|
||||
readme_data = rr.json()
|
||||
if readme_data.get("encoding") == "base64":
|
||||
readme_text = base64.b64decode(readme_data["content"]).decode("utf-8", errors="replace")
|
||||
|
||||
return ReadResult(
|
||||
title=f"{owner}/{repo}",
|
||||
content=readme_text or data.get("description", ""),
|
||||
url=url,
|
||||
author=owner,
|
||||
platform="github",
|
||||
extra={"stars": data.get("stargazers_count", 0), "forks": data.get("forks_count", 0),
|
||||
"language": data.get("language", ""), "description": data.get("description", "")},
|
||||
)
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
language = kwargs.get("language")
|
||||
limit = kwargs.get("limit", 5)
|
||||
|
||||
q = query
|
||||
if language:
|
||||
q += f" language:{language}"
|
||||
|
||||
resp = requests.get(
|
||||
f"{self.API}/search/repositories",
|
||||
headers=self._headers(config),
|
||||
params={"q": q, "sort": "stars", "per_page": min(limit, 30)},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
results = []
|
||||
for repo in resp.json().get("items", []):
|
||||
results.append(SearchResult(
|
||||
title=repo.get("full_name", ""),
|
||||
url=repo.get("html_url", ""),
|
||||
snippet=repo.get("description", ""),
|
||||
date=repo.get("updated_at", ""),
|
||||
extra={"stars": repo.get("stargazers_count", 0),
|
||||
"forks": repo.get("forks_count", 0),
|
||||
"language": repo.get("language", "")},
|
||||
))
|
||||
return results
|
||||
@@ -0,0 +1,96 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Reddit — via Reddit JSON API + optional proxy.
|
||||
|
||||
Backend: Reddit public JSON API (append .json to any URL)
|
||||
Swap to: any Reddit access method
|
||||
"""
|
||||
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class RedditChannel(Channel):
|
||||
name = "reddit"
|
||||
description = "Reddit posts and comments"
|
||||
backends = ["Reddit JSON API"]
|
||||
requires_config = ["reddit_proxy"]
|
||||
tier = 2
|
||||
|
||||
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "reddit.com" in domain or "redd.it" in domain
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
proxy = config.get("reddit_proxy") if config else None
|
||||
proxies = {"http": proxy, "https": proxy} if proxy else None
|
||||
|
||||
# Ensure URL ends with .json
|
||||
json_url = url.rstrip("/")
|
||||
if not json_url.endswith(".json"):
|
||||
json_url += ".json"
|
||||
|
||||
resp = requests.get(
|
||||
json_url,
|
||||
headers={"User-Agent": self.USER_AGENT},
|
||||
proxies=proxies,
|
||||
params={"limit": 50},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if isinstance(data, list) and len(data) >= 1:
|
||||
# Post page: [post_listing, comments_listing]
|
||||
post = data[0]["data"]["children"][0]["data"]
|
||||
title = post.get("title", "")
|
||||
author = post.get("author", "")
|
||||
selftext = post.get("selftext", "")
|
||||
score = post.get("score", 0)
|
||||
subreddit = post.get("subreddit", "")
|
||||
|
||||
# Extract comments
|
||||
comments_text = ""
|
||||
if len(data) >= 2:
|
||||
comments_text = self._extract_comments(data[1])
|
||||
|
||||
content = selftext
|
||||
if comments_text:
|
||||
content += f"\n\n---\n## Comments\n{comments_text}"
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=content,
|
||||
url=url,
|
||||
author=f"u/{author}",
|
||||
platform="reddit",
|
||||
extra={"subreddit": subreddit, "score": score},
|
||||
)
|
||||
|
||||
raise ValueError(f"Could not parse Reddit response for: {url}")
|
||||
|
||||
def _extract_comments(self, comments_data: dict, depth: int = 0, max_depth: int = 3) -> str:
|
||||
"""Recursively extract comments."""
|
||||
lines = []
|
||||
children = comments_data.get("data", {}).get("children", [])
|
||||
|
||||
for child in children:
|
||||
if child.get("kind") != "t1":
|
||||
continue
|
||||
data = child.get("data", {})
|
||||
author = data.get("author", "[deleted]")
|
||||
body = data.get("body", "")
|
||||
score = data.get("score", 0)
|
||||
indent = " " * depth
|
||||
|
||||
lines.append(f"{indent}**u/{author}** ({score} points):")
|
||||
lines.append(f"{indent}{body}")
|
||||
lines.append("")
|
||||
|
||||
# Recurse into replies
|
||||
if depth < max_depth and data.get("replies") and isinstance(data["replies"], dict):
|
||||
lines.append(self._extract_comments(data["replies"], depth + 1, max_depth))
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,57 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""RSS feeds — via feedparser (free, pip dependency).
|
||||
|
||||
Backend: feedparser (https://github.com/kurtmckee/feedparser)
|
||||
Swap to: any RSS parser
|
||||
"""
|
||||
|
||||
import feedparser
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class RSSChannel(Channel):
|
||||
name = "rss"
|
||||
description = "RSS and Atom feeds"
|
||||
backends = ["feedparser"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
lower = url.lower()
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return (lower.endswith(".xml") or "/rss" in lower or "/feed" in lower
|
||||
or "/atom" in lower or "rss" in domain)
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
feed = feedparser.parse(url)
|
||||
|
||||
if feed.bozo and not feed.entries:
|
||||
raise ValueError(f"Failed to parse RSS feed: {url}")
|
||||
|
||||
if not feed.entries:
|
||||
raise ValueError(f"No entries in RSS feed: {url}")
|
||||
|
||||
# Return latest entry
|
||||
entry = feed.entries[0]
|
||||
content = entry.get("summary", "") or entry.get("description", "")
|
||||
|
||||
# If multiple entries, summarize all
|
||||
if len(feed.entries) > 1:
|
||||
lines = [f"# {feed.feed.get('title', 'RSS Feed')}\n"]
|
||||
for i, e in enumerate(feed.entries[:20], 1):
|
||||
title = e.get("title", "Untitled")
|
||||
link = e.get("link", "")
|
||||
summary = e.get("summary", "")[:200]
|
||||
lines.append(f"## {i}. {title}")
|
||||
lines.append(f"🔗 {link}")
|
||||
if summary:
|
||||
lines.append(summary)
|
||||
lines.append("")
|
||||
content = "\n".join(lines)
|
||||
|
||||
return ReadResult(
|
||||
title=feed.feed.get("title", entry.get("title", url)),
|
||||
content=content,
|
||||
url=url,
|
||||
platform="rss",
|
||||
)
|
||||
@@ -0,0 +1,149 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Twitter/X — via birdx CLI (free) or Jina Reader fallback.
|
||||
|
||||
Backend: birdx (https://github.com/runesleo/birdx) for search/timeline
|
||||
Jina Reader for single tweets
|
||||
Swap to: any Twitter access tool
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from urllib.parse import urlparse
|
||||
from .base import Channel, ReadResult, SearchResult
|
||||
from typing import List
|
||||
import requests
|
||||
|
||||
|
||||
class TwitterChannel(Channel):
|
||||
name = "twitter"
|
||||
description = "Twitter/X posts, search, timelines"
|
||||
backends = ["birdx", "Jina Reader"]
|
||||
tier = 0 # Single tweet reading is zero-config
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "x.com" in domain or "twitter.com" in domain
|
||||
|
||||
def check(self, config=None):
|
||||
# Basic reading always works (Jina fallback)
|
||||
if shutil.which("birdx"):
|
||||
return "ok", "birdx (full: search + timeline + threads)"
|
||||
return "ok", "Jina Reader (single tweets only)"
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
# Try birdx first
|
||||
if shutil.which("birdx"):
|
||||
return await self._read_birdx(url)
|
||||
# Fallback: Jina Reader
|
||||
return await self._read_jina(url)
|
||||
|
||||
async def _read_birdx(self, url: str) -> ReadResult:
|
||||
result = subprocess.run(
|
||||
["birdx", "read", url],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return await self._read_jina(url)
|
||||
|
||||
text = result.stdout.strip()
|
||||
# Extract author from first line
|
||||
author = ""
|
||||
lines = text.split("\n")
|
||||
if lines and lines[0].startswith("@"):
|
||||
author = lines[0].split()[0]
|
||||
|
||||
return ReadResult(
|
||||
title=text[:100],
|
||||
content=text,
|
||||
url=url,
|
||||
author=author,
|
||||
platform="twitter",
|
||||
)
|
||||
|
||||
async def _read_jina(self, url: str) -> ReadResult:
|
||||
resp = requests.get(
|
||||
f"https://r.jina.ai/{url}",
|
||||
headers={"Accept": "text/markdown"},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.text
|
||||
title = text[:100] if text else url
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=text,
|
||||
url=url,
|
||||
platform="twitter",
|
||||
)
|
||||
|
||||
async def search(self, query: str, config=None, **kwargs) -> List[SearchResult]:
|
||||
limit = kwargs.get("limit", 10)
|
||||
|
||||
if shutil.which("birdx"):
|
||||
return await self._search_birdx(query, limit)
|
||||
|
||||
# Fallback to Exa
|
||||
return await self._search_exa(query, limit, config)
|
||||
|
||||
async def _search_birdx(self, query: str, limit: int) -> List[SearchResult]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["birdx", "search", query, "-n", str(limit)],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return []
|
||||
|
||||
return self._parse_birdx_output(result.stdout)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return []
|
||||
|
||||
def _parse_birdx_output(self, text: str) -> List[SearchResult]:
|
||||
"""Parse birdx text output into SearchResults."""
|
||||
results = []
|
||||
current = {}
|
||||
text_lines = []
|
||||
|
||||
for line in text.strip().split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("─"):
|
||||
if current:
|
||||
current["text"] = "\n".join(text_lines).strip()
|
||||
results.append(SearchResult(
|
||||
title=current.get("text", "")[:80],
|
||||
url=current.get("url", ""),
|
||||
snippet=current.get("text", ""),
|
||||
author=current.get("author", ""),
|
||||
date=current.get("date", ""),
|
||||
))
|
||||
current = {}
|
||||
text_lines = []
|
||||
continue
|
||||
if line.startswith("@") and line.endswith(":") and "(" in line:
|
||||
current["author"] = line.split()[0]
|
||||
continue
|
||||
if line.startswith("date:"):
|
||||
current["date"] = line[5:].strip()
|
||||
continue
|
||||
if line.startswith("url:"):
|
||||
current["url"] = line[4:].strip()
|
||||
continue
|
||||
if current is not None:
|
||||
text_lines.append(line)
|
||||
|
||||
if current and text_lines:
|
||||
current["text"] = "\n".join(text_lines).strip()
|
||||
results.append(SearchResult(
|
||||
title=current.get("text", "")[:80],
|
||||
url=current.get("url", ""),
|
||||
snippet=current.get("text", ""),
|
||||
author=current.get("author", ""),
|
||||
date=current.get("date", ""),
|
||||
))
|
||||
return results
|
||||
|
||||
async def _search_exa(self, query: str, limit: int, config=None) -> List[SearchResult]:
|
||||
from agent_eyes.channels.exa_search import ExaSearchChannel
|
||||
exa = ExaSearchChannel()
|
||||
return await exa.search(f"site:x.com {query}", config=config, limit=limit)
|
||||
@@ -0,0 +1,49 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Web pages — via Jina Reader API (free, no config needed).
|
||||
|
||||
Backend: Jina Reader (https://r.jina.ai)
|
||||
Swap to: Firecrawl, Trafilatura, or any other reader API
|
||||
"""
|
||||
|
||||
import requests
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class WebChannel(Channel):
|
||||
name = "web"
|
||||
description = "Web pages (any URL)"
|
||||
backends = ["Jina Reader API"]
|
||||
tier = 0
|
||||
|
||||
JINA_URL = "https://r.jina.ai/"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
# Fallback — handles any URL not matched by other channels
|
||||
return True
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
resp = requests.get(
|
||||
f"{self.JINA_URL}{url}",
|
||||
headers={"Accept": "text/markdown"},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.text
|
||||
|
||||
# Extract title from first markdown heading
|
||||
title = url
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("# "):
|
||||
title = line[2:].strip()
|
||||
break
|
||||
if line.startswith("Title:"):
|
||||
title = line[6:].strip()
|
||||
break
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=text,
|
||||
url=url,
|
||||
platform="web",
|
||||
)
|
||||
@@ -0,0 +1,94 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""YouTube — via yt-dlp (free, pip install yt-dlp).
|
||||
|
||||
Backend: yt-dlp (https://github.com/yt-dlp/yt-dlp)
|
||||
Swap to: any YouTube subtitle extractor
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from .base import Channel, ReadResult
|
||||
|
||||
|
||||
class YouTubeChannel(Channel):
|
||||
name = "youtube"
|
||||
description = "YouTube video transcripts"
|
||||
backends = ["yt-dlp"]
|
||||
requires_tools = ["yt-dlp"]
|
||||
tier = 0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
domain = urlparse(url).netloc.lower()
|
||||
return "youtube.com" in domain or "youtu.be" in domain
|
||||
|
||||
async def read(self, url: str, config=None) -> ReadResult:
|
||||
if not shutil.which("yt-dlp"):
|
||||
raise RuntimeError("yt-dlp not installed. Install: pip install yt-dlp")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Get video info
|
||||
info = self._get_info(url)
|
||||
title = info.get("title", url)
|
||||
author = info.get("uploader", "")
|
||||
|
||||
# Try to get subtitles
|
||||
transcript = self._get_subtitles(url, tmpdir)
|
||||
|
||||
if not transcript:
|
||||
transcript = f"[Video: {title}]\n[No subtitles available. Use Groq Whisper for transcription.]"
|
||||
|
||||
return ReadResult(
|
||||
title=title,
|
||||
content=transcript,
|
||||
url=url,
|
||||
author=author,
|
||||
platform="youtube",
|
||||
extra={
|
||||
"duration": info.get("duration"),
|
||||
"view_count": info.get("view_count"),
|
||||
"upload_date": info.get("upload_date"),
|
||||
},
|
||||
)
|
||||
|
||||
def _get_info(self, url: str) -> dict:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["yt-dlp", "--dump-json", "--no-download", url],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return json.loads(result.stdout)
|
||||
except (subprocess.TimeoutExpired, json.JSONDecodeError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _get_subtitles(self, url: str, tmpdir: str) -> str:
|
||||
"""Extract subtitles using yt-dlp."""
|
||||
try:
|
||||
subprocess.run(
|
||||
["yt-dlp", "--write-auto-sub", "--write-sub",
|
||||
"--sub-lang", "en,zh-Hans,zh",
|
||||
"--skip-download", "--sub-format", "vtt",
|
||||
"-o", f"{tmpdir}/%(id)s.%(ext)s", url],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
|
||||
# Find and read subtitle file
|
||||
for f in Path(tmpdir).glob("*.vtt"):
|
||||
text = f.read_text(errors="replace")
|
||||
# Strip VTT headers and timestamps
|
||||
lines = []
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("WEBVTT") or "-->" in line or line.isdigit():
|
||||
continue
|
||||
if line not in lines[-1:]: # deduplicate
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
return ""
|
||||
Reference in New Issue
Block a user