Panniantong
74c3df5c3d
BREAKING: Complete architectural rewrite.
Before: Copied x-reader's fetcher code into readers/ (1205 lines of borrowed code)
After: Pluggable channel system where each channel is a thin wrapper (~50 lines)
around the best external tool for that platform. Zero copied code.
Architecture:
- channels/base.py — Universal Channel interface (read, search, check)
- channels/web.py — Jina Reader API (swappable)
- channels/github.py — GitHub API (swappable)
- channels/twitter.py — birdx + Jina fallback (swappable)
- channels/youtube.py — yt-dlp (swappable)
- channels/reddit.py — Reddit JSON API + proxy (swappable)
- channels/rss.py — feedparser (swappable)
- channels/bilibili.py — Bilibili API (swappable)
- channels/exa_search.py — Exa semantic search (swappable)
Key design: every backend can be swapped by changing ONE file.
YouTube dies? Change youtube.py. Exa sucks? Swap exa_search.py for Tavily.
Nothing else changes.
Removed: reader.py, schema.py, readers/, search/, utils/ (all x-reader code)
Tests: 36/36 passing
50 lines
1.3 KiB
Python
50 lines
1.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Web pages — via Jina Reader API (free, no config needed).
|
|
|
|
Backend: Jina Reader (https://r.jina.ai)
|
|
Swap to: Firecrawl, Trafilatura, or any other reader API
|
|
"""
|
|
|
|
import requests
|
|
from .base import Channel, ReadResult
|
|
|
|
|
|
class WebChannel(Channel):
|
|
name = "web"
|
|
description = "Web pages (any URL)"
|
|
backends = ["Jina Reader API"]
|
|
tier = 0
|
|
|
|
JINA_URL = "https://r.jina.ai/"
|
|
|
|
def can_handle(self, url: str) -> bool:
|
|
# Fallback — handles any URL not matched by other channels
|
|
return True
|
|
|
|
async def read(self, url: str, config=None) -> ReadResult:
|
|
resp = requests.get(
|
|
f"{self.JINA_URL}{url}",
|
|
headers={"Accept": "text/markdown"},
|
|
timeout=15,
|
|
)
|
|
resp.raise_for_status()
|
|
text = resp.text
|
|
|
|
# Extract title from first markdown heading
|
|
title = url
|
|
for line in text.split("\n"):
|
|
line = line.strip()
|
|
if line.startswith("# "):
|
|
title = line[2:].strip()
|
|
break
|
|
if line.startswith("Title:"):
|
|
title = line[6:].strip()
|
|
break
|
|
|
|
return ReadResult(
|
|
title=title,
|
|
content=text,
|
|
url=url,
|
|
platform="web",
|
|
)
|