fix: import paths (fetchers→readers), schema field mismatch, RSS detection
Bugs found during fresh pip install testing: - readers/*.py still referenced agent_eyes.fetchers → fixed to agent_eyes.readers - reader.py passed 'author'/'metadata' to UnifiedContent which doesn't have those → use 'extra' field - RSS URL detection missed domains containing 'rss' (e.g. hnrss.org)
This commit is contained in:
@@ -15,7 +15,7 @@ import requests
|
||||
from loguru import logger
|
||||
from typing import Dict, Any
|
||||
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
from agent_eyes.readers.jina import fetch_via_jina
|
||||
|
||||
|
||||
OEMBED_URL = "https://publish.twitter.com/oembed"
|
||||
@@ -75,7 +75,7 @@ async def _fetch_via_playwright(url: str) -> Dict[str, Any]:
|
||||
" playwright install chromium"
|
||||
)
|
||||
|
||||
from agent_eyes.fetchers.browser import get_session_path
|
||||
from agent_eyes.readers.browser import get_session_path
|
||||
from pathlib import Path
|
||||
|
||||
session_path = get_session_path("twitter")
|
||||
|
||||
@@ -23,7 +23,7 @@ async def fetch_wechat(url: str) -> Dict[str, Any]:
|
||||
# Tier 1: Jina Reader
|
||||
try:
|
||||
logger.info(f"[WeChat] Tier 1 — Jina: {url}")
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
from agent_eyes.readers.jina import fetch_via_jina
|
||||
|
||||
data = fetch_via_jina(url)
|
||||
if data.get("content"):
|
||||
@@ -41,7 +41,7 @@ async def fetch_wechat(url: str) -> Dict[str, Any]:
|
||||
# Tier 2: Playwright headless (no session needed)
|
||||
try:
|
||||
logger.info(f"[WeChat] Tier 2 — Playwright headless: {url}")
|
||||
from agent_eyes.fetchers.browser import fetch_via_browser
|
||||
from agent_eyes.readers.browser import fetch_via_browser
|
||||
|
||||
data = await fetch_via_browser(url)
|
||||
return {
|
||||
|
||||
@@ -13,7 +13,7 @@ from loguru import logger
|
||||
from typing import Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
from agent_eyes.readers.jina import fetch_via_jina
|
||||
|
||||
|
||||
async def fetch_xhs(url: str) -> Dict[str, Any]:
|
||||
@@ -43,7 +43,7 @@ async def fetch_xhs(url: str) -> Dict[str, Any]:
|
||||
logger.warning(f"[XHS] Jina failed ({e}), falling back to browser")
|
||||
|
||||
# Tier 2: Playwright with session
|
||||
from agent_eyes.fetchers.browser import get_session_path, SESSION_DIR
|
||||
from agent_eyes.readers.browser import get_session_path, SESSION_DIR
|
||||
|
||||
session_path = get_session_path("xhs")
|
||||
if not Path(session_path).exists():
|
||||
@@ -56,7 +56,7 @@ async def fetch_xhs(url: str) -> Dict[str, Any]:
|
||||
|
||||
try:
|
||||
logger.info(f"[XHS] Tier 2 — Playwright with session: {url}")
|
||||
from agent_eyes.fetchers.browser import fetch_via_browser
|
||||
from agent_eyes.readers.browser import fetch_via_browser
|
||||
|
||||
data = await fetch_via_browser(url, storage_state=session_path)
|
||||
return {
|
||||
|
||||
@@ -17,7 +17,7 @@ import tempfile
|
||||
from loguru import logger
|
||||
from typing import Dict, Any
|
||||
|
||||
from agent_eyes.fetchers.jina import fetch_via_jina
|
||||
from agent_eyes.readers.jina import fetch_via_jina
|
||||
|
||||
|
||||
def _extract_video_id(url: str) -> str:
|
||||
|
||||
Reference in New Issue
Block a user