Agent Eyes v1.0.0 — search + read the entire internet

Based on x-reader by @runes_leo (MIT License). Extended with: - Reddit support (posts + comments, proxy support) - GitHub support (repos, issues, PRs) - Web search via Exa semantic search - Reddit search (bypasses IP blocks via Exa) - GitHub search (repos by stars) - Renamed package: x_reader → agent_eyes - New MCP tools: search, search_reddit, search_github - Agent-first positioning and documentation
2026-02-24 03:07:50 +01:00
parent ee2ad83b12
commit 3a3a0101cf
23 changed files with 659 additions and 231 deletions
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+"""
+Playwright browser fetcher — headless Chromium fallback for anti-scraping sites.
+
+Used when Jina Reader fails (403/451/timeout). Supports persistent login
+sessions via Playwright's storage_state for platforms requiring authentication.
+
+Install: pip install "x-reader[browser]" && playwright install chromium
+"""
+
+from loguru import logger
+from pathlib import Path
+
+SESSION_DIR = Path.home() / ".x-reader" / "sessions"
+TIMEOUT_MS = 30_000
+
+
+async def fetch_via_browser(url: str, storage_state: str = None) -> dict:
+    """
+    Fetch a URL using headless Chromium via Playwright.
+
+    Args:
+        url: Target URL to fetch.
+        storage_state: Path to a Playwright storage state JSON file (cookies/localStorage).
+                       If provided, the browser context will load this session.
+
+    Returns:
+        dict with keys: title, content, url, author
+    """
+    try:
+        from playwright.async_api import async_playwright
+    except ImportError:
+        raise RuntimeError(
+            "Playwright is not installed. Run:\n"
+            '  pip install "x-reader[browser]"\n'
+            "  playwright install chromium"
+        )
+
+    logger.info(f"Browser fetch: {url}")
+
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+
+        context_kwargs = {}
+        if storage_state and Path(storage_state).exists():
+            context_kwargs["storage_state"] = storage_state
+            logger.info(f"Using session: {storage_state}")
+
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                       "AppleWebKit/537.36 (KHTML, like Gecko) "
+                       "Chrome/120.0.0.0 Safari/537.36",
+            **context_kwargs,
+        )
+        page = await context.new_page()
+
+        try:
+            await page.goto(url, wait_until="domcontentloaded", timeout=TIMEOUT_MS)
+            # Extra wait for JS-heavy pages
+            await page.wait_for_timeout(2000)
+
+            title = await page.title()
+            # Extract main text content, stripping scripts/styles
+            content = await page.evaluate("""() => {
+                const el = document.querySelector('article')
+                    || document.querySelector('main')
+                    || document.querySelector('.content')
+                    || document.body;
+                return el ? el.innerText : '';
+            }""")
+
+            result = {
+                "title": (title or "").strip()[:200],
+                "content": (content or "").strip(),
+                "url": url,
+                "author": "",
+            }
+            logger.info(f"Browser fetch OK: {title[:60]}")
+            return result
+
+        finally:
+            await context.close()
+            await browser.close()
+
+
+def get_session_path(platform: str) -> str:
+    """Get the session file path for a platform."""
+    return str(SESSION_DIR / f"{platform}.json")