feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources
v3 rewrites the search engine from the ground up: - Intelligent pre-research: resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching - GitHub person-mode: PR velocity, top repos by stars, release notes - GitHub project-mode: live star counts, README, releases, top issues - ELI5 mode: plain language synthesis, no jargon - 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, Threads, Pinterest, Perplexity, Bluesky, Web - Free Reddit comments via public JSON (no API key needed) - Fun judge v2: humor scoring baked into narrative - Cookie consent before browser scanning - 10,000 free ScrapeCreators calls - 1,012 tests Thank you to the community contributors whose issues and PRs shaped v3: @uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116), @thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78), @zl190 (#115), @hnshah (#84, #85, #86) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""E2E comparison: run sample queries on both v3 (current branch) and v2.9.5 (main).
|
||||
|
||||
Usage:
|
||||
python3 tests/e2e_comparison.py [--v2-script PATH]
|
||||
|
||||
Outputs a markdown comparison table with per-query metrics.
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
V3_SCRIPT = str(REPO / "scripts" / "last30days.py")
|
||||
|
||||
# v2.9.5 from plugin cache (main branch equivalent)
|
||||
V2_SCRIPT = str(
|
||||
Path.home()
|
||||
/ ".claude/plugins/cache/last30days/last30days/2.9.5/scripts/last30days.py"
|
||||
)
|
||||
|
||||
EVAL_TOPICS_FILE = REPO / "fixtures" / "eval_topics.json"
|
||||
|
||||
|
||||
def _load_queries() -> list[tuple[str, str]]:
|
||||
if EVAL_TOPICS_FILE.exists():
|
||||
rows = json.loads(EVAL_TOPICS_FILE.read_text())
|
||||
return [(row["topic"], row["query_type"]) for row in rows]
|
||||
return [
|
||||
("openclaw vs nanoclaw vs ironclaw", "comparison"),
|
||||
("how to deploy on Fly.io", "how_to"),
|
||||
("kanye west", "breaking_news"),
|
||||
("odds of recession", "prediction"),
|
||||
("explain transformer architecture", "concept"),
|
||||
]
|
||||
|
||||
|
||||
QUERIES = _load_queries()
|
||||
|
||||
|
||||
def run_query(script: str, topic: str, timeout: int = 180) -> dict:
|
||||
"""Run a query and return parsed JSON + timing."""
|
||||
start = time.time()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, script, topic, "--emit=json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
elapsed = time.time() - start
|
||||
if result.returncode != 0:
|
||||
return {
|
||||
"error": result.stderr[:200],
|
||||
"elapsed": elapsed,
|
||||
"sources": 0,
|
||||
"candidates": 0,
|
||||
"intent": "error",
|
||||
"subqueries": 0,
|
||||
}
|
||||
data = json.loads(result.stdout)
|
||||
|
||||
# v3 shape
|
||||
if "query_plan" in data:
|
||||
items_by_source = data.get("items_by_source", {})
|
||||
return {
|
||||
"elapsed": elapsed,
|
||||
"sources": sum(1 for v in items_by_source.values() if v),
|
||||
"total_items": sum(len(v) for v in items_by_source.values()),
|
||||
"candidates": len(data.get("ranked_candidates", [])),
|
||||
"clusters": len(data.get("clusters", [])),
|
||||
"intent": data["query_plan"].get("intent", "?"),
|
||||
"subqueries": len(data["query_plan"].get("subqueries", [])),
|
||||
"errors": list(data.get("errors_by_source", {}).keys()),
|
||||
}
|
||||
|
||||
# v2 shape
|
||||
sources_with_items = 0
|
||||
total_items = 0
|
||||
for key in ["reddit", "x", "youtube", "tiktok", "instagram", "hackernews",
|
||||
"bluesky", "truthsocial", "polymarket", "web"]:
|
||||
items = data.get(key, [])
|
||||
if items:
|
||||
sources_with_items += 1
|
||||
total_items += len(items)
|
||||
return {
|
||||
"elapsed": elapsed,
|
||||
"sources": sources_with_items,
|
||||
"total_items": total_items,
|
||||
"candidates": total_items,
|
||||
"clusters": 0,
|
||||
"intent": data.get("mode", "?"),
|
||||
"subqueries": 0,
|
||||
"errors": [k for k in ["reddit_error", "x_error", "youtube_error",
|
||||
"tiktok_error", "instagram_error"]
|
||||
if data.get(k)],
|
||||
}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {
|
||||
"error": "timeout",
|
||||
"elapsed": timeout,
|
||||
"sources": 0,
|
||||
"candidates": 0,
|
||||
"intent": "timeout",
|
||||
"subqueries": 0,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"error": str(exc)[:200],
|
||||
"elapsed": time.time() - start,
|
||||
"sources": 0,
|
||||
"candidates": 0,
|
||||
"intent": "error",
|
||||
"subqueries": 0,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
v2_script = V2_SCRIPT
|
||||
if len(sys.argv) > 2 and sys.argv[1] == "--v2-script":
|
||||
v2_script = sys.argv[2]
|
||||
|
||||
if not Path(v2_script).exists():
|
||||
print(f"v2 script not found at {v2_script}", file=sys.stderr)
|
||||
print("Use --v2-script PATH to specify", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("# E2E Comparison: v3.0.0 (branch) vs v2.9.5 (main)")
|
||||
print()
|
||||
print(f"- v3 script: {V3_SCRIPT}")
|
||||
print(f"- v2 script: {v2_script}")
|
||||
print(f"- Queries: {len(QUERIES)}")
|
||||
print()
|
||||
|
||||
results = []
|
||||
for i, (topic, expected_intent) in enumerate(QUERIES, 1):
|
||||
print(f"[{i}/{len(QUERIES)}] {topic}", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
|
||||
print(f" v3...", end="", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
v3 = run_query(V3_SCRIPT, topic)
|
||||
print(f" {v3.get('elapsed', 0):.1f}s", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
|
||||
print(f" v2...", end="", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
v2 = run_query(v2_script, topic)
|
||||
print(f" {v2.get('elapsed', 0):.1f}s", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
|
||||
results.append({
|
||||
"topic": topic,
|
||||
"expected_intent": expected_intent,
|
||||
"v3": v3,
|
||||
"v2": v2,
|
||||
})
|
||||
|
||||
# Print comparison table
|
||||
print("| Query | Intent | v3 sources | v2 sources | v3 items | v2 items | v3 time | v2 time | v3 errors | v2 errors |")
|
||||
print("|-------|--------|-----------|-----------|---------|---------|---------|---------|-----------|-----------|")
|
||||
for r in results:
|
||||
v3, v2 = r["v3"], r["v2"]
|
||||
v3_err = ", ".join(v3.get("errors", [])) or "-"
|
||||
v2_err = ", ".join(v2.get("errors", [])) or "-"
|
||||
print(
|
||||
f"| {r['topic'][:45]} | {v3.get('intent', '?')} | "
|
||||
f"{v3.get('sources', 0)} | {v2.get('sources', 0)} | "
|
||||
f"{v3.get('total_items', 0)} | {v2.get('total_items', 0)} | "
|
||||
f"{v3.get('elapsed', 0):.1f}s | {v2.get('elapsed', 0):.1f}s | "
|
||||
f"{v3_err} | {v2_err} |"
|
||||
)
|
||||
|
||||
# Summary
|
||||
print()
|
||||
v3_total_sources = sum(r["v3"].get("sources", 0) for r in results)
|
||||
v2_total_sources = sum(r["v2"].get("sources", 0) for r in results)
|
||||
v3_total_items = sum(r["v3"].get("total_items", 0) for r in results)
|
||||
v2_total_items = sum(r["v2"].get("total_items", 0) for r in results)
|
||||
v3_total_time = sum(r["v3"].get("elapsed", 0) for r in results)
|
||||
v2_total_time = sum(r["v2"].get("elapsed", 0) for r in results)
|
||||
v3_errors = sum(len(r["v3"].get("errors", [])) for r in results)
|
||||
v2_errors = sum(len(r["v2"].get("errors", [])) for r in results)
|
||||
|
||||
print("## Summary")
|
||||
print()
|
||||
print(f"| Metric | v3.0.0 | v2.9.5 | Delta |")
|
||||
print(f"|--------|--------|--------|-------|")
|
||||
print(f"| Total sources with items | {v3_total_sources} | {v2_total_sources} | {v3_total_sources - v2_total_sources:+d} |")
|
||||
print(f"| Total items retrieved | {v3_total_items} | {v2_total_items} | {v3_total_items - v2_total_items:+d} |")
|
||||
print(f"| Total wall time | {v3_total_time:.1f}s | {v2_total_time:.1f}s | {v3_total_time - v2_total_time:+.1f}s |")
|
||||
print(f"| Source errors | {v3_errors} | {v2_errors} | {v3_errors - v2_errors:+d} |")
|
||||
print(f"| Avg sources/query | {v3_total_sources/len(results):.1f} | {v2_total_sources/len(results):.1f} | |")
|
||||
print(f"| Avg items/query | {v3_total_items/len(results):.1f} | {v2_total_items/len(results):.1f} | |")
|
||||
print(f"| Avg time/query | {v3_total_time/len(results):.1f}s | {v2_total_time/len(results):.1f}s | |")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user