Restructure as Codex plugin

2026-04-23 20:14:23 -04:00
parent 1f7e85a03f
commit 72495c1c14
156 changed files with 218 additions and 140 deletions
@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Run the v3 verification bundle for last30days."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+SKILL_ROOT = Path(__file__).resolve().parents[1]
+REPO_ROOT = Path(__file__).resolve().parents[3]
+PYTHON = sys.executable
+ENGINE = SKILL_ROOT / "scripts" / "last30days.py"
+EVALUATOR = SKILL_ROOT / "scripts" / "evaluate_search_quality.py"
+
+SMOKE_TOPIC = "openclaw skills"
+SMOKE_CASES = [
+    ("gemini", ["--quick", "--search=grounding,hackernews"]),
+    ("openai", ["--quick", "--search=reddit,hackernews"]),
+    ("xai", ["--quick", "--search=reddit,hackernews"]),
+    ("auto", ["--quick", "--search=reddit,grounding,hackernews"]),
+]
+
+LATENCY_TOPICS = [
+    "openclaw skills",
+    "codex vs claude code",
+    "anthropic odds",
+]
+LATENCY_PROFILES = [
+    ("quick", ["--quick", "--search=grounding,hackernews"]),
+    ("default", ["--search=grounding,hackernews"]),
+    ("deep", ["--deep", "--search=grounding,hackernews"]),
+]
+
+
+def run_command(cmd: list[str], *, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        cmd,
+        cwd=REPO_ROOT,
+        env=env,
+        text=True,
+        capture_output=True,
+        timeout=timeout,
+        check=True,
+    )
+
+
+def verify_unit() -> dict[str, str]:
+    run_command([PYTHON, "-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py"], timeout=600)
+    run_command(
+        [
+            PYTHON,
+            "-m",
+            "py_compile",
+            *subprocess.run(
+                [
+                    "rg",
+                    "--files",
+                    "skills/last30days/scripts",
+                    "tests",
+                    "-g",
+                    "*.py",
+                    "-g",
+                    "!skills/last30days/scripts/lib/vendor/**",
+                ],
+                cwd=REPO_ROOT,
+                text=True,
+                capture_output=True,
+                check=True,
+            ).stdout.split(),
+        ],
+        timeout=600,
+    )
+    return {"status": "ok"}
+
+
+def verify_diagnose() -> dict[str, object]:
+    result = run_command([PYTHON, str(ENGINE), "--diagnose"], timeout=120)
+    return json.loads(result.stdout)
+
+
+def verify_smoke() -> list[dict[str, object]]:
+    rows: list[dict[str, object]] = []
+    for provider, extra in SMOKE_CASES:
+        env = os.environ.copy()
+        env["LAST30DAYS_REASONING_PROVIDER"] = provider
+        start = time.time()
+        result = run_command(
+            [PYTHON, str(ENGINE), SMOKE_TOPIC, "--emit=json", *extra],
+            env=env,
+            timeout=240,
+        )
+        duration = round(time.time() - start, 2)
+        report = json.loads(result.stdout)
+        rows.append(
+            {
+                "provider": provider,
+                "duration_seconds": duration,
+                "reasoning_provider": (report.get("provider_runtime") or {}).get("reasoning_provider"),
+                "cluster_count": len(report.get("clusters") or []),
+                "candidate_count": len(report.get("ranked_candidates") or []),
+                "error_sources": sorted((report.get("errors_by_source") or {}).keys()),
+            }
+        )
+    return rows
+
+
+def verify_latency() -> dict[str, dict[str, object]]:
+    results: dict[str, dict[str, object]] = {}
+    for profile, extra in LATENCY_PROFILES:
+        timings = []
+        for topic in LATENCY_TOPICS:
+            start = time.time()
+            run_command(
+                [PYTHON, str(ENGINE), topic, "--emit=json", *extra],
+                timeout=300,
+            )
+            timings.append(time.time() - start)
+        results[profile] = {
+            "times": [round(value, 2) for value in timings],
+            "median_seconds": round(statistics.median(timings), 2),
+            "max_seconds": round(max(timings), 2),
+        }
+    return results
+
+
+def verify_eval(
+    *,
+    baseline: str,
+    candidate: str,
+    output_dir: str,
+    quick: bool,
+    limit: int,
+    timeout: int,
+) -> dict[str, object]:
+    cmd = [
+        PYTHON,
+        str(EVALUATOR),
+        f"--baseline={baseline}",
+        f"--candidate={candidate}",
+        f"--output-dir={output_dir}",
+        f"--limit={limit}",
+        f"--timeout={timeout}",
+    ]
+    if quick:
+        cmd.append("--quick")
+    run_command(cmd, timeout=max(timeout * 8, 600))
+    output = Path(output_dir)
+    metrics = json.loads((output / "metrics.json").read_text())
+    summary = (output / "summary.md").read_text()
+    return {"metrics": metrics, "summary": summary}
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Run the v3 verification bundle")
+    parser.add_argument("--skip-eval", action="store_true", help="Skip the judged evaluator")
+    parser.add_argument("--skip-latency", action="store_true", help="Skip live latency sampling")
+    parser.add_argument("--baseline", default="HEAD~1")
+    parser.add_argument("--candidate", default="WORKTREE")
+    parser.add_argument("--output-dir", default="/tmp/last30days-v3-verify")
+    parser.add_argument("--quick-eval", action="store_true", help="Use evaluator quick mode")
+    parser.add_argument("--eval-limit", type=int, default=20)
+    parser.add_argument("--eval-timeout", type=int, default=240)
+    return parser
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+    summary: dict[str, object] = {}
+    summary["unit"] = verify_unit()
+    summary["diagnose"] = verify_diagnose()
+    summary["smoke"] = verify_smoke()
+    if not args.skip_latency:
+        summary["latency"] = verify_latency()
+    if not args.skip_eval:
+        summary["eval"] = verify_eval(
+            baseline=args.baseline,
+            candidate=args.candidate,
+            output_dir=args.output_dir,
+            quick=args.quick_eval,
+            limit=args.eval_limit,
+            timeout=args.eval_timeout,
+        )
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())