Add local search quality evaluation harness
Add an optional local evaluator that compares a baseline revision against a candidate checkout, computes deterministic stability metrics, and can call Gemini for judged ranking metrics when configured. The harness isolates child runs with a temporary HOME and a node-free PATH so historical revisions cannot trigger Bird browser-cookie auth during evaluation. Validation: uv run python -m unittest and local smoke/full deterministic eval runs.
This commit is contained in:
Jeffrey Sperling
@@ -0,0 +1,586 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run local search-quality evaluations across fixed topics.
|
||||
|
||||
This is an optional local gate, not a required CI job. It compares a baseline
|
||||
revision against a candidate checkout, computes deterministic regression
|
||||
metrics, and optionally calls Gemini as a judge for graded relevance labels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from lib import env as envlib
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_TOPICS: List[Tuple[str, str]] = [
|
||||
("nano banana pro prompting", "product"),
|
||||
("codex vs claude code", "comparison"),
|
||||
("anthropic odds", "prediction"),
|
||||
("kanye west", "breaking_news"),
|
||||
("remotion animations for Claude Code", "how_to"),
|
||||
]
|
||||
DEFAULT_SEARCH = "reddit,x,youtube,hn,polymarket"
|
||||
SOURCE_KEYS = [
|
||||
"reddit",
|
||||
"x",
|
||||
"youtube",
|
||||
"tiktok",
|
||||
"instagram",
|
||||
"hackernews",
|
||||
"bluesky",
|
||||
"truthsocial",
|
||||
"polymarket",
|
||||
"websearch",
|
||||
]
|
||||
DEFAULT_JUDGE_MODEL = "gemini-3-pro-preview"
|
||||
GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||||
|
||||
|
||||
def slugify(topic: str) -> str:
|
||||
return "".join(c.lower() if c.isalnum() else "-" for c in topic).strip("-")
|
||||
|
||||
|
||||
def path_without_node(path_value: str) -> str:
|
||||
parts = []
|
||||
for entry in path_value.split(os.pathsep):
|
||||
if not entry:
|
||||
continue
|
||||
if (Path(entry) / "node").exists():
|
||||
continue
|
||||
parts.append(entry)
|
||||
return os.pathsep.join(parts)
|
||||
|
||||
|
||||
def stable_item_key(source: str, item: Dict[str, Any]) -> str:
|
||||
url = str(item.get("url") or "").strip()
|
||||
if url:
|
||||
return url
|
||||
item_id = str(item.get("id") or "").strip()
|
||||
text = item_text(source, item)
|
||||
return f"{source}:{item_id}:{text[:120]}"
|
||||
|
||||
|
||||
def item_text(source: str, item: Dict[str, Any]) -> str:
|
||||
if source in {"x", "bluesky", "truthsocial"}:
|
||||
return str(item.get("text") or "").strip()
|
||||
if source == "polymarket":
|
||||
return str(item.get("question") or item.get("title") or "").strip()
|
||||
return str(item.get("title") or "").strip()
|
||||
|
||||
|
||||
def build_ranked_items(report: Dict[str, Any], per_source_limit: int) -> List[Dict[str, Any]]:
|
||||
ranked: List[Dict[str, Any]] = []
|
||||
for source in SOURCE_KEYS:
|
||||
items = list(report.get(source) or [])[:per_source_limit]
|
||||
for item in items:
|
||||
ranked.append({
|
||||
"source": source,
|
||||
"key": stable_item_key(source, item),
|
||||
"url": str(item.get("url") or "").strip(),
|
||||
"text": item_text(source, item),
|
||||
"score": float(item.get("score") or 0),
|
||||
"relevance": float(item.get("relevance") or 0),
|
||||
"date": item.get("date"),
|
||||
})
|
||||
ranked.sort(key=lambda item: (-item["score"], item["source"], item["key"]))
|
||||
return ranked
|
||||
|
||||
|
||||
def url_sets_by_source(report: Dict[str, Any]) -> Dict[str, set[str]]:
|
||||
result: Dict[str, set[str]] = {}
|
||||
for source in SOURCE_KEYS:
|
||||
items = report.get(source) or []
|
||||
urls = {
|
||||
stable_item_key(source, item)
|
||||
for item in items
|
||||
}
|
||||
result[source] = urls
|
||||
return result
|
||||
|
||||
|
||||
def jaccard(left: Iterable[str], right: Iterable[str]) -> float:
|
||||
left_set = set(left)
|
||||
right_set = set(right)
|
||||
if not left_set and not right_set:
|
||||
return 1.0
|
||||
union = left_set | right_set
|
||||
if not union:
|
||||
return 1.0
|
||||
return len(left_set & right_set) / len(union)
|
||||
|
||||
|
||||
def retention(left: Iterable[str], right: Iterable[str]) -> float:
|
||||
left_set = set(left)
|
||||
right_set = set(right)
|
||||
if not left_set:
|
||||
return 1.0
|
||||
return len(left_set & right_set) / len(left_set)
|
||||
|
||||
|
||||
def precision_at_k(ranking: List[Dict[str, Any]], judgments: Dict[str, int], k: int) -> float:
|
||||
top = ranking[:k]
|
||||
if not top:
|
||||
return 0.0
|
||||
hits = sum(1 for item in top if judgments.get(item["key"], 0) >= 2)
|
||||
return hits / len(top)
|
||||
|
||||
|
||||
def ndcg_at_k(ranking: List[Dict[str, Any]], judgments: Dict[str, int], k: int) -> float:
|
||||
top = ranking[:k]
|
||||
if not top:
|
||||
return 0.0
|
||||
|
||||
def dcg(grades: List[int]) -> float:
|
||||
total = 0.0
|
||||
for index, grade in enumerate(grades, start=1):
|
||||
total += (2**grade - 1) / math.log2(index + 1)
|
||||
return total
|
||||
|
||||
actual = [judgments.get(item["key"], 0) for item in top]
|
||||
ideal = sorted(actual, reverse=True)
|
||||
ideal_score = dcg(ideal)
|
||||
if ideal_score == 0:
|
||||
return 0.0
|
||||
return dcg(actual) / ideal_score
|
||||
|
||||
|
||||
def source_coverage_recall(
|
||||
ranking: List[Dict[str, Any]],
|
||||
judged_pool: List[Dict[str, Any]],
|
||||
judgments: Dict[str, int],
|
||||
) -> float:
|
||||
good_sources = {item["source"] for item in judged_pool if judgments.get(item["key"], 0) >= 2}
|
||||
if not good_sources:
|
||||
return 1.0
|
||||
hit_sources = {
|
||||
item["source"]
|
||||
for item in ranking
|
||||
if judgments.get(item["key"], 0) >= 2
|
||||
}
|
||||
return len(hit_sources & good_sources) / len(good_sources)
|
||||
|
||||
|
||||
def create_eval_env(include_web: bool) -> Tuple[Dict[str, str], Path]:
|
||||
config = envlib.get_config()
|
||||
eval_home = Path(tempfile.mkdtemp(prefix="last30days-eval-home-"))
|
||||
(eval_home / ".config").mkdir(parents=True, exist_ok=True)
|
||||
passthrough = {
|
||||
"HOME": str(eval_home),
|
||||
"XDG_CONFIG_HOME": str(eval_home / ".config"),
|
||||
"PATH": path_without_node(os.environ.get("PATH", "")),
|
||||
"LANG": os.environ.get("LANG", "en_US.UTF-8"),
|
||||
"LC_ALL": os.environ.get("LC_ALL", ""),
|
||||
"TMPDIR": os.environ.get("TMPDIR", ""),
|
||||
"PYTHONUTF8": "1",
|
||||
"LAST30DAYS_CONFIG_DIR": "",
|
||||
"BIRD_DISABLE_BROWSER_COOKIES": "1",
|
||||
"LAST30DAYS_DISABLE_BROWSER_COOKIES": "1",
|
||||
}
|
||||
for key in ("OPENAI_API_KEY", "XAI_API_KEY", "SCRAPECREATORS_API_KEY"):
|
||||
value = config.get(key)
|
||||
if value:
|
||||
passthrough[key] = value
|
||||
if include_web:
|
||||
for key in ("PARALLEL_API_KEY", "BRAVE_API_KEY", "OPENROUTER_API_KEY"):
|
||||
value = config.get(key)
|
||||
if value:
|
||||
passthrough[key] = value
|
||||
return passthrough, eval_home
|
||||
|
||||
|
||||
def run_last30days(
|
||||
repo_dir: Path,
|
||||
topic: str,
|
||||
*,
|
||||
search: str,
|
||||
timeout_seconds: int,
|
||||
include_web: bool,
|
||||
env: Dict[str, str],
|
||||
) -> Tuple[Dict[str, Any], str]:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"scripts/last30days.py",
|
||||
topic,
|
||||
"--emit",
|
||||
"json",
|
||||
"--search",
|
||||
search,
|
||||
"--timeout",
|
||||
str(timeout_seconds),
|
||||
]
|
||||
if not include_web:
|
||||
cmd.append("--no-native-web")
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=repo_dir,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout_seconds + 30,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"{repo_dir.name} failed for '{topic}' with exit {result.returncode}\n{result.stderr.strip()}"
|
||||
)
|
||||
return json.loads(result.stdout), result.stderr
|
||||
|
||||
|
||||
def create_worktree(rev: str) -> Path:
|
||||
worktree_dir = Path(tempfile.mkdtemp(prefix="last30days-eval-"))
|
||||
subprocess.run(
|
||||
["git", "worktree", "add", "--detach", str(worktree_dir), rev],
|
||||
cwd=REPO_ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return worktree_dir
|
||||
|
||||
|
||||
def remove_worktree(path: Path) -> None:
|
||||
subprocess.run(
|
||||
["git", "worktree", "remove", "--force", str(path)],
|
||||
cwd=REPO_ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
|
||||
def extract_gemini_text(payload: Dict[str, Any]) -> str:
|
||||
for candidate in payload.get("candidates", []):
|
||||
content = candidate.get("content") or {}
|
||||
for part in content.get("parts", []):
|
||||
text = part.get("text")
|
||||
if text:
|
||||
return text
|
||||
raise ValueError("Gemini response did not contain text")
|
||||
|
||||
|
||||
def call_gemini_judge(api_key: str, model: str, prompt: str) -> Dict[str, Any]:
|
||||
body = {
|
||||
"contents": [{"parts": [{"text": prompt}]}],
|
||||
"generationConfig": {
|
||||
"temperature": 0,
|
||||
"responseMimeType": "application/json",
|
||||
},
|
||||
}
|
||||
url = GEMINI_API_URL.format(model=model, api_key=api_key)
|
||||
request = Request(
|
||||
url,
|
||||
data=json.dumps(body).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=120) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"Gemini HTTP {exc.code}: {detail}") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"Gemini request failed: {exc}") from exc
|
||||
return json.loads(extract_gemini_text(payload))
|
||||
|
||||
|
||||
def build_judge_prompt(
|
||||
*,
|
||||
topic: str,
|
||||
query_type: str,
|
||||
items: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
item_lines = []
|
||||
for item in items:
|
||||
item_lines.append(
|
||||
"\n".join([
|
||||
f"- id: {item['key']}",
|
||||
f" source: {item['source']}",
|
||||
f" title: {item['text'][:220]}",
|
||||
f" url: {item['url']}",
|
||||
f" date: {item.get('date') or 'unknown'}",
|
||||
])
|
||||
)
|
||||
joined = "\n".join(item_lines)
|
||||
return textwrap.dedent(
|
||||
f"""
|
||||
Judge search-result relevance for a last-30-days research tool.
|
||||
|
||||
Topic: {topic}
|
||||
Query type: {query_type}
|
||||
|
||||
Score each item on this 0-3 scale:
|
||||
- 0 = off-topic or clearly bad
|
||||
- 1 = weak or tangential
|
||||
- 2 = relevant and useful
|
||||
- 3 = highly relevant, one of the best results
|
||||
|
||||
Focus on actual user intent, not just token overlap. Penalize items that
|
||||
only match generic words like "odds", "review", or "tips" without
|
||||
matching the real entity or subject. Favor items that would genuinely
|
||||
help answer the topic in the context of recent discussion.
|
||||
|
||||
Return strict JSON with this shape:
|
||||
{{
|
||||
"judgments": [
|
||||
{{"id": "ITEM_ID", "grade": 0, "reason": "short reason"}}
|
||||
]
|
||||
}}
|
||||
|
||||
Items:
|
||||
{joined}
|
||||
"""
|
||||
).strip()
|
||||
|
||||
|
||||
def get_judgments(
|
||||
*,
|
||||
output_dir: Path,
|
||||
slug: str,
|
||||
topic: str,
|
||||
query_type: str,
|
||||
items: List[Dict[str, Any]],
|
||||
judge_model: str,
|
||||
gemini_api_key: Optional[str],
|
||||
) -> Dict[str, int]:
|
||||
cache_file = output_dir / "judgments" / f"{slug}.json"
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
if cache_file.exists():
|
||||
cached = json.loads(cache_file.read_text())
|
||||
return {entry["id"]: int(entry["grade"]) for entry in cached.get("judgments", [])}
|
||||
|
||||
if not gemini_api_key:
|
||||
return {}
|
||||
|
||||
prompt = build_judge_prompt(topic=topic, query_type=query_type, items=items)
|
||||
payload = call_gemini_judge(gemini_api_key, judge_model, prompt)
|
||||
cache_file.write_text(json.dumps(payload, indent=2))
|
||||
return {entry["id"]: int(entry["grade"]) for entry in payload.get("judgments", [])}
|
||||
|
||||
|
||||
def summarize_topic(
|
||||
*,
|
||||
topic: str,
|
||||
query_type: str,
|
||||
baseline_report: Dict[str, Any],
|
||||
candidate_report: Dict[str, Any],
|
||||
judged_pool: List[Dict[str, Any]],
|
||||
judgments: Dict[str, int],
|
||||
per_source_limit: int,
|
||||
) -> Dict[str, Any]:
|
||||
baseline_ranked = build_ranked_items(baseline_report, per_source_limit)
|
||||
candidate_ranked = build_ranked_items(candidate_report, per_source_limit)
|
||||
|
||||
baseline_sets = url_sets_by_source(baseline_report)
|
||||
candidate_sets = url_sets_by_source(candidate_report)
|
||||
|
||||
metrics = {
|
||||
"topic": topic,
|
||||
"query_type": query_type,
|
||||
"baseline": {
|
||||
"precision_at_5": precision_at_k(baseline_ranked, judgments, 5),
|
||||
"ndcg_at_5": ndcg_at_k(baseline_ranked, judgments, 5),
|
||||
"source_coverage_recall": source_coverage_recall(baseline_ranked, judged_pool, judgments),
|
||||
},
|
||||
"candidate": {
|
||||
"precision_at_5": precision_at_k(candidate_ranked, judgments, 5),
|
||||
"ndcg_at_5": ndcg_at_k(candidate_ranked, judgments, 5),
|
||||
"source_coverage_recall": source_coverage_recall(candidate_ranked, judged_pool, judgments),
|
||||
},
|
||||
"stability": {
|
||||
"overall_jaccard": jaccard(
|
||||
set().union(*baseline_sets.values()),
|
||||
set().union(*candidate_sets.values()),
|
||||
),
|
||||
"overall_retention_vs_baseline": retention(
|
||||
set().union(*baseline_sets.values()),
|
||||
set().union(*candidate_sets.values()),
|
||||
),
|
||||
"per_source": {
|
||||
source: {
|
||||
"baseline_count": len(baseline_sets[source]),
|
||||
"candidate_count": len(candidate_sets[source]),
|
||||
"jaccard": jaccard(baseline_sets[source], candidate_sets[source]),
|
||||
"retention_vs_baseline": retention(baseline_sets[source], candidate_sets[source]),
|
||||
}
|
||||
for source in SOURCE_KEYS
|
||||
},
|
||||
},
|
||||
}
|
||||
return metrics
|
||||
|
||||
|
||||
def write_markdown_summary(
|
||||
output_dir: Path,
|
||||
baseline_label: str,
|
||||
candidate_label: str,
|
||||
topic_summaries: List[Dict[str, Any]],
|
||||
) -> None:
|
||||
lines = [
|
||||
f"# Search Quality Evaluation",
|
||||
"",
|
||||
f"- Baseline: `{baseline_label}`",
|
||||
f"- Candidate: `{candidate_label}`",
|
||||
f"- Generated: {datetime.now().isoformat(timespec='seconds')}",
|
||||
"",
|
||||
"## Topic Metrics",
|
||||
"",
|
||||
"| Topic | Base P@5 | Cand P@5 | Base nDCG@5 | Cand nDCG@5 | Jaccard | Retention |",
|
||||
"|---|---:|---:|---:|---:|---:|---:|",
|
||||
]
|
||||
for summary in topic_summaries:
|
||||
lines.append(
|
||||
"| {topic} | {bp:.2f} | {cp:.2f} | {bn:.2f} | {cn:.2f} | {jac:.2f} | {ret:.2f} |".format(
|
||||
topic=summary["topic"],
|
||||
bp=summary["baseline"]["precision_at_5"],
|
||||
cp=summary["candidate"]["precision_at_5"],
|
||||
bn=summary["baseline"]["ndcg_at_5"],
|
||||
cn=summary["candidate"]["ndcg_at_5"],
|
||||
jac=summary["stability"]["overall_jaccard"],
|
||||
ret=summary["stability"]["overall_retention_vs_baseline"],
|
||||
)
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("## Notes")
|
||||
lines.append("")
|
||||
lines.append("- `Precision@5` and `nDCG@5` depend on the judged union pool, not a full gold corpus.")
|
||||
lines.append("- `Source coverage recall` measures whether a run surfaced at least one judged-good result from the good sources in the judged pool.")
|
||||
lines.append("- `Jaccard` and `retention` are stability guards against baseline drift, not truth metrics.")
|
||||
(output_dir / "summary.md").write_text("\n".join(lines))
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Evaluate last30days search quality locally")
|
||||
parser.add_argument("--baseline-rev", default="origin/main", help="Git revision for the baseline run")
|
||||
parser.add_argument("--candidate-rev", default=None, help="Optional git revision for the candidate run")
|
||||
parser.add_argument("--no-default-topics", action="store_true", help="Do not include the built-in 5-topic suite")
|
||||
parser.add_argument("--topic", action="append", default=[], help="Extra topic to evaluate (repeatable)")
|
||||
parser.add_argument("--search", default=DEFAULT_SEARCH, help="Comma-separated sources passed to --search")
|
||||
parser.add_argument("--timeout", type=int, default=180, help="Per-topic timeout passed to last30days")
|
||||
parser.add_argument("--per-source-limit", type=int, default=5, help="Items per source to judge")
|
||||
parser.add_argument("--include-web", action="store_true", help="Include web-search keys and native web backends")
|
||||
parser.add_argument("--judge-model", default=None, help="Gemini judge model override")
|
||||
parser.add_argument("--judge-provider", choices=["auto", "gemini", "none"], default="auto")
|
||||
parser.add_argument("--keep-worktrees", action="store_true", help="Leave temporary baseline/candidate worktrees on disk")
|
||||
parser.add_argument("--output-dir", default=None, help="Output directory (default: docs/test-results/search-quality-<timestamp>)")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
output_dir = Path(args.output_dir) if args.output_dir else REPO_ROOT / "docs" / "test-results" / f"search-quality-{timestamp}"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
topics = [] if args.no_default_topics else list(DEFAULT_TOPICS)
|
||||
topics.extend((topic, "custom") for topic in args.topic)
|
||||
if not topics:
|
||||
raise SystemExit("No topics configured. Use the default suite or pass --topic.")
|
||||
|
||||
judge_config = envlib.get_config()
|
||||
judge_provider = args.judge_provider
|
||||
gemini_api_key = judge_config.get("GEMINI_API_KEY")
|
||||
judge_model = args.judge_model or judge_config.get("GEMINI_MODEL") or DEFAULT_JUDGE_MODEL
|
||||
if judge_provider == "auto":
|
||||
judge_provider = "gemini" if gemini_api_key else "none"
|
||||
if judge_provider == "none":
|
||||
gemini_api_key = None
|
||||
|
||||
eval_env, eval_home = create_eval_env(include_web=args.include_web)
|
||||
baseline_dir = create_worktree(args.baseline_rev)
|
||||
candidate_dir = create_worktree(args.candidate_rev) if args.candidate_rev else REPO_ROOT
|
||||
|
||||
baseline_label = args.baseline_rev
|
||||
candidate_label = args.candidate_rev or "working-tree"
|
||||
topic_summaries: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
for topic, query_type in topics:
|
||||
slug = slugify(topic)
|
||||
baseline_report, baseline_stderr = run_last30days(
|
||||
baseline_dir,
|
||||
topic,
|
||||
search=args.search,
|
||||
timeout_seconds=args.timeout,
|
||||
include_web=args.include_web,
|
||||
env=eval_env,
|
||||
)
|
||||
candidate_report, candidate_stderr = run_last30days(
|
||||
candidate_dir,
|
||||
topic,
|
||||
search=args.search,
|
||||
timeout_seconds=args.timeout,
|
||||
include_web=args.include_web,
|
||||
env=eval_env,
|
||||
)
|
||||
|
||||
topic_dir = output_dir / slug
|
||||
topic_dir.mkdir(parents=True, exist_ok=True)
|
||||
(topic_dir / "baseline.json").write_text(json.dumps(baseline_report, indent=2))
|
||||
(topic_dir / "candidate.json").write_text(json.dumps(candidate_report, indent=2))
|
||||
(topic_dir / "baseline.stderr.txt").write_text(baseline_stderr)
|
||||
(topic_dir / "candidate.stderr.txt").write_text(candidate_stderr)
|
||||
|
||||
baseline_ranked = build_ranked_items(baseline_report, args.per_source_limit)
|
||||
candidate_ranked = build_ranked_items(candidate_report, args.per_source_limit)
|
||||
union_map = {item["key"]: item for item in baseline_ranked + candidate_ranked}
|
||||
judgments = get_judgments(
|
||||
output_dir=output_dir,
|
||||
slug=slug,
|
||||
topic=topic,
|
||||
query_type=query_type,
|
||||
items=list(union_map.values()),
|
||||
judge_model=judge_model,
|
||||
gemini_api_key=gemini_api_key,
|
||||
)
|
||||
|
||||
summary = summarize_topic(
|
||||
topic=topic,
|
||||
query_type=query_type,
|
||||
baseline_report=baseline_report,
|
||||
candidate_report=candidate_report,
|
||||
judged_pool=list(union_map.values()),
|
||||
judgments=judgments,
|
||||
per_source_limit=args.per_source_limit,
|
||||
)
|
||||
topic_summaries.append(summary)
|
||||
|
||||
payload = {
|
||||
"baseline": baseline_label,
|
||||
"candidate": candidate_label,
|
||||
"judge_provider": judge_provider,
|
||||
"judge_model": judge_model if gemini_api_key else None,
|
||||
"topics": topic_summaries,
|
||||
}
|
||||
(output_dir / "summary.json").write_text(json.dumps(payload, indent=2))
|
||||
write_markdown_summary(output_dir, baseline_label, candidate_label, topic_summaries)
|
||||
print(output_dir)
|
||||
return 0
|
||||
finally:
|
||||
if not args.keep_worktrees:
|
||||
remove_worktree(baseline_dir)
|
||||
if args.candidate_rev:
|
||||
remove_worktree(candidate_dir)
|
||||
shutil.rmtree(eval_home, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -243,10 +243,12 @@ def get_config() -> Dict[str, Any]:
|
||||
|
||||
keys = [
|
||||
('XAI_API_KEY', None),
|
||||
('GEMINI_API_KEY', None),
|
||||
('OPENROUTER_API_KEY', None),
|
||||
('PARALLEL_API_KEY', None),
|
||||
('BRAVE_API_KEY', None),
|
||||
('XIAOHONGSHU_API_BASE', None),
|
||||
('GEMINI_MODEL', None),
|
||||
('OPENAI_MODEL_POLICY', 'auto'),
|
||||
('OPENAI_MODEL_PIN', None),
|
||||
('XAI_MODEL_POLICY', 'latest'),
|
||||
|
||||
Reference in New Issue
Block a user