16gagent/scripts/e2e-benchmark-v1.mjs

#!/usr/bin/env node

/**
 * End-to-End Generator Benchmark v1
 *
 * 完整链路：需求 → 前端 → 后端 → 全栈 → Electron → Release
 * 10 个领域全量验证，输出 end-to-end-benchmark-v1.md
 */

import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync } from "node:fs";
import { resolve, dirname, join } from "node:path";
import { execSync } from "node:child_process";
import { fileURLToPath } from "node:url";

const __dirname = dirname(resolve(fileURLToPath(import.meta.url)));
const WORKSPACE = resolve(__dirname, "..");
const BENCH_ROOT = resolve(WORKSPACE, ".benchmark");
const SCRIPTS = resolve(WORKSPACE, "scripts");
const REPORT_PATH = resolve(WORKSPACE, "end-to-end-benchmark-v1.md");

const DOMAINS = [
  { id: "petcare", input: "做一个宠物护理管理平台，宠物主人可以管理宠物档案、健康日程、日常记录和成长相册" },
  { id: "crm", input: "做一个客户关系管理系统，支持客户管理、销售漏斗、跟进记录和数据分析仪表盘" },
  { id: "inventory", input: "做一个库存管理系统，支持商品入库出库、库存盘点、供应商管理和库存预警" },
  { id: "ticket", input: "做一个工单系统，支持工单创建、分配、处理流程、优先级管理和工单归档" },
  { id: "blog-cms", input: "做一个博客内容管理系统，支持文章发布、分类标签、评论管理和媒体库" },
  { id: "project-mgmt", input: "做一个项目管理系统，支持项目看板、任务分配、甘特图和团队协作" },
  { id: "hr", input: "做一个人力资源管理系统，支持员工档案、考勤管理、招聘流程和绩效评估" },
  { id: "asset", input: "做一个固定资产管理系统，支持资产登记、领用归还、折旧计算和盘点统计" },
  { id: "course", input: "做一个在线课程管理系统，支持课程发布、章节管理、学员进度和作业批改" },
  { id: "appointment", input: "做一个预约管理系统，支持服务项目、时间段预约、客户通知和预约统计" },
];

// ═══════════════════════════════════════════════════════
// Helpers
// ═══════════════════════════════════════════════════════

function cmd(c, opts = {}) {
  try {
    return execSync(c, { cwd: WORKSPACE, encoding: "utf8", ...opts, stdio: opts.stdio ?? "pipe" });
  } catch (e) {
    return { error: e.message, stderr: e.stderr?.toString() || "", stdout: e.stdout?.toString() || "" };
  }
}

function runAgent(script, args) {
  const t0 = Date.now();
  const result = cmd(`node ${join(SCRIPTS, script)} ${args}`);
  const ms = Date.now() - t0;
  if (result.error) {
    try { return { ms, ...JSON.parse(result.stdout?.trim() || "{}") }; } catch { return { ms, error: result.error }; }
  }
  try { return { ms, ...JSON.parse(result.trim()) }; } catch { return { ms, raw: result }; }
}

function countFiles(dir) {
  try {
    const r = execSync(`find ${dir} -type f 2>/dev/null | wc -l`, { encoding: "utf8", cwd: WORKSPACE });
    return parseInt(r.trim(), 10);
  } catch { return 0; }
}

function loadJSON(path) {
  try { return JSON.parse(readFileSync(path, "utf8")); } catch { return {}; }
}

// ═══════════════════════════════════════════════════════
// Validation
// ═══════════════════════════════════════════════════════

function validateFrontend(dir) {
  if (!existsSync(dir)) return { pass: false, reason: "dir missing" };
  const checks = {
    packageJson: existsSync(join(dir, "package.json")),
    appDir: existsSync(join(dir, "app")),
    services: existsSync(join(dir, "services")) || existsSync(join(dir, "src", "services")),
    types: existsSync(join(dir, "types")) || existsSync(join(dir, "src", "types")),
  };
  const fileCount = countFiles(dir);
  const pass = checks.packageJson && checks.appDir && fileCount > 10;
  return { pass, fileCount, ...checks };
}

function validateBackend(dir) {
  if (!existsSync(dir)) return { pass: false, reason: "dir missing" };
  const checks = {
    packageJson: existsSync(join(dir, "package.json")),
    routes: existsSync(join(dir, "src", "routes")),
    services: existsSync(join(dir, "src", "services")),
    auth: existsSync(join(dir, "src", "routes", "auth.ts")) || existsSync(join(dir, "src", "routes", "auth-route.ts")),
    db: existsSync(join(dir, "src", "db")) || existsSync(join(dir, "src", "schema")),
  };
  const fileCount = countFiles(dir);
  const pass = checks.packageJson && checks.routes && checks.services && fileCount > 10;
  return { pass, fileCount, ...checks };
}

function validateFullstack(dir) {
  if (!existsSync(dir)) return { pass: false, reason: "dir missing" };
  const checks = {
    packageJson: existsSync(join(dir, "package.json")),
    web: existsSync(join(dir, "apps", "web")),
    api: existsSync(join(dir, "apps", "api")),
    shared: existsSync(join(dir, "packages")),
  };
  const fileCount = countFiles(dir);
  const pass = checks.packageJson && checks.web && checks.api && fileCount > 30;
  return { pass, fileCount, ...checks };
}

function validateElectron(dir) {
  if (!existsSync(dir)) return { pass: false, reason: "dir missing" };
  const checks = {
    packageJson: existsSync(join(dir, "package.json")),
    main: existsSync(join(dir, "electron", "main.ts")),
    preload: existsSync(join(dir, "electron", "preload.ts")),
    ipc: existsSync(join(dir, "electron", "ipc.ts")),
    yml: existsSync(join(dir, "electron-builder.yml")),
  };
  const fileCount = countFiles(dir);
  const pass = checks.packageJson && checks.main && checks.preload && fileCount >= 8;
  return { pass, fileCount, ...checks };
}

function validateRelease(dir) {
  if (!existsSync(dir)) return { pass: false, reason: "dir missing" };
  const rDir = join(dir, "release");
  const checks = {
    versionJson: existsSync(join(rDir, "version.json")),
    manifest: existsSync(join(rDir, "manifests", "manifest.json")),
    checksums: existsSync(join(rDir, "checksums", "checksums.txt")),
    releaseNotes: existsSync(join(rDir, "release-notes", "release-notes.md")),
    buildInfo: existsSync(join(rDir, "build-info.json")),
    windows: existsSync(join(rDir, "windows")),
    macos: existsSync(join(rDir, "macos")),
    linux: existsSync(join(rDir, "linux")),
  };
  // Validate content
  if (checks.versionJson) {
    try {
      const v = JSON.parse(readFileSync(join(rDir, "version.json"), "utf8"));
      checks.versionValid = !!(v.name && v.version && v.platforms);
    } catch { checks.versionValid = false; }
  }
  if (checks.manifest) {
    try {
      const m = JSON.parse(readFileSync(join(rDir, "manifests", "manifest.json"), "utf8"));
      checks.manifestValid = !!(m.project && m.files?.length > 0);
    } catch { checks.manifestValid = false; }
  }
  if (checks.checksums) {
    const content = readFileSync(join(rDir, "checksums", "checksums.txt"), "utf8");
    const lines = content.trim().split("\n").filter(l => l.length > 0);
    checks.checksumCount = lines.length;
    checks.checksumsValid = lines.every(l => l.split(/\s+/)[0]?.length === 64);
  }
  if (checks.releaseNotes) {
    const notes = readFileSync(join(rDir, "release-notes", "release-notes.md"), "utf8");
    checks.notesValid = notes.includes("Version") && notes.includes("Installation");
  }
  if (checks.buildInfo) {
    try {
      const bi = JSON.parse(readFileSync(join(rDir, "build-info.json"), "utf8"));
      checks.buildInfoValid = !!(bi.nodeVersion && bi.generatorVersion);
    } catch { checks.buildInfoValid = false; }
  }
  // Platform file counts
  for (const p of ["windows", "macos", "linux"]) {
    if (checks[p]) {
      try { checks[p + "Count"] = readdirSync(join(rDir, p)).length; } catch { checks[p + "Count"] = 0; }
    }
  }
  const fileCount = countFiles(dir);
  const pass = Object.values(checks).every(v => v !== false && v !== 0);
  return { pass, fileCount, ...checks };
}

// ═══════════════════════════════════════════════════════
// Pipeline
// ═══════════════════════════════════════════════════════

function runPipeline(domain) {
  const B = join(BENCH_ROOT, domain.id);
  mkdirSync(B, { recursive: true });

  const result = { id: domain.id, input: domain.input, stages: {} };

  // SF-01: PRD
  const s1 = runAgent("project-intake-agent.mjs", `--input "${domain.input}" --output "${join(B, "prd.json")}"`);
  result.stages.prd = { ms: s1.ms, projectName: s1.projectName, domain: s1.domain, error: s1.error };
  result.projectName = s1.projectName;

  // SF-02: Architecture
  const s2 = runAgent("architecture-agent.mjs", `--input "${join(B, "prd.json")}" --output "${join(B, "arch.json")}"`);
  result.stages.arch = { ms: s2.ms, modules: s2.moduleCount, error: s2.error };

  // SF-03: Frontend
  const feOut = join(B, "frontend");
  const s3 = runAgent("frontend-builder-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${feOut}"`);
  const feVal = validateFrontend(feOut);
  result.stages.frontend = { ms: s3.ms, files: s3.stats?.totalFiles || feVal.fileCount, pass: feVal.pass, error: s3.error };

  // SF-04: Backend
  const beOut = join(B, "backend");
  const s4 = runAgent("backend-builder-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${beOut}"`);
  const beVal = validateBackend(beOut);
  result.stages.backend = { ms: s4.ms, files: s4.stats?.totalFiles || beVal.fileCount, pass: beVal.pass, error: s4.error };

  // SF-05: Fullstack
  const fsOut = join(B, "fullstack");
  const s5 = runAgent("fullstack-composer-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${fsOut}"`);
  const fsVal = validateFullstack(fsOut);
  result.stages.fullstack = { ms: s5.ms, files: s5.stats?.totalFiles || fsVal.fileCount, pass: fsVal.pass, error: s5.error };

  // SF-06: Electron
  const elOut = join(B, "electron");
  const s6 = runAgent("electron-builder-agent.mjs", `--input "${fsOut}" --output "${elOut}" --prd "${join(B, "prd.json")}"`);
  const elVal = validateElectron(elOut);
  result.stages.electron = { ms: s6.ms, files: s6.stats?.totalFiles || elVal.fileCount, pass: elVal.pass, error: s6.error };

  // SF-07: Release
  const rlOut = join(B, "release");
  const s7 = runAgent("release-builder-agent.mjs", `--input "${fsOut}" --output "${rlOut}"`);
  const rlVal = validateRelease(rlOut);
  result.stages.release = { ms: s7.ms, files: s7.stats?.totalFiles || rlVal.fileCount, pass: rlVal.pass, error: s7.error };

  // Totals
  result.totalMs = Object.values(result.stages).reduce((s, v) => s + (v.ms || 0), 0);
  result.totalFiles = Object.values(result.stages).reduce((s, v) => s + (v.files || 0), 0);
  result.allPass = Object.values(result.stages).every(v => v.pass !== false);

  return result;
}

// ═══════════════════════════════════════════════════════
// Report
// ═══════════════════════════════════════════════════════

function generateReport(allResults) {
  const L = [];
  const h = (t) => { L.push(t); return L; };
  const now = new Date();

  h(`# End-to-End Generator Benchmark v1`);
  h(``);
  h(`> ${now.toISOString()}`);
  h(`> 链路: 需求 → 前端 → 后端 → 全栈 → Electron → Release`);
  h(`> 领域: ${allResults.length} 个`);
  h(``);

  // ── Summary ──
  const passCount = allResults.filter(r => r.allPass).length;
  const totalFiles = allResults.reduce((s, r) => s + r.totalFiles, 0);
  const avgMs = (allResults.reduce((s, r) => s + r.totalMs, 0) / allResults.length).toFixed(0);

  h(`## Summary`);
  h(``);
  h(`| Metric | Value |`);
  h(`|--------|-------|`);
  h(`| Domains | ${allResults.length} |`);
  h(`| PASS | ${passCount}/${allResults.length} |`);
  h(`| FAIL | ${allResults.length - passCount}/${allResults.length} |`);
  h(`| Total Files Generated | ${totalFiles} |`);
  h(`| Avg Pipeline Time | ${avgMs}ms |`);
  h(``);

  // ── Domain Results ──
  h(`## Domain Results`);
  h(``);
  h(`| Domain | PRD | Arch | Frontend | Backend | Fullstack | Electron | Release | Files | Time | Result |`);
  h(`|--------|-----|------|----------|---------|-----------|----------|---------|-------|------|--------|`);

  for (const r of allResults) {
    const s = r.stages;
    const p = (v) => v.pass === false ? "❌" : "✅";
    const t = (v) => v.ms ? `${v.ms}ms` : "—";
    h(`| ${r.id} | ${t(s.prd)} | ${t(s.arch)} | ${p(s.frontend)} ${s.frontend.files || 0}f | ${p(s.backend)} ${s.backend.files || 0}f | ${p(s.fullstack)} ${s.fullstack.files || 0}f | ${p(s.electron)} ${s.electron.files || 0}f | ${p(s.release)} ${s.release.files || 0}f | ${r.totalFiles} | ${r.totalMs}ms | **${r.allPass ? "PASS" : "FAIL"}** |`);
  }
  h(``);

  // ── Per-Stage Stats ──
  h(`## Per-Stage Statistics`);
  h(``);
  const stages = ["prd", "arch", "frontend", "backend", "fullstack", "electron", "release"];
  const stageNames = ["SF-01 PRD", "SF-02 Arch", "SF-03 Frontend", "SF-04 Backend", "SF-05 Fullstack", "SF-06 Electron", "SF-07 Release"];

  h(`| Stage | Avg Time | Min | Max | Avg Files | Pass |`);
  h(`|-------|----------|-----|-----|-----------|------|`);
  for (let i = 0; i < stages.length; i++) {
    const key = stages[i];
    const times = allResults.map(r => r.stages[key]?.ms || 0);
    const files = allResults.map(r => r.stages[key]?.files || 0);
    const passes = allResults.filter(r => r.stages[key]?.pass !== false).length;
    const avg = (times.reduce((a, b) => a + b, 0) / times.length).toFixed(0);
    const min = Math.min(...times);
    const max = Math.max(...times);
    const avgF = (files.reduce((a, b) => a + b, 0) / files.length).toFixed(0);
    h(`| ${stageNames[i]} | ${avg}ms | ${min}ms | ${max}ms | ${avgF} | ${passes}/${allResults.length} |`);
  }
  h(``);

  // ── File Generation ──
  h(`## File Generation`);
  h(``);
  h(`| Domain | Frontend | Backend | Fullstack | Electron | Release | Total |`);
  h(`|--------|----------|---------|-----------|----------|---------|-------|`);
  for (const r of allResults) {
    const s = r.stages;
    h(`| ${r.id} | ${s.frontend.files || 0} | ${s.backend.files || 0} | ${s.fullstack.files || 0} | ${s.electron.files || 0} | ${s.release.files || 0} | ${r.totalFiles} |`);
  }
  h(``);

  // ── Failure Details ──
  const failures = allResults.filter(r => !r.allPass);
  if (failures.length > 0) {
    h(`## Failure Details`);
    h(``);
    for (const r of failures) {
      h(`### ${r.id}`);
      h(``);
      for (const [stage, data] of Object.entries(r.stages)) {
        if (data.pass === false) {
          h(`- **${stage}:** FAIL${data.error ? " — " + data.error : ""}`);
        }
      }
      h(``);
    }
  }

  // ── Final Status ──
  h(`## Final Status`);
  h(``);
  if (passCount === allResults.length) {
    h(`🎉 **End-to-End Generator — PASS** (${passCount}/${allResults.length})`);
    h(``);
    h(`完整链路 需求 → 前端 → 后端 → 全栈 → Electron → Release 全部通过。`);
    h(`系统具备跨领域泛化能力，可作为通用 Web 全栈 + 桌面应用生成器。`);
  } else {
    h(`⚠️ **End-to-End Generator — PARTIAL** (${passCount}/${allResults.length})`);
    h(``);
    h(`Failed: ${failures.map(r => r.id).join(", ")}`);
  }
  h(``);

  h(`---`);
  h(`*End-to-End Generator Benchmark v1 — ${now.toISOString()}*`);

  return L.join("\n");
}

// ═══════════════════════════════════════════════════════
// Main
// ═══════════════════════════════════════════════════════

async function main() {
  console.log(`\n🧪 End-to-End Generator Benchmark v1`);
  console.log(`   ${DOMAINS.length} domains × 7 stages = ${DOMAINS.length * 7} checks`);
  console.log(`   链路: 需求 → 前端 → 后端 → 全栈 → Electron → Release\n`);

  const allResults = [];

  for (const domain of DOMAINS) {
    console.log(`${"─".repeat(50)}`);
    console.log(`🏗️  ${domain.id}`);

    const result = runPipeline(domain);
    allResults.push(result);

    const status = result.allPass ? "✅ PASS" : "❌ FAIL";
    console.log(`   ${result.projectName || "?"} | ${result.totalFiles} files | ${result.totalMs}ms | ${status}`);
  }

  const report = generateReport(allResults);
  writeFileSync(REPORT_PATH, report, "utf8");

  const passCount = allResults.filter(r => r.allPass).length;
  console.log(`\n${"═".repeat(50)}`);
  console.log(`📊 ${passCount}/${DOMAINS.length} PASS`);
  console.log(`📄 ${REPORT_PATH}`);
}

main().catch(e => {
  console.error("Benchmark failed:", e);
  process.exit(1);
});