#!/usr/bin/env node /** * Domain Benchmark Suite * * 验证现有系统的泛化能力: * project-intake-agent → architecture-agent * → frontend-builder-agent → backend-builder-agent → fullstack-composer-agent * * 对 10 个领域生成完整项目,验证 Build + API,输出 benchmark-report.md * * Usage: * node scripts/domain-benchmark.mjs [--domain ] [--skip-build] [--skip-api] */ import { readFileSync, writeFileSync, existsSync, mkdirSync, rmSync } from "node:fs"; import { resolve, dirname, join } from "node:path"; import { execSync } from "node:child_process"; import { fileURLToPath } from "node:url"; const __dirname = dirname(resolve(fileURLToPath(import.meta.url))); const WORKSPACE = resolve(__dirname, ".."); const BENCH_ROOT = resolve(WORKSPACE, ".benchmark"); const SCRIPTS = resolve(WORKSPACE, "scripts"); const REPORT_PATH = resolve(WORKSPACE, "benchmark-report.md"); // ═══════════════════════════════════════════════════════ // Domain Definitions // ═══════════════════════════════════════════════════════ const DOMAINS = [ { id: "petcare", input: "做一个宠物护理管理平台,宠物主人可以管理宠物档案、健康日程、日常记录和成长相册" }, { id: "crm", input: "做一个客户关系管理系统,支持客户管理、销售漏斗、跟进记录和数据分析仪表盘" }, { id: "inventory", input: "做一个库存管理系统,支持商品入库出库、库存盘点、供应商管理和库存预警" }, { id: "ticket", input: "做一个工单系统,支持工单创建、分配、处理流程、优先级管理和工单归档" }, { id: "blog-cms", input: "做一个博客内容管理系统,支持文章发布、分类标签、评论管理和媒体库" }, { id: "project-mgmt", input: "做一个项目管理系统,支持项目看板、任务分配、甘特图和团队协作" }, { id: "hr", input: "做一个人力资源管理系统,支持员工档案、考勤管理、招聘流程和绩效评估" }, { id: "asset", input: "做一个固定资产管理系统,支持资产登记、领用归还、折旧计算和盘点统计" }, { id: "course", input: "做一个在线课程管理系统,支持课程发布、章节管理、学员进度和作业批改" }, { id: "appointment", input: "做一个预约管理系统,支持服务项目、时间段预约、客户通知和预约统计" }, ]; // ═══════════════════════════════════════════════════════ // Helpers // ═══════════════════════════════════════════════════════ function cmd(c, opts = {}) { try { return execSync(c, { cwd: WORKSPACE, encoding: "utf8", ...opts, stdio: opts.stdio ?? "pipe" }); } catch (e) { return { error: e.message, stderr: e.stderr?.toString() || "", stdout: e.stdout?.toString() || "" }; } } function runAgent(agentPath, args) { const result = cmd(`node ${agentPath} ${args}`); if (result.error) { // Try to extract JSON from stdout even on error try { return JSON.parse(result.stdout?.trim() || "{}"); } catch { return { error: result.error, stderr: result.stderr }; } } try { return JSON.parse(result.trim()); } catch { return { raw: result }; } } function rmDirSafe(dir) { try { if (existsSync(dir)) rmSync(dir, { recursive: true, force: true }); } catch {} } function listFilesRecursive(dir, acc = []) { if (!existsSync(dir)) return acc; const entries = readFileSync(dir)?.toString?.(); // not for dirs const { readdirSync, statSync } = require("node:fs") || {}; // Use shell for simplicity const result = execSync(`find ${dir} -type f | head -500`, { encoding: "utf8", cwd: WORKSPACE }); return result.trim().split("\n").filter(Boolean); } function countFiles(dir) { try { const r = execSync(`find ${dir} -type f 2>/dev/null | wc -l`, { encoding: "utf8", cwd: WORKSPACE }); return parseInt(r.trim(), 10); } catch { return 0; } } function fileExists(dir, pattern) { try { const r = execSync(`find ${dir} -path "*${pattern}*" -type f 2>/dev/null | head -1`, { encoding: "utf8", cwd: WORKSPACE }); return r.trim().length > 0; } catch { return false; } } // ═══════════════════════════════════════════════════════ // Step 1: RequirementPackage (SF-01 + SF-02) // ═══════════════════════════════════════════════════════ function loadJSON(path) { try { return JSON.parse(readFileSync(path, "utf8")); } catch { return {}; } } function generateRequirementPackage(domain) { console.log(`\n📋 [${domain.id}] Generating RequirementPackage...`); const t0 = Date.now(); // SF-01: PRD const prdOut = join(BENCH_ROOT, domain.id, "prd.json"); mkdirSync(dirname(prdOut), { recursive: true }); runAgent( join(SCRIPTS, "project-intake-agent.mjs"), `--input "${domain.input}" --output "${prdOut}"` ); const prd = loadJSON(prdOut); // SF-02: Architecture const archOut = join(BENCH_ROOT, domain.id, "arch.json"); runAgent( join(SCRIPTS, "architecture-agent.mjs"), `--input "${prdOut}" --output "${archOut}"` ); const arch = loadJSON(archOut); const elapsed = ((Date.now() - t0) / 1000).toFixed(1); console.log(` ✅ PRD + Architecture generated (${elapsed}s)`); console.log(` Project: ${prd.projectName || "?"}, Domain: ${prd.domain || "?"}`); console.log(` Features: ${prd.features?.length || 0}, Pages: ${prd.pages?.length || 0}, APIs: ${prd.apiRequirements?.length || 0}`); return { prd, arch, elapsed: parseFloat(elapsed), prdPath: prdOut, archPath: archOut }; } // ═══════════════════════════════════════════════════════ // Step 2: Frontend Builder // ═══════════════════════════════════════════════════════ function generateFrontend(domain, prdPath, archPath) { console.log(` 🎨 Generating Frontend...`); const t0 = Date.now(); const outDir = join(BENCH_ROOT, domain.id, "frontend"); const result = runAgent( join(SCRIPTS, "frontend-builder-agent.mjs"), `--prd "${prdPath}" --arch "${archPath}" --output "${outDir}"` ); const elapsed = ((Date.now() - t0) / 1000).toFixed(1); const fileCount = countFiles(outDir); // Validation checks const checks = { hasPackageJson: fileExists(outDir, "package.json"), hasNextConfig: fileExists(outDir, "next.config.*"), hasAppPage: fileExists(outDir, "app/page.*"), hasEntityPages: fileExists(outDir, "app/") && countFiles(join(outDir, "app")) > 2, hasServices: fileExists(outDir, "services/"), hasTypes: fileExists(outDir, "types/"), fileCount, }; const allPassed = checks.hasPackageJson && checks.hasNextConfig && checks.hasAppPage && checks.hasEntityPages; console.log(` ${allPassed ? "✅" : "❌"} Frontend generated (${elapsed}s, ${fileCount} files)`); if (!allPassed) { console.log(` Missing: ${Object.entries(checks).filter(([,v]) => !v).map(([k]) => k).join(", ")}`); } return { ...checks, elapsed: parseFloat(elapsed), allPassed, result }; } // ═══════════════════════════════════════════════════════ // Step 3: Backend Builder // ═══════════════════════════════════════════════════════ function generateBackend(domain, prdPath, archPath) { console.log(` ⚙️ Generating Backend...`); const t0 = Date.now(); const outDir = join(BENCH_ROOT, domain.id, "backend"); const result = runAgent( join(SCRIPTS, "backend-builder-agent.mjs"), `--prd "${prdPath}" --arch "${archPath}" --output "${outDir}"` ); const elapsed = ((Date.now() - t0) / 1000).toFixed(1); const fileCount = countFiles(outDir); const checks = { hasPackageJson: fileExists(outDir, "package.json"), hasDbSchema: fileExists(outDir, "schema") || fileExists(outDir, "db"), hasRoutes: fileExists(outDir, "routes/"), hasServices: fileExists(outDir, "services/"), hasAuth: fileExists(outDir, "auth"), hasMiddleware: fileExists(outDir, "middleware"), hasTypes: fileExists(outDir, "types/"), fileCount, }; const allPassed = checks.hasPackageJson && checks.hasRoutes && checks.hasServices && checks.hasAuth; console.log(` ${allPassed ? "✅" : "❌"} Backend generated (${elapsed}s, ${fileCount} files)`); if (!allPassed) { console.log(` Missing: ${Object.entries(checks).filter(([,v]) => !v).map(([k]) => k).join(", ")}`); } return { ...checks, elapsed: parseFloat(elapsed), allPassed, result }; } // ═══════════════════════════════════════════════════════ // Step 4: Fullstack Composer // ═══════════════════════════════════════════════════════ function generateFullstack(domain, prdPath, archPath) { console.log(` 🔗 Composing Fullstack...`); const t0 = Date.now(); const outDir = join(BENCH_ROOT, domain.id, "fullstack"); const result = runAgent( join(SCRIPTS, "fullstack-composer-agent.mjs"), `--prd "${prdPath}" --arch "${archPath}" --output "${outDir}"` ); const elapsed = ((Date.now() - t0) / 1000).toFixed(1); const fileCount = countFiles(outDir); const checks = { hasWeb: fileExists(outDir, "apps/web/"), hasApi: fileExists(outDir, "apps/api/"), hasSharedTypes: fileExists(outDir, "packages/shared-types/"), hasSharedConfig: fileExists(outDir, "packages/shared-config/"), hasRootPackageJson: fileExists(outDir, "package.json"), hasReadme: fileExists(outDir, "README.md"), fileCount, }; const allPassed = checks.hasWeb && checks.hasApi && checks.hasSharedTypes && checks.hasRootPackageJson; console.log(` ${allPassed ? "✅" : "❌"} Fullstack composed (${elapsed}s, ${fileCount} files)`); if (!allPassed) { console.log(` Missing: ${Object.entries(checks).filter(([,v]) => !v).map(([k]) => k).join(", ")}`); } return { ...checks, elapsed: parseFloat(elapsed), allPassed, result }; } // ═══════════════════════════════════════════════════════ // Step 5: Build Validation // ═══════════════════════════════════════════════════════ function validateBuild(domain) { console.log(` 🔨 Build Validation...`); const results = { frontend: null, backend: null, fullstack: null }; // Frontend build const feDir = join(BENCH_ROOT, domain.id, "frontend"); if (existsSync(join(feDir, "package.json"))) { console.log(` Frontend: npm install...`); const install = cmd("npm install --legacy-peer-deps 2>&1", { cwd: feDir, timeout: 120_000 }); if (install.error) { results.frontend = { build: "FAIL", reason: `npm install failed: ${install.stderr?.slice(0, 200) || install.error}` }; console.log(` ❌ npm install failed`); } else { console.log(` Frontend: npm run build...`); // Next.js build may need specific config; try build, if no build script, try next build const buildResult = cmd("npm run build 2>&1 || npx next build 2>&1", { cwd: feDir, timeout: 180_000 }); const buildFailed = buildResult.error || (typeof buildResult === 'string' && buildResult.includes('error')); const buildErrText = typeof buildResult === 'string' ? buildResult.split('\n').filter(l => l.includes('error') || l.includes('Error') || l.includes('⨯')).slice(0,5).join(' || ') : (buildResult.stderr?.slice(0, 500) || buildResult.error); results.frontend = { build: buildFailed ? "FAIL" : "PASS", error: buildFailed ? buildErrText : null }; console.log(` ${buildFailed ? "❌" : "✅"} Frontend build ${results.frontend.build}`); } } else { results.frontend = { build: "SKIP", reason: "No package.json" }; } // Backend build const beDir = join(BENCH_ROOT, domain.id, "backend"); if (existsSync(join(beDir, "package.json"))) { console.log(` Backend: npm install...`); const install = cmd("npm install --legacy-peer-deps 2>&1", { cwd: beDir, timeout: 120_000 }); if (install.error) { results.backend = { build: "FAIL", reason: `npm install failed: ${install.stderr?.slice(0, 200) || install.error}` }; console.log(` ❌ npm install failed`); } else { console.log(` Backend: tsc --noEmit...`); const buildResult = cmd("npx tsc --noEmit 2>&1", { cwd: beDir, timeout: 120_000 }); const buildFailed = buildResult.error || (typeof buildResult === 'string' && buildResult.includes('error TS')); const buildErrText = typeof buildResult === 'string' ? buildResult.split('\n').filter(l => l.includes('error TS')).slice(0,5).join(' || ') : (buildResult.stderr?.slice(0, 500) || buildResult.error); results.backend = { build: buildFailed ? "FAIL" : "PASS", error: buildFailed ? buildErrText : null }; console.log(` ${buildFailed ? "❌" : "✅"} Backend typecheck ${results.backend.build}`); } } else { results.backend = { build: "SKIP", reason: "No package.json" }; } // Fullstack build (just verify structure, don't run full monorepo build) const fsDir = join(BENCH_ROOT, domain.id, "fullstack"); if (existsSync(join(fsDir, "package.json"))) { results.fullstack = { build: "PASS", reason: "Structure validated (monorepo)" }; console.log(` ✅ Fullstack structure validated`); } else { results.fullstack = { build: "FAIL", reason: "No root package.json" }; } return results; } // ═══════════════════════════════════════════════════════ // Step 6: API Validation // ═══════════════════════════════════════════════════════ function validateAPI(domain) { console.log(` 🧪 API Validation...`); const beDir = join(BENCH_ROOT, domain.id, "backend"); const results = {}; if (!existsSync(join(beDir, "package.json"))) { return { error: "No backend package.json" }; } // Check route files for expected endpoints const routesDir = join(beDir, "src", "routes"); const routeFiles = []; try { const find = execSync(`find ${routesDir} -name "*.ts" -type f 2>/dev/null`, { encoding: "utf8", cwd: WORKSPACE }); routeFiles.push(...find.trim().split("\n").filter(Boolean)); } catch {} // Check for auth routes const hasAuthRoute = routeFiles.some(f => f.includes("auth")); const hasCRUDRoutes = routeFiles.filter(f => !f.includes("auth")).length; // Check for health endpoint const indexFile = join(beDir, "src", "index.ts"); let hasHealth = false; try { hasHealth = readFileSync(indexFile, "utf8").includes("health"); } catch {} // Check CRUD route content const crudOps = { create: false, read: false, update: false, delete: false }; for (const f of routeFiles) { try { const content = readFileSync(f, "utf8"); if (content.includes(".post(") || content.includes("app.post") || content.includes("router.post")) crudOps.create = true; if (content.includes(".get(") || content.includes("app.get") || content.includes("router.get")) crudOps.read = true; if (content.includes(".put(") || content.includes("app.put") || content.includes("router.put") || content.includes(".patch(") || content.includes("app.patch") || content.includes("router.patch")) crudOps.update = true; if (content.includes(".delete(") || content.includes("app.delete") || content.includes("router.delete")) crudOps.delete = true; } catch {} } // Check auth route content let authOps = { register: false, login: false, me: false }; const authFile = routeFiles.find(f => f.includes("auth")); if (authFile) { try { const content = readFileSync(authFile, "utf8"); authOps.register = content.includes("register") || content.includes("Register"); authOps.login = content.includes("login") || content.includes("Login"); authOps.me = content.includes("/me") || content.includes("'me'") || content.includes('"me"'); } catch {} } const crudPass = Object.values(crudOps).every(Boolean); const authPass = authOps.register && authOps.login; results.health = hasHealth; results.register = authOps.register; results.login = authOps.login; results.me = authOps.me; results.create = crudOps.create; results.read = crudOps.read; results.update = crudOps.update; results.delete = crudOps.delete; results.crudAllPass = crudPass; results.authAllPass = authPass; results.routeCount = routeFiles.length; console.log(` Health: ${hasHealth ? "✅" : "❌"} | Auth: ${authPass ? "✅" : "❌"} | CRUD: ${crudPass ? "✅" : "❌"} | Routes: ${routeFiles.length}`); return results; } // ═══════════════════════════════════════════════════════ // Step 7: Report Generation // ═══════════════════════════════════════════════════════ function generateReport(allResults) { console.log("\n\n📊 Generating Benchmark Report...\n"); const lines = []; const h = (text) => { lines.push(text); return lines; }; h(`# Domain Benchmark Report`); h(``); h(`> Generated: ${new Date().toISOString()}`); h(`> System: project-intake-agent → architecture-agent → frontend-builder → backend-builder → fullstack-composer`); h(``); // Summary metrics const total = allResults.length; const fePass = allResults.filter(r => r.frontend?.allPassed).length; const bePass = allResults.filter(r => r.backend?.allPassed).length; const fsPass = allResults.filter(r => r.fullstack?.allPassed).length; const buildPass = allResults.filter(r => r.build?.frontend?.build === "PASS").length; const typecheckPass = allResults.filter(r => r.build?.backend?.build === "PASS").length; const crudPass = allResults.filter(r => r.api?.crudAllPass).length; const authPass = allResults.filter(r => r.api?.authAllPass).length; h(`## Summary`); h(``); h(`| Metric | Value |`); h(`|--------|-------|`); h(`| Total Domains | ${total} |`); h(`| Frontend Generated | ${fePass}/${total} |`); h(`| Backend Generated | ${bePass}/${total} |`); h(`| Fullstack Composed | ${fsPass}/${total} |`); h(`| Frontend Build PASS | ${buildPass}/${total} |`); h(`| Backend Typecheck PASS | ${typecheckPass}/${total} |`); h(`| CRUD PASS | ${crudPass}/${total} |`); h(`| Auth PASS | ${authPass}/${total} |`); h(``); const avgFrontendFiles = allResults.reduce((s, r) => s + (r.frontend?.fileCount || 0), 0) / total; const avgBackendFiles = allResults.reduce((s, r) => s + (r.backend?.fileCount || 0), 0) / total; const avgFullstackFiles = allResults.reduce((s, r) => s + (r.fullstack?.fileCount || 0), 0) / total; const avgReqTime = allResults.reduce((s, r) => s + (r.reqTime || 0), 0) / total; const avgFeTime = allResults.reduce((s, r) => s + (r.frontend?.elapsed || 0), 0) / total; const avgBeTime = allResults.reduce((s, r) => s + (r.backend?.elapsed || 0), 0) / total; const avgFsTime = allResults.reduce((s, r) => s + (r.fullstack?.elapsed || 0), 0) / total; const avgTotalTime = allResults.reduce((s, r) => s + (r.totalTime || 0), 0) / total; h(`## Timing`); h(``); h(`| Phase | Avg Time |`); h(`|-------|----------|`); h(`| RequirementPackage | ${avgReqTime.toFixed(1)}s |`); h(`| Frontend Builder | ${avgFeTime.toFixed(1)}s |`); h(`| Backend Builder | ${avgBeTime.toFixed(1)}s |`); h(`| Fullstack Composer | ${avgFsTime.toFixed(1)}s |`); h(`| **Total Pipeline** | **${avgTotalTime.toFixed(1)}s** |`); h(``); h(`## Files Generated`); h(``); h(`| Layer | Avg Files |`); h(`|-------|-----------|`); h(`| Frontend | ${avgFrontendFiles.toFixed(0)} |`); h(`| Backend | ${avgBackendFiles.toFixed(0)} |`); h(`| Fullstack | ${avgFullstackFiles.toFixed(0)} |`); h(``); // Domain Results Table h(`## Domain Results`); h(``); h(`| Domain | FE Gen | BE Gen | FS Gen | FE Build | BE Typecheck | CRUD | Auth | Total Files | Time |`); h(`|--------|--------|--------|--------|----------|-------------|------|------|-------------|------|`); for (const r of allResults) { const feOk = r.frontend?.allPassed ? "✅" : "❌"; const beOk = r.backend?.allPassed ? "✅" : "❌"; const fsOk = r.fullstack?.allPassed ? "✅" : "❌"; const feBuild = r.build?.frontend?.build === "PASS" ? "✅" : r.build?.frontend?.build === "SKIP" ? "—" : "❌"; const beTC = r.build?.backend?.build === "PASS" ? "✅" : r.build?.backend?.build === "SKIP" ? "—" : "❌"; const crudOk = r.api?.crudAllPass ? "✅" : "❌"; const authOk = r.api?.authAllPass ? "✅" : "❌"; const totalFiles = (r.frontend?.fileCount || 0) + (r.backend?.fileCount || 0) + (r.fullstack?.fileCount || 0); h(`| ${r.id} | ${feOk} | ${beOk} | ${fsOk} | ${feBuild} | ${beTC} | ${crudOk} | ${authOk} | ${totalFiles} | ${(r.totalTime || 0).toFixed(1)}s |`); } h(``); // Detailed Domain Results h(`## Detailed Results`); h(``); for (const r of allResults) { h(`### ${r.id}`); h(``); h(`**Input:** "${r.input}"`); h(`**Project:** ${r.projectName || "N/A"} | **Domain:** ${r.domain || "N/A"}`); h(``); h(`#### Frontend`); if (r.frontend?.allPassed === false) { const missing = Object.entries(r.frontend || {}) .filter(([k, v]) => k.startsWith("has") && !v) .map(([k]) => k.replace("has", "")); h(`❌ Missing: ${missing.join(", ")}`); } else { h(`✅ ${r.frontend?.fileCount || 0} files generated`); } h(``); h(`#### Backend`); if (r.backend?.allPassed === false) { const missing = Object.entries(r.backend || {}) .filter(([k, v]) => k.startsWith("has") && !v) .map(([k]) => k.replace("has", "")); h(`❌ Missing: ${missing.join(", ")}`); } else { h(`✅ ${r.backend?.fileCount || 0} files generated`); } h(``); h(`#### Fullstack`); if (r.fullstack?.allPassed === false) { const missing = Object.entries(r.fullstack || {}) .filter(([k, v]) => k.startsWith("has") && !v) .map(([k]) => k.replace("has", "")); h(`❌ Missing: ${missing.join(", ")}`); } else { h(`✅ ${r.fullstack?.fileCount || 0} files generated`); } h(``); h(`#### Build`); if (r.build?.frontend?.build === "FAIL") { h(`❌ Frontend build failed: ${r.build.frontend.reason || r.build.frontend.error || ""}`); } else { h(`✅ Frontend build: ${r.build?.frontend?.build || "SKIP"}`); } if (r.build?.backend?.build === "FAIL") { h(`❌ Backend typecheck failed: ${r.build.backend.reason || r.build.backend.error || ""}`); } else { h(`✅ Backend typecheck: ${r.build?.backend?.build || "SKIP"}`); } h(``); h(`#### API`); if (r.api) { h(`| Endpoint | Status |`); h(`|----------|--------|`); h(`| Health | ${r.api.health ? "✅" : "❌"} |`); h(`| Register | ${r.api.register ? "✅" : "❌"} |`); h(`| Login | ${r.api.login ? "✅" : "❌"} |`); h(`| Me | ${r.api.me ? "✅" : "❌"} |`); h(`| Create | ${r.api.create ? "✅" : "❌"} |`); h(`| Read | ${r.api.read ? "✅" : "❌"} |`); h(`| Update | ${r.api.update ? "✅" : "❌"} |`); h(`| Delete | ${r.api.delete ? "✅" : "❌"} |`); } h(``); } // Failure Analysis h(`## Failure Analysis`); h(``); const failures = allResults.filter(r => !r.frontend?.allPassed || !r.backend?.allPassed || !r.fullstack?.allPassed || r.build?.frontend?.build === "FAIL" || r.build?.backend?.build === "FAIL" || !r.api?.crudAllPass || !r.api?.authAllPass ); if (failures.length === 0) { h(`✅ **All domains passed all checks!**`); h(``); } else { h(`### Failure Categories`); h(``); // Categorize failures const feGenFails = allResults.filter(r => !r.frontend?.allPassed); const beGenFails = allResults.filter(r => !r.backend?.allPassed); const fsGenFails = allResults.filter(r => !r.fullstack?.allPassed); const feBuildFails = allResults.filter(r => r.build?.frontend?.build === "FAIL"); const beBuildFails = allResults.filter(r => r.build?.backend?.build === "FAIL"); const crudFails = allResults.filter(r => !r.api?.crudAllPass); const authFails = allResults.filter(r => !r.api?.authAllPass); h(`| Category | Count | Domains |`); h(`|----------|-------|---------|`); h(`| Frontend Generation | ${feGenFails.length} | ${feGenFails.map(r => r.id).join(", ") || "—"} |`); h(`| Backend Generation | ${beGenFails.length} | ${beGenFails.map(r => r.id).join(", ") || "—"} |`); h(`| Fullstack Composition | ${fsGenFails.length} | ${fsGenFails.map(r => r.id).join(", ") || "—"} |`); h(`| Frontend Build | ${feBuildFails.length} | ${feBuildFails.map(r => r.id).join(", ") || "—"} |`); h(`| Backend Typecheck | ${beBuildFails.length} | ${beBuildFails.map(r => r.id).join(", ") || "—"} |`); h(`| CRUD Missing | ${crudFails.length} | ${crudFails.map(r => r.id).join(", ") || "—"} |`); h(`| Auth Missing | ${authFails.length} | ${authFails.map(r => r.id).join(", ") || "—"} |`); h(``); // Detailed failure reasons h(`### Failure Details`); h(``); for (const r of allResults) { const issues = []; if (!r.frontend?.allPassed) { const missing = Object.entries(r.frontend || {}).filter(([k, v]) => k.startsWith("has") && !v).map(([k]) => k.replace("has", "")); issues.push(`Frontend: missing ${missing.join(", ")}`); } if (!r.backend?.allPassed) { const missing = Object.entries(r.backend || {}).filter(([k, v]) => k.startsWith("has") && !v).map(([k]) => k.replace("has", "")); issues.push(`Backend: missing ${missing.join(", ")}`); } if (!r.fullstack?.allPassed) { const missing = Object.entries(r.fullstack || {}).filter(([k, v]) => k.startsWith("has") && !v).map(([k]) => k.replace("has", "")); issues.push(`Fullstack: missing ${missing.join(", ")}`); } if (r.build?.frontend?.build === "FAIL") issues.push(`Frontend build: ${r.build.frontend.error || r.build.frontend.reason}`); if (r.build?.backend?.build === "FAIL") issues.push(`Backend typecheck: ${r.build.backend.error || r.build.backend.reason}`); if (!r.api?.crudAllPass) { const missing = Object.entries({create: r.api?.create, read: r.api?.read, update: r.api?.update, delete: r.api?.delete}) .filter(([,v]) => !v).map(([k]) => k); issues.push(`CRUD: missing ${missing.join(", ")}`); } if (!r.api?.authAllPass) { const missing = Object.entries({register: r.api?.register, login: r.api?.login, me: r.api?.me}) .filter(([,v]) => !v).map(([k]) => k); issues.push(`Auth: missing ${missing.join(", ")}`); } if (issues.length > 0) { h(`**${r.id}:**`); for (const i of issues) h(`- ${i}`); h(``); } } } // Pattern Analysis h(`## Pattern Analysis`); h(``); // Best domains const scored = allResults.map(r => { let score = 0; if (r.frontend?.allPassed) score += 2; if (r.backend?.allPassed) score += 2; if (r.fullstack?.allPassed) score += 1; if (r.build?.frontend?.build === "PASS") score += 2; if (r.build?.backend?.build === "PASS") score += 2; if (r.api?.crudAllPass) score += 2; if (r.api?.authAllPass) score += 2; return { ...r, score }; }).sort((a, b) => b.score - a.score); h(`### Best Performing Domains`); h(``); h(`| Domain | Score | Key Strength |`); h(`|--------|-------|-------------|`); for (const r of scored.slice(0, 3)) { h(`| ${r.id} | ${r.score}/13 | ${r.domain || "N/A"} |`); } h(``); h(`### Weakest Domains`); h(``); h(`| Domain | Score | Key Issue |`); h(`|--------|-------|-----------|`); for (const r of scored.slice(-3).reverse()) { const issues = []; if (!r.frontend?.allPassed) issues.push("FE gen"); if (!r.backend?.allPassed) issues.push("BE gen"); if (r.build?.frontend?.build === "FAIL") issues.push("FE build"); if (r.build?.backend?.build === "FAIL") issues.push("BE typecheck"); if (!r.api?.crudAllPass) issues.push("CRUD"); if (!r.api?.authAllPass) issues.push("Auth"); h(`| ${r.id} | ${r.score}/13 | ${issues.join(", ") || "—"} |`); } h(``); h(`### Domain Type Analysis`); h(``); h(`| Domain Type | Count | Avg Score |`); h(`|-------------|-------|-----------|`); const byDomain = {}; for (const r of allResults) { const d = r.domain || "unknown"; if (!byDomain[d]) byDomain[d] = { count: 0, scores: [] }; byDomain[d].count++; byDomain[d].scores.push(r.score); } for (const [domain, data] of Object.entries(byDomain)) { const avg = (data.scores.reduce((a,b) => a + b, 0) / data.scores.length).toFixed(1); h(`| ${domain} | ${data.count} | ${avg} |`); } h(``); // Failure categories (defined above already, but recompute for Fixes section) const feGenFailsR = allResults.filter(r => !r.frontend?.allPassed); const beGenFailsR = allResults.filter(r => !r.backend?.allPassed); const fsGenFailsR = allResults.filter(r => !r.fullstack?.allPassed); const feBuildFailsR = allResults.filter(r => r.build?.frontend?.build === "FAIL"); const beBuildFailsR = allResults.filter(r => r.build?.backend?.build === "FAIL"); const crudFailsR = allResults.filter(r => !r.api?.crudAllPass); const authFailsR = allResults.filter(r => !r.api?.authAllPass); // Required Fixes h(`## Required Fixes`); h(``); const criticalFixes = []; const highFixes = []; const mediumFixes = []; // Analyze patterns if (feBuildFailsR.length > 0) { criticalFixes.push(`**Frontend Build Failures** (${feBuildFailsR.length}/${total}): Fix Next.js build issues in frontend-builder-agent — check tsconfig, module resolution, component imports`); } if (beBuildFailsR.length > 0) { criticalFixes.push(`**Backend Typecheck Failures** (${beBuildFailsR.length}/${total}): Fix TypeScript errors in backend-builder-agent — check type definitions, import paths`); } if (beGenFailsR.length > 0) { highFixes.push(`**Backend Generation Gaps** (${beGenFailsR.length}/${total}): Missing routes/services/auth in some domains`); } if (crudFailsR.length > 0) { highFixes.push(`**CRUD Completeness** (${crudFailsR.length}/${total}): Some domains missing full CRUD operations`); } if (authFailsR.length > 0) { highFixes.push(`**Auth Endpoint Coverage** (${authFailsR.length}/${total}): Missing register/login/me in some domains`); } if (feGenFailsR.length > 0) { mediumFixes.push(`**Frontend Generation Gaps** (${feGenFailsR.length}/${total}): Missing pages or services in some domains`); } h(`### P0 — Critical`); h(``); if (criticalFixes.length === 0) { h(`✅ No critical issues found.`); } else { for (const f of criticalFixes) h(`1. ${f}`); } h(``); h(`### P1 — High`); h(``); if (highFixes.length === 0) { h(`✅ No high-priority issues found.`); } else { for (let i = 0; i < highFixes.length; i++) h(`${i + 1}. ${highFixes[i]}`); } h(``); h(`### P2 — Medium`); h(``); if (mediumFixes.length === 0) { h(`✅ No medium-priority issues found.`); } else { for (let i = 0; i < mediumFixes.length; i++) h(`${i + 1}. ${mediumFixes[i]}`); } h(``); // Conclusion h(`## Conclusion`); h(``); const passRate = allResults.filter(r => r.score >= 11).length; if (passRate === total) { h(`🎉 **All ${total} domains passed!** The system demonstrates robust generalization capability across diverse business domains.`); h(``); h(`The RequirementPackage → Frontend Builder → Backend Builder → Fullstack Composer pipeline is production-ready for Web Fullstack generation.`); h(``); h(`**Next: Electron Builder can now proceed.**`); } else { h(`⚠️ **${passRate}/${total} domains fully passed.**`); h(``); h(`The pipeline shows partial generalization. ${total - passRate} domains need fixes before the system can be considered a truly general Web Fullstack Generator.`); h(``); h(`**Electron Builder should wait until fixes above are addressed.**`); } h(``); h(`---`); h(`*Report generated by Domain Benchmark Suite*`); return lines.join("\n"); } // ═══════════════════════════════════════════════════════ // Main // ═══════════════════════════════════════════════════════ async function main() { const args = process.argv.slice(2); const filterDomain = args[0] === "--domain" ? args[1] : null; const skipBuild = args.includes("--skip-build"); const skipAPI = args.includes("--skip-api"); // Filter domains if specified const domains = filterDomain ? DOMAINS.filter(d => d.id === filterDomain) : DOMAINS; if (domains.length === 0) { console.error(`Unknown domain: ${filterDomain}`); console.error(`Available: ${DOMAINS.map(d => d.id).join(", ")}`); process.exit(1); } console.log(`\n🧪 Domain Benchmark Suite`); console.log(` Testing ${domains.length} domain(s)`); console.log(` Pipeline: project-intake → architecture → frontend → backend → fullstack`); if (skipBuild) console.log(` ⏭️ Skipping build validation`); if (skipAPI) console.log(` ⏭️ Skipping API validation`); console.log(``); const allResults = []; for (const domain of domains) { console.log(`\n${"=".repeat(60)}`); console.log(`🏗️ Domain: ${domain.id}`); console.log(`${"=".repeat(60)}`); const tTotal = Date.now(); // Step 1: RequirementPackage const { prd, arch, elapsed: reqTime, prdPath, archPath } = generateRequirementPackage(domain); // Step 2: Frontend const frontend = generateFrontend(domain, prdPath, archPath); // Step 3: Backend const backend = generateBackend(domain, prdPath, archPath); // Step 4: Fullstack const fullstack = generateFullstack(domain, prdPath, archPath); // Step 5: Build (optional) let build = null; if (!skipBuild) { build = validateBuild(domain); } // Step 6: API (optional) let api = null; if (!skipAPI) { api = validateAPI(domain); } const totalTime = ((Date.now() - tTotal) / 1000).toFixed(1); allResults.push({ id: domain.id, input: domain.input, projectName: prd.projectName, domain: prd.domain, reqTime, frontend, backend, fullstack, build, api, totalTime: parseFloat(totalTime), }); console.log(`\n ⏱️ Total: ${totalTime}s`); } // Generate report const report = generateReport(allResults); writeFileSync(REPORT_PATH, report, "utf8"); console.log(`\n📄 Report written to: ${REPORT_PATH}`); // Print summary const passCount = allResults.filter(r => { return r.frontend?.allPassed && r.backend?.allPassed && r.fullstack?.allPassed; }).length; console.log(`\n📊 Summary: ${passCount}/${domains.length} domains generated successfully`); // Also output JSON for programmatic consumption const jsonPath = resolve(WORKSPACE, "benchmark-results.json"); writeFileSync(jsonPath, JSON.stringify(allResults, null, 2), "utf8"); console.log(`📄 JSON results: ${jsonPath}`); } main().catch(e => { console.error("Benchmark failed:", e); process.exit(1); });