大海
874 lines
36 KiB
JavaScript
874 lines
36 KiB
JavaScript
#!/usr/bin/env node
|
||
|
||
/**
|
||
* Domain Benchmark Suite
|
||
*
|
||
* 验证现有系统的泛化能力：
|
||
* project-intake-agent → architecture-agent
|
||
* → frontend-builder-agent → backend-builder-agent → fullstack-composer-agent
|
||
*
|
||
* 对 10 个领域生成完整项目，验证 Build + API，输出 benchmark-report.md
|
||
*
|
||
* Usage:
|
||
* node scripts/domain-benchmark.mjs [--domain <name>] [--skip-build] [--skip-api]
|
||
*/
|
||
|
||
import { readFileSync, writeFileSync, existsSync, mkdirSync, rmSync } from "node:fs";
|
||
import { resolve, dirname, join } from "node:path";
|
||
import { execSync } from "node:child_process";
|
||
import { fileURLToPath } from "node:url";
|
||
|
||
const __dirname = dirname(resolve(fileURLToPath(import.meta.url)));
|
||
const WORKSPACE = resolve(__dirname, "..");
|
||
const BENCH_ROOT = resolve(WORKSPACE, ".benchmark");
|
||
const SCRIPTS = resolve(WORKSPACE, "scripts");
|
||
|
||
const REPORT_PATH = resolve(WORKSPACE, "benchmark-report.md");
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Domain Definitions
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
const DOMAINS = [
|
||
{ id: "petcare", input: "做一个宠物护理管理平台，宠物主人可以管理宠物档案、健康日程、日常记录和成长相册" },
|
||
{ id: "crm", input: "做一个客户关系管理系统，支持客户管理、销售漏斗、跟进记录和数据分析仪表盘" },
|
||
{ id: "inventory", input: "做一个库存管理系统，支持商品入库出库、库存盘点、供应商管理和库存预警" },
|
||
{ id: "ticket", input: "做一个工单系统，支持工单创建、分配、处理流程、优先级管理和工单归档" },
|
||
{ id: "blog-cms", input: "做一个博客内容管理系统，支持文章发布、分类标签、评论管理和媒体库" },
|
||
{ id: "project-mgmt", input: "做一个项目管理系统，支持项目看板、任务分配、甘特图和团队协作" },
|
||
{ id: "hr", input: "做一个人力资源管理系统，支持员工档案、考勤管理、招聘流程和绩效评估" },
|
||
{ id: "asset", input: "做一个固定资产管理系统，支持资产登记、领用归还、折旧计算和盘点统计" },
|
||
{ id: "course", input: "做一个在线课程管理系统，支持课程发布、章节管理、学员进度和作业批改" },
|
||
{ id: "appointment", input: "做一个预约管理系统，支持服务项目、时间段预约、客户通知和预约统计" },
|
||
];
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Helpers
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function cmd(c, opts = {}) {
|
||
try {
|
||
return execSync(c, { cwd: WORKSPACE, encoding: "utf8", ...opts, stdio: opts.stdio ?? "pipe" });
|
||
} catch (e) {
|
||
return { error: e.message, stderr: e.stderr?.toString() || "", stdout: e.stdout?.toString() || "" };
|
||
}
|
||
}
|
||
|
||
function runAgent(agentPath, args) {
|
||
const result = cmd(`node ${agentPath} ${args}`);
|
||
if (result.error) {
|
||
// Try to extract JSON from stdout even on error
|
||
try { return JSON.parse(result.stdout?.trim() || "{}"); } catch { return { error: result.error, stderr: result.stderr }; }
|
||
}
|
||
try { return JSON.parse(result.trim()); } catch { return { raw: result }; }
|
||
}
|
||
|
||
function rmDirSafe(dir) {
|
||
try { if (existsSync(dir)) rmSync(dir, { recursive: true, force: true }); } catch {}
|
||
}
|
||
|
||
function listFilesRecursive(dir, acc = []) {
|
||
if (!existsSync(dir)) return acc;
|
||
const entries = readFileSync(dir)?.toString?.(); // not for dirs
|
||
const { readdirSync, statSync } = require("node:fs") || {};
|
||
// Use shell for simplicity
|
||
const result = execSync(`find ${dir} -type f | head -500`, { encoding: "utf8", cwd: WORKSPACE });
|
||
return result.trim().split("\n").filter(Boolean);
|
||
}
|
||
|
||
function countFiles(dir) {
|
||
try {
|
||
const r = execSync(`find ${dir} -type f 2>/dev/null | wc -l`, { encoding: "utf8", cwd: WORKSPACE });
|
||
return parseInt(r.trim(), 10);
|
||
} catch { return 0; }
|
||
}
|
||
|
||
function fileExists(dir, pattern) {
|
||
try {
|
||
const r = execSync(`find ${dir} -path "*${pattern}*" -type f 2>/dev/null | head -1`, { encoding: "utf8", cwd: WORKSPACE });
|
||
return r.trim().length > 0;
|
||
} catch { return false; }
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Step 1: RequirementPackage (SF-01 + SF-02)
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function loadJSON(path) {
|
||
try { return JSON.parse(readFileSync(path, "utf8")); } catch { return {}; }
|
||
}
|
||
|
||
function generateRequirementPackage(domain) {
|
||
console.log(`\n📋 [${domain.id}] Generating RequirementPackage...`);
|
||
const t0 = Date.now();
|
||
|
||
// SF-01: PRD
|
||
const prdOut = join(BENCH_ROOT, domain.id, "prd.json");
|
||
mkdirSync(dirname(prdOut), { recursive: true });
|
||
runAgent(
|
||
join(SCRIPTS, "project-intake-agent.mjs"),
|
||
`--input "${domain.input}" --output "${prdOut}"`
|
||
);
|
||
const prd = loadJSON(prdOut);
|
||
|
||
// SF-02: Architecture
|
||
const archOut = join(BENCH_ROOT, domain.id, "arch.json");
|
||
runAgent(
|
||
join(SCRIPTS, "architecture-agent.mjs"),
|
||
`--input "${prdOut}" --output "${archOut}"`
|
||
);
|
||
const arch = loadJSON(archOut);
|
||
|
||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||
console.log(` ✅ PRD + Architecture generated (${elapsed}s)`);
|
||
console.log(` Project: ${prd.projectName || "?"}, Domain: ${prd.domain || "?"}`);
|
||
console.log(` Features: ${prd.features?.length || 0}, Pages: ${prd.pages?.length || 0}, APIs: ${prd.apiRequirements?.length || 0}`);
|
||
|
||
return { prd, arch, elapsed: parseFloat(elapsed), prdPath: prdOut, archPath: archOut };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Step 2: Frontend Builder
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function generateFrontend(domain, prdPath, archPath) {
|
||
console.log(` 🎨 Generating Frontend...`);
|
||
const t0 = Date.now();
|
||
const outDir = join(BENCH_ROOT, domain.id, "frontend");
|
||
|
||
const result = runAgent(
|
||
join(SCRIPTS, "frontend-builder-agent.mjs"),
|
||
`--prd "${prdPath}" --arch "${archPath}" --output "${outDir}"`
|
||
);
|
||
|
||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||
const fileCount = countFiles(outDir);
|
||
|
||
// Validation checks
|
||
const checks = {
|
||
hasPackageJson: fileExists(outDir, "package.json"),
|
||
hasNextConfig: fileExists(outDir, "next.config.*"),
|
||
hasAppPage: fileExists(outDir, "app/page.*"),
|
||
hasEntityPages: fileExists(outDir, "app/") && countFiles(join(outDir, "app")) > 2,
|
||
hasServices: fileExists(outDir, "services/"),
|
||
hasTypes: fileExists(outDir, "types/"),
|
||
fileCount,
|
||
};
|
||
|
||
const allPassed = checks.hasPackageJson && checks.hasNextConfig && checks.hasAppPage && checks.hasEntityPages;
|
||
|
||
console.log(` ${allPassed ? "✅" : "❌"} Frontend generated (${elapsed}s, ${fileCount} files)`);
|
||
if (!allPassed) {
|
||
console.log(` Missing: ${Object.entries(checks).filter(([,v]) => !v).map(([k]) => k).join(", ")}`);
|
||
}
|
||
|
||
return { ...checks, elapsed: parseFloat(elapsed), allPassed, result };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Step 3: Backend Builder
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function generateBackend(domain, prdPath, archPath) {
|
||
console.log(` ⚙️ Generating Backend...`);
|
||
const t0 = Date.now();
|
||
const outDir = join(BENCH_ROOT, domain.id, "backend");
|
||
|
||
const result = runAgent(
|
||
join(SCRIPTS, "backend-builder-agent.mjs"),
|
||
`--prd "${prdPath}" --arch "${archPath}" --output "${outDir}"`
|
||
);
|
||
|
||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||
const fileCount = countFiles(outDir);
|
||
|
||
const checks = {
|
||
hasPackageJson: fileExists(outDir, "package.json"),
|
||
hasDbSchema: fileExists(outDir, "schema") || fileExists(outDir, "db"),
|
||
hasRoutes: fileExists(outDir, "routes/"),
|
||
hasServices: fileExists(outDir, "services/"),
|
||
hasAuth: fileExists(outDir, "auth"),
|
||
hasMiddleware: fileExists(outDir, "middleware"),
|
||
hasTypes: fileExists(outDir, "types/"),
|
||
fileCount,
|
||
};
|
||
|
||
const allPassed = checks.hasPackageJson && checks.hasRoutes && checks.hasServices && checks.hasAuth;
|
||
|
||
console.log(` ${allPassed ? "✅" : "❌"} Backend generated (${elapsed}s, ${fileCount} files)`);
|
||
if (!allPassed) {
|
||
console.log(` Missing: ${Object.entries(checks).filter(([,v]) => !v).map(([k]) => k).join(", ")}`);
|
||
}
|
||
|
||
return { ...checks, elapsed: parseFloat(elapsed), allPassed, result };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Step 4: Fullstack Composer
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function generateFullstack(domain, prdPath, archPath) {
|
||
console.log(` 🔗 Composing Fullstack...`);
|
||
const t0 = Date.now();
|
||
const outDir = join(BENCH_ROOT, domain.id, "fullstack");
|
||
|
||
const result = runAgent(
|
||
join(SCRIPTS, "fullstack-composer-agent.mjs"),
|
||
`--prd "${prdPath}" --arch "${archPath}" --output "${outDir}"`
|
||
);
|
||
|
||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||
const fileCount = countFiles(outDir);
|
||
|
||
const checks = {
|
||
hasWeb: fileExists(outDir, "apps/web/"),
|
||
hasApi: fileExists(outDir, "apps/api/"),
|
||
hasSharedTypes: fileExists(outDir, "packages/shared-types/"),
|
||
hasSharedConfig: fileExists(outDir, "packages/shared-config/"),
|
||
hasRootPackageJson: fileExists(outDir, "package.json"),
|
||
hasReadme: fileExists(outDir, "README.md"),
|
||
fileCount,
|
||
};
|
||
|
||
const allPassed = checks.hasWeb && checks.hasApi && checks.hasSharedTypes && checks.hasRootPackageJson;
|
||
|
||
console.log(` ${allPassed ? "✅" : "❌"} Fullstack composed (${elapsed}s, ${fileCount} files)`);
|
||
if (!allPassed) {
|
||
console.log(` Missing: ${Object.entries(checks).filter(([,v]) => !v).map(([k]) => k).join(", ")}`);
|
||
}
|
||
|
||
return { ...checks, elapsed: parseFloat(elapsed), allPassed, result };
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Step 5: Build Validation
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function validateBuild(domain) {
|
||
console.log(` 🔨 Build Validation...`);
|
||
|
||
const results = { frontend: null, backend: null, fullstack: null };
|
||
|
||
// Frontend build
|
||
const feDir = join(BENCH_ROOT, domain.id, "frontend");
|
||
if (existsSync(join(feDir, "package.json"))) {
|
||
console.log(` Frontend: npm install...`);
|
||
const install = cmd("npm install --legacy-peer-deps 2>&1", { cwd: feDir, timeout: 120_000 });
|
||
if (install.error) {
|
||
results.frontend = { build: "FAIL", reason: `npm install failed: ${install.stderr?.slice(0, 200) || install.error}` };
|
||
console.log(` ❌ npm install failed`);
|
||
} else {
|
||
console.log(` Frontend: npm run build...`);
|
||
// Next.js build may need specific config; try build, if no build script, try next build
|
||
const buildResult = cmd("npm run build 2>&1 || npx next build 2>&1", { cwd: feDir, timeout: 180_000 });
|
||
const buildFailed = buildResult.error || (typeof buildResult === 'string' && buildResult.includes('error'));
|
||
const buildErrText = typeof buildResult === 'string' ? buildResult.split('\n').filter(l => l.includes('error') || l.includes('Error') || l.includes('⨯')).slice(0,5).join(' || ') : (buildResult.stderr?.slice(0, 500) || buildResult.error);
|
||
results.frontend = {
|
||
build: buildFailed ? "FAIL" : "PASS",
|
||
error: buildFailed ? buildErrText : null
|
||
};
|
||
console.log(` ${buildFailed ? "❌" : "✅"} Frontend build ${results.frontend.build}`);
|
||
}
|
||
} else {
|
||
results.frontend = { build: "SKIP", reason: "No package.json" };
|
||
}
|
||
|
||
// Backend build
|
||
const beDir = join(BENCH_ROOT, domain.id, "backend");
|
||
if (existsSync(join(beDir, "package.json"))) {
|
||
console.log(` Backend: npm install...`);
|
||
const install = cmd("npm install --legacy-peer-deps 2>&1", { cwd: beDir, timeout: 120_000 });
|
||
if (install.error) {
|
||
results.backend = { build: "FAIL", reason: `npm install failed: ${install.stderr?.slice(0, 200) || install.error}` };
|
||
console.log(` ❌ npm install failed`);
|
||
} else {
|
||
console.log(` Backend: tsc --noEmit...`);
|
||
const buildResult = cmd("npx tsc --noEmit 2>&1", { cwd: beDir, timeout: 120_000 });
|
||
const buildFailed = buildResult.error || (typeof buildResult === 'string' && buildResult.includes('error TS'));
|
||
const buildErrText = typeof buildResult === 'string' ? buildResult.split('\n').filter(l => l.includes('error TS')).slice(0,5).join(' || ') : (buildResult.stderr?.slice(0, 500) || buildResult.error);
|
||
results.backend = {
|
||
build: buildFailed ? "FAIL" : "PASS",
|
||
error: buildFailed ? buildErrText : null
|
||
};
|
||
console.log(` ${buildFailed ? "❌" : "✅"} Backend typecheck ${results.backend.build}`);
|
||
}
|
||
} else {
|
||
results.backend = { build: "SKIP", reason: "No package.json" };
|
||
}
|
||
|
||
// Fullstack build (just verify structure, don't run full monorepo build)
|
||
const fsDir = join(BENCH_ROOT, domain.id, "fullstack");
|
||
if (existsSync(join(fsDir, "package.json"))) {
|
||
results.fullstack = { build: "PASS", reason: "Structure validated (monorepo)" };
|
||
console.log(` ✅ Fullstack structure validated`);
|
||
} else {
|
||
results.fullstack = { build: "FAIL", reason: "No root package.json" };
|
||
}
|
||
|
||
return results;
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Step 6: API Validation
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function validateAPI(domain) {
|
||
console.log(` 🧪 API Validation...`);
|
||
|
||
const beDir = join(BENCH_ROOT, domain.id, "backend");
|
||
const results = {};
|
||
|
||
if (!existsSync(join(beDir, "package.json"))) {
|
||
return { error: "No backend package.json" };
|
||
}
|
||
|
||
// Check route files for expected endpoints
|
||
const routesDir = join(beDir, "src", "routes");
|
||
const routeFiles = [];
|
||
try {
|
||
const find = execSync(`find ${routesDir} -name "*.ts" -type f 2>/dev/null`, { encoding: "utf8", cwd: WORKSPACE });
|
||
routeFiles.push(...find.trim().split("\n").filter(Boolean));
|
||
} catch {}
|
||
|
||
// Check for auth routes
|
||
const hasAuthRoute = routeFiles.some(f => f.includes("auth"));
|
||
const hasCRUDRoutes = routeFiles.filter(f => !f.includes("auth")).length;
|
||
|
||
// Check for health endpoint
|
||
const indexFile = join(beDir, "src", "index.ts");
|
||
let hasHealth = false;
|
||
try {
|
||
hasHealth = readFileSync(indexFile, "utf8").includes("health");
|
||
} catch {}
|
||
|
||
// Check CRUD route content
|
||
const crudOps = { create: false, read: false, update: false, delete: false };
|
||
for (const f of routeFiles) {
|
||
try {
|
||
const content = readFileSync(f, "utf8");
|
||
if (content.includes(".post(") || content.includes("app.post") || content.includes("router.post")) crudOps.create = true;
|
||
if (content.includes(".get(") || content.includes("app.get") || content.includes("router.get")) crudOps.read = true;
|
||
if (content.includes(".put(") || content.includes("app.put") || content.includes("router.put") || content.includes(".patch(") || content.includes("app.patch") || content.includes("router.patch")) crudOps.update = true;
|
||
if (content.includes(".delete(") || content.includes("app.delete") || content.includes("router.delete")) crudOps.delete = true;
|
||
} catch {}
|
||
}
|
||
|
||
// Check auth route content
|
||
let authOps = { register: false, login: false, me: false };
|
||
const authFile = routeFiles.find(f => f.includes("auth"));
|
||
if (authFile) {
|
||
try {
|
||
const content = readFileSync(authFile, "utf8");
|
||
authOps.register = content.includes("register") || content.includes("Register");
|
||
authOps.login = content.includes("login") || content.includes("Login");
|
||
authOps.me = content.includes("/me") || content.includes("'me'") || content.includes('"me"');
|
||
} catch {}
|
||
}
|
||
|
||
const crudPass = Object.values(crudOps).every(Boolean);
|
||
const authPass = authOps.register && authOps.login;
|
||
|
||
results.health = hasHealth;
|
||
results.register = authOps.register;
|
||
results.login = authOps.login;
|
||
results.me = authOps.me;
|
||
results.create = crudOps.create;
|
||
results.read = crudOps.read;
|
||
results.update = crudOps.update;
|
||
results.delete = crudOps.delete;
|
||
results.crudAllPass = crudPass;
|
||
results.authAllPass = authPass;
|
||
results.routeCount = routeFiles.length;
|
||
|
||
console.log(` Health: ${hasHealth ? "✅" : "❌"} | Auth: ${authPass ? "✅" : "❌"} | CRUD: ${crudPass ? "✅" : "❌"} | Routes: ${routeFiles.length}`);
|
||
|
||
return results;
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Step 7: Report Generation
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
function generateReport(allResults) {
|
||
console.log("\n\n📊 Generating Benchmark Report...\n");
|
||
|
||
const lines = [];
|
||
const h = (text) => { lines.push(text); return lines; };
|
||
|
||
h(`# Domain Benchmark Report`);
|
||
h(``);
|
||
h(`> Generated: ${new Date().toISOString()}`);
|
||
h(`> System: project-intake-agent → architecture-agent → frontend-builder → backend-builder → fullstack-composer`);
|
||
h(``);
|
||
|
||
// Summary metrics
|
||
const total = allResults.length;
|
||
const fePass = allResults.filter(r => r.frontend?.allPassed).length;
|
||
const bePass = allResults.filter(r => r.backend?.allPassed).length;
|
||
const fsPass = allResults.filter(r => r.fullstack?.allPassed).length;
|
||
const buildPass = allResults.filter(r => r.build?.frontend?.build === "PASS").length;
|
||
const typecheckPass = allResults.filter(r => r.build?.backend?.build === "PASS").length;
|
||
const crudPass = allResults.filter(r => r.api?.crudAllPass).length;
|
||
const authPass = allResults.filter(r => r.api?.authAllPass).length;
|
||
|
||
h(`## Summary`);
|
||
h(``);
|
||
h(`| Metric | Value |`);
|
||
h(`|--------|-------|`);
|
||
h(`| Total Domains | ${total} |`);
|
||
h(`| Frontend Generated | ${fePass}/${total} |`);
|
||
h(`| Backend Generated | ${bePass}/${total} |`);
|
||
h(`| Fullstack Composed | ${fsPass}/${total} |`);
|
||
h(`| Frontend Build PASS | ${buildPass}/${total} |`);
|
||
h(`| Backend Typecheck PASS | ${typecheckPass}/${total} |`);
|
||
h(`| CRUD PASS | ${crudPass}/${total} |`);
|
||
h(`| Auth PASS | ${authPass}/${total} |`);
|
||
h(``);
|
||
|
||
const avgFrontendFiles = allResults.reduce((s, r) => s + (r.frontend?.fileCount || 0), 0) / total;
|
||
const avgBackendFiles = allResults.reduce((s, r) => s + (r.backend?.fileCount || 0), 0) / total;
|
||
const avgFullstackFiles = allResults.reduce((s, r) => s + (r.fullstack?.fileCount || 0), 0) / total;
|
||
const avgReqTime = allResults.reduce((s, r) => s + (r.reqTime || 0), 0) / total;
|
||
const avgFeTime = allResults.reduce((s, r) => s + (r.frontend?.elapsed || 0), 0) / total;
|
||
const avgBeTime = allResults.reduce((s, r) => s + (r.backend?.elapsed || 0), 0) / total;
|
||
const avgFsTime = allResults.reduce((s, r) => s + (r.fullstack?.elapsed || 0), 0) / total;
|
||
const avgTotalTime = allResults.reduce((s, r) => s + (r.totalTime || 0), 0) / total;
|
||
|
||
h(`## Timing`);
|
||
h(``);
|
||
h(`| Phase | Avg Time |`);
|
||
h(`|-------|----------|`);
|
||
h(`| RequirementPackage | ${avgReqTime.toFixed(1)}s |`);
|
||
h(`| Frontend Builder | ${avgFeTime.toFixed(1)}s |`);
|
||
h(`| Backend Builder | ${avgBeTime.toFixed(1)}s |`);
|
||
h(`| Fullstack Composer | ${avgFsTime.toFixed(1)}s |`);
|
||
h(`| **Total Pipeline** | **${avgTotalTime.toFixed(1)}s** |`);
|
||
h(``);
|
||
|
||
h(`## Files Generated`);
|
||
h(``);
|
||
h(`| Layer | Avg Files |`);
|
||
h(`|-------|-----------|`);
|
||
h(`| Frontend | ${avgFrontendFiles.toFixed(0)} |`);
|
||
h(`| Backend | ${avgBackendFiles.toFixed(0)} |`);
|
||
h(`| Fullstack | ${avgFullstackFiles.toFixed(0)} |`);
|
||
h(``);
|
||
|
||
// Domain Results Table
|
||
h(`## Domain Results`);
|
||
h(``);
|
||
h(`| Domain | FE Gen | BE Gen | FS Gen | FE Build | BE Typecheck | CRUD | Auth | Total Files | Time |`);
|
||
h(`|--------|--------|--------|--------|----------|-------------|------|------|-------------|------|`);
|
||
|
||
for (const r of allResults) {
|
||
const feOk = r.frontend?.allPassed ? "✅" : "❌";
|
||
const beOk = r.backend?.allPassed ? "✅" : "❌";
|
||
const fsOk = r.fullstack?.allPassed ? "✅" : "❌";
|
||
const feBuild = r.build?.frontend?.build === "PASS" ? "✅" : r.build?.frontend?.build === "SKIP" ? "—" : "❌";
|
||
const beTC = r.build?.backend?.build === "PASS" ? "✅" : r.build?.backend?.build === "SKIP" ? "—" : "❌";
|
||
const crudOk = r.api?.crudAllPass ? "✅" : "❌";
|
||
const authOk = r.api?.authAllPass ? "✅" : "❌";
|
||
const totalFiles = (r.frontend?.fileCount || 0) + (r.backend?.fileCount || 0) + (r.fullstack?.fileCount || 0);
|
||
h(`| ${r.id} | ${feOk} | ${beOk} | ${fsOk} | ${feBuild} | ${beTC} | ${crudOk} | ${authOk} | ${totalFiles} | ${(r.totalTime || 0).toFixed(1)}s |`);
|
||
}
|
||
h(``);
|
||
|
||
// Detailed Domain Results
|
||
h(`## Detailed Results`);
|
||
h(``);
|
||
|
||
for (const r of allResults) {
|
||
h(`### ${r.id}`);
|
||
h(``);
|
||
h(`**Input:** "${r.input}"`);
|
||
h(`**Project:** ${r.projectName || "N/A"} | **Domain:** ${r.domain || "N/A"}`);
|
||
h(``);
|
||
|
||
h(`#### Frontend`);
|
||
if (r.frontend?.allPassed === false) {
|
||
const missing = Object.entries(r.frontend || {})
|
||
.filter(([k, v]) => k.startsWith("has") && !v)
|
||
.map(([k]) => k.replace("has", ""));
|
||
h(`❌ Missing: ${missing.join(", ")}`);
|
||
} else {
|
||
h(`✅ ${r.frontend?.fileCount || 0} files generated`);
|
||
}
|
||
h(``);
|
||
|
||
h(`#### Backend`);
|
||
if (r.backend?.allPassed === false) {
|
||
const missing = Object.entries(r.backend || {})
|
||
.filter(([k, v]) => k.startsWith("has") && !v)
|
||
.map(([k]) => k.replace("has", ""));
|
||
h(`❌ Missing: ${missing.join(", ")}`);
|
||
} else {
|
||
h(`✅ ${r.backend?.fileCount || 0} files generated`);
|
||
}
|
||
h(``);
|
||
|
||
h(`#### Fullstack`);
|
||
if (r.fullstack?.allPassed === false) {
|
||
const missing = Object.entries(r.fullstack || {})
|
||
.filter(([k, v]) => k.startsWith("has") && !v)
|
||
.map(([k]) => k.replace("has", ""));
|
||
h(`❌ Missing: ${missing.join(", ")}`);
|
||
} else {
|
||
h(`✅ ${r.fullstack?.fileCount || 0} files generated`);
|
||
}
|
||
h(``);
|
||
|
||
h(`#### Build`);
|
||
if (r.build?.frontend?.build === "FAIL") {
|
||
h(`❌ Frontend build failed: ${r.build.frontend.reason || r.build.frontend.error || ""}`);
|
||
} else {
|
||
h(`✅ Frontend build: ${r.build?.frontend?.build || "SKIP"}`);
|
||
}
|
||
if (r.build?.backend?.build === "FAIL") {
|
||
h(`❌ Backend typecheck failed: ${r.build.backend.reason || r.build.backend.error || ""}`);
|
||
} else {
|
||
h(`✅ Backend typecheck: ${r.build?.backend?.build || "SKIP"}`);
|
||
}
|
||
h(``);
|
||
|
||
h(`#### API`);
|
||
if (r.api) {
|
||
h(`| Endpoint | Status |`);
|
||
h(`|----------|--------|`);
|
||
h(`| Health | ${r.api.health ? "✅" : "❌"} |`);
|
||
h(`| Register | ${r.api.register ? "✅" : "❌"} |`);
|
||
h(`| Login | ${r.api.login ? "✅" : "❌"} |`);
|
||
h(`| Me | ${r.api.me ? "✅" : "❌"} |`);
|
||
h(`| Create | ${r.api.create ? "✅" : "❌"} |`);
|
||
h(`| Read | ${r.api.read ? "✅" : "❌"} |`);
|
||
h(`| Update | ${r.api.update ? "✅" : "❌"} |`);
|
||
h(`| Delete | ${r.api.delete ? "✅" : "❌"} |`);
|
||
}
|
||
h(``);
|
||
}
|
||
|
||
// Failure Analysis
|
||
h(`## Failure Analysis`);
|
||
h(``);
|
||
|
||
const failures = allResults.filter(r =>
|
||
!r.frontend?.allPassed || !r.backend?.allPassed || !r.fullstack?.allPassed ||
|
||
r.build?.frontend?.build === "FAIL" || r.build?.backend?.build === "FAIL" ||
|
||
!r.api?.crudAllPass || !r.api?.authAllPass
|
||
);
|
||
|
||
if (failures.length === 0) {
|
||
h(`✅ **All domains passed all checks!**`);
|
||
h(``);
|
||
} else {
|
||
h(`### Failure Categories`);
|
||
h(``);
|
||
|
||
// Categorize failures
|
||
const feGenFails = allResults.filter(r => !r.frontend?.allPassed);
|
||
const beGenFails = allResults.filter(r => !r.backend?.allPassed);
|
||
const fsGenFails = allResults.filter(r => !r.fullstack?.allPassed);
|
||
const feBuildFails = allResults.filter(r => r.build?.frontend?.build === "FAIL");
|
||
const beBuildFails = allResults.filter(r => r.build?.backend?.build === "FAIL");
|
||
const crudFails = allResults.filter(r => !r.api?.crudAllPass);
|
||
const authFails = allResults.filter(r => !r.api?.authAllPass);
|
||
|
||
h(`| Category | Count | Domains |`);
|
||
h(`|----------|-------|---------|`);
|
||
h(`| Frontend Generation | ${feGenFails.length} | ${feGenFails.map(r => r.id).join(", ") || "—"} |`);
|
||
h(`| Backend Generation | ${beGenFails.length} | ${beGenFails.map(r => r.id).join(", ") || "—"} |`);
|
||
h(`| Fullstack Composition | ${fsGenFails.length} | ${fsGenFails.map(r => r.id).join(", ") || "—"} |`);
|
||
h(`| Frontend Build | ${feBuildFails.length} | ${feBuildFails.map(r => r.id).join(", ") || "—"} |`);
|
||
h(`| Backend Typecheck | ${beBuildFails.length} | ${beBuildFails.map(r => r.id).join(", ") || "—"} |`);
|
||
h(`| CRUD Missing | ${crudFails.length} | ${crudFails.map(r => r.id).join(", ") || "—"} |`);
|
||
h(`| Auth Missing | ${authFails.length} | ${authFails.map(r => r.id).join(", ") || "—"} |`);
|
||
h(``);
|
||
|
||
// Detailed failure reasons
|
||
h(`### Failure Details`);
|
||
h(``);
|
||
for (const r of allResults) {
|
||
const issues = [];
|
||
if (!r.frontend?.allPassed) {
|
||
const missing = Object.entries(r.frontend || {}).filter(([k, v]) => k.startsWith("has") && !v).map(([k]) => k.replace("has", ""));
|
||
issues.push(`Frontend: missing ${missing.join(", ")}`);
|
||
}
|
||
if (!r.backend?.allPassed) {
|
||
const missing = Object.entries(r.backend || {}).filter(([k, v]) => k.startsWith("has") && !v).map(([k]) => k.replace("has", ""));
|
||
issues.push(`Backend: missing ${missing.join(", ")}`);
|
||
}
|
||
if (!r.fullstack?.allPassed) {
|
||
const missing = Object.entries(r.fullstack || {}).filter(([k, v]) => k.startsWith("has") && !v).map(([k]) => k.replace("has", ""));
|
||
issues.push(`Fullstack: missing ${missing.join(", ")}`);
|
||
}
|
||
if (r.build?.frontend?.build === "FAIL") issues.push(`Frontend build: ${r.build.frontend.error || r.build.frontend.reason}`);
|
||
if (r.build?.backend?.build === "FAIL") issues.push(`Backend typecheck: ${r.build.backend.error || r.build.backend.reason}`);
|
||
if (!r.api?.crudAllPass) {
|
||
const missing = Object.entries({create: r.api?.create, read: r.api?.read, update: r.api?.update, delete: r.api?.delete})
|
||
.filter(([,v]) => !v).map(([k]) => k);
|
||
issues.push(`CRUD: missing ${missing.join(", ")}`);
|
||
}
|
||
if (!r.api?.authAllPass) {
|
||
const missing = Object.entries({register: r.api?.register, login: r.api?.login, me: r.api?.me})
|
||
.filter(([,v]) => !v).map(([k]) => k);
|
||
issues.push(`Auth: missing ${missing.join(", ")}`);
|
||
}
|
||
|
||
if (issues.length > 0) {
|
||
h(`**${r.id}:**`);
|
||
for (const i of issues) h(`- ${i}`);
|
||
h(``);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Pattern Analysis
|
||
h(`## Pattern Analysis`);
|
||
h(``);
|
||
|
||
// Best domains
|
||
const scored = allResults.map(r => {
|
||
let score = 0;
|
||
if (r.frontend?.allPassed) score += 2;
|
||
if (r.backend?.allPassed) score += 2;
|
||
if (r.fullstack?.allPassed) score += 1;
|
||
if (r.build?.frontend?.build === "PASS") score += 2;
|
||
if (r.build?.backend?.build === "PASS") score += 2;
|
||
if (r.api?.crudAllPass) score += 2;
|
||
if (r.api?.authAllPass) score += 2;
|
||
return { ...r, score };
|
||
}).sort((a, b) => b.score - a.score);
|
||
|
||
h(`### Best Performing Domains`);
|
||
h(``);
|
||
h(`| Domain | Score | Key Strength |`);
|
||
h(`|--------|-------|-------------|`);
|
||
for (const r of scored.slice(0, 3)) {
|
||
h(`| ${r.id} | ${r.score}/13 | ${r.domain || "N/A"} |`);
|
||
}
|
||
h(``);
|
||
|
||
h(`### Weakest Domains`);
|
||
h(``);
|
||
h(`| Domain | Score | Key Issue |`);
|
||
h(`|--------|-------|-----------|`);
|
||
for (const r of scored.slice(-3).reverse()) {
|
||
const issues = [];
|
||
if (!r.frontend?.allPassed) issues.push("FE gen");
|
||
if (!r.backend?.allPassed) issues.push("BE gen");
|
||
if (r.build?.frontend?.build === "FAIL") issues.push("FE build");
|
||
if (r.build?.backend?.build === "FAIL") issues.push("BE typecheck");
|
||
if (!r.api?.crudAllPass) issues.push("CRUD");
|
||
if (!r.api?.authAllPass) issues.push("Auth");
|
||
h(`| ${r.id} | ${r.score}/13 | ${issues.join(", ") || "—"} |`);
|
||
}
|
||
h(``);
|
||
|
||
h(`### Domain Type Analysis`);
|
||
h(``);
|
||
h(`| Domain Type | Count | Avg Score |`);
|
||
h(`|-------------|-------|-----------|`);
|
||
const byDomain = {};
|
||
for (const r of allResults) {
|
||
const d = r.domain || "unknown";
|
||
if (!byDomain[d]) byDomain[d] = { count: 0, scores: [] };
|
||
byDomain[d].count++;
|
||
byDomain[d].scores.push(r.score);
|
||
}
|
||
for (const [domain, data] of Object.entries(byDomain)) {
|
||
const avg = (data.scores.reduce((a,b) => a + b, 0) / data.scores.length).toFixed(1);
|
||
h(`| ${domain} | ${data.count} | ${avg} |`);
|
||
}
|
||
h(``);
|
||
|
||
// Failure categories (defined above already, but recompute for Fixes section)
|
||
const feGenFailsR = allResults.filter(r => !r.frontend?.allPassed);
|
||
const beGenFailsR = allResults.filter(r => !r.backend?.allPassed);
|
||
const fsGenFailsR = allResults.filter(r => !r.fullstack?.allPassed);
|
||
const feBuildFailsR = allResults.filter(r => r.build?.frontend?.build === "FAIL");
|
||
const beBuildFailsR = allResults.filter(r => r.build?.backend?.build === "FAIL");
|
||
const crudFailsR = allResults.filter(r => !r.api?.crudAllPass);
|
||
const authFailsR = allResults.filter(r => !r.api?.authAllPass);
|
||
|
||
// Required Fixes
|
||
h(`## Required Fixes`);
|
||
h(``);
|
||
|
||
const criticalFixes = [];
|
||
const highFixes = [];
|
||
const mediumFixes = [];
|
||
|
||
// Analyze patterns
|
||
if (feBuildFailsR.length > 0) {
|
||
criticalFixes.push(`**Frontend Build Failures** (${feBuildFailsR.length}/${total}): Fix Next.js build issues in frontend-builder-agent — check tsconfig, module resolution, component imports`);
|
||
}
|
||
if (beBuildFailsR.length > 0) {
|
||
criticalFixes.push(`**Backend Typecheck Failures** (${beBuildFailsR.length}/${total}): Fix TypeScript errors in backend-builder-agent — check type definitions, import paths`);
|
||
}
|
||
if (beGenFailsR.length > 0) {
|
||
highFixes.push(`**Backend Generation Gaps** (${beGenFailsR.length}/${total}): Missing routes/services/auth in some domains`);
|
||
}
|
||
if (crudFailsR.length > 0) {
|
||
highFixes.push(`**CRUD Completeness** (${crudFailsR.length}/${total}): Some domains missing full CRUD operations`);
|
||
}
|
||
if (authFailsR.length > 0) {
|
||
highFixes.push(`**Auth Endpoint Coverage** (${authFailsR.length}/${total}): Missing register/login/me in some domains`);
|
||
}
|
||
if (feGenFailsR.length > 0) {
|
||
mediumFixes.push(`**Frontend Generation Gaps** (${feGenFailsR.length}/${total}): Missing pages or services in some domains`);
|
||
}
|
||
|
||
h(`### P0 — Critical`);
|
||
h(``);
|
||
if (criticalFixes.length === 0) {
|
||
h(`✅ No critical issues found.`);
|
||
} else {
|
||
for (const f of criticalFixes) h(`1. ${f}`);
|
||
}
|
||
h(``);
|
||
|
||
h(`### P1 — High`);
|
||
h(``);
|
||
if (highFixes.length === 0) {
|
||
h(`✅ No high-priority issues found.`);
|
||
} else {
|
||
for (let i = 0; i < highFixes.length; i++) h(`${i + 1}. ${highFixes[i]}`);
|
||
}
|
||
h(``);
|
||
|
||
h(`### P2 — Medium`);
|
||
h(``);
|
||
if (mediumFixes.length === 0) {
|
||
h(`✅ No medium-priority issues found.`);
|
||
} else {
|
||
for (let i = 0; i < mediumFixes.length; i++) h(`${i + 1}. ${mediumFixes[i]}`);
|
||
}
|
||
h(``);
|
||
|
||
// Conclusion
|
||
h(`## Conclusion`);
|
||
h(``);
|
||
const passRate = allResults.filter(r => r.score >= 11).length;
|
||
if (passRate === total) {
|
||
h(`🎉 **All ${total} domains passed!** The system demonstrates robust generalization capability across diverse business domains.`);
|
||
h(``);
|
||
h(`The RequirementPackage → Frontend Builder → Backend Builder → Fullstack Composer pipeline is production-ready for Web Fullstack generation.`);
|
||
h(``);
|
||
h(`**Next: Electron Builder can now proceed.**`);
|
||
} else {
|
||
h(`⚠️ **${passRate}/${total} domains fully passed.**`);
|
||
h(``);
|
||
h(`The pipeline shows partial generalization. ${total - passRate} domains need fixes before the system can be considered a truly general Web Fullstack Generator.`);
|
||
h(``);
|
||
h(`**Electron Builder should wait until fixes above are addressed.**`);
|
||
}
|
||
h(``);
|
||
|
||
h(`---`);
|
||
h(`*Report generated by Domain Benchmark Suite*`);
|
||
|
||
return lines.join("\n");
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════════
|
||
// Main
|
||
// ═══════════════════════════════════════════════════════
|
||
|
||
async function main() {
|
||
const args = process.argv.slice(2);
|
||
const filterDomain = args[0] === "--domain" ? args[1] : null;
|
||
const skipBuild = args.includes("--skip-build");
|
||
const skipAPI = args.includes("--skip-api");
|
||
|
||
// Filter domains if specified
|
||
const domains = filterDomain
|
||
? DOMAINS.filter(d => d.id === filterDomain)
|
||
: DOMAINS;
|
||
|
||
if (domains.length === 0) {
|
||
console.error(`Unknown domain: ${filterDomain}`);
|
||
console.error(`Available: ${DOMAINS.map(d => d.id).join(", ")}`);
|
||
process.exit(1);
|
||
}
|
||
|
||
console.log(`\n🧪 Domain Benchmark Suite`);
|
||
console.log(` Testing ${domains.length} domain(s)`);
|
||
console.log(` Pipeline: project-intake → architecture → frontend → backend → fullstack`);
|
||
if (skipBuild) console.log(` ⏭️ Skipping build validation`);
|
||
if (skipAPI) console.log(` ⏭️ Skipping API validation`);
|
||
console.log(``);
|
||
|
||
const allResults = [];
|
||
|
||
for (const domain of domains) {
|
||
console.log(`\n${"=".repeat(60)}`);
|
||
console.log(`🏗️ Domain: ${domain.id}`);
|
||
console.log(`${"=".repeat(60)}`);
|
||
|
||
const tTotal = Date.now();
|
||
|
||
// Step 1: RequirementPackage
|
||
const { prd, arch, elapsed: reqTime, prdPath, archPath } = generateRequirementPackage(domain);
|
||
|
||
// Step 2: Frontend
|
||
const frontend = generateFrontend(domain, prdPath, archPath);
|
||
|
||
// Step 3: Backend
|
||
const backend = generateBackend(domain, prdPath, archPath);
|
||
|
||
// Step 4: Fullstack
|
||
const fullstack = generateFullstack(domain, prdPath, archPath);
|
||
|
||
// Step 5: Build (optional)
|
||
let build = null;
|
||
if (!skipBuild) {
|
||
build = validateBuild(domain);
|
||
}
|
||
|
||
// Step 6: API (optional)
|
||
let api = null;
|
||
if (!skipAPI) {
|
||
api = validateAPI(domain);
|
||
}
|
||
|
||
const totalTime = ((Date.now() - tTotal) / 1000).toFixed(1);
|
||
|
||
allResults.push({
|
||
id: domain.id,
|
||
input: domain.input,
|
||
projectName: prd.projectName,
|
||
domain: prd.domain,
|
||
reqTime,
|
||
frontend,
|
||
backend,
|
||
fullstack,
|
||
build,
|
||
api,
|
||
totalTime: parseFloat(totalTime),
|
||
});
|
||
|
||
console.log(`\n ⏱️ Total: ${totalTime}s`);
|
||
}
|
||
|
||
// Generate report
|
||
const report = generateReport(allResults);
|
||
writeFileSync(REPORT_PATH, report, "utf8");
|
||
console.log(`\n📄 Report written to: ${REPORT_PATH}`);
|
||
|
||
// Print summary
|
||
const passCount = allResults.filter(r => {
|
||
return r.frontend?.allPassed && r.backend?.allPassed && r.fullstack?.allPassed;
|
||
}).length;
|
||
|
||
console.log(`\n📊 Summary: ${passCount}/${domains.length} domains generated successfully`);
|
||
|
||
// Also output JSON for programmatic consumption
|
||
const jsonPath = resolve(WORKSPACE, "benchmark-results.json");
|
||
writeFileSync(jsonPath, JSON.stringify(allResults, null, 2), "utf8");
|
||
console.log(`📄 JSON results: ${jsonPath}`);
|
||
}
|
||
|
||
main().catch(e => {
|
||
console.error("Benchmark failed:", e);
|
||
process.exit(1);
|
||
});
|