/** * E2E Regression Test Suite * * 固化所有 benchmark 为回归测试。 * 每个领域验证完整链路:需求 → PRD → 架构 → 前端 → 后端 → 全栈 → Electron → Release * * 运行: node --test test/e2e-regression.test.mjs */ import { describe, it, before } from "node:test"; import assert from "node:assert/strict"; import { resolve, join } from "node:path"; import { fileURLToPath } from "node:url"; import { existsSync, readFileSync, mkdirSync, readdirSync } from "node:fs"; import { execSync } from "node:child_process"; const __dirname = resolve(fileURLToPath(new URL(".", import.meta.url))); const WORKSPACE = resolve(__dirname, ".."); const BENCH_ROOT = resolve(WORKSPACE, ".benchmark"); const SCRIPTS = resolve(WORKSPACE, "scripts"); // ═══════════════════════════════════════════════════════ // Domain Definitions // ═══════════════════════════════════════════════════════ const DOMAINS = [ { id: "petcare", input: "做一个宠物护理管理平台,宠物主人可以管理宠物档案、健康日程、日常记录和成长相册", minFeatures: 5, minPages: 4, minApis: 3 }, { id: "crm", input: "做一个客户关系管理系统,支持客户管理、销售漏斗、跟进记录和数据分析仪表盘", minFeatures: 4, minPages: 3, minApis: 3 }, { id: "inventory", input: "做一个库存管理系统,支持商品入库出库、库存盘点、供应商管理和库存预警", minFeatures: 4, minPages: 3, minApis: 3 }, { id: "ticket", input: "做一个工单系统,支持工单创建、分配、处理流程、优先级管理和工单归档", minFeatures: 4, minPages: 3, minApis: 3 }, { id: "blog-cms", input: "做一个博客内容管理系统,支持文章发布、分类标签、评论管理和媒体库", minFeatures: 4, minPages: 2, minApis: 2 }, { id: "project-mgmt", input: "做一个项目管理系统,支持项目看板、任务分配、甘特图和团队协作", minFeatures: 3, minPages: 2, minApis: 3 }, { id: "hr", input: "做一个人力资源管理系统,支持员工档案、考勤管理、招聘流程和绩效评估", minFeatures: 4, minPages: 3, minApis: 3 }, { id: "asset", input: "做一个固定资产管理系统,支持资产登记、领用归还、折旧计算和盘点统计", minFeatures: 3, minPages: 2, minApis: 3 }, { id: "course", input: "做一个在线课程管理系统,支持课程发布、章节管理、学员进度和作业批改", minFeatures: 4, minPages: 3, minApis: 3 }, { id: "appointment", input: "做一个预约管理系统,支持服务项目、时间段预约、客户通知和预约统计", minFeatures: 3, minPages: 2, minApis: 3 }, ]; // ═══════════════════════════════════════════════════════ // Helpers // ═══════════════════════════════════════════════════════ function runAgent(script, args) { try { const result = execSync(`node ${join(SCRIPTS, script)} ${args}`, { cwd: WORKSPACE, encoding: "utf8", stdio: "pipe", timeout: 30_000, }); return JSON.parse(result.trim()); } catch (e) { try { return JSON.parse(e.stdout?.trim() || "{}"); } catch { return { error: e.message }; } } } function countFiles(dir) { try { return parseInt(execSync(`find ${dir} -type f | wc -l`, { encoding: "utf8" }).trim()); } catch { return 0; } } function loadJSON(path) { try { return JSON.parse(readFileSync(path, "utf8")); } catch { return {}; } } function scanDir(dir) { try { return readdirSync(dir, { withFileTypes: true }); } catch { return []; } } // ═══════════════════════════════════════════════════════ // Full Pipeline Runner // ═══════════════════════════════════════════════════════ function runFullPipeline(domain) { const B = join(BENCH_ROOT, domain.id); mkdirSync(B, { recursive: true }); const result = { stages: {} }; // SF-01 const s1 = runAgent("project-intake-agent.mjs", `--input "${domain.input}" --output "${join(B, "prd.json")}"`); result.stages.prd = { ok: !s1.error && !!s1.projectName, projectName: s1.projectName }; // SF-02 const s2 = runAgent("architecture-agent.mjs", `--input "${join(B, "prd.json")}" --output "${join(B, "arch.json")}"`); result.stages.arch = { ok: !s2.error }; // SF-03 const feOut = join(B, "frontend"); const s3 = runAgent("frontend-builder-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${feOut}"`); result.stages.frontend = { ok: !s3.error, files: countFiles(feOut) }; // SF-04 const beOut = join(B, "backend"); const s4 = runAgent("backend-builder-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${beOut}"`); result.stages.backend = { ok: !s4.error, files: countFiles(beOut) }; // SF-05 const fsOut = join(B, "fullstack"); const s5 = runAgent("fullstack-composer-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${fsOut}"`); result.stages.fullstack = { ok: !s5.error, files: countFiles(fsOut) }; // SF-06 const elOut = join(B, "electron"); const s6 = runAgent("electron-builder-agent.mjs", `--input "${fsOut}" --output "${elOut}" --prd "${join(B, "prd.json")}"`); result.stages.electron = { ok: !s6.error, files: countFiles(elOut) }; // SF-07 const rlOut = join(B, "release"); const s7 = runAgent("release-builder-agent.mjs", `--input "${fsOut}" --output "${rlOut}"`); result.stages.release = { ok: !s7.error, files: countFiles(rlOut) }; return result; } // ═══════════════════════════════════════════════════════ // Validation Helpers // ═══════════════════════════════════════════════════════ function validatePRD(domain) { const prd = loadJSON(join(BENCH_ROOT, domain.id, "prd.json")); return { hasProjectName: !!prd.projectName, hasDomain: !!prd.domain, featureCount: prd.features?.length || 0, pageCount: prd.pages?.length || 0, apiCount: prd.apiRequirements?.length || 0, pass: !!prd.projectName && !!prd.domain && (prd.features?.length || 0) >= domain.minFeatures, }; } function validateFrontend(domain) { const dir = join(BENCH_ROOT, domain.id, "frontend"); const checks = { packageJson: existsSync(join(dir, "package.json")), appDir: existsSync(join(dir, "app")), appPages: scanDir(join(dir, "app")).filter(e => e.isDirectory() && !e.name.startsWith("_") && !e.name.startsWith(".")).length, services: existsSync(join(dir, "services")) || existsSync(join(dir, "src", "services")), types: existsSync(join(dir, "types")) || existsSync(join(dir, "src", "types")), fileCount: countFiles(dir), }; checks.pass = checks.packageJson && checks.appDir && checks.fileCount > 10 && checks.appPages >= domain.minPages; return checks; } function validateBackend(domain) { const dir = join(BENCH_ROOT, domain.id, "backend"); const routesDir = join(dir, "src", "routes"); const routeFiles = scanDir(routesDir).filter(e => e.isFile() && e.name.endsWith(".ts") && e.name !== "index.ts"); const checks = { packageJson: existsSync(join(dir, "package.json")), routes: existsSync(routesDir), routeCount: routeFiles.length, services: existsSync(join(dir, "src", "services")), auth: routeFiles.some(e => e.name.includes("auth")), db: existsSync(join(dir, "src", "db")) || existsSync(join(dir, "src", "schema")), fileCount: countFiles(dir), }; checks.pass = checks.packageJson && checks.routes && checks.services && checks.routeCount >= domain.minApis; return checks; } function validateFullstack(domain) { const dir = join(BENCH_ROOT, domain.id, "fullstack"); const checks = { packageJson: existsSync(join(dir, "package.json")), web: existsSync(join(dir, "apps", "web")), api: existsSync(join(dir, "apps", "api")), shared: existsSync(join(dir, "packages")), fileCount: countFiles(dir), }; checks.pass = checks.packageJson && checks.web && checks.api && checks.fileCount > 30; return checks; } function validateElectron(domain) { const dir = join(BENCH_ROOT, domain.id, "electron"); const checks = { packageJson: existsSync(join(dir, "package.json")), main: existsSync(join(dir, "electron", "main.ts")), preload: existsSync(join(dir, "electron", "preload.ts")), ipc: existsSync(join(dir, "electron", "ipc.ts")), yml: existsSync(join(dir, "electron-builder.yml")), fileCount: countFiles(dir), }; checks.pass = checks.packageJson && checks.main && checks.preload && checks.fileCount >= 8; return checks; } function validateRelease(domain) { const dir = join(BENCH_ROOT, domain.id, "release", "release"); const checks = { versionJson: existsSync(join(dir, "version.json")), manifest: existsSync(join(dir, "manifests", "manifest.json")), checksums: existsSync(join(dir, "checksums", "checksums.txt")), releaseNotes: existsSync(join(dir, "release-notes", "release-notes.md")), buildInfo: existsSync(join(dir, "build-info.json")), windows: existsSync(join(dir, "windows")), macos: existsSync(join(dir, "macos")), linux: existsSync(join(dir, "linux")), }; // Validate content if (checks.versionJson) { const v = loadJSON(join(dir, "version.json")); checks.versionValid = !!(v.name && v.version && v.platforms); } if (checks.manifest) { const m = loadJSON(join(dir, "manifests", "manifest.json")); checks.manifestValid = !!(m.project && m.files?.length > 0); } if (checks.checksums) { const content = readFileSync(join(dir, "checksums", "checksums.txt"), "utf8"); const lines = content.trim().split("\n").filter(l => l.length > 0); checks.checksumCount = lines.length; checks.checksumsValid = lines.every(l => l.split(/\s+/)[0]?.length === 64); } if (checks.releaseNotes) { const notes = readFileSync(join(dir, "release-notes", "release-notes.md"), "utf8"); checks.notesValid = notes.includes("Version") && notes.includes("Installation"); } if (checks.buildInfo) { const bi = loadJSON(join(dir, "build-info.json")); checks.buildInfoValid = !!(bi.nodeVersion && bi.generatorVersion); } for (const p of ["windows", "macos", "linux"]) { if (checks[p]) { try { checks[p + "Count"] = readdirSync(join(dir, p)).length; } catch { checks[p + "Count"] = 0; } } } checks.pass = checks.versionJson && checks.versionValid && checks.manifest && checks.manifestValid && checks.checksums && checks.checksumsValid && checks.releaseNotes && checks.notesValid && checks.buildInfo && checks.buildInfoValid && checks.windows && checks.macos && checks.linux; return checks; } // ═══════════════════════════════════════════════════════ // Tests // ═══════════════════════════════════════════════════════ describe("E2E Regression — 全链路生成", () => { for (const domain of DOMAINS) { describe(`${domain.id}`, () => { let pipeline; before(() => { pipeline = runFullPipeline(domain); }); it("SF-01 PRD 生成成功", () => { assert.ok(pipeline.stages.prd.ok, `${domain.id}: PRD generation failed`); }); it("SF-02 Architecture 生成成功", () => { assert.ok(pipeline.stages.arch.ok, `${domain.id}: Architecture generation failed`); }); it("SF-03 Frontend 生成成功", () => { assert.ok(pipeline.stages.frontend.ok, `${domain.id}: Frontend generation failed`); assert.ok(pipeline.stages.frontend.files > 10, `${domain.id}: Too few frontend files`); }); it("SF-04 Backend 生成成功", () => { assert.ok(pipeline.stages.backend.ok, `${domain.id}: Backend generation failed`); assert.ok(pipeline.stages.backend.files > 10, `${domain.id}: Too few backend files`); }); it("SF-05 Fullstack 组装成功", () => { assert.ok(pipeline.stages.fullstack.ok, `${domain.id}: Fullstack composition failed`); assert.ok(pipeline.stages.fullstack.files > 30, `${domain.id}: Too few fullstack files`); }); it("SF-06 Electron 封装成功", () => { assert.ok(pipeline.stages.electron.ok, `${domain.id}: Electron wrapper failed`); assert.ok(pipeline.stages.electron.files >= 8, `${domain.id}: Too few electron files`); }); it("SF-07 Release 打包成功", () => { assert.ok(pipeline.stages.release.ok, `${domain.id}: Release packaging failed`); }); }); } }); describe("E2E Regression — PRD 质量", () => { for (const domain of DOMAINS) { it(`${domain.id}: PRD 包含必要的字段和最低特性数`, () => { const v = validatePRD(domain); assert.ok(v.hasProjectName, `${domain.id}: missing projectName`); assert.ok(v.hasDomain, `${domain.id}: missing domain`); assert.ok(v.featureCount >= domain.minFeatures, `${domain.id}: ${v.featureCount} features < ${domain.minFeatures}`); }); } }); describe("E2E Regression — Frontend 结构", () => { for (const domain of DOMAINS) { it(`${domain.id}: Frontend 包含 package.json, app/, services/, types/`, () => { const v = validateFrontend(domain); assert.ok(v.pass, `${domain.id}: FE validation failed — pages:${v.appPages} files:${v.fileCount}`); }); } }); describe("E2E Regression — Backend 结构", () => { for (const domain of DOMAINS) { it(`${domain.id}: Backend 包含 routes, services, auth, db`, () => { const v = validateBackend(domain); assert.ok(v.pass, `${domain.id}: BE validation failed — routes:${v.routeCount} files:${v.fileCount}`); }); } }); describe("E2E Regression — Fullstack 结构", () => { for (const domain of DOMAINS) { it(`${domain.id}: Fullstack 包含 apps/web, apps/api, packages/`, () => { const v = validateFullstack(domain); assert.ok(v.pass, `${domain.id}: FS validation failed — files:${v.fileCount}`); }); } }); describe("E2E Regression — Electron 结构", () => { for (const domain of DOMAINS) { it(`${domain.id}: Electron 包含 main.ts, preload.ts, ipc.ts`, () => { const v = validateElectron(domain); assert.ok(v.pass, `${domain.id}: EL validation failed — files:${v.fileCount}`); }); } }); describe("E2E Regression — Release 结构", () => { for (const domain of DOMAINS) { it(`${domain.id}: Release 包含 version, manifest, checksums, notes, build-info, 3 platforms`, () => { const v = validateRelease(domain); assert.ok(v.pass, `${domain.id}: Release validation failed`); }); } }); describe("E2E Regression — Release 内容完整性", () => { for (const domain of DOMAINS) { it(`${domain.id}: version.json 包含 name, version, platforms`, () => { const dir = join(BENCH_ROOT, domain.id, "release", "release"); const v = loadJSON(join(dir, "version.json")); assert.ok(v.name, `${domain.id}: missing version.name`); assert.ok(v.version, `${domain.id}: missing version.version`); assert.deepEqual(v.platforms, ["windows", "macos", "linux"]); }); it(`${domain.id}: manifest.json 文件列表与 checksums.txt 一致`, () => { const dir = join(BENCH_ROOT, domain.id, "release", "release"); const m = loadJSON(join(dir, "manifests", "manifest.json")); const c = readFileSync(join(dir, "checksums", "checksums.txt"), "utf8"); const cLines = c.trim().split("\n").filter(l => l.length > 0); assert.equal(m.files.length, cLines.length, `${domain.id}: manifest/checksums count mismatch`); }); it(`${domain.id}: SHA256 格式正确 (64 hex chars)`, () => { const dir = join(BENCH_ROOT, domain.id, "release", "release"); const c = readFileSync(join(dir, "checksums", "checksums.txt"), "utf8"); for (const line of c.trim().split("\n").filter(l => l.length > 0)) { const sha = line.split(/\s+/)[0]; assert.match(sha, /^[0-9a-f]{64}$/, `${domain.id}: invalid SHA256: ${sha}`); } }); it(`${domain.id}: Platform 目录包含占位文件`, () => { const dir = join(BENCH_ROOT, domain.id, "release", "release"); for (const p of ["windows", "macos", "linux"]) { const files = readdirSync(join(dir, p)); assert.ok(files.length >= 2, `${domain.id}: ${p}/ has ${files.length} files, expected >= 2`); } }); } });