大海
374 lines
17 KiB
JavaScript
374 lines
17 KiB
JavaScript
/**
|
|
* E2E Regression Test Suite
|
|
*
|
|
* 固化所有 benchmark 为回归测试。
|
|
* 每个领域验证完整链路：需求 → PRD → 架构 → 前端 → 后端 → 全栈 → Electron → Release
|
|
*
|
|
* 运行: node --test test/e2e-regression.test.mjs
|
|
*/
|
|
|
|
import { describe, it, before } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { resolve, join } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { existsSync, readFileSync, mkdirSync, readdirSync } from "node:fs";
|
|
import { execSync } from "node:child_process";
|
|
|
|
const __dirname = resolve(fileURLToPath(new URL(".", import.meta.url)));
|
|
const WORKSPACE = resolve(__dirname, "..");
|
|
const BENCH_ROOT = resolve(WORKSPACE, ".benchmark");
|
|
const SCRIPTS = resolve(WORKSPACE, "scripts");
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Domain Definitions
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
const DOMAINS = [
|
|
{ id: "petcare", input: "做一个宠物护理管理平台，宠物主人可以管理宠物档案、健康日程、日常记录和成长相册", minFeatures: 5, minPages: 4, minApis: 3 },
|
|
{ id: "crm", input: "做一个客户关系管理系统，支持客户管理、销售漏斗、跟进记录和数据分析仪表盘", minFeatures: 4, minPages: 3, minApis: 3 },
|
|
{ id: "inventory", input: "做一个库存管理系统，支持商品入库出库、库存盘点、供应商管理和库存预警", minFeatures: 4, minPages: 3, minApis: 3 },
|
|
{ id: "ticket", input: "做一个工单系统，支持工单创建、分配、处理流程、优先级管理和工单归档", minFeatures: 4, minPages: 3, minApis: 3 },
|
|
{ id: "blog-cms", input: "做一个博客内容管理系统，支持文章发布、分类标签、评论管理和媒体库", minFeatures: 4, minPages: 2, minApis: 2 },
|
|
{ id: "project-mgmt", input: "做一个项目管理系统，支持项目看板、任务分配、甘特图和团队协作", minFeatures: 3, minPages: 2, minApis: 3 },
|
|
{ id: "hr", input: "做一个人力资源管理系统，支持员工档案、考勤管理、招聘流程和绩效评估", minFeatures: 4, minPages: 3, minApis: 3 },
|
|
{ id: "asset", input: "做一个固定资产管理系统，支持资产登记、领用归还、折旧计算和盘点统计", minFeatures: 3, minPages: 2, minApis: 3 },
|
|
{ id: "course", input: "做一个在线课程管理系统，支持课程发布、章节管理、学员进度和作业批改", minFeatures: 4, minPages: 3, minApis: 3 },
|
|
{ id: "appointment", input: "做一个预约管理系统，支持服务项目、时间段预约、客户通知和预约统计", minFeatures: 3, minPages: 2, minApis: 3 },
|
|
];
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Helpers
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
function runAgent(script, args) {
|
|
try {
|
|
const result = execSync(`node ${join(SCRIPTS, script)} ${args}`, {
|
|
cwd: WORKSPACE, encoding: "utf8", stdio: "pipe", timeout: 30_000,
|
|
});
|
|
return JSON.parse(result.trim());
|
|
} catch (e) {
|
|
try { return JSON.parse(e.stdout?.trim() || "{}"); } catch { return { error: e.message }; }
|
|
}
|
|
}
|
|
|
|
function countFiles(dir) {
|
|
try {
|
|
return parseInt(execSync(`find ${dir} -type f | wc -l`, { encoding: "utf8" }).trim());
|
|
} catch { return 0; }
|
|
}
|
|
|
|
function loadJSON(path) {
|
|
try { return JSON.parse(readFileSync(path, "utf8")); } catch { return {}; }
|
|
}
|
|
|
|
function scanDir(dir) {
|
|
try { return readdirSync(dir, { withFileTypes: true }); } catch { return []; }
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Full Pipeline Runner
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
function runFullPipeline(domain) {
|
|
const B = join(BENCH_ROOT, domain.id);
|
|
mkdirSync(B, { recursive: true });
|
|
|
|
const result = { stages: {} };
|
|
|
|
// SF-01
|
|
const s1 = runAgent("project-intake-agent.mjs", `--input "${domain.input}" --output "${join(B, "prd.json")}"`);
|
|
result.stages.prd = { ok: !s1.error && !!s1.projectName, projectName: s1.projectName };
|
|
|
|
// SF-02
|
|
const s2 = runAgent("architecture-agent.mjs", `--input "${join(B, "prd.json")}" --output "${join(B, "arch.json")}"`);
|
|
result.stages.arch = { ok: !s2.error };
|
|
|
|
// SF-03
|
|
const feOut = join(B, "frontend");
|
|
const s3 = runAgent("frontend-builder-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${feOut}"`);
|
|
result.stages.frontend = { ok: !s3.error, files: countFiles(feOut) };
|
|
|
|
// SF-04
|
|
const beOut = join(B, "backend");
|
|
const s4 = runAgent("backend-builder-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${beOut}"`);
|
|
result.stages.backend = { ok: !s4.error, files: countFiles(beOut) };
|
|
|
|
// SF-05
|
|
const fsOut = join(B, "fullstack");
|
|
const s5 = runAgent("fullstack-composer-agent.mjs", `--prd "${join(B, "prd.json")}" --arch "${join(B, "arch.json")}" --output "${fsOut}"`);
|
|
result.stages.fullstack = { ok: !s5.error, files: countFiles(fsOut) };
|
|
|
|
// SF-06
|
|
const elOut = join(B, "electron");
|
|
const s6 = runAgent("electron-builder-agent.mjs", `--input "${fsOut}" --output "${elOut}" --prd "${join(B, "prd.json")}"`);
|
|
result.stages.electron = { ok: !s6.error, files: countFiles(elOut) };
|
|
|
|
// SF-07
|
|
const rlOut = join(B, "release");
|
|
const s7 = runAgent("release-builder-agent.mjs", `--input "${fsOut}" --output "${rlOut}"`);
|
|
result.stages.release = { ok: !s7.error, files: countFiles(rlOut) };
|
|
|
|
return result;
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Validation Helpers
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
function validatePRD(domain) {
|
|
const prd = loadJSON(join(BENCH_ROOT, domain.id, "prd.json"));
|
|
return {
|
|
hasProjectName: !!prd.projectName,
|
|
hasDomain: !!prd.domain,
|
|
featureCount: prd.features?.length || 0,
|
|
pageCount: prd.pages?.length || 0,
|
|
apiCount: prd.apiRequirements?.length || 0,
|
|
pass: !!prd.projectName && !!prd.domain && (prd.features?.length || 0) >= domain.minFeatures,
|
|
};
|
|
}
|
|
|
|
function validateFrontend(domain) {
|
|
const dir = join(BENCH_ROOT, domain.id, "frontend");
|
|
const checks = {
|
|
packageJson: existsSync(join(dir, "package.json")),
|
|
appDir: existsSync(join(dir, "app")),
|
|
appPages: scanDir(join(dir, "app")).filter(e => e.isDirectory() && !e.name.startsWith("_") && !e.name.startsWith(".")).length,
|
|
services: existsSync(join(dir, "services")) || existsSync(join(dir, "src", "services")),
|
|
types: existsSync(join(dir, "types")) || existsSync(join(dir, "src", "types")),
|
|
fileCount: countFiles(dir),
|
|
};
|
|
checks.pass = checks.packageJson && checks.appDir && checks.fileCount > 10 && checks.appPages >= domain.minPages;
|
|
return checks;
|
|
}
|
|
|
|
function validateBackend(domain) {
|
|
const dir = join(BENCH_ROOT, domain.id, "backend");
|
|
const routesDir = join(dir, "src", "routes");
|
|
const routeFiles = scanDir(routesDir).filter(e => e.isFile() && e.name.endsWith(".ts") && e.name !== "index.ts");
|
|
const checks = {
|
|
packageJson: existsSync(join(dir, "package.json")),
|
|
routes: existsSync(routesDir),
|
|
routeCount: routeFiles.length,
|
|
services: existsSync(join(dir, "src", "services")),
|
|
auth: routeFiles.some(e => e.name.includes("auth")),
|
|
db: existsSync(join(dir, "src", "db")) || existsSync(join(dir, "src", "schema")),
|
|
fileCount: countFiles(dir),
|
|
};
|
|
checks.pass = checks.packageJson && checks.routes && checks.services && checks.routeCount >= domain.minApis;
|
|
return checks;
|
|
}
|
|
|
|
function validateFullstack(domain) {
|
|
const dir = join(BENCH_ROOT, domain.id, "fullstack");
|
|
const checks = {
|
|
packageJson: existsSync(join(dir, "package.json")),
|
|
web: existsSync(join(dir, "apps", "web")),
|
|
api: existsSync(join(dir, "apps", "api")),
|
|
shared: existsSync(join(dir, "packages")),
|
|
fileCount: countFiles(dir),
|
|
};
|
|
checks.pass = checks.packageJson && checks.web && checks.api && checks.fileCount > 30;
|
|
return checks;
|
|
}
|
|
|
|
function validateElectron(domain) {
|
|
const dir = join(BENCH_ROOT, domain.id, "electron");
|
|
const checks = {
|
|
packageJson: existsSync(join(dir, "package.json")),
|
|
main: existsSync(join(dir, "electron", "main.ts")),
|
|
preload: existsSync(join(dir, "electron", "preload.ts")),
|
|
ipc: existsSync(join(dir, "electron", "ipc.ts")),
|
|
yml: existsSync(join(dir, "electron-builder.yml")),
|
|
fileCount: countFiles(dir),
|
|
};
|
|
checks.pass = checks.packageJson && checks.main && checks.preload && checks.fileCount >= 8;
|
|
return checks;
|
|
}
|
|
|
|
function validateRelease(domain) {
|
|
const dir = join(BENCH_ROOT, domain.id, "release", "release");
|
|
const checks = {
|
|
versionJson: existsSync(join(dir, "version.json")),
|
|
manifest: existsSync(join(dir, "manifests", "manifest.json")),
|
|
checksums: existsSync(join(dir, "checksums", "checksums.txt")),
|
|
releaseNotes: existsSync(join(dir, "release-notes", "release-notes.md")),
|
|
buildInfo: existsSync(join(dir, "build-info.json")),
|
|
windows: existsSync(join(dir, "windows")),
|
|
macos: existsSync(join(dir, "macos")),
|
|
linux: existsSync(join(dir, "linux")),
|
|
};
|
|
// Validate content
|
|
if (checks.versionJson) {
|
|
const v = loadJSON(join(dir, "version.json"));
|
|
checks.versionValid = !!(v.name && v.version && v.platforms);
|
|
}
|
|
if (checks.manifest) {
|
|
const m = loadJSON(join(dir, "manifests", "manifest.json"));
|
|
checks.manifestValid = !!(m.project && m.files?.length > 0);
|
|
}
|
|
if (checks.checksums) {
|
|
const content = readFileSync(join(dir, "checksums", "checksums.txt"), "utf8");
|
|
const lines = content.trim().split("\n").filter(l => l.length > 0);
|
|
checks.checksumCount = lines.length;
|
|
checks.checksumsValid = lines.every(l => l.split(/\s+/)[0]?.length === 64);
|
|
}
|
|
if (checks.releaseNotes) {
|
|
const notes = readFileSync(join(dir, "release-notes", "release-notes.md"), "utf8");
|
|
checks.notesValid = notes.includes("Version") && notes.includes("Installation");
|
|
}
|
|
if (checks.buildInfo) {
|
|
const bi = loadJSON(join(dir, "build-info.json"));
|
|
checks.buildInfoValid = !!(bi.nodeVersion && bi.generatorVersion);
|
|
}
|
|
for (const p of ["windows", "macos", "linux"]) {
|
|
if (checks[p]) {
|
|
try { checks[p + "Count"] = readdirSync(join(dir, p)).length; } catch { checks[p + "Count"] = 0; }
|
|
}
|
|
}
|
|
checks.pass = checks.versionJson && checks.versionValid && checks.manifest && checks.manifestValid &&
|
|
checks.checksums && checks.checksumsValid && checks.releaseNotes && checks.notesValid &&
|
|
checks.buildInfo && checks.buildInfoValid && checks.windows && checks.macos && checks.linux;
|
|
return checks;
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Tests
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
describe("E2E Regression — 全链路生成", () => {
|
|
for (const domain of DOMAINS) {
|
|
describe(`${domain.id}`, () => {
|
|
let pipeline;
|
|
|
|
before(() => {
|
|
pipeline = runFullPipeline(domain);
|
|
});
|
|
|
|
it("SF-01 PRD 生成成功", () => {
|
|
assert.ok(pipeline.stages.prd.ok, `${domain.id}: PRD generation failed`);
|
|
});
|
|
|
|
it("SF-02 Architecture 生成成功", () => {
|
|
assert.ok(pipeline.stages.arch.ok, `${domain.id}: Architecture generation failed`);
|
|
});
|
|
|
|
it("SF-03 Frontend 生成成功", () => {
|
|
assert.ok(pipeline.stages.frontend.ok, `${domain.id}: Frontend generation failed`);
|
|
assert.ok(pipeline.stages.frontend.files > 10, `${domain.id}: Too few frontend files`);
|
|
});
|
|
|
|
it("SF-04 Backend 生成成功", () => {
|
|
assert.ok(pipeline.stages.backend.ok, `${domain.id}: Backend generation failed`);
|
|
assert.ok(pipeline.stages.backend.files > 10, `${domain.id}: Too few backend files`);
|
|
});
|
|
|
|
it("SF-05 Fullstack 组装成功", () => {
|
|
assert.ok(pipeline.stages.fullstack.ok, `${domain.id}: Fullstack composition failed`);
|
|
assert.ok(pipeline.stages.fullstack.files > 30, `${domain.id}: Too few fullstack files`);
|
|
});
|
|
|
|
it("SF-06 Electron 封装成功", () => {
|
|
assert.ok(pipeline.stages.electron.ok, `${domain.id}: Electron wrapper failed`);
|
|
assert.ok(pipeline.stages.electron.files >= 8, `${domain.id}: Too few electron files`);
|
|
});
|
|
|
|
it("SF-07 Release 打包成功", () => {
|
|
assert.ok(pipeline.stages.release.ok, `${domain.id}: Release packaging failed`);
|
|
});
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("E2E Regression — PRD 质量", () => {
|
|
for (const domain of DOMAINS) {
|
|
it(`${domain.id}: PRD 包含必要的字段和最低特性数`, () => {
|
|
const v = validatePRD(domain);
|
|
assert.ok(v.hasProjectName, `${domain.id}: missing projectName`);
|
|
assert.ok(v.hasDomain, `${domain.id}: missing domain`);
|
|
assert.ok(v.featureCount >= domain.minFeatures, `${domain.id}: ${v.featureCount} features < ${domain.minFeatures}`);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("E2E Regression — Frontend 结构", () => {
|
|
for (const domain of DOMAINS) {
|
|
it(`${domain.id}: Frontend 包含 package.json, app/, services/, types/`, () => {
|
|
const v = validateFrontend(domain);
|
|
assert.ok(v.pass, `${domain.id}: FE validation failed — pages:${v.appPages} files:${v.fileCount}`);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("E2E Regression — Backend 结构", () => {
|
|
for (const domain of DOMAINS) {
|
|
it(`${domain.id}: Backend 包含 routes, services, auth, db`, () => {
|
|
const v = validateBackend(domain);
|
|
assert.ok(v.pass, `${domain.id}: BE validation failed — routes:${v.routeCount} files:${v.fileCount}`);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("E2E Regression — Fullstack 结构", () => {
|
|
for (const domain of DOMAINS) {
|
|
it(`${domain.id}: Fullstack 包含 apps/web, apps/api, packages/`, () => {
|
|
const v = validateFullstack(domain);
|
|
assert.ok(v.pass, `${domain.id}: FS validation failed — files:${v.fileCount}`);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("E2E Regression — Electron 结构", () => {
|
|
for (const domain of DOMAINS) {
|
|
it(`${domain.id}: Electron 包含 main.ts, preload.ts, ipc.ts`, () => {
|
|
const v = validateElectron(domain);
|
|
assert.ok(v.pass, `${domain.id}: EL validation failed — files:${v.fileCount}`);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("E2E Regression — Release 结构", () => {
|
|
for (const domain of DOMAINS) {
|
|
it(`${domain.id}: Release 包含 version, manifest, checksums, notes, build-info, 3 platforms`, () => {
|
|
const v = validateRelease(domain);
|
|
assert.ok(v.pass, `${domain.id}: Release validation failed`);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("E2E Regression — Release 内容完整性", () => {
|
|
for (const domain of DOMAINS) {
|
|
it(`${domain.id}: version.json 包含 name, version, platforms`, () => {
|
|
const dir = join(BENCH_ROOT, domain.id, "release", "release");
|
|
const v = loadJSON(join(dir, "version.json"));
|
|
assert.ok(v.name, `${domain.id}: missing version.name`);
|
|
assert.ok(v.version, `${domain.id}: missing version.version`);
|
|
assert.deepEqual(v.platforms, ["windows", "macos", "linux"]);
|
|
});
|
|
|
|
it(`${domain.id}: manifest.json 文件列表与 checksums.txt 一致`, () => {
|
|
const dir = join(BENCH_ROOT, domain.id, "release", "release");
|
|
const m = loadJSON(join(dir, "manifests", "manifest.json"));
|
|
const c = readFileSync(join(dir, "checksums", "checksums.txt"), "utf8");
|
|
const cLines = c.trim().split("\n").filter(l => l.length > 0);
|
|
assert.equal(m.files.length, cLines.length, `${domain.id}: manifest/checksums count mismatch`);
|
|
});
|
|
|
|
it(`${domain.id}: SHA256 格式正确 (64 hex chars)`, () => {
|
|
const dir = join(BENCH_ROOT, domain.id, "release", "release");
|
|
const c = readFileSync(join(dir, "checksums", "checksums.txt"), "utf8");
|
|
for (const line of c.trim().split("\n").filter(l => l.length > 0)) {
|
|
const sha = line.split(/\s+/)[0];
|
|
assert.match(sha, /^[0-9a-f]{64}$/, `${domain.id}: invalid SHA256: ${sha}`);
|
|
}
|
|
});
|
|
|
|
it(`${domain.id}: Platform 目录包含占位文件`, () => {
|
|
const dir = join(BENCH_ROOT, domain.id, "release", "release");
|
|
for (const p of ["windows", "macos", "linux"]) {
|
|
const files = readdirSync(join(dir, p));
|
|
assert.ok(files.length >= 2, `${domain.id}: ${p}/ has ${files.length} files, expected >= 2`);
|
|
}
|
|
});
|
|
}
|
|
});
|