大海
250 lines
11 KiB
JavaScript
250 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* audit-agents.mjs — AGENTS.md Execution Protocol Audit
|
|
*
|
|
* Checks:
|
|
* 1. Task Tree exists (§1)
|
|
* 2. Agent Package isolation requirements
|
|
* 3. No XL Package (context budget forbids it)
|
|
* 4. Injected Risks mechanism (§1e / §3b)
|
|
* 5. Quality Gate completeness (§5)
|
|
*
|
|
* Usage:
|
|
* node scripts/audit-agents.mjs ← standalone
|
|
* import { runAgentAudit } from './audit-agents.mjs' ← via orchestrator
|
|
*/
|
|
|
|
import { fileURLToPath } from 'url';
|
|
import path from 'path';
|
|
import fs from 'fs';
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = path.dirname(__filename);
|
|
const WORKSPACE = path.resolve(__dirname, '..');
|
|
const SELF = process.argv[1].replace(/\\/g, '/');
|
|
|
|
/**
|
|
* @typedef {Object} CheckResult
|
|
* @property {string} id
|
|
* @property {string} name
|
|
* @property {'PASS'|'WARN'|'FAIL'} status
|
|
* @property {string} evidence
|
|
* @property {string} [suggestion]
|
|
*/
|
|
|
|
/** @returns {CheckResult[]} */
|
|
export function runAgentAudit(workspaceDir) {
|
|
const root = workspaceDir || WORKSPACE;
|
|
const checks = [];
|
|
|
|
// ---- Helper ----
|
|
function readAgents() {
|
|
const p = path.join(root, 'AGENTS.md');
|
|
if (!fs.existsSync(p)) return null;
|
|
return fs.readFileSync(p, 'utf8');
|
|
}
|
|
|
|
function check(prefix, id, name, condition, evidence, suggestion) {
|
|
checks.push({
|
|
id: `${prefix}-${id}`,
|
|
name,
|
|
status: condition ? 'PASS' : 'FAIL',
|
|
evidence: condition ? evidence : `File found but missing: ${evidence}`,
|
|
...(condition ? {} : { suggestion: suggestion || `Ensure §${id} is properly defined in AGENTS.md` }),
|
|
});
|
|
}
|
|
|
|
function warn(prefix, id, name, evidence, suggestion) {
|
|
checks.push({
|
|
id: `${prefix}-${id}`,
|
|
name,
|
|
status: 'WARN',
|
|
evidence,
|
|
suggestion,
|
|
});
|
|
}
|
|
|
|
// ---- Read AGENTS.md ----
|
|
const agents = readAgents();
|
|
if (!agents) {
|
|
checks.push({
|
|
id: 'AG-001',
|
|
name: 'AGENTS.md exists',
|
|
status: 'FAIL',
|
|
evidence: 'AGENTS.md not found in workspace root',
|
|
suggestion: 'Create AGENTS.md with the Execution Protocol',
|
|
});
|
|
return checks;
|
|
}
|
|
checks.push({
|
|
id: 'AG-001',
|
|
name: 'AGENTS.md exists',
|
|
status: 'PASS',
|
|
evidence: `AGENTS.md found (${agents.length} bytes)`,
|
|
});
|
|
|
|
// ---- §0 Complexity Classification ----
|
|
const hasClassify = /###\s*0\.\s*Complexity Classification/.test(agents);
|
|
check('AG', '0', '§0 Complexity Classification defined', hasClassify,
|
|
'Found "### 0. Complexity Classification" with Simple/Complex criteria',
|
|
'Add "### 0. Complexity Classification" with Simple→Direct / Complex→Task Tree decision rules');
|
|
|
|
// Detect if the "if uncertain → treat as Complex" rule exists
|
|
const hasUncertainRule = /if uncertain → treat as Complex/i.test(agents) || /if uncertain.*complex/i.test(agents);
|
|
check('AG', '0b', '§0 Uncertainty fallback ("if uncertain → treat as Complex")', hasUncertainRule,
|
|
'Uncertainty fallback rule found',
|
|
'Add decision rule: "if uncertain → treat as Complex"');
|
|
|
|
// ---- §1 Task Tree ----
|
|
const hasTaskTree = /###\s*1\.\s*Task Tree/.test(agents);
|
|
check('AG', '1', '§1 Task Tree Construction defined', hasTaskTree,
|
|
'Found "### 1. Task Tree Construction" with Task Tree definition',
|
|
'Add "### 1. Task Tree Construction" defining the execution tree');
|
|
|
|
// Task Tree Quality Gate (7 checks)
|
|
const hasTaskTreeGate = /Task Tree Quality Gate/.test(agents);
|
|
check('AG', '1f', '§1f Task Tree Quality Gate (7 checks)', hasTaskTreeGate,
|
|
'Found Task Tree Quality Gate with 7 verification items',
|
|
'Add §1f Quality Gate checklist (Root Task / No orphan / No circular / No XL / No unjust serial / Risks populated / No human dep)');
|
|
|
|
// ---- §2 PLAN ----
|
|
const hasPlan = /###\s*2\.\s*Planning Protocol/.test(agents);
|
|
check('AG', '2', '§2 Planning Protocol defined', hasPlan,
|
|
'Found "### 2. Planning Protocol" with Goal/Subtasks/Risks/Verify',
|
|
'Add "### 2. Planning Protocol" section');
|
|
|
|
// ---- §3 Pre-flight ----
|
|
const hasPreflight = /###\s*3\.\s*Pre-flight/.test(agents);
|
|
check('AG', '3', '§3 Pre-flight Check defined', hasPreflight,
|
|
'Found "### 3. Pre-flight Check" with Dependency Scan / Learning Context / Domain Checklists',
|
|
'Add "### 3. Pre-flight Check" section');
|
|
|
|
// 3a Dependency Scan
|
|
const hasDepScan = /####\s*3a\.\s*Dependency Scan/.test(agents);
|
|
check('AG', '3a', '§3a Dependency Scan defined', hasDepScan,
|
|
'Found §3a Dependency Scan',
|
|
'Add §3a Dependency Scan (files/tables, dependencies, blast radius)');
|
|
|
|
// 3b Learning Context Injector
|
|
const hasLearningCtx = /####\s*3b\.\s*Learning Context Injector/.test(agents);
|
|
check('AG', '3b', '§3b Learning Context Injector (3 searches)', hasLearningCtx,
|
|
'Found §3b Learning Context Injector with 3 mandatory searches',
|
|
'Add §3b with 3 searches: similar failures, best practices, playbooks');
|
|
|
|
// 3c Domain Checklists
|
|
const hasDomainChecklists = /####\s*3c\.\s*Domain Checklist Injection/.test(agents);
|
|
check('AG', '3c', '§3c Domain Checklists with ⚠️ Hard Rules', hasDomainChecklists,
|
|
'Found §3c Domain Checklist Injection with SQL/JS/Next.js/Backend/Generator/Release checklists',
|
|
'Add §3c with at least SQL, JS/TS, Next.js, Backend, Generator, Release checklists');
|
|
|
|
// Hard Rules exist
|
|
const hardRuleCount = (agents.match(/⚠️\s*HARD RULE/g) || []).length;
|
|
warn('AG', '3c-hard', `§3c Hard Rules count: ${hardRuleCount} (target: ≥5)`,
|
|
`Found ${hardRuleCount} ⚠️ HARD RULE markers`,
|
|
'Promote recurring patterns to Hard Rules. Target ≥5.');
|
|
|
|
// ---- §4 Failure Replanning ----
|
|
const hasReplan = /###\s*4\.\s*Failure Replanning Protocol/.test(agents);
|
|
check('AG', '4', '§4 Failure Replanning Protocol defined', hasReplan,
|
|
'Found "### 4. Failure Replanning Protocol" with local/structural replan logic',
|
|
'Add "### 4. Failure Replanning Protocol"');
|
|
|
|
// ---- §5 Verification Protocol ----
|
|
const hasVerification = /###\s*5\.\s*Verification Protocol/.test(agents);
|
|
check('AG', '5', '§5 Verification Protocol defined', hasVerification,
|
|
'Found "### 5. Verification Protocol" with Build/Tests/Lint/Regression/Manual checks',
|
|
'Add "### 5. Verification Protocol" with the verification block format');
|
|
|
|
// Quality Gate in Verification
|
|
const hasQualityGate = /Pre-flight Compliance Check/.test(agents) || /Quality Gate/.test(agents);
|
|
check('AG', '5b', '§5 Pre-flight Compliance / Quality Gate', hasQualityGate,
|
|
'Found "Pre-flight Compliance Check" with 5 mandatory gates',
|
|
'Add Quality Gate checklist within §5: PLAN has Injected Risks, ≥2 sources searched, domain checklist, reflection, 7-field reflection');
|
|
|
|
// ---- §6 Governance Compliance ----
|
|
const hasGovernance = /###\s*6\.\s*Governance Compliance/.test(agents);
|
|
check('AG', '6', '§6 Governance Compliance defined', hasGovernance,
|
|
'Found "### 6. Governance Compliance" with State Machine / Heartbeat / Quality Gate',
|
|
'Add "### 6. Governance Compliance" referencing GOVERNANCE.md');
|
|
|
|
// ---- §7 Auto-Capture ----
|
|
const hasAutoCapture = /###\s*7\.\s*Auto-Capture Trigger/.test(agents);
|
|
check('AG', '7', '§7 Auto-Capture Trigger defined', hasAutoCapture,
|
|
'Found "### 7. Auto-Capture Trigger" with T1-T7 trigger conditions',
|
|
'Add "### 7. Auto-Capture Trigger" with 7 trigger conditions and skip rules');
|
|
|
|
// ---- Learning Loop ----
|
|
const hasLearningLoop = /## Learning Loop/.test(agents);
|
|
check('AG', 'LL', 'Learning Loop (🔄) defined', hasLearningLoop,
|
|
'Found "## Learning Loop 🔄" with Loop Architecture / Promotion Rules / Reflection Quality Gate / Storage Locations',
|
|
'Add "## Learning Loop 🔄" section');
|
|
|
|
// Promotion Rules (1→2→3→4→5)
|
|
const hasPromotion = /Promotion Rules/.test(agents) && /1st.*Reflection/.test(agents) && /5th.*Hard Rule/.test(agents);
|
|
check('AG', 'LL-promo', 'Promotion Rules (1→2→3→4→5 appearances)', hasPromotion,
|
|
'Found Promotion Rules chart with 1st=Reflection through 5th=Hard Rule',
|
|
'Add Promotion Rules table mapping appearance count to promotion destination');
|
|
|
|
// ---- Context Budget ----
|
|
const hasBudget = /## Agent Package Context Budget/.test(agents);
|
|
check('AG', 'CB', 'Agent Package Context Budget defined', hasBudget,
|
|
'Found "## Agent Package Context Budget ⚠️ HARD RULE" with size→token map',
|
|
'Add "## Agent Package Context Budget" section');
|
|
|
|
// No XL
|
|
const forbidsXL = /XL.*FORBIDDEN/.test(agents) || /XL.*❌/.test(agents);
|
|
check('AG', 'CB-noxl', 'XL Package explicitly forbidden', forbidsXL,
|
|
'Found "XL | ❌ FORBIDDEN" in Context Budget table',
|
|
'Add "XL | ❌ FORBIDDEN" row to the Size→Token Budget Mapping table');
|
|
|
|
// Input Size Pre-check
|
|
const hasPrecheck = /Input Size Pre-check/.test(agents);
|
|
check('AG', 'CB-precheck', 'Input Size Pre-check (Context Budget Declaration)', hasPrecheck,
|
|
'Found "Input Size Pre-check" with Context Budget Declaration template',
|
|
'Add "Input Size Pre-check" section with file inventory / token estimate / budget check');
|
|
|
|
// ---- Cross-reference integrity ----
|
|
const refsGovernance = (agents.match(/GOVERNANCE\.md/g) || []).length;
|
|
warn('AG', 'XR-gov', `Cross-references to GOVERNANCE.md: ${refsGovernance} (target: ≥3)`,
|
|
`Found ${refsGovernance} references to GOVERNANCE.md`,
|
|
'Add cross-references from AGENTS.md to GOVERNANCE.md in §6');
|
|
|
|
const refsPortfolio = (agents.match(/PORTFOLIO\.md/g) || []).length;
|
|
warn('AG', 'XR-port', `Cross-references to PORTFOLIO.md: ${refsPortfolio} (target: ≥1)`,
|
|
`Found ${refsPortfolio} references to PORTFOLIO.md`,
|
|
'Add cross-reference to PORTFOLIO.md in §6e Reference section');
|
|
|
|
// ---- Injected Risks (target check) ----
|
|
const hasInjectedRisks = /Injected Risks/.test(agents);
|
|
check('AG', 'IR', 'Injected Risks mechanism defined', hasInjectedRisks,
|
|
'Found "### Injected Risks" / "### Per-Task Injected Risks"',
|
|
'Add "### Injected Risks (from Learning Context)" format in §1e and §3b');
|
|
|
|
// ---- Summary ----
|
|
const passCount = checks.filter(c => c.status === 'PASS').length;
|
|
const warnCount = checks.filter(c => c.status === 'WARN').length;
|
|
const failCount = checks.filter(c => c.status === 'FAIL').length;
|
|
checks.push({
|
|
id: 'AG-SUMMARY',
|
|
name: `Summary — ${passCount} PASS / ${warnCount} WARN / ${failCount} FAIL`,
|
|
status: failCount > 0 ? 'FAIL' : warnCount > 0 ? 'WARN' : 'PASS',
|
|
evidence: `${passCount} passed, ${warnCount} warnings, ${failCount} failures`,
|
|
});
|
|
|
|
return checks;
|
|
}
|
|
|
|
// ---- Standalone runner ----
|
|
if (SELF.endsWith('/audit-agents.mjs') || SELF.endsWith('\\audit-agents.mjs')) {
|
|
const results = runAgentAudit(WORKSPACE);
|
|
console.log(`\n=== AGENTS.md Audit ===\n`);
|
|
for (const r of results) {
|
|
const icon = r.status === 'PASS' ? '✅' : r.status === 'WARN' ? '⚠️' : '❌';
|
|
console.log(` ${icon} [${r.id}] ${r.name}`);
|
|
console.log(` ${r.evidence}`);
|
|
if (r.suggestion) console.log(` → ${r.suggestion}`);
|
|
console.log();
|
|
}
|
|
}
|