16gagent/test/memory-core/search-shadow.test.mjs

/**
 * PR-3: Search Shadow Migration Test
 * 
 * Run: node --test test/memory-core/search-shadow.test.mjs
 * 
 * Tests the shadow comparison tool:
 *   1. Shadow tool is runnable
 *   2. All 22 queries complete
 *   3. Shape compatibility reported for each
 *   4. Empty query handled gracefully
 *   5. Production check still passes
 */

import { describe, it, before } from "node:test";
import assert from "node:assert";
import { join, resolve } from "node:path";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";

const __dirname = join(fileURLToPath(import.meta.url), "..", "..", "..");
const ROOT = resolve(__dirname);
const SHADOW_TOOL = join(ROOT, "scripts", "memory-search-shadow.mjs");

describe("PR-3: Search Shadow Migration Test", () => {
  
  let shadowResult;

  before(async () => {
    // Run shadow tool with --json for structured output
    const result = spawnSync("node", [SHADOW_TOOL, "--json"], {
      cwd: ROOT,
      encoding: "utf8",
      timeout: 60000,
      maxBuffer: 2 * 1024 * 1024,
    });

    const output = result.stdout + result.stderr;
    
    // Extract JSON from output (may have other text)
    const jsonMatch = output.match(/\{[\s\S]*"results"[\s\S]*\}/);
    if (jsonMatch) {
      try {
        shadowResult = JSON.parse(jsonMatch[0]);
      } catch {
        // Fall back to running without --json and parsing text
        shadowResult = null;
      }
    }
    
    console.log(output.slice(-500));
  });

  // ─── Test 1 ──────────────────────────────────────────
  it("shadow tool is runnable and produces results", () => {
    assert.ok(shadowResult, "Shadow tool should produce JSON output");
    assert.ok(typeof shadowResult.totalQueries === "number", "Should have totalQueries");
    assert.ok(shadowResult.totalQueries >= 20, `Expected >=20 queries, got ${shadowResult.totalQueries}`);
  });

  // ─── Test 2 ──────────────────────────────────────────
  it("all queries complete without crash", () => {
    assert.ok(Array.isArray(shadowResult.results), "results should be array");
    assert.equal(shadowResult.results.length, shadowResult.totalQueries,
      "Results count should match total queries");
  });

  // ─── Test 3 ──────────────────────────────────────────
  it("each query reports shapeCompatible", () => {
    for (const r of shadowResult.results) {
      assert.ok(typeof r.shapeCompatible === "boolean",
        `shapeCompatible should be boolean for query "${r.query}"`);
    }
  });

  // ─── Test 4 ──────────────────────────────────────────
  it("empty query handled gracefully (documented as known difference)", () => {
    const emptyResult = shadowResult.results.find(r => r.query === "");
    assert.ok(emptyResult, "Should have empty query test case");
    
    // Empty query: legacy FTS5 returns all, facade rejects
    // This is a KNOWN difference — facade is more correct
    assert.equal(emptyResult.legacyOk, true,
      "Legacy treats empty as match-all (known quirk)");
    assert.equal(emptyResult.facadeOk, false,
      "Facade rejects empty query (correct behavior)");
    // shapeCompatible may be false here — that's OK, documented
  });

  // ─── Test 5 ──────────────────────────────────────────
  it("high shape compatibility rate (>90%)", () => {
    const rate = shadowResult.passed / shadowResult.totalQueries;
    assert.ok(rate >= 0.9,
      `Shape compatibility rate ${Math.round(rate*100)}% below 90% threshold`);
  });

  // ─── Test 6 ──────────────────────────────────────────
  it("results have required fields", () => {
    const requiredFields = [
      "query", "legacyOk", "facadeOk", "legacyCount", "facadeCount",
      "topOverlap", "shapeCompatible", "latencyLegacyMs", "latencyFacadeMs",
    ];
    
    for (const r of shadowResult.results) {
      for (const field of requiredFields) {
        assert.ok(field in r, `Missing field "${field}" in result for "${r.query}"`);
      }
    }
  });

  // ─── Test 7 ──────────────────────────────────────────
  it("production check still passes after shadow", { timeout: 180000 }, () => {
    const result = spawnSync("bash", [join(ROOT, "scripts", "production-check.sh")], {
      cwd: ROOT,
      encoding: "utf8",
      timeout: 170000,
      maxBuffer: 2 * 1024 * 1024,
    });
    
    const output = result.stdout + result.stderr;
    
    assert.equal(result.status, 0,
      `Production check failed (exit ${result.status})`);
    assert.ok(output.includes("PRODUCTION CHECK PASSED"),
      "Production check should report PASSED");
  });

  // ─── Test 8 ──────────────────────────────────────────
  it("shadow compare does not modify any memory", async () => {
    const { readFileSync } = await import("node:fs");
    const memContent = readFileSync(join(ROOT, "MEMORY.md"), "utf8");
    
    // Re-run shadow tool
    spawnSync("node", [SHADOW_TOOL], {
      cwd: ROOT,
      encoding: "utf8",
      timeout: 60000,
    });

    // Verify MEMORY.md unchanged
    const memAfter = readFileSync(join(ROOT, "MEMORY.md"), "utf8");
    assert.equal(memAfter, memContent, "MEMORY.md must not be modified by shadow");
  });
});