#!/usr/bin/env node
/**
 * OpenClaw Context — SmartCrusher
 *
 * Intelligent JSON array compression with query-aware row selection.
 * Migrated from Headroom `crates/headroom-core/src/transforms/smart_crusher/`
 *
 * Algorithm (per Headroom crush_array pipeline):
 *   1. computeOptimalK() → adaptive retention count
 *   2. Lossless compaction (CSV format if efficient enough)
 *   3. Analyze array → detect pattern & recommend strategy
 *   4. Skip gate: no signal → passthrough
 *   5. Plan → select strategy (SmartSample / TopN / ClusterSample / TimeSeries)
 *   6. Execute → extract keep_indices subset
 *   7. CCR marker → <<ccr:HASH N_rows_offloaded>> for reversible retrieval
 *
 * Usage:
 *   node src/compression/smart-crusher.js < input.json
 *   → { compressed, strategy, savings, ccr_hash, ... }
 */

const crypto = require('crypto');

// ─── Configuration ───────────────────────────────────────────────

const DEFAULT_CONFIG = {
  maxK: 200,                          // max rows to retain
  minK: 3,                            // min rows to retain
  losslessMinSavingsRatio: 0.30,      // lossless must save 30%+
  outlierSigma: 2.5,                  // numerical outlier threshold
  varianceThreshold: 3.0,             // change point sensitivity
  changePointWindow: 2,               // ±window around change points
  clusterMaxPerClass: 2,              // max representatives per cluster
  anchorFirst: 3,                     // keep first N rows
  anchorLast: 2,                      // keep last N rows
  anchorDistributed: 3,               // evenly distributed anchors
  enableCcr: true,                    // insert CCR markers
  enableCompaction: true,             // try lossless first
  dedupContent: true,                 // deduplicate identical rows
  bias: 0.5,                          // 0=conservative, 1=aggressive
};

// ─── Adaptive K Calculation ──────────────────────────────────────

/**
 * Compute optimal retention count based on array size and bias.
 * Formula: min(maxK, max(minK, ceil(n * (0.3 + bias * 0.3))))
 */
function computeOptimalK(n, config) {
  const { maxK, minK, bias } = config;
  // bias=0 → keep 30%, bias=0.5 → keep 45%, bias=1 → keep 60%
  const ratio = 0.25 + (bias || 0.5) * 0.35;
  const k = Math.ceil(n * ratio);
  return Math.max(minK, Math.min(maxK, k));
}

// ─── Content Hashing (for dedup & CCR) ───────────────────────────

function contentHash(obj) {
  const str = typeof obj === 'string' ? obj : JSON.stringify(obj);
  return crypto.createHash('sha256').update(str).digest('hex').substring(0, 12);
}

function blake3ish(content) {
  // Simplified: use SHA-256 as BLAKE3 replacement (universally available in Node)
  return crypto.createHash('sha256').update(content).digest('hex').substring(0, 24);
}

// ─── Array Analysis ──────────────────────────────────────────────

/**
 * Analyze JSON array to detect pattern and recommend strategy.
 */
function analyzeArray(items, config) {
  const n = items.length;
  if (n === 0) return { pattern: 'empty', recommendedStrategy: 'Skip', fields: [] };

  // Collect field statistics
  const firstItem = items[0];
  if (typeof firstItem !== 'object' || firstItem === null) {
    return { pattern: 'primitive', recommendedStrategy: 'SmartSample', fields: [] };
  }

  const fieldNames = Object.keys(firstItem);
  const fieldStats = {};

  for (const field of fieldNames) {
    const values = items.map(item => {
      const v = item[field];
      return v === null || v === undefined ? null : v;
    });

    const nonNull = values.filter(v => v !== null);
    const uniqueValues = new Set(nonNull.map(v => String(v)));

    fieldStats[field] = {
      nullCount: values.length - nonNull.length,
      uniqueRatio: nonNull.length > 0 ? uniqueValues.size / nonNull.length : 1,
      type: nonNull.length > 0 ? typeof nonNull[0] : 'null',
      isNumeric: nonNull.every(v => typeof v === 'number'),
      isConstant: uniqueValues.size <= 1,
      isHighCardinality: uniqueValues.size > n * 0.9,
    };
  }

  // Detect pattern
  const hasScoreField = fieldNames.some(f =>
    /^(score|rank|rating|relevance|confidence|probability|weight)$/i.test(f)
    && fieldStats[f].isNumeric
  );

  const hasMessageField = fieldNames.some(f =>
    /^(message|msg|text|description|error|body|summary)$/i.test(f)
    && fieldStats[f].isHighCardinality
  );

  const hasTimestamp = fieldNames.some(f =>
    /^(time|timestamp|date|created_at|updated_at|ts)$/i.test(f)
  );

  const totalUniquePerRow = items.map(item => {
    const vals = Object.values(item).map(v => String(v));
    return new Set(vals).size;
  });
  const avgUniqueness = totalUniquePerRow.reduce((a, b) => a + b, 0) / n;

  let pattern, strategy;

  if (hasScoreField && !hasMessageField) {
    pattern = 'ranked';
    strategy = 'TopN';
  } else if (hasMessageField && fieldStats[fieldNames.find(f => /message|msg|text|error/i.test(f))]?.uniqueRatio > 0.5) {
    pattern = 'log';
    strategy = 'ClusterSample';
  } else if (hasTimestamp && n > 20) {
    pattern = 'time_series';
    strategy = 'TimeSeries';
  } else if (avgUniqueness > 0.8 && n > 10) {
    pattern = 'diverse';
    strategy = 'SmartSample';
  } else {
    pattern = 'generic';
    strategy = 'SmartSample';
  }

  return {
    pattern,
    recommendedStrategy: strategy,
    fields: Object.entries(fieldStats).map(([name, stats]) => ({ name, ...stats })),
    n,
    avgUniqueness,
    hasScoreField,
    hasMessageField,
    hasTimestamp,
  };
}

// ─── Lossless Compaction ─────────────────────────────────────────

/**
 * Try lossless compaction: convert array of dicts to CSV+schema format.
 * Only applied if savings ratio >= losslessMinSavingsRatio.
 */
function tryLosslessCompaction(items, config) {
  if (!config.enableCompaction || items.length < 2) return null;
  if (typeof items[0] !== 'object' || items[0] === null) return null;

  const keys = Object.keys(items[0]);
  // Only compact if all items have same keys
  const allSameKeys = items.every(item =>
    typeof item === 'object' && item !== null &&
    Object.keys(item).length === keys.length &&
    keys.every(k => k in item)
  );
  if (!allSameKeys) return null;

  // Build CSV-like format
  const types = keys.map(k => {
    const vals = items.map(i => typeof i[k]);
    const unique = new Set(vals);
    return unique.size === 1 ? vals[0] : 'mixed';
  });

  const schema = `[${items.length}]{${keys.map((k, i) => `${k}:${types[i]}`).join(',')}}`;
  const rows = items.map((item, idx) =>
    idx + ',' + keys.map(k => {
      const v = item[k];
      if (typeof v === 'string' && v.includes(',')) return `"${v}"`;
      return String(v ?? '');
    }).join(',')
  );

  const compacted = schema + '\n' + rows.join('\n');
  const original = JSON.stringify(items);
  const savings = 1 - (compacted.length / original.length);

  if (savings >= config.losslessMinSavingsRatio) {
    return {
      compacted,
      savings: Math.round(savings * 10000) / 100,
      kind: 'lossless:csv',
    };
  }

  return null;
}

// ─── Strategy: SmartSample ───────────────────────────────────────

/**
 * SmartSample: keep anchors (first/last/distributed) + constraints (errors/outliers) + query matches.
 */
function planSmartSample(analysis, items, queryContext, config) {
  const n = items.length;
  const keepIndices = new Set();

  // Anchors: first + last
  for (let i = 0; i < Math.min(config.anchorFirst, n); i++) keepIndices.add(i);
  for (let i = n - 1; i >= Math.max(0, n - config.anchorLast); i--) keepIndices.add(i);

  // Distributed anchors: evenly spaced through middle
  if (config.anchorDistributed > 0 && n > config.anchorFirst + config.anchorLast + 2) {
    const midStart = config.anchorFirst;
    const midEnd = n - config.anchorLast - 1;
    const midLen = midEnd - midStart + 1;
    const step = Math.max(1, Math.floor(midLen / (config.anchorDistributed + 1)));
    for (let i = midStart + step; i < midEnd; i += step) {
      if (keepIndices.size < config.maxK) keepIndices.add(i);
    }
  }

  // Constraints: keep rows with errors/outliers
  for (let i = 0; i < n; i++) {
    const item = items[i];
    if (typeof item !== 'object' || item === null) continue;

    // Error detection
    const str = JSON.stringify(item).toLowerCase();
    const hasError = /\b(error|fail|fatal|exception|crash|panic|timeout|rejected)\b/.test(str);
    const hasHighPriority = /high|critical|urgent|blocker/i.test(str);

    if (hasError || hasHighPriority) {
      keepIndices.add(i);
    }
  }

  // Numerical outlier detection (for fields with numbers)
  if (analysis.fields) {
    for (const field of analysis.fields) {
      if (!field.isNumeric) continue;
      const values = items.map(item => item[field.name]).filter(v => typeof v === 'number');
      if (values.length < 5) continue;

      const mean = values.reduce((a, b) => a + b, 0) / values.length;
      const variance = values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length;
      const std = Math.sqrt(variance);

      if (std > 0) {
        for (let i = 0; i < n; i++) {
          const v = items[i]?.[field.name];
          if (typeof v === 'number' && Math.abs(v - mean) > config.outlierSigma * std) {
            keepIndices.add(i);
            // Keep ±1 window around outlier
            if (i > 0) keepIndices.add(i - 1);
            if (i < n - 1) keepIndices.add(i + 1);
          }
        }
      }
    }
  }

  // Query-based matching
  if (queryContext) {
    const queryTerms = queryContext.toLowerCase().split(/\s+/).filter(t => t.length > 1);
    for (let i = 0; i < n; i++) {
      if (keepIndices.has(i)) continue;
      const str = JSON.stringify(items[i]).toLowerCase();
      const matchCount = queryTerms.filter(term => str.includes(term)).length;
      if (matchCount >= Math.max(1, queryTerms.length * 0.5)) {
        keepIndices.add(i);
      }
    }
  }

  // Dedup: remove duplicate content
  if (config.dedupContent) {
    const seen = new Set();
    const deduped = new Set();
    for (const i of keepIndices) {
      const h = contentHash(items[i]);
      if (seen.has(h)) {
        deduped.delete(i);
      } else {
        seen.add(h);
        deduped.add(i);
      }
    }
    keepIndices.clear();
    for (const i of deduped) keepIndices.add(i);
  }

  return { keepIndices: [...keepIndices].sort((a, b) => a - b) };
}

// ─── Strategy: TopN ──────────────────────────────────────────────

function planTopN(analysis, items, config) {
  const n = items.length;
  const scoreField = analysis.fields.find(f =>
    /^(score|rank|rating|relevance|confidence|probability|weight)$/i.test(f.name) && f.isNumeric
  );

  if (!scoreField) {
    return planSmartSample(analysis, items, null, config);
  }

  const scored = items.map((item, idx) => ({
    idx,
    score: item[scoreField.name] ?? 0,
  }));

  scored.sort((a, b) => b.score - a.score);

  const k = computeOptimalK(n, config);
  const keepIndices = scored.slice(0, k).map(s => s.idx);

  return { keepIndices: keepIndices.sort((a, b) => a - b) };
}

// ─── Strategy: ClusterSample ─────────────────────────────────────

function planClusterSample(analysis, items, config) {
  const n = items.length;
  // Find message field
  const msgField = analysis.fields.find(f =>
    /^(message|msg|text|description|error|body|summary)$/i.test(f.name)
  );

  if (!msgField) {
    return planSmartSample(analysis, items, null, config);
  }

  // Cluster by message content hash (MD5-like)
  const clusters = new Map();
  for (let i = 0; i < n; i++) {
    const msg = String(items[i][msgField.name] ?? '');
    const key = contentHash(msg).substring(0, 8);
    if (!clusters.has(key)) clusters.set(key, []);
    clusters.get(key).push(i);
  }

  // Take up to config.clusterMaxPerClass representatives per cluster
  const keepIndices = [];
  for (const indices of clusters.values()) {
    keepIndices.push(...indices.slice(0, config.clusterMaxPerClass));
  }

  return { keepIndices: keepIndices.sort((a, b) => a - b) };
}

// ─── Strategy: TimeSeries ────────────────────────────────────────

function planTimeSeries(analysis, items, config) {
  const n = items.length;
  const keepIndices = new Set();

  // Anchors
  for (let i = 0; i < Math.min(config.anchorFirst, n); i++) keepIndices.add(i);
  for (let i = n - 1; i >= Math.max(0, n - config.anchorLast); i--) keepIndices.add(i);

  // Error/outlier detection (same as SmartSample)
  for (let i = 0; i < n; i++) {
    const str = JSON.stringify(items[i]).toLowerCase();
    if (/\b(error|fail|fatal|exception|crash|panic|timeout|rejected)\b/.test(str) ||
        /high|critical|urgent|blocker/i.test(str)) {
      keepIndices.add(i);
      if (i > 0) keepIndices.add(i - 1);
      if (i < n - 1) keepIndices.add(i + 1);
    }
  }

  // Change point detection: find points with significant value change
  const valueField = analysis.fields.find(f =>
    f.isNumeric && !/^(time|timestamp|date|ts|created|updated)$/i.test(f.name)
  );

  if (valueField && n > 5) {
    const values = items.map(item => item[valueField.name]).filter(v => typeof v === 'number');
    if (values.length > 5) {
      const mean = values.reduce((a, b) => a + b, 0) / values.length;
      const diffs = [];
      for (let i = 1; i < values.length; i++) {
        if (typeof values[i] === 'number' && typeof values[i-1] === 'number') {
          diffs.push(Math.abs(values[i] - values[i-1]));
        }
      }
      const avgDiff = diffs.reduce((a, b) => a + b, 0) / diffs.length;

      for (let i = 1; i < n - 1; i++) {
        const prev = items[i-1]?.[valueField.name];
        const curr = items[i]?.[valueField.name];
        const next = items[i+1]?.[valueField.name];
        if (typeof prev === 'number' && typeof curr === 'number' && avgDiff > 0) {
          const change = Math.abs(curr - prev) / avgDiff;
          if (change > config.varianceThreshold) {
            // Keep ±window around change point
            const w = config.changePointWindow;
            for (let j = Math.max(0, i - w); j <= Math.min(n - 1, i + w); j++) {
              keepIndices.add(j);
            }
          }
        }
      }
    }
  }

  return { keepIndices: [...keepIndices].sort((a, b) => a - b) };
}

// ─── Main crushArray Pipeline ────────────────────────────────────

/**
 * Main entry point: compress JSON array with query-aware strategy.
 */
function crushArray(items, options = {}) {
  const config = { ...DEFAULT_CONFIG, ...options };
  const n = items.length;

  // 1. Compute adaptive K
  const adaptiveK = computeOptimalK(n, config);

  // 2. Small array → passthrough
  if (n <= adaptiveK) {
    return {
      compressed: items,
      strategy: 'passthrough:small_array',
      savings: 0,
      originalCount: n,
      keptCount: n,
    };
  }

  // 3. Try lossless compaction first
  if (config.enableCompaction) {
    const compacted = tryLosslessCompaction(items, config);
    if (compacted) {
      return {
        compressed: compacted.compacted,
        strategy: compacted.kind,
        savings: compacted.savings,
        originalCount: n,
        keptCount: n,
      };
    }
  }

  // 4. Analyze array
  const analysis = analyzeArray(items, config);

  // 5. Skip gate
  if (analysis.recommendedStrategy === 'Skip') {
    return {
      compressed: items,
      strategy: 'passthrough:no_signal',
      savings: 0,
      originalCount: n,
      keptCount: n,
    };
  }

  // 6. Plan: select strategy
  let plan;
  const query = config.query;
  switch (analysis.recommendedStrategy) {
    case 'TopN':
      plan = planTopN(analysis, items, config);
      break;
    case 'ClusterSample':
      plan = planClusterSample(analysis, items, config);
      break;
    case 'TimeSeries':
      plan = planTimeSeries(analysis, items, config);
      break;
    case 'SmartSample':
    default:
      plan = planSmartSample(analysis, items, query, config);
  }

  // 7. Execute: extract subset
  const keepIndices = plan.keepIndices;
  const kept = keepIndices.map(i => items[i]);

  // 8. CCR marker
  let ccrHash = null;
  let ccrMarker = null;
  if (config.enableCcr && kept.length < n) {
    const original = JSON.stringify(items);
    ccrHash = blake3ish(original);
    ccrMarker = `{"_ccr_offloaded": "<<ccr:${ccrHash} ${n - kept.length}_rows_offloaded>>"}`;
  }

  // 9. Build result
  const compressed = ccrMarker
    ? [...kept, JSON.parse(ccrMarker)]
    : kept;

  const originalSize = JSON.stringify(items).length;
  const compressedSize = JSON.stringify(compressed).length;
  const savings = originalSize > 0
    ? Math.round((1 - compressedSize / originalSize) * 10000) / 100
    : 0;

  return {
    compressed,
    strategy: analysis.recommendedStrategy,
    savings,
    originalCount: n,
    keptCount: kept.length,
    ccrHash,
    ccrMarker,
    analysis: {
      pattern: analysis.pattern,
      avgUniqueness: analysis.avgUniqueness,
    },
  };
}

// ─── Convenience: compress text input ────────────────────────────

function compressContent(text, options = {}) {
  // Try parsing as JSON array
  let parsed;
  try {
    parsed = JSON.parse(text);
  } catch {
    return { compressed: text, strategy: 'passthrough:not_json', savings: 0 };
  }

  if (Array.isArray(parsed)) {
    return crushArray(parsed, options);
  }

  if (typeof parsed === 'object' && parsed !== null) {
    return { compressed: parsed, strategy: 'passthrough:json_object', savings: 0 };
  }

  return { compressed: text, strategy: 'passthrough:primitive', savings: 0 };
}

// ─── Module exports ──────────────────────────────────────────────

module.exports = {
  crushArray,
  compressContent,
  analyzeArray,
  computeOptimalK,
  tryLosslessCompaction,
  planSmartSample,
  planTopN,
  planClusterSample,
  planTimeSeries,
  contentHash,
  blake3ish,
  DEFAULT_CONFIG,
};

// ─── CLI ─────────────────────────────────────────────────────────

function main() {
  const args = process.argv.slice(2);

  if (args.includes('--help') || args.includes('-h')) {
    console.log(`SmartCrusher — JSON Array Compressor

Usage:
  node smart-crusher.js [--bias 0.7] [--query "search terms"] < input.json

Options:
  --bias N       Compression aggressiveness (0=conservative, 1=aggressive, default 0.5)
  --query "..."  Query context for relevance-based selection
  --json         Output full result JSON

Output: compressed JSON array`);
    return;
  }

  const biasIdx = args.indexOf('--bias');
  const bias = biasIdx >= 0 ? parseFloat(args[biasIdx + 1]) : 0.5;
  const queryIdx = args.indexOf('--query');
  const query = queryIdx >= 0 ? args[queryIdx + 1] : null;
  const jsonOutput = args.includes('--json');

  let input = '';
  process.stdin.setEncoding('utf8');
  process.stdin.on('data', chunk => input += chunk);
  process.stdin.on('end', () => {
    const result = compressContent(input, { bias, query });

    if (jsonOutput) {
      // Output metadata only (not the full compressed data)
      const { compressed, ...meta } = result;
      console.log(JSON.stringify({
        ...meta,
        savings: result.savings + '%',
        compressed_sample: Array.isArray(compressed)
          ? `[${compressed.length} items]`
          : typeof compressed === 'string'
            ? compressed.substring(0, 200)
            : compressed,
      }, null, 2));
    } else {
      // Output compressed content
      if (typeof result.compressed === 'string') {
        console.log(result.compressed);
      } else {
        console.log(JSON.stringify(result.compressed));
      }
    }
  });
}

if (require.main === module) {
  main();
}
