mtgodot-poc/script/validate_playable_report.mjs

#!/usr/bin/env node
/* INF-02 / REL-01 report gate.
 *
 * Run mode (called by run_client_gate.sh after the child exited):
 *   --report client-report.json --output report.json --log client.log
 *   --process-code N --timed-out 0|1 --run-id ID --suite S
 *   --required-cases required-cases.json --build build.json --events events.jsonl
 *   --started-ms MS [--redactor-code N] [--require-pass]
 *   Seals the final report. The client can never mark a run PASS by itself.
 *
 * Release mode:
 *   --release-dir DIR   reads DIR/release-manifest.json, checks every explicitly
 *   listed sealed run (same candidate hashes, PASS, counts), writes
 *   DIR/release-report.json. History is never scanned.
 *
 * exit: 0 PASS, 1 FAIL, 2 BLOCKED (only BLOCKED/SKIP findings, clean process+log).
 * Credential literals are read from MT_ACCOUNT/MT_PASSWORD and never printed. */
import fs from 'node:fs';
import path from 'node:path';

const VALID = new Set(['PASS', 'FAIL', 'BLOCKED', 'SKIP']);
const SUITES = new Set(['playable', 'full', 'soak', 'forest_render']);
// Same rule as script/package_render_test.sh, plus parse errors. No RID allow-list.
const ERROR_RE = /SCRIPT ERROR|Parse Error|^ERROR:|leaked at exit|shaders of type .* were never freed/;
const EVENT_KEYS = ['monotonic_us', 'run_id', 'case_id', 'connection_epoch', 'stage', 'kind', 'actor_vid', 'target_vid', 'payload'];
const HASH_KEYS = ['engine_sha256', 'extension_sha256', 'pck_sha256'];
const HEX64 = /^[0-9a-f]{64}$/;
const SECRET_MIN_SCAN = 4;
const MTIME_SLACK_MS = 1000;

function usage() {
  console.error('usage: validate_playable_report.mjs --report PATH --output PATH --log PATH --process-code N --run-id ID '
    + '--suite S --required-cases PATH --build PATH --events PATH --started-ms MS [--timed-out 0|1] [--require-pass]\n'
    + '       validate_playable_report.mjs --release-dir DIR');
}

function argsOf(argv) {
  const out = { require_pass: false };
  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === '--require-pass') out.require_pass = true;
    else if (arg === '--help') out.help = true;
    else if (arg.startsWith('--')) {
      const key = arg.slice(2).replaceAll('-', '_');
      if (i + 1 >= argv.length) throw new Error(`missing value for ${arg}`);
      out[key] = argv[++i];
    } else throw new Error(`unknown argument: ${arg}`);
  }
  if (out.help) return out;
  if (!out.release_dir) {
    for (const key of ['report', 'output', 'log', 'process_code', 'run_id', 'suite', 'required_cases', 'build', 'events', 'started_ms']) {
      if (out[key] === undefined) throw new Error(`--${key.replaceAll('_', '-')} is required`);
    }
  }
  return out;
}

function readJson(file) {
  try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
  catch (error) { return { __read_error: `${path.basename(file)}: ${error.code || 'invalid JSON'}` }; }
}

function secretLiterals() {
  return ['MT_ACCOUNT', 'MT_PASSWORD'].map((name) => process.env[name] || '').filter((v) => v.length >= SECRET_MIN_SCAN);
}

function scanSecrets(label, text, errors) {
  for (const literal of secretLiterals()) {
    if (text.includes(literal)) errors.fail.push(`${label} contains a credential literal`);
  }
}

class Findings {
  constructor() { this.fail = []; this.blocked = []; }
  get all() { return [...new Set([...this.fail, ...this.blocked])]; }
}

function checkBuild(build, expected, prefix, errors) {
  for (const key of HASH_KEYS) {
    if (typeof build?.[key] !== 'string' || !HEX64.test(build[key])) errors.fail.push(`${prefix}build.${key} is missing or not sha256`);
    else if (expected && build[key] !== expected[key]) errors.fail.push(`${prefix}build.${key} differs from the candidate`);
  }
  if (typeof build?.arch !== 'string' || build.arch.trim() === '') errors.fail.push(`${prefix}build.arch is missing`);
  else if (expected && build.arch !== expected.arch) errors.fail.push(`${prefix}build.arch differs from the candidate`);
}

function validateReport(report, options, errors) {
  if (report.__read_error) {
    errors.fail.push(`client report unreadable: ${report.__read_error}`);
    return;
  }
  if (!report || typeof report !== 'object' || Array.isArray(report)) {
    errors.fail.push('report root must be an object');
    return;
  }
  if (report.schema_version !== 1) errors.fail.push('schema_version must be 1');
  if (typeof report.run_id !== 'string' || report.run_id.trim() === '') errors.fail.push('run_id is missing');
  else if (report.run_id !== options.run_id) errors.fail.push('run_id does not match this run (stale or foreign report)');
  if (report.suite !== options.suite) errors.fail.push(`suite ${report.suite} does not match ${options.suite}`);
  if (!VALID.has(report.status)) errors.fail.push(`invalid report status: ${report.status}`);
  const expectedBuild = readJson(options.build);
  if (expectedBuild.__read_error) errors.fail.push(`build.json unreadable: ${expectedBuild.__read_error}`);
  checkBuild(report.build, expectedBuild.__read_error ? null : expectedBuild, '', errors);

  const required = readJson(options.required_cases);
  const requiredIds = Array.isArray(required?.cases) ? required.cases : null;
  if (!requiredIds || requiredIds.length === 0) errors.fail.push('required-cases.json is missing or empty');
  else if (required.suite !== options.suite) errors.fail.push('required-cases.json suite mismatch');
  const cases = Array.isArray(report.cases) ? report.cases : [];
  if (cases.length === 0) errors.fail.push('cases must be non-empty');
  const seen = new Map();
  let passed = 0;
  for (const item of cases) {
    if (!item || typeof item !== 'object' || typeof item.id !== 'string' || item.id.trim() === '') {
      errors.fail.push('case id is missing');
      continue;
    }
    if (seen.has(item.id)) errors.fail.push(`${item.id}: duplicated case`);
    seen.set(item.id, item);
    if (!VALID.has(item.status)) errors.fail.push(`${item.id}: invalid status`);
    if (item.status === 'PASS') passed += 1;
    if (item.status === 'FAIL') errors.fail.push(`${item.id}: FAIL ${item.reason || ''}`.trim());
    if (item.status === 'BLOCKED' || item.status === 'SKIP') errors.blocked.push(`${item.id}: ${item.status} is not releasable (${item.reason || 'no reason'})`);
    if (!Array.isArray(item.evidence)) errors.fail.push(`${item.id}: evidence must be an array`);
    for (const evidence of (Array.isArray(item.evidence) ? item.evidence : [])) {
      const resolved = typeof evidence === 'string' ? path.resolve(path.dirname(options.report), evidence) : '';
      if (!resolved || !resolved.startsWith(path.resolve(path.dirname(options.report)) + path.sep)) errors.fail.push(`${item.id}: evidence path escapes the run directory`);
      else if (!fs.existsSync(resolved)) errors.fail.push(`${item.id}: missing evidence ${evidence}`);
    }
  }
  if (requiredIds) {
    for (const id of requiredIds) if (!seen.has(id)) errors.fail.push(`${id}: required case missing from report`);
    for (const id of seen.keys()) if (!requiredIds.includes(id)) errors.fail.push(`${id}: case is not in the required list`);
    if (JSON.stringify(report.required_cases) !== JSON.stringify(requiredIds)) errors.fail.push('report required_cases differs from the runner list');
  }
  const coverage = report.coverage;
  if (!coverage || coverage.required !== cases.length || coverage.passed !== passed
      || (requiredIds && coverage.required !== requiredIds.length)) errors.fail.push('coverage does not match cases');
  if (!Array.isArray(report.failures) || !Array.isArray(report.blocked)) errors.fail.push('failures and blocked must be arrays');
  else if (report.failures.length > 0) errors.fail.push('client report contains failures');
  if (report.status === 'PASS' && (passed !== cases.length || report.blocked?.length)) errors.fail.push('client PASS contradicts its cases');

  // Staleness: the client report must have been written during this run.
  try {
    const mtime = fs.statSync(options.report).mtimeMs;
    if (mtime + MTIME_SLACK_MS < Number(options.started_ms)) errors.fail.push('client report predates this run (stale)');
  } catch { /* unreadable already reported */ }

  // Event contract.
  let lines = [];
  if (fs.existsSync(options.events)) {
    const text = fs.readFileSync(options.events, 'utf8');
    scanSecrets('events', text, errors);
    lines = text.split('\n').filter((line) => line.trim() !== '');
  }
  if (report.status === 'PASS' && lines.length === 0) errors.fail.push('PASS report has no events');
  if (Number(report.event_count ?? 0) !== lines.length) errors.fail.push(`event_count ${report.event_count} does not match events lines ${lines.length}`);
  let badEvents = 0;
  let lastUs = -Infinity;
  for (const line of lines) {
    let event;
    try { event = JSON.parse(line); } catch { badEvents += 1; continue; }
    if (!EVENT_KEYS.every((key) => Object.hasOwn(event, key)) || event.run_id !== options.run_id
        || typeof event.payload !== 'object' || event.payload === null || Array.isArray(event.payload)
        || !Number.isInteger(event.connection_epoch) || event.monotonic_us < lastUs) badEvents += 1;
    else lastUs = event.monotonic_us;
  }
  if (badEvents > 0) errors.fail.push(`${badEvents} events violate the contract (keys/run_id/order)`);
}

function validateProcess(options, errors) {
  const code = Number(options.process_code);
  if (options.timed_out === '1') errors.fail.push(`wall-clock timeout (raw exit ${code})`);
  if (code !== 0) errors.fail.push(`child process exit code ${code}`);
  if (options.redactor_code !== undefined && Number(options.redactor_code) !== 0) errors.fail.push('log redactor did not finish cleanly');
  if (!fs.existsSync(options.log)) {
    errors.fail.push('client log does not exist');
    return;
  }
  const text = fs.readFileSync(options.log, 'utf8');
  scanSecrets('client log', text, errors);
  const hits = text.split(/\r?\n/).map((line, index) => [line, index]).filter(([line]) => ERROR_RE.test(line));
  // Report the line numbers and the matched rule, not arbitrary log text.
  for (const [line, index] of hits.slice(0, 50)) errors.fail.push(`log:${index + 1}: ${line.match(ERROR_RE)[0]}`);
  if (hits.length > 50) errors.fail.push(`log: ${hits.length - 50} more error lines`);
}

function sealRun(options) {
  const errors = new Findings();
  const input = readJson(options.report);
  validateReport(input, options, errors);
  validateProcess(options, errors);
  const report = input.__read_error ? {
    schema_version: 1, run_id: options.run_id, suite: options.suite, status: 'FAIL',
    build: readJson(options.build), cases: [], failures: [input.__read_error], blocked: [],
    coverage: { required: 0, passed: 0 },
  } : input;
  let status = 'PASS';
  if (errors.fail.length > 0) status = 'FAIL';
  else if (errors.blocked.length > 0) status = 'BLOCKED';
  if (options.require_pass && status === 'PASS' && report.status !== 'PASS') { status = 'FAIL'; errors.fail.push(`client status ${report.status}`); }
  report.status = status;
  report.exit_gate = {
    checked: true,
    process_code: Number(options.process_code),
    timed_out: options.timed_out === '1',
    errors: errors.all,
  };
  const text = `${JSON.stringify(report, null, 2)}\n`;
  const leak = new Findings();
  scanSecrets('final report', text, leak);
  const output = options.output;
  fs.mkdirSync(path.dirname(path.resolve(output)), { recursive: true });
  if (leak.fail.length > 0) {
    // Never persist a report that would carry the secret; keep only the verdict.
    fs.writeFileSync(output, `${JSON.stringify({ schema_version: 1, run_id: options.run_id, suite: options.suite, status: 'FAIL',
      exit_gate: { checked: true, process_code: Number(options.process_code), timed_out: options.timed_out === '1', errors: leak.fail } }, null, 2)}\n`);
    console.error('PLAYABLE REPORT: FAIL (credential literal in report)');
    return 1;
  }
  fs.writeFileSync(output, text);
  if (status !== 'PASS') {
    console.error(`PLAYABLE REPORT: ${status} (${errors.all.length} findings)`);
    errors.all.slice(0, 40).forEach((error) => console.error(`- ${error}`));
    return status === 'BLOCKED' ? 2 : 1;
  }
  console.log(`PLAYABLE REPORT: PASS ${output}`);
  return 0;
}

function sealRelease(dir) {
  const errors = new Findings();
  const root = path.resolve(dir);
  const manifest = readJson(path.join(root, 'release-manifest.json'));
  const summary = { schema_version: 1, status: 'FAIL', candidate: null, runs: [], counts: {}, errors: [] };
  if (manifest.__read_error) errors.fail.push(`release-manifest.json unreadable: ${manifest.__read_error}`);
  else {
    if (manifest.schema_version !== 1) errors.fail.push('manifest schema_version must be 1');
    checkBuild(manifest.candidate, null, 'manifest.', errors);
    summary.candidate = manifest.candidate;
    const runs = Array.isArray(manifest.runs) ? manifest.runs : [];
    if (runs.length === 0) errors.fail.push('manifest lists no runs');
    const ids = new Set();
    const passedBySuite = {};
    const blockedBySuite = {};
    for (const run of runs) {
      const label = `run ${run?.run_id ?? '?'}`;
      if (typeof run?.run_id !== 'string' || typeof run?.path !== 'string' || !SUITES.has(run?.suite)) {
        errors.fail.push(`${label}: run_id/path/suite missing`);
        continue;
      }
      if (ids.has(run.run_id)) errors.fail.push(`${label}: listed twice`);
      ids.add(run.run_id);
      const reportPath = path.resolve(root, run.path, 'report.json');
      if (!reportPath.startsWith(root + path.sep)) {
        errors.fail.push(`${label}: path escapes the release directory`);
        continue;
      }
      const report = readJson(reportPath);
      const entry = { run_id: run.run_id, suite: run.suite, path: run.path, status: report.status ?? 'MISSING' };
      summary.runs.push(entry);
      if (report.__read_error) {
        // run_client_gate.sh records precondition stops in gate.log before any client starts.
        const gateLog = path.resolve(root, run.path, 'gate.log');
        const preconditionBlocked = fs.existsSync(gateLog) && /^BLOCKED /m.test(fs.readFileSync(gateLog, 'utf8'));
        if (preconditionBlocked) {
          entry.status = 'BLOCKED';
          blockedBySuite[run.suite] = (blockedBySuite[run.suite] || 0) + 1;
          errors.blocked.push(`${label}: precondition BLOCKED before launch`);
        } else errors.fail.push(`${label}: sealed report missing`);
        continue;
      }
      if (report.run_id !== run.run_id) errors.fail.push(`${label}: report run_id mismatch`);
      if (report.suite !== run.suite) errors.fail.push(`${label}: report suite mismatch`);
      if (report.exit_gate?.checked !== true || report.exit_gate?.process_code !== 0 || report.exit_gate?.timed_out !== false) errors.fail.push(`${label}: report not sealed by a clean exit gate`);
      checkBuild(report.build, manifest.candidate, `${label}: `, errors);
      if (report.status === 'PASS') passedBySuite[run.suite] = (passedBySuite[run.suite] || 0) + 1;
      else if (report.status === 'BLOCKED') {
        blockedBySuite[run.suite] = (blockedBySuite[run.suite] || 0) + 1;
        errors.blocked.push(`${label}: BLOCKED`);
      }
      else errors.fail.push(`${label}: ${report.status}`);
    }
    const requiredRuns = manifest.required_runs && typeof manifest.required_runs === 'object' ? manifest.required_runs : {};
    if (Object.keys(requiredRuns).length === 0) errors.fail.push('manifest required_runs is empty');
    for (const [suite, count] of Object.entries(requiredRuns)) {
      const got = passedBySuite[suite] || 0;
      summary.counts[suite] = { required: count, passed: got };
      if (!Number.isInteger(count) || count < 1) errors.fail.push(`required_runs.${suite} must be a positive integer`);
      else if (got < count) {
        // Missing passes explained by BLOCKED runs stay BLOCKED; anything else is a FAIL.
        const message = `${suite}: ${got}/${count} passing runs`;
        if (got + (blockedBySuite[suite] || 0) >= count) errors.blocked.push(message);
        else errors.fail.push(message);
      }
    }
  }
  summary.status = errors.fail.length ? 'FAIL' : (errors.blocked.length ? 'BLOCKED' : 'PASS');
  summary.errors = errors.all;
  fs.writeFileSync(path.join(root, 'release-report.json'), `${JSON.stringify(summary, null, 2)}\n`);
  if (summary.status !== 'PASS') {
    console.error(`PLAYABLE RELEASE: ${summary.status}`);
    summary.errors.forEach((error) => console.error(`- ${error}`));
    return summary.status === 'BLOCKED' ? 2 : 1;
  }
  console.log(`PLAYABLE RELEASE: PASS ${path.join(root, 'release-report.json')}`);
  return 0;
}

function main() {
  let options;
  try { options = argsOf(process.argv.slice(2)); }
  catch (error) { usage(); console.error(error.message); return 2; }
  if (options.help) { usage(); return 0; }
  if (options.release_dir) return sealRelease(options.release_dir);
  return sealRun(options);
}

process.exitCode = main();