mtgodot-poc/script/playable_soak_metrics_test.mjs

#!/usr/bin/env node
/* STB-01 §9.1 memory gate regression. Synthetic RSS/rest-window data proves the
 * leak rule (5 consecutive growing low waters AND last-5 median above
 * max(50MiB, 5%) of the first-5 median), that missing data is BLOCKED rather
 * than PASS, and that the external sampler records KiB for one PID only. */
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawn, spawnSync } from 'node:child_process';
import { parseRss, restWindows, roundLowWaters, leakVerdict, median, sealMemory } from './playable_soak_metrics.mjs';

const script = path.join(path.dirname(new URL(import.meta.url).pathname), 'playable_soak_metrics.mjs');
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mt-soak-metrics.'));
let failures = 0;
const check = (ok, label) => { if (ok) console.log(`ok - ${label}`); else { failures += 1; console.error(`not ok - ${label}`); } };

const MIB = 1024;
const event = (round, state, wallMs, us) => JSON.stringify({ monotonic_us: us, run_id: 'r', case_id: 'STB-MEMORY-01', connection_epoch: 1,
  stage: 'REST', kind: 'soak_rest', actor_vid: 0, target_vid: 0, payload: { round, state, wall_ms: wallMs } });

/** Build rounds of rest windows (30 s each, 1 Hz samples) whose low water follows `lows` (KiB). */
function fixture(lows, { restMs = 30000, gapMs = 60000, dropSamplesInRound = -1 } = {}) {
  const events = [];
  const rss = [];
  let t = 1_700_000_000_000;
  lows.forEach((low, round) => {
    for (let s = 1; s < gapMs / 1000; s += 1) rss.push({ wall_ms: t + s * 1000, rss_kib: low + 80 * MIB });
    t += gapMs;
    events.push(event(round, 'start', t, round * 10 + 1));
    for (let s = 0; s <= restMs / 1000; s += 1) {
      if (round !== dropSamplesInRound) rss.push({ wall_ms: t + s * 1000, rss_kib: low + (s === 20 ? 0 : 3 * MIB) });
    }
    t += restMs;
    events.push(event(round, 'end', t, round * 10 + 2));
  });
  return { eventsText: `${events.join('\n')}\n`, rssText: `${rss.map((r) => JSON.stringify(r)).join('\n')}\n` };
}

const flat = Array.from({ length: 12 }, (_, i) => 900 * MIB + (i % 2) * MIB);
const leaking = Array.from({ length: 12 }, (_, i) => 900 * MIB + (i < 6 ? 0 : (i - 5) * 20 * MIB));
const sawtooth = Array.from({ length: 12 }, (_, i) => 900 * MIB + i * 30 * MIB - (i % 3 === 0 ? 40 * MIB : 0));
const smallCreep = Array.from({ length: 12 }, (_, i) => 900 * MIB + i * 2 * MIB);

check(median([3, 1, 2]) === 2 && median([4, 1, 3, 2]) === 2.5, 'median of odd and even lists');
{
  const rows = parseRss('{"wall_ms":1,"rss_kib":10}\nnot json\n{"wall_ms":2,"rss_kib":null}\n');
  check(rows.samples.length === 2 && rows.bad_lines === 1 && rows.samples[1].rss_kib === null, 'rss parser keeps null samples and counts bad lines');
}
{
  const { eventsText, rssText } = fixture(flat);
  const windows = restWindows(eventsText);
  check(windows.windows.length === 12 && windows.windows[0].round === 0 && windows.windows[0].rest_ms === 30000, 'rest windows come from soak_rest start/end events');
  const lows = roundLowWaters(windows.windows, parseRss(rssText).samples);
  check(lows[3].low_water_kib === flat[3] && lows[3].samples === 31, 'low water is the minimum RSS inside the rest window');
  const verdict = leakVerdict(lows, { warmupRounds: 1 });
  check(verdict.status === 'PASS' && verdict.rounds_analysed === 11, `flat memory passes (${verdict.status} ${verdict.reason})`);
  check(/不能证明长期无泄漏/.test(verdict.note), 'a pass still says it cannot prove no long-term leak');
  check(verdict.unit === 'KiB' && Number.isInteger(verdict.first5_median_kib) && Number.isInteger(verdict.last5_median_kib), 'first/last 5 medians are recorded in KiB');
}
{
  const { eventsText, rssText } = fixture(leaking);
  const verdict = leakVerdict(roundLowWaters(restWindows(eventsText).windows, parseRss(rssText).samples), { warmupRounds: 1 });
  check(verdict.status === 'FAIL' && verdict.max_growth_streak >= 5 && verdict.growth_kib > verdict.threshold_kib, `steady growth fails (${JSON.stringify(verdict)})`);
  check(verdict.threshold_kib === 50 * MIB, 'threshold is max(50MiB, 5%) — 50MiB for a 900MiB baseline');
}
{
  // Large growth without 5 consecutive rising rounds is reported but does not fail the gate.
  const verdict = leakVerdict(sawtooth.map((low, round) => ({ round, low_water_kib: low, samples: 31, rest_ms: 30000 })), { warmupRounds: 1 });
  check(verdict.status === 'PASS' && verdict.max_growth_streak < 5 && verdict.growth_kib > verdict.threshold_kib && verdict.warnings.length === 1,
    `median growth without a 5-round streak is a warning (${JSON.stringify(verdict)})`);
}
{
  const verdict = leakVerdict(smallCreep.map((low, round) => ({ round, low_water_kib: low, samples: 31, rest_ms: 30000 })), { warmupRounds: 1 });
  check(verdict.status === 'PASS' && verdict.max_growth_streak >= 5 && verdict.growth_kib <= verdict.threshold_kib, 'a monotonic creep below the threshold passes with the note');
}
{
  const big = Array.from({ length: 12 }, (_, i) => 4096 * MIB + i * 60 * MIB);
  const verdict = leakVerdict(big.map((low, round) => ({ round, low_water_kib: low, samples: 31, rest_ms: 30000 })), { warmupRounds: 1 });
  check(verdict.threshold_kib > 50 * MIB && verdict.threshold_kib === Math.ceil(0.05 * verdict.first5_median_kib), '5% wins for a large baseline');
}
{
  const verdict = leakVerdict(flat.slice(0, 10).map((low, round) => ({ round, low_water_kib: low, samples: 31, rest_ms: 30000 })), { warmupRounds: 1 });
  check(verdict.status === 'BLOCKED' && /10/.test(verdict.reason), 'fewer than 10 rounds after warm-up is BLOCKED');
}
{
  const { eventsText, rssText } = fixture(flat, { dropSamplesInRound: 4 });
  const verdict = leakVerdict(roundLowWaters(restWindows(eventsText).windows, parseRss(rssText).samples), { warmupRounds: 1 });
  check(verdict.status === 'BLOCKED' && /round 4/.test(verdict.reason), `a rest window without RSS samples is BLOCKED (${verdict.reason})`);
}
{
  const { eventsText, rssText } = fixture(flat, { restMs: 10000 });
  const verdict = leakVerdict(roundLowWaters(restWindows(eventsText).windows, parseRss(rssText).samples), { warmupRounds: 1 });
  check(verdict.status === 'FAIL' && /30/.test(verdict.reason), 'a rest window shorter than 30 s violates the contract');
}
{
  const unmatched = `${event(0, 'start', 1000, 1)}\n${event(1, 'start', 2000, 2)}\n`;
  const windows = restWindows(unmatched);
  check(windows.windows.length === 0 && windows.errors.length === 2, 'unmatched rest events are errors, not windows');
}

// Seal: the memory verdict downgrades the sealed report and never upgrades it.
{
  const { eventsText, rssText } = fixture(leaking);
  const base = { schema_version: 1, run_id: 'r', suite: 'soak', status: 'PASS', exit_gate: { checked: true, process_code: 0, timed_out: false, errors: [] } };
  const sealed = sealMemory(structuredClone(base), { eventsText, rssText, warmupRounds: 1 });
  check(sealed.report.status === 'FAIL' && sealed.code === 1, 'leak downgrades PASS to FAIL');
  check(sealed.report.runner_cases?.[0]?.id === 'STB-MEMORY-01' && sealed.report.runner_cases[0].status === 'FAIL', 'runner case STB-MEMORY-01 is recorded');
  check(sealed.report.memory.gpu_memory_kib === null && sealed.report.memory.gpu_memory_status === 'unavailable', 'GPU memory stays null/unavailable');
  check(sealed.report.exit_gate.errors.some((e) => e.startsWith('STB-MEMORY-01')), 'memory verdict reaches exit_gate.errors');
  const blocked = sealMemory(structuredClone(base), { eventsText: '', rssText, warmupRounds: 1 });
  check(blocked.report.status === 'BLOCKED' && blocked.code === 2, 'no rest windows downgrades PASS to BLOCKED');
  const failed = sealMemory({ ...structuredClone(base), status: 'FAIL' }, { ...fixture(flat), warmupRounds: 1 });
  check(failed.report.status === 'FAIL' && failed.code === 1 && failed.report.runner_cases[0].status === 'PASS', 'a clean memory verdict never upgrades a FAIL');
}

// CLI seal writes in place; unknown input is a usage error.
{
  const { eventsText, rssText } = fixture(flat);
  const dir = path.join(root, 'seal');
  fs.mkdirSync(dir);
  fs.writeFileSync(path.join(dir, 'events.jsonl'), eventsText);
  fs.writeFileSync(path.join(dir, 'rss.jsonl'), rssText);
  fs.writeFileSync(path.join(dir, 'report.json'), JSON.stringify({ schema_version: 1, run_id: 'r', suite: 'soak', status: 'PASS', exit_gate: { checked: true, errors: [] } }));
  const run = spawnSync(process.execPath, [script, '--seal', '--report', path.join(dir, 'report.json'), '--rss', path.join(dir, 'rss.jsonl'),
    '--events', path.join(dir, 'events.jsonl')], { encoding: 'utf8' });
  const out = JSON.parse(fs.readFileSync(path.join(dir, 'report.json'), 'utf8'));
  check(run.status === 0 && out.status === 'PASS' && out.memory.rounds.length === 12, `CLI seal PASS (${run.status} ${run.stderr})`);
  const bad = spawnSync(process.execPath, [script, '--seal', '--report', path.join(dir, 'report.json')], { encoding: 'utf8' });
  check(bad.status === 2, 'missing seal arguments exit 2');
  const help = spawnSync(process.execPath, [script, '--help'], { encoding: 'utf8' });
  check(help.status === 0 && /--sample-rss/.test(help.stdout + help.stderr), '--help documents the sampler');
}

// Sampler: 1 Hz ps samples of one PID, KiB integers, stops by itself when that PID exits.
{
  const target = spawn('/bin/sleep', ['3'], { stdio: 'ignore' });
  const output = path.join(root, 'sampler.jsonl');
  const started = Date.now();
  const sampler = spawnSync(process.execPath, [script, '--sample-rss', '--pid', String(target.pid), '--output', output, '--interval-ms', '500'],
    { encoding: 'utf8', timeout: 15000 });
  const rows = parseRss(fs.readFileSync(output, 'utf8')).samples;
  check(sampler.status === 0 && Date.now() - started < 12000, `sampler exits after the PID is gone (${sampler.status} ${sampler.stderr})`);
  check(rows.length >= 3 && rows.every((r) => Number.isInteger(r.rss_kib) && r.rss_kib > 0 && Number.isInteger(r.wall_ms)), `sampler writes KiB rows (${rows.length})`);
  const refused = spawnSync(process.execPath, [script, '--sample-rss', '--pid', '0', '--output', output], { encoding: 'utf8' });
  check(refused.status === 2, 'sampler refuses a non-positive PID');
}

fs.rmSync(root, { recursive: true, force: true });
console.log(failures === 0 ? 'PASS: playable_soak_metrics_test' : `FAIL: playable_soak_metrics_test (${failures})`);
process.exitCode = failures === 0 ? 0 : 1;