fix(playable): close release validation lifecycle gaps

This commit is contained in:
shen
2026-09-11 21:23:44 +08:00
parent f39a55fdd5
commit cc5573ddc1
13 changed files with 230 additions and 7 deletions
+2
View File
@@ -26,6 +26,7 @@ timeout_seconds=900
required_arch="${MT_PLAYABLE_REQUIRED_ARCH:-arm64}"
kill_grace_seconds=10
suite="forest_render"
source script/playable_process_cleanup.sh
usage() {
cat <<'EOF'
@@ -181,6 +182,7 @@ while kill -0 "$child_pid" 2>/dev/null; do
done
wait "$child_pid"
process_code=$?
child_pid=""
node script/validate_playable_report.mjs --report "$client_report" --output "$final_report" \
--log "$log" --process-code "$process_code" --timed-out "$timed_out" --run-id "$run_id" \
+44
View File
@@ -27,6 +27,7 @@ fixture="$repo/test/playable"
validate_fixture() { # report log process_code timed_out [extra...]
local report="$1" log="$2" code="$3" timed="$4"
shift 4
rm -f "$tmp_dir/sealed.json"
node script/validate_playable_report.mjs --report "$report" --output "$tmp_dir/sealed.json" --log "$log" \
--process-code "$code" --timed-out "$timed" --run-id fixture-run --suite playable \
--required-cases "$tmp_dir/v/required-cases.json" --build "$tmp_dir/v/build.json" \
@@ -54,6 +55,17 @@ cp "$fixture/report.blocked.json" "$tmp_dir/v/client-report.json"
expect_code 2 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: BLOCKED case is not releasable"
reset_fixture
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 1 0)" "validator: nonzero child exit"
for root_value in null '[]' '"invalid"' 42; do
reset_fixture
printf '%s\n' "$root_value" >"$tmp_dir/v/client-report.json"
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: invalid root $root_value"
if [ -f "$tmp_dir/sealed.json" ]; then pass "validator: invalid root sealed as FAIL"; else fail "validator: invalid root crashed before sealing"; fi
done
reset_fixture
printf 'null\n' >"$tmp_dir/v/events.jsonl"
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: null event"
[ -f "$tmp_dir/sealed.json" ] && pass "validator: null event sealed as FAIL" || fail "validator: null event crashed before sealing"
reset_fixture
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 1)" "validator: timed out"
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/rid-warning.log" 0 0)" "validator: RID warning in log"
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$tmp_dir/missing.log" 0 0)" "validator: missing log"
@@ -171,6 +183,36 @@ run_gate() { # scenario output [extra args...]
}
report_status() { node -e 'try{process.stdout.write(require(process.argv[1]).status)}catch{process.stdout.write("MISSING")}' "$1/report.json"; }
cancel_gate() { # script output [runner args...]
local runner="$1" out="$2" owner child="" code pid_file
shift 2
MT_FAKE_SCENARIO=cancel bash "$runner" --app "$app" --output "$out" "$@" >"$out.stdout" 2>&1 &
owner=$!
for ((attempt=0; attempt<100; attempt++)); do
pid_file="$(find "$out" -name fake-child.pid -print -quit 2>/dev/null || true)"
if [ -n "$pid_file" ] && [ -s "$pid_file" ]; then child="$(<"$pid_file")"; break; fi
kill -0 "$owner" 2>/dev/null || break
sleep 0.1
done
kill -TERM "$owner" 2>/dev/null || true
set +e
wait "$owner"
code=$?
set -e
expect_code 143 "$code" "cancel: $runner preserves signal exit"
if [ -n "$child" ] && ! kill -0 "$child" 2>/dev/null; then
pass "cancel: $runner reaps its client"
else
fail "cancel: $runner left a client alive or never started one"
if [ -n "$child" ]; then kill -KILL "$child" 2>/dev/null || true; fi
fi
}
cancel_gate script/run_client_gate.sh "$tmp_dir/run-cancel" --config "$tmp_dir/scenario.local.json" \
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt"
cancel_gate script/forest_map_render_test.sh "$tmp_dir/forest-cancel" --maps "$map_key"
cancel_gate script/playable_test.sh "$tmp_dir/batch-cancel" --config "$tmp_dir/scenario.local.json" \
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt"
expect_code 0 "$(code_of run_gate pass "$tmp_dir/run-pass")" "runner: PASS"
[ "$(report_status "$tmp_dir/run-pass")" = "PASS" ] && pass "runner: sealed report PASS" || fail "runner: sealed report not PASS"
expect_code 0 "$(code_of run_gate pass "$tmp_dir/run-pass-2")" "runner: second independent PASS"
@@ -365,6 +407,8 @@ expect_code 2 "$(code_of run_soak_gate soak-pass "$tmp_dir/soak-proxy-down" "$tm
[ ! -e "$tmp_dir/soak-proxy-down/client.log" ] && pass "soak runner: client not started when the upstream is down" || fail "soak runner: client started with upstream down"
# ---------- STB-01 public entry: script/playable_soak.sh ----------
cancel_gate script/playable_soak.sh "$tmp_dir/soak-cancel" --config "$tmp_dir/soak.local.json" \
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" --allow-gameplay
run_soak() { # scenario output [extra args...]
local scenario="$1" out="$2"
shift 2
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Sourced by runners. Only signal children whose PIDs this runner still owns.
# Clear each PID immediately after wait, so EXIT cannot signal a reused PID.
child_pid=""
redactor_pid=""
sampler_pid=""
proxy_pid=""
cleanup_playable_processes() {
local status=$? pid round alive
trap - EXIT
trap '' HUP INT TERM
for pid in "$child_pid" "$redactor_pid" "$sampler_pid" "$proxy_pid"; do
if [ -n "$pid" ]; then kill -TERM "$pid" 2>/dev/null || true; fi
done
for ((round=0; round<${process_cleanup_grace_seconds:-10}; round++)); do
alive=0
for pid in "$child_pid" "$redactor_pid" "$sampler_pid" "$proxy_pid"; do
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then alive=1; fi
done
[ "$alive" -eq 1 ] || break
sleep 1
done
for pid in "$child_pid" "$redactor_pid" "$sampler_pid" "$proxy_pid"; do
if [ -n "$pid" ]; then
kill -KILL "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
fi
done
if [ -n "${fifo:-}" ] && [ -p "$fifo" ]; then rm -f "$fifo"; fi
return "$status"
}
trap cleanup_playable_processes EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env node
// Release aggregation must reject the intermediate report before the RSS seal.
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mt-playable-release-'));
const write = (name, value) => fs.writeFileSync(path.join(root, name), JSON.stringify(value));
const fixture = JSON.parse(fs.readFileSync(path.join(repo, 'test/playable/report.valid.json'), 'utf8'));
const manifest = { schema_version: 1, candidate: fixture.build, required_runs: { soak: 1 },
runs: [{ run_id: fixture.run_id, suite: 'soak', path: 'run-1' }] };
const report = { ...fixture, suite: 'soak', exit_gate: { checked: true, process_code: 0, timed_out: false, errors: [] } };
const baseline = process.argv.includes('--baseline')
? spawnSync('git', ['show', 'f39a55fd:script/validate_playable_report.mjs'], { cwd: repo, encoding: 'utf8' }).stdout : null;
function check(expected, label) {
const args = baseline ? ['--input-type=module', '-'] : [path.join(repo, 'script/validate_playable_report.mjs')];
const result = spawnSync(process.execPath, [...args, '--release-dir', root], { input: baseline, encoding: 'utf8' });
assert.equal(result.status, expected, `${label}: ${result.stdout} ${result.stderr}`);
assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'release-report.json'))).status, expected === 0 ? 'PASS' : 'FAIL');
console.log(`ok - ${label}`);
}
try {
fs.mkdirSync(path.join(root, 'run-1'));
write('release-manifest.json', manifest);
write('run-1/report.json', report);
check(1, 'soak PASS without external memory seal is rejected');
report.runner_cases = [{ id: 'STB-MEMORY-01', status: 'PASS' }];
write('run-1/report.json', report);
check(1, 'memory case alone cannot replace the verdict');
report.memory = { verdict: { status: 'FAIL' } };
write('run-1/report.json', report);
check(1, 'contradictory memory verdict is rejected');
report.memory.verdict.status = 'PASS';
write('run-1/report.json', report);
check(0, 'clean exit and external memory PASS can be aggregated');
write('run-1/report.json', null);
check(1, 'null run report produces a sealed release failure');
write('release-manifest.json', null);
check(1, 'null manifest produces a sealed release failure');
} finally {
fs.rmSync(root, { recursive: true, force: true });
}
+6 -1
View File
@@ -22,6 +22,8 @@ set -euo pipefail
set +x
cd "$(dirname "$0")/.."
repo="$PWD"
process_cleanup_grace_seconds=15 # Allow the child runner's 10-second cleanup to finish.
source script/playable_process_cleanup.sh
app="${MT_PLAYABLE_APP:-}"
config=""
output=""
@@ -152,8 +154,11 @@ run_one() { # name suite timeout
if [ -n "$assets" ]; then args+=(--assets "$assets"); fi
if [ -n "$serverlist" ]; then args+=(--serverlist "$serverlist"); fi
set +e
bash script/run_client_gate.sh "${args[@]}" </dev/null
bash script/run_client_gate.sh "${args[@]}" </dev/null &
child_pid=$!
wait "$child_pid"
code=$?
child_pid=""
set -e
if [ "$code" -eq 124 ]; then timed_out=1; fi
if [ "$code" -ne 0 ] && [ "$run_suite" = "playable" ]; then stop_reason="$name exit $code"; fi
+6 -1
View File
@@ -13,6 +13,8 @@ set -euo pipefail
set +x
cd "$(dirname "$0")/.."
repo="$PWD"
process_cleanup_grace_seconds=15
source script/playable_process_cleanup.sh
app="${MT_PLAYABLE_APP:-}"
config=""
output=""
@@ -92,8 +94,11 @@ for index in $(seq 1 "$repeat"); do
if [ -n "$serverlist" ]; then args+=(--serverlist "$serverlist"); fi
if [ "$allow" -eq 1 ]; then args+=(--allow-gameplay); fi
set +e
bash script/run_client_gate.sh "${args[@]}" </dev/null
bash script/run_client_gate.sh "${args[@]}" </dev/null &
child_pid=$!
wait "$child_pid"
code=$?
child_pid=""
set -e
if [ "$code" -eq 124 ]; then timed_out=1; fi
# A run that never produced a sealed report is still listed so the release fails loudly.
+4 -1
View File
@@ -31,6 +31,7 @@ allow_gameplay=0
required_arch="${MT_PLAYABLE_REQUIRED_ARCH:-arm64}"
godot_bin="${MT_GODOT:-godot}"
kill_grace_seconds=10
source script/playable_process_cleanup.sh
usage() {
cat <<'EOF'
@@ -162,7 +163,6 @@ stop_proxy() {
proxy_pid=""
fi
}
trap stop_proxy EXIT
if [ "$fault_status" = "confirmed" ]; then
# Loopback proxy in front of the test server for this client only. server.* points at it, so
# playable exit runs sharing the scenario go through it too; only the soak client requests faults.
@@ -255,6 +255,7 @@ while kill -0 "$child_pid" 2>/dev/null; do
done
wait "$child_pid"
process_code=$?
child_pid=""
# The redactor ends at EOF; bound the wait in case a grandchild kept the FIFO open.
redactor_wait=0
while kill -0 "$redactor_pid" 2>/dev/null && [ "$redactor_wait" -lt 10 ]; do
@@ -267,6 +268,7 @@ if kill -0 "$redactor_pid" 2>/dev/null; then
fi
wait "$redactor_pid"
redactor_code=$?
redactor_pid=""
rm -f "$fifo"
if [ -n "$sampler_pid" ]; then
# The sampler stops by itself once the child is gone; bound the wait anyway.
@@ -277,6 +279,7 @@ if [ -n "$sampler_pid" ]; then
done
if kill -0 "$sampler_pid" 2>/dev/null; then kill -TERM "$sampler_pid" 2>/dev/null; fi
wait "$sampler_pid"
sampler_pid=""
fi
stop_proxy
+19 -2
View File
@@ -56,10 +56,16 @@ function argsOf(argv) {
}
function readJson(file) {
try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
try {
const value = JSON.parse(fs.readFileSync(file, 'utf8'));
if (!isObject(value)) return { __read_error: `${path.basename(file)}: root must be an object` };
return value;
}
catch (error) { return { __read_error: `${path.basename(file)}: ${error.code || 'invalid JSON'}` }; }
}
function isObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); }
function secretLiterals() {
return ['MT_ACCOUNT', 'MT_PASSWORD'].map((name) => process.env[name] || '').filter((v) => v.length >= SECRET_MIN_SCAN);
}
@@ -160,7 +166,8 @@ function validateReport(report, options, errors) {
for (const line of lines) {
let event;
try { event = JSON.parse(line); } catch { badEvents += 1; continue; }
if (!EVENT_KEYS.every((key) => Object.hasOwn(event, key)) || event.run_id !== options.run_id
if (!isObject(event) || !EVENT_KEYS.every((key) => Object.hasOwn(event, key)) || event.run_id !== options.run_id
|| !Number.isSafeInteger(event.monotonic_us) || event.monotonic_us < 0
|| typeof event.payload !== 'object' || event.payload === null || Array.isArray(event.payload)
|| !Number.isInteger(event.connection_epoch) || event.monotonic_us < lastUs) badEvents += 1;
else lastUs = event.monotonic_us;
@@ -273,6 +280,16 @@ function sealRelease(dir) {
if (report.run_id !== run.run_id) errors.fail.push(`${label}: report run_id mismatch`);
if (report.suite !== run.suite) errors.fail.push(`${label}: report suite mismatch`);
if (report.exit_gate?.checked !== true || report.exit_gate?.process_code !== 0 || report.exit_gate?.timed_out !== false) errors.fail.push(`${label}: report not sealed by a clean exit gate`);
// The client exit gate runs before the external RSS seal. An interrupted
// or crashed memory sealer must not leave a releasable intermediate PASS.
if (run.suite === 'soak' && report.status === 'PASS') {
const memoryCases = Array.isArray(report.runner_cases)
? report.runner_cases.filter((item) => item?.id === 'STB-MEMORY-01') : [];
if (memoryCases.length !== 1 || memoryCases[0].status !== 'PASS'
|| report.memory?.verdict?.status !== 'PASS') {
errors.fail.push(`${label}: STB-MEMORY-01 external memory seal missing or not PASS`);
}
}
checkBuild(report.build, manifest.candidate, `${label}: `, errors);
if (report.status === 'PASS') passedBySuite[run.suite] = (passedBySuite[run.suite] || 0) + 1;
else if (report.status === 'BLOCKED') {