fix(playable): close release validation lifecycle gaps
This commit is contained in:
@@ -26,6 +26,7 @@ timeout_seconds=900
|
||||
required_arch="${MT_PLAYABLE_REQUIRED_ARCH:-arm64}"
|
||||
kill_grace_seconds=10
|
||||
suite="forest_render"
|
||||
source script/playable_process_cleanup.sh
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
@@ -181,6 +182,7 @@ while kill -0 "$child_pid" 2>/dev/null; do
|
||||
done
|
||||
wait "$child_pid"
|
||||
process_code=$?
|
||||
child_pid=""
|
||||
|
||||
node script/validate_playable_report.mjs --report "$client_report" --output "$final_report" \
|
||||
--log "$log" --process-code "$process_code" --timed-out "$timed_out" --run-id "$run_id" \
|
||||
|
||||
@@ -27,6 +27,7 @@ fixture="$repo/test/playable"
|
||||
validate_fixture() { # report log process_code timed_out [extra...]
|
||||
local report="$1" log="$2" code="$3" timed="$4"
|
||||
shift 4
|
||||
rm -f "$tmp_dir/sealed.json"
|
||||
node script/validate_playable_report.mjs --report "$report" --output "$tmp_dir/sealed.json" --log "$log" \
|
||||
--process-code "$code" --timed-out "$timed" --run-id fixture-run --suite playable \
|
||||
--required-cases "$tmp_dir/v/required-cases.json" --build "$tmp_dir/v/build.json" \
|
||||
@@ -54,6 +55,17 @@ cp "$fixture/report.blocked.json" "$tmp_dir/v/client-report.json"
|
||||
expect_code 2 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: BLOCKED case is not releasable"
|
||||
reset_fixture
|
||||
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 1 0)" "validator: nonzero child exit"
|
||||
for root_value in null '[]' '"invalid"' 42; do
|
||||
reset_fixture
|
||||
printf '%s\n' "$root_value" >"$tmp_dir/v/client-report.json"
|
||||
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: invalid root $root_value"
|
||||
if [ -f "$tmp_dir/sealed.json" ]; then pass "validator: invalid root sealed as FAIL"; else fail "validator: invalid root crashed before sealing"; fi
|
||||
done
|
||||
reset_fixture
|
||||
printf 'null\n' >"$tmp_dir/v/events.jsonl"
|
||||
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: null event"
|
||||
[ -f "$tmp_dir/sealed.json" ] && pass "validator: null event sealed as FAIL" || fail "validator: null event crashed before sealing"
|
||||
reset_fixture
|
||||
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 1)" "validator: timed out"
|
||||
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/rid-warning.log" 0 0)" "validator: RID warning in log"
|
||||
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$tmp_dir/missing.log" 0 0)" "validator: missing log"
|
||||
@@ -171,6 +183,36 @@ run_gate() { # scenario output [extra args...]
|
||||
}
|
||||
report_status() { node -e 'try{process.stdout.write(require(process.argv[1]).status)}catch{process.stdout.write("MISSING")}' "$1/report.json"; }
|
||||
|
||||
cancel_gate() { # script output [runner args...]
|
||||
local runner="$1" out="$2" owner child="" code pid_file
|
||||
shift 2
|
||||
MT_FAKE_SCENARIO=cancel bash "$runner" --app "$app" --output "$out" "$@" >"$out.stdout" 2>&1 &
|
||||
owner=$!
|
||||
for ((attempt=0; attempt<100; attempt++)); do
|
||||
pid_file="$(find "$out" -name fake-child.pid -print -quit 2>/dev/null || true)"
|
||||
if [ -n "$pid_file" ] && [ -s "$pid_file" ]; then child="$(<"$pid_file")"; break; fi
|
||||
kill -0 "$owner" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
kill -TERM "$owner" 2>/dev/null || true
|
||||
set +e
|
||||
wait "$owner"
|
||||
code=$?
|
||||
set -e
|
||||
expect_code 143 "$code" "cancel: $runner preserves signal exit"
|
||||
if [ -n "$child" ] && ! kill -0 "$child" 2>/dev/null; then
|
||||
pass "cancel: $runner reaps its client"
|
||||
else
|
||||
fail "cancel: $runner left a client alive or never started one"
|
||||
if [ -n "$child" ]; then kill -KILL "$child" 2>/dev/null || true; fi
|
||||
fi
|
||||
}
|
||||
cancel_gate script/run_client_gate.sh "$tmp_dir/run-cancel" --config "$tmp_dir/scenario.local.json" \
|
||||
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt"
|
||||
cancel_gate script/forest_map_render_test.sh "$tmp_dir/forest-cancel" --maps "$map_key"
|
||||
cancel_gate script/playable_test.sh "$tmp_dir/batch-cancel" --config "$tmp_dir/scenario.local.json" \
|
||||
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt"
|
||||
|
||||
expect_code 0 "$(code_of run_gate pass "$tmp_dir/run-pass")" "runner: PASS"
|
||||
[ "$(report_status "$tmp_dir/run-pass")" = "PASS" ] && pass "runner: sealed report PASS" || fail "runner: sealed report not PASS"
|
||||
expect_code 0 "$(code_of run_gate pass "$tmp_dir/run-pass-2")" "runner: second independent PASS"
|
||||
@@ -365,6 +407,8 @@ expect_code 2 "$(code_of run_soak_gate soak-pass "$tmp_dir/soak-proxy-down" "$tm
|
||||
[ ! -e "$tmp_dir/soak-proxy-down/client.log" ] && pass "soak runner: client not started when the upstream is down" || fail "soak runner: client started with upstream down"
|
||||
|
||||
# ---------- STB-01 public entry: script/playable_soak.sh ----------
|
||||
cancel_gate script/playable_soak.sh "$tmp_dir/soak-cancel" --config "$tmp_dir/soak.local.json" \
|
||||
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" --allow-gameplay
|
||||
run_soak() { # scenario output [extra args...]
|
||||
local scenario="$1" out="$2"
|
||||
shift 2
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# Sourced by runners. Only signal children whose PIDs this runner still owns.
|
||||
# Clear each PID immediately after wait, so EXIT cannot signal a reused PID.
|
||||
child_pid=""
|
||||
redactor_pid=""
|
||||
sampler_pid=""
|
||||
proxy_pid=""
|
||||
|
||||
cleanup_playable_processes() {
|
||||
local status=$? pid round alive
|
||||
trap - EXIT
|
||||
trap '' HUP INT TERM
|
||||
for pid in "$child_pid" "$redactor_pid" "$sampler_pid" "$proxy_pid"; do
|
||||
if [ -n "$pid" ]; then kill -TERM "$pid" 2>/dev/null || true; fi
|
||||
done
|
||||
for ((round=0; round<${process_cleanup_grace_seconds:-10}; round++)); do
|
||||
alive=0
|
||||
for pid in "$child_pid" "$redactor_pid" "$sampler_pid" "$proxy_pid"; do
|
||||
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then alive=1; fi
|
||||
done
|
||||
[ "$alive" -eq 1 ] || break
|
||||
sleep 1
|
||||
done
|
||||
for pid in "$child_pid" "$redactor_pid" "$sampler_pid" "$proxy_pid"; do
|
||||
if [ -n "$pid" ]; then
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
if [ -n "${fifo:-}" ] && [ -p "$fifo" ]; then rm -f "$fifo"; fi
|
||||
return "$status"
|
||||
}
|
||||
|
||||
trap cleanup_playable_processes EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env node
|
||||
// Release aggregation must reject the intermediate report before the RSS seal.
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mt-playable-release-'));
|
||||
const write = (name, value) => fs.writeFileSync(path.join(root, name), JSON.stringify(value));
|
||||
const fixture = JSON.parse(fs.readFileSync(path.join(repo, 'test/playable/report.valid.json'), 'utf8'));
|
||||
const manifest = { schema_version: 1, candidate: fixture.build, required_runs: { soak: 1 },
|
||||
runs: [{ run_id: fixture.run_id, suite: 'soak', path: 'run-1' }] };
|
||||
const report = { ...fixture, suite: 'soak', exit_gate: { checked: true, process_code: 0, timed_out: false, errors: [] } };
|
||||
const baseline = process.argv.includes('--baseline')
|
||||
? spawnSync('git', ['show', 'f39a55fd:script/validate_playable_report.mjs'], { cwd: repo, encoding: 'utf8' }).stdout : null;
|
||||
function check(expected, label) {
|
||||
const args = baseline ? ['--input-type=module', '-'] : [path.join(repo, 'script/validate_playable_report.mjs')];
|
||||
const result = spawnSync(process.execPath, [...args, '--release-dir', root], { input: baseline, encoding: 'utf8' });
|
||||
assert.equal(result.status, expected, `${label}: ${result.stdout} ${result.stderr}`);
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'release-report.json'))).status, expected === 0 ? 'PASS' : 'FAIL');
|
||||
console.log(`ok - ${label}`);
|
||||
}
|
||||
try {
|
||||
fs.mkdirSync(path.join(root, 'run-1'));
|
||||
write('release-manifest.json', manifest);
|
||||
write('run-1/report.json', report);
|
||||
check(1, 'soak PASS without external memory seal is rejected');
|
||||
report.runner_cases = [{ id: 'STB-MEMORY-01', status: 'PASS' }];
|
||||
write('run-1/report.json', report);
|
||||
check(1, 'memory case alone cannot replace the verdict');
|
||||
report.memory = { verdict: { status: 'FAIL' } };
|
||||
write('run-1/report.json', report);
|
||||
check(1, 'contradictory memory verdict is rejected');
|
||||
report.memory.verdict.status = 'PASS';
|
||||
write('run-1/report.json', report);
|
||||
check(0, 'clean exit and external memory PASS can be aggregated');
|
||||
write('run-1/report.json', null);
|
||||
check(1, 'null run report produces a sealed release failure');
|
||||
write('release-manifest.json', null);
|
||||
check(1, 'null manifest produces a sealed release failure');
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -22,6 +22,8 @@ set -euo pipefail
|
||||
set +x
|
||||
cd "$(dirname "$0")/.."
|
||||
repo="$PWD"
|
||||
process_cleanup_grace_seconds=15 # Allow the child runner's 10-second cleanup to finish.
|
||||
source script/playable_process_cleanup.sh
|
||||
app="${MT_PLAYABLE_APP:-}"
|
||||
config=""
|
||||
output=""
|
||||
@@ -152,8 +154,11 @@ run_one() { # name suite timeout
|
||||
if [ -n "$assets" ]; then args+=(--assets "$assets"); fi
|
||||
if [ -n "$serverlist" ]; then args+=(--serverlist "$serverlist"); fi
|
||||
set +e
|
||||
bash script/run_client_gate.sh "${args[@]}" </dev/null
|
||||
bash script/run_client_gate.sh "${args[@]}" </dev/null &
|
||||
child_pid=$!
|
||||
wait "$child_pid"
|
||||
code=$?
|
||||
child_pid=""
|
||||
set -e
|
||||
if [ "$code" -eq 124 ]; then timed_out=1; fi
|
||||
if [ "$code" -ne 0 ] && [ "$run_suite" = "playable" ]; then stop_reason="$name exit $code"; fi
|
||||
|
||||
@@ -13,6 +13,8 @@ set -euo pipefail
|
||||
set +x
|
||||
cd "$(dirname "$0")/.."
|
||||
repo="$PWD"
|
||||
process_cleanup_grace_seconds=15
|
||||
source script/playable_process_cleanup.sh
|
||||
app="${MT_PLAYABLE_APP:-}"
|
||||
config=""
|
||||
output=""
|
||||
@@ -92,8 +94,11 @@ for index in $(seq 1 "$repeat"); do
|
||||
if [ -n "$serverlist" ]; then args+=(--serverlist "$serverlist"); fi
|
||||
if [ "$allow" -eq 1 ]; then args+=(--allow-gameplay); fi
|
||||
set +e
|
||||
bash script/run_client_gate.sh "${args[@]}" </dev/null
|
||||
bash script/run_client_gate.sh "${args[@]}" </dev/null &
|
||||
child_pid=$!
|
||||
wait "$child_pid"
|
||||
code=$?
|
||||
child_pid=""
|
||||
set -e
|
||||
if [ "$code" -eq 124 ]; then timed_out=1; fi
|
||||
# A run that never produced a sealed report is still listed so the release fails loudly.
|
||||
|
||||
@@ -31,6 +31,7 @@ allow_gameplay=0
|
||||
required_arch="${MT_PLAYABLE_REQUIRED_ARCH:-arm64}"
|
||||
godot_bin="${MT_GODOT:-godot}"
|
||||
kill_grace_seconds=10
|
||||
source script/playable_process_cleanup.sh
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
@@ -162,7 +163,6 @@ stop_proxy() {
|
||||
proxy_pid=""
|
||||
fi
|
||||
}
|
||||
trap stop_proxy EXIT
|
||||
if [ "$fault_status" = "confirmed" ]; then
|
||||
# Loopback proxy in front of the test server for this client only. server.* points at it, so
|
||||
# playable exit runs sharing the scenario go through it too; only the soak client requests faults.
|
||||
@@ -255,6 +255,7 @@ while kill -0 "$child_pid" 2>/dev/null; do
|
||||
done
|
||||
wait "$child_pid"
|
||||
process_code=$?
|
||||
child_pid=""
|
||||
# The redactor ends at EOF; bound the wait in case a grandchild kept the FIFO open.
|
||||
redactor_wait=0
|
||||
while kill -0 "$redactor_pid" 2>/dev/null && [ "$redactor_wait" -lt 10 ]; do
|
||||
@@ -267,6 +268,7 @@ if kill -0 "$redactor_pid" 2>/dev/null; then
|
||||
fi
|
||||
wait "$redactor_pid"
|
||||
redactor_code=$?
|
||||
redactor_pid=""
|
||||
rm -f "$fifo"
|
||||
if [ -n "$sampler_pid" ]; then
|
||||
# The sampler stops by itself once the child is gone; bound the wait anyway.
|
||||
@@ -277,6 +279,7 @@ if [ -n "$sampler_pid" ]; then
|
||||
done
|
||||
if kill -0 "$sampler_pid" 2>/dev/null; then kill -TERM "$sampler_pid" 2>/dev/null; fi
|
||||
wait "$sampler_pid"
|
||||
sampler_pid=""
|
||||
fi
|
||||
stop_proxy
|
||||
|
||||
|
||||
@@ -56,10 +56,16 @@ function argsOf(argv) {
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
|
||||
try {
|
||||
const value = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
if (!isObject(value)) return { __read_error: `${path.basename(file)}: root must be an object` };
|
||||
return value;
|
||||
}
|
||||
catch (error) { return { __read_error: `${path.basename(file)}: ${error.code || 'invalid JSON'}` }; }
|
||||
}
|
||||
|
||||
function isObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); }
|
||||
|
||||
function secretLiterals() {
|
||||
return ['MT_ACCOUNT', 'MT_PASSWORD'].map((name) => process.env[name] || '').filter((v) => v.length >= SECRET_MIN_SCAN);
|
||||
}
|
||||
@@ -160,7 +166,8 @@ function validateReport(report, options, errors) {
|
||||
for (const line of lines) {
|
||||
let event;
|
||||
try { event = JSON.parse(line); } catch { badEvents += 1; continue; }
|
||||
if (!EVENT_KEYS.every((key) => Object.hasOwn(event, key)) || event.run_id !== options.run_id
|
||||
if (!isObject(event) || !EVENT_KEYS.every((key) => Object.hasOwn(event, key)) || event.run_id !== options.run_id
|
||||
|| !Number.isSafeInteger(event.monotonic_us) || event.monotonic_us < 0
|
||||
|| typeof event.payload !== 'object' || event.payload === null || Array.isArray(event.payload)
|
||||
|| !Number.isInteger(event.connection_epoch) || event.monotonic_us < lastUs) badEvents += 1;
|
||||
else lastUs = event.monotonic_us;
|
||||
@@ -273,6 +280,16 @@ function sealRelease(dir) {
|
||||
if (report.run_id !== run.run_id) errors.fail.push(`${label}: report run_id mismatch`);
|
||||
if (report.suite !== run.suite) errors.fail.push(`${label}: report suite mismatch`);
|
||||
if (report.exit_gate?.checked !== true || report.exit_gate?.process_code !== 0 || report.exit_gate?.timed_out !== false) errors.fail.push(`${label}: report not sealed by a clean exit gate`);
|
||||
// The client exit gate runs before the external RSS seal. An interrupted
|
||||
// or crashed memory sealer must not leave a releasable intermediate PASS.
|
||||
if (run.suite === 'soak' && report.status === 'PASS') {
|
||||
const memoryCases = Array.isArray(report.runner_cases)
|
||||
? report.runner_cases.filter((item) => item?.id === 'STB-MEMORY-01') : [];
|
||||
if (memoryCases.length !== 1 || memoryCases[0].status !== 'PASS'
|
||||
|| report.memory?.verdict?.status !== 'PASS') {
|
||||
errors.fail.push(`${label}: STB-MEMORY-01 external memory seal missing or not PASS`);
|
||||
}
|
||||
}
|
||||
checkBuild(report.build, manifest.candidate, `${label}: `, errors);
|
||||
if (report.status === 'PASS') passedBySuite[run.suite] = (passedBySuite[run.suite] || 0) + 1;
|
||||
else if (report.status === 'BLOCKED') {
|
||||
|
||||
Reference in New Issue
Block a user