454 lines
34 KiB
Bash
Executable File
454 lines
34 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# INF-02 runner/exit-gate regression. Builds a throw-away ad-hoc signed arm64
|
|
# app whose executable execs test/playable/fake_client.sh, listens on a local
|
|
# 127.0.0.1 port, and drives script/run_client_gate.sh through PASS and every
|
|
# failure path. Never starts the real client and never contacts the test server.
|
|
set -euo pipefail
|
|
set +x
|
|
cd "$(dirname "$0")/.."
|
|
repo="$PWD"
|
|
tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/mt-playable-gate.XXXXXX")"
|
|
listener_pid=""
|
|
cleanup() {
|
|
if [ -n "$listener_pid" ]; then kill "$listener_pid" 2>/dev/null || true; fi
|
|
if [ -z "${MT_GATE_TEST_KEEP:-}" ]; then rm -R "$tmp_dir"; else echo "kept $tmp_dir"; fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
failures=0
|
|
pass() { echo "ok - $*"; }
|
|
fail() { echo "not ok - $*" >&2; failures=$((failures + 1)); }
|
|
expect_code() { # expected actual label
|
|
if [ "$1" -eq "$2" ]; then pass "$3 (exit $2)"; else fail "$3: expected exit $1, got $2"; fi
|
|
}
|
|
|
|
# ---------- validator unit checks on static fixtures ----------
|
|
fixture="$repo/test/playable"
|
|
validate_fixture() { # report log process_code timed_out [extra...]
|
|
local report="$1" log="$2" code="$3" timed="$4"
|
|
shift 4
|
|
rm -f "$tmp_dir/sealed.json"
|
|
node script/validate_playable_report.mjs --report "$report" --output "$tmp_dir/sealed.json" --log "$log" \
|
|
--process-code "$code" --timed-out "$timed" --run-id fixture-run --suite playable \
|
|
--required-cases "$tmp_dir/v/required-cases.json" --build "$tmp_dir/v/build.json" \
|
|
--events "$tmp_dir/v/events.jsonl" --started-ms "$(($(date +%s) * 1000))" --require-pass "$@" >/dev/null 2>&1
|
|
}
|
|
reset_fixture() {
|
|
rm -rf "$tmp_dir/v"
|
|
mkdir -p "$tmp_dir/v"
|
|
cp "$fixture/report.valid.json" "$tmp_dir/v/client-report.json"
|
|
cp "$fixture/required-cases.fixture.json" "$tmp_dir/v/required-cases.json"
|
|
cp "$fixture/build.fixture.json" "$tmp_dir/v/build.json"
|
|
cp "$fixture/events.valid.jsonl" "$tmp_dir/v/events.jsonl"
|
|
}
|
|
mutate() { # js expression body operating on `r`
|
|
node -e 'const fs=require("fs");const f=process.argv[1];const r=JSON.parse(fs.readFileSync(f,"utf8"));'"$1"';fs.writeFileSync(f,JSON.stringify(r));' "$tmp_dir/v/client-report.json"
|
|
}
|
|
code_of() { set +e; "$@"; local c=$?; set -e; echo "$c"; }
|
|
quiet() { "$@" >/dev/null 2>&1; }
|
|
|
|
reset_fixture
|
|
expect_code 0 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: valid fixture"
|
|
sealed_checked="$(node -e 'const r=require(process.argv[1]);process.stdout.write(String(r.exit_gate.checked===true&&r.exit_gate.process_code===0))' "$tmp_dir/sealed.json")"
|
|
[ "$sealed_checked" = "true" ] && pass "validator: exit_gate sealed" || fail "validator: exit_gate not sealed"
|
|
cp "$fixture/report.blocked.json" "$tmp_dir/v/client-report.json"
|
|
expect_code 2 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: BLOCKED case is not releasable"
|
|
reset_fixture
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 1 0)" "validator: nonzero child exit"
|
|
for root_value in null '[]' '"invalid"' 42; do
|
|
reset_fixture
|
|
printf '%s\n' "$root_value" >"$tmp_dir/v/client-report.json"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: invalid root $root_value"
|
|
if [ -f "$tmp_dir/sealed.json" ]; then pass "validator: invalid root sealed as FAIL"; else fail "validator: invalid root crashed before sealing"; fi
|
|
done
|
|
reset_fixture
|
|
printf 'null\n' >"$tmp_dir/v/events.jsonl"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: null event"
|
|
[ -f "$tmp_dir/sealed.json" ] && pass "validator: null event sealed as FAIL" || fail "validator: null event crashed before sealing"
|
|
reset_fixture
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 1)" "validator: timed out"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/rid-warning.log" 0 0)" "validator: RID warning in log"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$tmp_dir/missing.log" 0 0)" "validator: missing log"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/missing.json" "$fixture/clean.log" 0 0)" "validator: missing client report"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0 --redactor-code 143)" "validator: redactor killed"
|
|
mutate 'r.schema_version=2'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: unknown schema"
|
|
reset_fixture; mutate 'r.suite="full"'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: suite mismatch"
|
|
reset_fixture; mutate 'r.cases=[];r.required_cases=[];r.coverage={required:0,passed:0}'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: empty cases"
|
|
reset_fixture; mutate 'r.cases.push(r.cases[0]);r.coverage.required=2;r.coverage.passed=2'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: duplicate case"
|
|
reset_fixture; mutate 'r.build.pck_sha256=""'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: missing hash"
|
|
reset_fixture; mutate 'r.build.engine_sha256="d".repeat(64)'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: build differs from runner build.json"
|
|
reset_fixture; mutate 'r.cases[0].evidence=["shot.png"]'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: missing evidence file"
|
|
reset_fixture; mutate 'r.cases[0].evidence=["../../etc/hosts"]'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: evidence path escape"
|
|
reset_fixture; mutate 'r.event_count=5'
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: event_count mismatch"
|
|
reset_fixture; sed 's/fixture-run/other-run/' "$fixture/events.valid.jsonl" >"$tmp_dir/v/events.jsonl"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: foreign run_id in events"
|
|
reset_fixture; node -e 'require("fs").writeFileSync(process.argv[1], JSON.stringify({schema_version:1,suite:"playable",cases:["FIXTURE-01","FIXTURE-02"]}))' "$tmp_dir/v/required-cases.json"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: required count differs from case list"
|
|
reset_fixture; touch -t 200001010000 "$tmp_dir/v/client-report.json"
|
|
expect_code 1 "$(code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)" "validator: stale report mtime"
|
|
reset_fixture; mutate 'r.cases[0].reason="fixture-secret-value"'
|
|
code="$(MT_PASSWORD=fixture-secret-value code_of validate_fixture "$tmp_dir/v/client-report.json" "$fixture/clean.log" 0 0)"
|
|
expect_code 1 "$code" "validator: credential literal in client report"
|
|
if grep -q "fixture-secret-value" "$tmp_dir/sealed.json"; then fail "validator: secret copied into sealed report"; else pass "validator: secret not persisted"; fi
|
|
|
|
# ---------- log redactor ----------
|
|
redacted="$(printf 'user fixture-acct pass fixture-acct-pw\n' | MT_ACCOUNT=fixture-acct MT_PASSWORD=fixture-acct-pw node script/redact_stream.mjs)"
|
|
[ "$redacted" = "user [redacted] pass [redacted]" ] && pass "redactor: longest literal first" || fail "redactor output: $redacted"
|
|
|
|
# ---------- scripts never trace ----------
|
|
if grep -nE '^[[:space:]]*set[[:space:]]+-[a-zA-Z]*x' script/run_client_gate.sh script/playable_test.sh script/playable_soak.sh script/live_smoke_test.sh script/forest_map_render_test.sh 2>/dev/null; then
|
|
fail "a launch script enables set -x"
|
|
else
|
|
pass "launch scripts never enable set -x"
|
|
fi
|
|
|
|
# ---------- example config is rejected ----------
|
|
if MT_PLAYABLE_VALIDATE_CONFIG="$repo/test/playable/scenario.example.json" MT_PLAYABLE_ASSETS="$tmp_dir" \
|
|
"${MT_GODOT:-godot}" --headless --path project --script playable_config_gate.gd >/dev/null 2>&1; then
|
|
fail "empty example scenario accepted"
|
|
else
|
|
pass "empty example scenario rejected"
|
|
fi
|
|
|
|
# ---------- fake signed arm64 app ----------
|
|
app="$tmp_dir/Fake.app"
|
|
mkdir -p "$app/Contents/MacOS" "$app/Contents/Frameworks" "$app/Contents/Resources"
|
|
cat >"$tmp_dir/fake.c" <<'EOF'
|
|
#include <stdlib.h>
|
|
#include <unistd.h>
|
|
int main(void) {
|
|
const char *script = getenv("MT_FAKE_CHILD_SCRIPT");
|
|
if (script == NULL) return 64;
|
|
execl("/bin/sh", "sh", script, (char *)0);
|
|
return 127;
|
|
}
|
|
EOF
|
|
printf 'int fake_extension(void) { return 1; }\n' >"$tmp_dir/ext.c"
|
|
cc -arch arm64 -o "$app/Contents/MacOS/mtgodot-poc" "$tmp_dir/fake.c"
|
|
cc -arch arm64 -dynamiclib -o "$app/Contents/Frameworks/libmtgodot.macos.template_release.dylib" "$tmp_dir/ext.c"
|
|
printf 'fake pck\n' >"$app/Contents/Resources/mtgodot-poc.pck"
|
|
cat >"$app/Contents/Info.plist" <<'EOF'
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
<plist version="1.0"><dict>
|
|
<key>CFBundleExecutable</key><string>mtgodot-poc</string>
|
|
<key>CFBundleIdentifier</key><string>test.mt.playable-gate</string>
|
|
<key>CFBundlePackageType</key><string>APPL</string>
|
|
</dict></plist>
|
|
EOF
|
|
codesign --force -s - "$app/Contents/Frameworks/libmtgodot.macos.template_release.dylib" >/dev/null 2>&1
|
|
codesign --force -s - "$app" >/dev/null 2>&1
|
|
|
|
# Local listener so the TCP precheck has a real endpoint; nothing reads from it.
|
|
node -e '
|
|
const server = require("node:net").createServer((socket) => socket.destroy());
|
|
server.listen(0, "127.0.0.1", () => console.log(server.address().port));
|
|
' >"$tmp_dir/port" &
|
|
listener_pid=$!
|
|
for _ in 1 2 3 4 5 6 7 8 9 10; do [ -s "$tmp_dir/port" ] && break; sleep 0.2; done
|
|
port="$(tr -d '\n' <"$tmp_dir/port")"
|
|
printf 'fixture\t127.0.0.1\t%s\t127.0.0.1\t%s\t1\t10\t0\n' "$port" "$port" >"$tmp_dir/serverlist.txt"
|
|
map_key="outdoortrent/metin2_map_trent"
|
|
mkdir -p "$tmp_dir/assets/$map_key"
|
|
printf 'ScriptType\tMapSetting\nBasePosition\t0\t0\nMapSize\t2\t2\n' >"$tmp_dir/assets/$map_key/Setting.txt"
|
|
node -e '
|
|
const [file, port, key] = process.argv.slice(1);
|
|
require("node:fs").writeFileSync(file, JSON.stringify({
|
|
schema_version: 1, scenario_id: "gate-fixture", protocol: "classic",
|
|
server: { server_index: 0, channel: 1, auth_host: "127.0.0.1", auth_port: Number(port), game_host: "127.0.0.1", game_port: Number(port) },
|
|
character_slot: 0, map_key: key, waypoints_cm: [[1000, 2000], [1600, 2000]],
|
|
allowed_mob_vnums: [101], allowed_drop_vnums: [19],
|
|
skill_cases: [{ case_id: "strike", skill_id: 5, target: "enemy", repeats: 2, required_evidence: ["cast_started", "damage"] }],
|
|
resolution: [1280, 720], loops: 1, timeout_seconds: 900,
|
|
}, null, 2));
|
|
' "$tmp_dir/scenario.local.json" "$port" "$map_key"
|
|
|
|
export MT_ACCOUNT="gate-fixture-account"
|
|
export MT_PASSWORD="gate-fixture-password"
|
|
export MT_FAKE_CHILD_SCRIPT="$repo/test/playable/fake_client.sh"
|
|
run_gate() { # scenario output [extra args...]
|
|
local scenario="$1" out="$2"
|
|
shift 2
|
|
MT_FAKE_SCENARIO="$scenario" bash script/run_client_gate.sh --app "$app" --config "$tmp_dir/scenario.local.json" \
|
|
--output "$out" --assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" "$@" >"$out.stdout" 2>&1
|
|
}
|
|
report_status() { node -e 'try{process.stdout.write(require(process.argv[1]).status)}catch{process.stdout.write("MISSING")}' "$1/report.json"; }
|
|
|
|
cancel_gate() { # script output [runner args...]
|
|
local runner="$1" out="$2" owner child="" code pid_file
|
|
shift 2
|
|
MT_FAKE_SCENARIO=cancel bash "$runner" --app "$app" --output "$out" "$@" >"$out.stdout" 2>&1 &
|
|
owner=$!
|
|
for ((attempt=0; attempt<100; attempt++)); do
|
|
pid_file="$(find "$out" -name fake-child.pid -print -quit 2>/dev/null || true)"
|
|
if [ -n "$pid_file" ] && [ -s "$pid_file" ]; then child="$(<"$pid_file")"; break; fi
|
|
kill -0 "$owner" 2>/dev/null || break
|
|
sleep 0.1
|
|
done
|
|
kill -TERM "$owner" 2>/dev/null || true
|
|
set +e
|
|
wait "$owner"
|
|
code=$?
|
|
set -e
|
|
expect_code 143 "$code" "cancel: $runner preserves signal exit"
|
|
if [ -n "$child" ] && ! kill -0 "$child" 2>/dev/null; then
|
|
pass "cancel: $runner reaps its client"
|
|
else
|
|
fail "cancel: $runner left a client alive or never started one"
|
|
if [ -n "$child" ]; then kill -KILL "$child" 2>/dev/null || true; fi
|
|
fi
|
|
}
|
|
cancel_gate script/run_client_gate.sh "$tmp_dir/run-cancel" --config "$tmp_dir/scenario.local.json" \
|
|
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt"
|
|
cancel_gate script/forest_map_render_test.sh "$tmp_dir/forest-cancel" --maps "$map_key"
|
|
cancel_gate script/playable_test.sh "$tmp_dir/batch-cancel" --config "$tmp_dir/scenario.local.json" \
|
|
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt"
|
|
|
|
expect_code 0 "$(code_of run_gate pass "$tmp_dir/run-pass")" "runner: PASS"
|
|
[ "$(report_status "$tmp_dir/run-pass")" = "PASS" ] && pass "runner: sealed report PASS" || fail "runner: sealed report not PASS"
|
|
expect_code 0 "$(code_of run_gate pass "$tmp_dir/run-pass-2")" "runner: second independent PASS"
|
|
expect_code 0 "$(code_of run_gate pass "$tmp_dir/run-full" --suite full)" "runner: full suite PASS"
|
|
expect_code 1 "$(code_of run_gate exit3 "$tmp_dir/run-exit3")" "runner: exit 3 after client PASS"
|
|
[ "$(report_status "$tmp_dir/run-exit3")" = "FAIL" ] && pass "runner: exit 3 sealed FAIL" || fail "runner: exit 3 not sealed FAIL"
|
|
expect_code 1 "$(code_of run_gate signal "$tmp_dir/run-signal")" "runner: child killed by signal"
|
|
expect_code 1 "$(code_of run_gate stale "$tmp_dir/run-stale")" "runner: stale run_id"
|
|
expect_code 1 "$(code_of run_gate no-report "$tmp_dir/run-noreport")" "runner: missing client report"
|
|
expect_code 1 "$(code_of run_gate rid "$tmp_dir/run-rid")" "runner: RID warning after PASS"
|
|
expect_code 1 "$(code_of run_gate leak "$tmp_dir/run-leak")" "runner: leak warning after PASS"
|
|
expect_code 1 "$(code_of run_gate missing-case "$tmp_dir/run-missing-case")" "runner: missing required case"
|
|
expect_code 1 "$(code_of run_gate missing-evidence "$tmp_dir/run-missing-evidence")" "runner: missing evidence file"
|
|
expect_code 2 "$(code_of run_gate blocked "$tmp_dir/run-blocked")" "runner: BLOCKED cases"
|
|
expect_code 0 "$(code_of run_gate print-secret "$tmp_dir/run-secret")" "runner: printed credentials are redacted before disk"
|
|
if grep -rqF -e "$MT_ACCOUNT" -e "$MT_PASSWORD" "$tmp_dir/run-secret" "$tmp_dir/run-secret.stdout"; then
|
|
fail "runner: credential literal reached the run directory"
|
|
else
|
|
pass "runner: no credential literal in run directory"
|
|
fi
|
|
|
|
expect_code 124 "$(code_of run_gate hang "$tmp_dir/run-hang" --timeout-seconds 2)" "runner: TERM-ignoring child is killed"
|
|
if grep -qE '^TIMEOUT term_grace_s=1[0-9] killed=1$' "$tmp_dir/run-hang/gate.log"; then
|
|
pass "runner: TERM grace honoured before KILL"
|
|
else
|
|
fail "runner: timeout sequence: $(tr '\n' ' ' <"$tmp_dir/run-hang/gate.log" 2>/dev/null)"
|
|
fi
|
|
[ "$(report_status "$tmp_dir/run-hang")" = "FAIL" ] && pass "runner: timeout sealed FAIL" || fail "runner: timeout not sealed FAIL"
|
|
|
|
expect_code 1 "$(code_of run_gate fifo-holder "$tmp_dir/run-fifo")" "runner: grandchild holding the log pipe"
|
|
|
|
set +e
|
|
MT_FAKE_SCENARIO=exit3 bash script/run_client_gate.sh --app "$app" --config "$tmp_dir/scenario.local.json" \
|
|
--output "$tmp_dir/run-pipe" --assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" 2>&1 | cat >/dev/null
|
|
piped=$?
|
|
set -e
|
|
expect_code 1 "$piped" "runner: exit code survives | cat under pipefail"
|
|
|
|
mkdir -p "$tmp_dir/run-nocreds"
|
|
code="$(code_of quiet env -u MT_ACCOUNT -u MT_PASSWORD MT_FAKE_SCENARIO=pass bash script/run_client_gate.sh --app "$app" \
|
|
--config "$tmp_dir/scenario.local.json" --output "$tmp_dir/run-nocreds" --assets "$tmp_dir/assets" \
|
|
--serverlist "$tmp_dir/serverlist.txt" </dev/null)"
|
|
expect_code 2 "$code" "runner: no credentials and no TTY"
|
|
[ ! -e "$tmp_dir/run-nocreds/client.log" ] && pass "runner: client not started without credentials" || fail "runner: client started without credentials"
|
|
|
|
expect_code 2 "$(code_of run_gate pass "$tmp_dir/run-pass")" "runner: refuses a non-empty output directory"
|
|
expect_code 2 "$(code_of env MT_PLAYABLE_REQUIRED_ARCH=x86_64 bash -c 'MT_FAKE_SCENARIO=pass bash script/run_client_gate.sh --app "$1" --config "$2" --output "$3" --assets "$4" --serverlist "$5" >/dev/null 2>&1' _ "$app" "$tmp_dir/scenario.local.json" "$tmp_dir/run-arch" "$tmp_dir/assets" "$tmp_dir/serverlist.txt")" "runner: missing required architecture"
|
|
printf '0\t127.0.0.1\t1\t127.0.0.1\t1\t1\t10\t0\n' >"$tmp_dir/serverlist-drift.txt"
|
|
expect_code 2 "$(code_of env bash -c 'MT_FAKE_SCENARIO=pass bash script/run_client_gate.sh --app "$1" --config "$2" --output "$3" --assets "$4" --serverlist "$5" >/dev/null 2>&1' _ "$app" "$tmp_dir/scenario.local.json" "$tmp_dir/run-drift" "$tmp_dir/assets" "$tmp_dir/serverlist-drift.txt")" "runner: config address differs from serverlist"
|
|
[ ! -e "$tmp_dir/run-drift/client.log" ] && pass "runner: client not started on address drift" || fail "runner: client started on address drift"
|
|
cp -R "$app" "$tmp_dir/Tampered.app"
|
|
printf 'tampered\n' >>"$tmp_dir/Tampered.app/Contents/Resources/mtgodot-poc.pck"
|
|
expect_code 2 "$(code_of env bash -c 'MT_FAKE_SCENARIO=pass bash script/run_client_gate.sh --app "$1" --config "$2" --output "$3" --assets "$4" --serverlist "$5" >/dev/null 2>&1' _ "$tmp_dir/Tampered.app" "$tmp_dir/scenario.local.json" "$tmp_dir/run-tampered" "$tmp_dir/assets" "$tmp_dir/serverlist.txt")" "runner: broken signature"
|
|
|
|
# ---------- release aggregation: explicit run list only ----------
|
|
release="$tmp_dir/release"
|
|
mkdir -p "$release"
|
|
cp -R "$tmp_dir/run-pass" "$release/run-1"
|
|
cp -R "$tmp_dir/run-pass-2" "$release/run-2"
|
|
cp -R "$tmp_dir/run-exit3" "$release/run-bad"
|
|
bash script/playable_build_info.sh "$app" >"$tmp_dir/candidate.json"
|
|
write_manifest() { # required runs-json [candidate-override-js]
|
|
node -e '
|
|
const fs = require("node:fs");
|
|
const [dir, required, runs, candidateFile, tweak] = process.argv.slice(1);
|
|
const candidate = JSON.parse(fs.readFileSync(candidateFile, "utf8"));
|
|
if (tweak) eval(tweak);
|
|
const list = JSON.parse(runs).map((name) => ({ run_id: JSON.parse(fs.readFileSync(`${dir}/${name}/report.json`, "utf8")).run_id, suite: "playable", path: name }));
|
|
fs.writeFileSync(`${dir}/release-manifest.json`, JSON.stringify({ schema_version: 1, candidate, required_runs: { playable: Number(required) }, runs: list }));
|
|
' "$release" "$1" "$2" "$tmp_dir/candidate.json" "${3:-}"
|
|
}
|
|
release_code() { code_of quiet node script/validate_playable_report.mjs --release-dir "$release"; }
|
|
write_manifest 2 '["run-1","run-2"]'
|
|
expect_code 0 "$(release_code)" "release: two listed PASS runs"
|
|
write_manifest 2 '["run-1"]'
|
|
expect_code 1 "$(release_code)" "release: unlisted PASS runs are never counted"
|
|
write_manifest 2 '["run-1","run-2","run-bad"]'
|
|
expect_code 1 "$(release_code)" "release: a listed FAIL run fails the release"
|
|
write_manifest 2 '["run-1","run-2"]' 'candidate.pck_sha256 = "e".repeat(64)'
|
|
expect_code 1 "$(release_code)" "release: runs from another candidate"
|
|
write_manifest 2 '["run-1","run-1"]'
|
|
expect_code 1 "$(release_code)" "release: the same run listed twice"
|
|
node -e 'const fs=require("fs");const f=process.argv[1];const m=JSON.parse(fs.readFileSync(f));m.runs[0].path="../run-pass";fs.writeFileSync(f,JSON.stringify(m));' "$release/release-manifest.json"
|
|
expect_code 1 "$(release_code)" "release: run path escapes the release directory"
|
|
|
|
mkdir -p "$release/run-blocked"
|
|
printf 'BLOCKED 缺少 MT_ACCOUNT/MT_PASSWORD，且不是交互终端\n' >"$release/run-blocked/gate.log"
|
|
node -e 'const fs=require("fs");const f=process.argv[1];const m=JSON.parse(fs.readFileSync(f));m.runs=[m.runs[1],{run_id:"run-blocked-unsealed",suite:"playable",path:"run-blocked"}];m.required_runs={playable:2};fs.writeFileSync(f,JSON.stringify(m));' "$release/release-manifest.json"
|
|
expect_code 2 "$(release_code)" "release: precondition-blocked run is BLOCKED, not PASS"
|
|
|
|
# ---------- batch entry: manifest written from the runs it just made ----------
|
|
expect_code 0 "$(code_of quiet env MT_FAKE_SCENARIO=pass bash script/playable_test.sh --app "$app" --config "$tmp_dir/scenario.local.json" \
|
|
--output "$tmp_dir/batch" --suite playable --repeat 2 --assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt")" "batch: two runs aggregate to PASS"
|
|
batch_runs="$(node -e 'const m=require(process.argv[1]);process.stdout.write(m.runs.map((r)=>r.path).join(","))' "$tmp_dir/batch/release-manifest.json")"
|
|
[ "$batch_runs" = "run-1,run-2" ] && pass "batch: manifest lists exactly its own runs" || fail "batch: manifest runs $batch_runs"
|
|
expect_code 1 "$(code_of quiet env MT_FAKE_SCENARIO=rid bash script/playable_test.sh --app "$app" --config "$tmp_dir/scenario.local.json" \
|
|
--output "$tmp_dir/batch-rid" --suite playable --assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt")" "batch: a failing run fails the batch"
|
|
expect_code 2 "$(code_of quiet env -u MT_ACCOUNT -u MT_PASSWORD bash script/playable_test.sh --app "$app" --config "$tmp_dir/scenario.local.json" \
|
|
--output "$tmp_dir/batch-nocreds" </dev/null)" "batch: no credentials and no TTY"
|
|
|
|
# ---------- MAP-02 forest render runner (offline, same exit gate) ----------
|
|
run_forest() { # scenario output [extra args...]
|
|
local scenario="$1" out="$2"
|
|
shift 2
|
|
MT_FAKE_SCENARIO="$scenario" bash script/forest_map_render_test.sh --app "$app" --maps "$map_key" --output "$out" "$@" >"$out.stdout" 2>&1
|
|
}
|
|
expect_code 0 "$(code_of run_forest no-creds "$tmp_dir/forest-pass")" "forest: PASS without credentials in the child"
|
|
forest_cases="$(node -e 'process.stdout.write(require(process.argv[1]).cases.join(","))' "$tmp_dir/forest-pass/required-cases.json")"
|
|
[ "$forest_cases" = "MAP-FOREST-CONFIG,MAP-FOREST-LOAD-metin2_map_trent,MAP-FOREST-VP-metin2_map_trent-flat,MAP-FOREST-VP-metin2_map_trent-slope,MAP-FOREST-VP-metin2_map_trent-dense,MAP-FOREST-VP-metin2_map_trent-warp,MAP-FOREST-MOTION-metin2_map_trent" ] \
|
|
&& pass "forest: required cases follow required_case_ids()" || fail "forest: required cases $forest_cases"
|
|
grep -q "人工签核" "$tmp_dir/forest-pass.stdout" && pass "forest: manual screenshot sign-off stays pending" || fail "forest: sign-off reminder missing"
|
|
expect_code 2 "$(code_of run_forest blocked "$tmp_dir/forest-blocked")" "forest: BLOCKED viewpoints are not releasable"
|
|
expect_code 1 "$(code_of run_forest missing-case "$tmp_dir/forest-missing-case")" "forest: missing required case"
|
|
expect_code 1 "$(code_of run_forest leak "$tmp_dir/forest-leak")" "forest: leak warning after PASS"
|
|
expect_code 124 "$(code_of run_forest hang "$tmp_dir/forest-hang" --timeout-seconds 2)" "forest: TERM-ignoring child is killed"
|
|
mkdir -p "$tmp_dir/forest-nomap"
|
|
expect_code 2 "$(code_of quiet env MT_FAKE_SCENARIO=pass bash script/forest_map_render_test.sh --app "$app" --output "$tmp_dir/forest-nomap")" "forest: no map key"
|
|
[ ! -e "$tmp_dir/forest-nomap/client.log" ] && pass "forest: client not started without a map" || fail "forest: client started without a map"
|
|
expect_code 2 "$(code_of run_forest pass "$tmp_dir/forest-races" --races 2301,abc)" "forest: invalid race list"
|
|
expect_code 2 "$(code_of run_forest pass "$tmp_dir/forest-pass")" "forest: refuses a non-empty output directory"
|
|
|
|
# ---------- STB-01 soak runner (offline; fake client, no real 2 h run) ----------
|
|
soak_config() { # output-file server-port [faults-js]
|
|
node -e '
|
|
const fs = require("node:fs");
|
|
const [base, out, port, faults] = process.argv.slice(1);
|
|
const c = JSON.parse(fs.readFileSync(base, "utf8"));
|
|
c.timeout_seconds = 8100;
|
|
c.server.auth_port = Number(port); c.server.game_port = Number(port);
|
|
c.soak = { duration_seconds: 7200, rest_seconds: 30, warmup_rounds: 1, min_rounds: 10, reconnects: 10, exits: 10,
|
|
resolutions: { sizes: [[1280, 720], [1440, 900], [1600, 1000]], switches_per_size: 10 },
|
|
warp: { status: "unconfirmed" }, faults: { status: "unconfirmed" } };
|
|
if (faults) c.soak.faults = JSON.parse(faults);
|
|
fs.writeFileSync(out, JSON.stringify(c, null, 2));
|
|
' "$tmp_dir/scenario.local.json" "$1" "$2" "${3:-}"
|
|
}
|
|
soak_config "$tmp_dir/soak.local.json" "$port"
|
|
run_soak_gate() { # scenario output config serverlist [extra args...]
|
|
local scenario="$1" out="$2" config="$3" list="$4"
|
|
shift 4
|
|
MT_FAKE_SCENARIO="$scenario" bash script/run_client_gate.sh --app "$app" --config "$config" --output "$out" \
|
|
--assets "$tmp_dir/assets" --serverlist "$list" --suite soak --allow-gameplay "$@" >"$out.stdout" 2>&1
|
|
}
|
|
mkdir -p "$tmp_dir/soak-short"
|
|
expect_code 2 "$(code_of run_soak_gate soak-pass "$tmp_dir/soak-short" "$tmp_dir/soak.local.json" "$tmp_dir/serverlist.txt" --timeout-seconds 900)" "soak runner: timeout below duration+900"
|
|
[ ! -e "$tmp_dir/soak-short/client.log" ] && pass "soak runner: client not started with a short timeout" || fail "soak runner: client started with a short timeout"
|
|
mkdir -p "$tmp_dir/soak-noblock"
|
|
expect_code 2 "$(code_of run_soak_gate soak-pass "$tmp_dir/soak-noblock" "$tmp_dir/scenario.local.json" "$tmp_dir/serverlist.txt" --timeout-seconds 8100)" "soak runner: scenario without a soak block"
|
|
[ ! -e "$tmp_dir/soak-noblock/client.log" ] && pass "soak runner: client not started without a soak block" || fail "soak runner: client started without a soak block"
|
|
|
|
expect_code 2 "$(code_of run_soak_gate soak-pass "$tmp_dir/soak-run" "$tmp_dir/soak.local.json" "$tmp_dir/serverlist.txt" --timeout-seconds 8100)" "soak runner: fake client without soak rounds is BLOCKED, not PASS"
|
|
soak_seal="$(node -e '
|
|
const fs = require("node:fs");
|
|
const dir = process.argv[1];
|
|
const r = JSON.parse(fs.readFileSync(`${dir}/report.json`, "utf8"));
|
|
const memory = (r.runner_cases || []).find((c) => c.id === "STB-MEMORY-01");
|
|
const rows = fs.readFileSync(`${dir}/rss.jsonl`, "utf8").trim().split("\n").filter(Boolean).length;
|
|
process.stdout.write([r.status, memory ? memory.status : "none", rows > 0 ? "rss" : "no-rss", r.exit_gate.checked].join(","));
|
|
' "$tmp_dir/soak-run" 2>/dev/null || echo unreadable)"
|
|
[ "$soak_seal" = "BLOCKED,BLOCKED,rss,true" ] && pass "soak runner: RSS sampled for the child and STB-MEMORY-01 sealed" || fail "soak runner: seal $soak_seal"
|
|
[ ! -e "$tmp_dir/soak-run/fault-proxy.jsonl" ] && pass "soak runner: no fault proxy while faults are unconfirmed" || fail "soak runner: proxy started for unconfirmed faults"
|
|
|
|
# Confirmed faults: client points at a loopback proxy port, proxy forwards to the local listener.
|
|
proxy_port="$(node -e 'const s=require("node:net").createServer();s.listen(0,"127.0.0.1",()=>{const p=s.address().port;s.close(()=>process.stdout.write(String(p)));})')"
|
|
printf 'fixture\t127.0.0.1\t%s\t127.0.0.1\t%s\t1\t10\t0\n' "$proxy_port" "$proxy_port" >"$tmp_dir/serverlist-proxy.txt"
|
|
soak_config "$tmp_dir/soak-proxy.local.json" "$proxy_port" "{\"status\":\"confirmed\",\"mode\":\"local_proxy\",\"per_type\":1,\"unreachable_seconds\":5,\"upstream\":{\"auth_host\":\"127.0.0.1\",\"auth_port\":$port,\"game_host\":\"127.0.0.1\",\"game_port\":$port}}"
|
|
expect_code 2 "$(code_of run_soak_gate soak-pass "$tmp_dir/soak-proxy" "$tmp_dir/soak-proxy.local.json" "$tmp_dir/serverlist-proxy.txt" --timeout-seconds 8100)" "soak runner: confirmed faults run through the loopback proxy"
|
|
proxy_actions="$(node -e 'const rows=require("node:fs").readFileSync(process.argv[1],"utf8").trim().split("\n").map((l)=>JSON.parse(l).action);process.stdout.write(String(rows[0]==="listening"&&rows.at(-1)==="stopped"))' "$tmp_dir/soak-proxy/fault-proxy.jsonl" 2>/dev/null || echo false)"
|
|
[ "$proxy_actions" = "true" ] && pass "soak runner: proxy listened before the client and stopped after it" || fail "soak runner: proxy log incomplete"
|
|
proxy_pid="$(sed -n 's/^FAULT PROXY pid=\([0-9]*\) .*/\1/p' "$tmp_dir/soak-proxy/gate.log")"
|
|
if [ -n "$proxy_pid" ] && ! kill -0 "$proxy_pid" 2>/dev/null; then pass "soak runner: proxy process is gone after the run"; else fail "soak runner: proxy pid [$proxy_pid] still alive or unknown"; fi
|
|
proxy_playable() { # output: an exit run (suite playable) with the same confirmed-faults config still reaches the server
|
|
MT_FAKE_SCENARIO=pass bash script/run_client_gate.sh --app "$app" --config "$tmp_dir/soak-proxy.local.json" --output "$1" \
|
|
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist-proxy.txt" --allow-gameplay >"$1.stdout" 2>&1
|
|
}
|
|
expect_code 0 "$(code_of proxy_playable "$tmp_dir/soak-proxy-exit")" "soak runner: playable exit run behind the confirmed proxy"
|
|
grep -q '"action":"stopped"' "$tmp_dir/soak-proxy-exit/fault-proxy.jsonl" 2>/dev/null && [ ! -e "$tmp_dir/soak-proxy-exit/rss.jsonl" ] \
|
|
&& pass "soak runner: exit run forwards through the proxy without soak sampling" || fail "soak runner: exit run proxy/sampler state"
|
|
node -e 'const fs=require("fs");const c=JSON.parse(fs.readFileSync(process.argv[1]));c.timeout_seconds=900;fs.writeFileSync(process.argv[2],JSON.stringify(c));' "$tmp_dir/soak.local.json" "$tmp_dir/soak-short-timeout.local.json"
|
|
expect_code 0 "$(code_of quiet env MT_FAKE_SCENARIO=pass bash script/run_client_gate.sh --app "$app" --config "$tmp_dir/soak-short-timeout.local.json" \
|
|
--output "$tmp_dir/soak-block-playable" --assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt")" "soak runner: playable suite ignores the soak-only timeout margin"
|
|
bad_soak="$tmp_dir/soak-bad-playable.local.json"
|
|
node -e 'const fs=require("fs");const c=JSON.parse(fs.readFileSync(process.argv[1]));c.soak.faults.upstream.auth_port=70000;fs.writeFileSync(process.argv[2],JSON.stringify(c));' "$tmp_dir/soak-proxy.local.json" "$bad_soak"
|
|
mkdir -p "$tmp_dir/soak-bad-playable"
|
|
expect_code 2 "$(code_of quiet env MT_FAKE_SCENARIO=pass bash script/run_client_gate.sh --app "$app" --config "$bad_soak" --output "$tmp_dir/soak-bad-playable" \
|
|
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist-proxy.txt")" "soak runner: an invalid soak block is rejected for every suite"
|
|
grep -q "soak.faults.upstream.auth_port" "$tmp_dir/soak-bad-playable/config.log" 2>/dev/null && [ ! -e "$tmp_dir/soak-bad-playable/client.log" ] \
|
|
&& pass "soak runner: config gate names the invalid upstream before any client" || fail "soak runner: invalid soak block not caught by the config gate"
|
|
mkdir -p "$tmp_dir/soak-proxy-down"
|
|
soak_config "$tmp_dir/soak-proxy-down.local.json" "$proxy_port" "{\"status\":\"confirmed\",\"mode\":\"local_proxy\",\"per_type\":1,\"unreachable_seconds\":5,\"upstream\":{\"auth_host\":\"127.0.0.1\",\"auth_port\":1,\"game_host\":\"127.0.0.1\",\"game_port\":1}}"
|
|
expect_code 2 "$(code_of run_soak_gate soak-pass "$tmp_dir/soak-proxy-down" "$tmp_dir/soak-proxy-down.local.json" "$tmp_dir/serverlist-proxy.txt" --timeout-seconds 8100)" "soak runner: unreachable fault-proxy upstream"
|
|
[ ! -e "$tmp_dir/soak-proxy-down/client.log" ] && pass "soak runner: client not started when the upstream is down" || fail "soak runner: client started with upstream down"
|
|
|
|
# ---------- STB-01 public entry: script/playable_soak.sh ----------
|
|
cancel_gate script/playable_soak.sh "$tmp_dir/soak-cancel" --config "$tmp_dir/soak.local.json" \
|
|
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" --allow-gameplay
|
|
run_soak() { # scenario output [extra args...]
|
|
local scenario="$1" out="$2"
|
|
shift 2
|
|
MT_FAKE_SCENARIO="$scenario" bash script/playable_soak.sh --app "$app" --config "$tmp_dir/soak.local.json" --output "$out" \
|
|
--assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" "$@" </dev/null >"$out.stdout" 2>&1
|
|
}
|
|
expect_code 0 "$(code_of quiet bash script/playable_soak.sh --help)" "soak: --help"
|
|
expect_code 2 "$(code_of quiet bash script/playable_soak.sh --app "$app" </dev/null)" "soak: no config"
|
|
expect_code 2 "$(code_of run_soak soak-pass "$tmp_dir/soak-nogameplay")" "soak: refuses to start without --allow-gameplay"
|
|
expect_code 2 "$(code_of run_soak soak-pass "$tmp_dir/soak-duration" --allow-gameplay --duration-seconds 60)" "soak: --duration-seconds must match the config"
|
|
expect_code 2 "$(code_of run_soak soak-pass "$tmp_dir/soak-exits" --allow-gameplay --exits 9)" "soak: fewer than 10 exit runs"
|
|
expect_code 2 "$(code_of run_soak soak-pass "$tmp_dir/soak-timeout" --allow-gameplay --timeout-seconds 7300)" "soak: timeout below duration+900"
|
|
expect_code 2 "$(code_of quiet env MT_FAKE_SCENARIO=soak-pass bash script/playable_soak.sh --app "$app" --config "$tmp_dir/scenario.local.json" \
|
|
--output "$tmp_dir/soak-noblock-entry" --assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" --allow-gameplay </dev/null)" "soak: config without a soak block"
|
|
for dir in soak-nogameplay soak-duration soak-exits soak-timeout soak-noblock-entry; do
|
|
if [ -n "$(find "$tmp_dir/$dir" -name client.log 2>/dev/null)" ]; then fail "soak: $dir started a client"; fi
|
|
done
|
|
expect_code 2 "$(code_of quiet env -u MT_ACCOUNT -u MT_PASSWORD MT_FAKE_SCENARIO=soak-pass bash script/playable_soak.sh --app "$app" \
|
|
--config "$tmp_dir/soak.local.json" --output "$tmp_dir/soak-nocreds" --assets "$tmp_dir/assets" --serverlist "$tmp_dir/serverlist.txt" --allow-gameplay </dev/null)" "soak: no credentials and no TTY"
|
|
|
|
expect_code 1 "$(code_of run_soak rid "$tmp_dir/soak-exitfail" --allow-gameplay)" "soak: a failing exit run fails the release"
|
|
[ ! -e "$tmp_dir/soak-exitfail/soak-1/client.log" ] && grep -q "^BLOCKED not started: exit-1" "$tmp_dir/soak-exitfail/soak-1/gate.log" 2>/dev/null \
|
|
&& pass "soak: 2 h soak not started after a failed exit run" || fail "soak: soak run started after a failed exit run"
|
|
|
|
expect_code 2 "$(code_of run_soak soak-pass "$tmp_dir/soak-batch" --allow-gameplay)" "soak: fake batch stays BLOCKED without real soak rounds"
|
|
soak_runs="$(node -e 'const m=require(process.argv[1]);process.stdout.write(m.runs.map((r)=>`${r.path}:${r.suite}`).join(",")+"|"+JSON.stringify(m.required_runs))' "$tmp_dir/soak-batch/release-manifest.json" 2>/dev/null || echo missing)"
|
|
expected_runs="$(for i in 1 2 3 4 5 6 7 8 9 10; do printf 'exit-%s:playable,' "$i"; done)soak-1:soak|{\"playable\":10,\"soak\":1}"
|
|
[ "$soak_runs" = "$expected_runs" ] && pass "soak: manifest lists 10 exit runs and one soak run" || fail "soak: manifest $soak_runs"
|
|
exit_passes="$(node -e 'const r=require(process.argv[1]);process.stdout.write(String(r.counts.playable.passed)+"/"+r.status)' "$tmp_dir/soak-batch/release-report.json" 2>/dev/null || echo missing)"
|
|
[ "$exit_passes" = "10/BLOCKED" ] && pass "soak: 10 independent exit runs PASS, release BLOCKED by memory evidence" || fail "soak: release $exit_passes"
|
|
if grep -rqF -e "$MT_ACCOUNT" -e "$MT_PASSWORD" "$tmp_dir/soak-batch" "$tmp_dir/soak-batch.stdout"; then
|
|
fail "soak: credential literal reached the batch directory"
|
|
else
|
|
pass "soak: no credential literal in the batch directory"
|
|
fi
|
|
expect_code 2 "$(code_of run_soak soak-pass "$tmp_dir/soak-batch" --allow-gameplay)" "soak: refuses a non-empty output directory"
|
|
|
|
if [ "$failures" -ne 0 ]; then
|
|
echo "playable_gate_test: FAIL ($failures)" >&2
|
|
exit 1
|
|
fi
|
|
echo "playable_gate_test: PASS"
|