351 lines
21 KiB
Python
Executable File
351 lines
21 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run MeshPort's local regression suites and write CI-friendly reports."""
|
|
import argparse
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
import xml.etree.ElementTree as ET
|
|
import performance
|
|
import coexistence
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SUITES = ("all", "runner", "sdk", "engine", "client", "macos", "throughput", "speed", "combined")
|
|
|
|
|
|
@dataclass
|
|
class Step:
|
|
name: str
|
|
command: list
|
|
cwd: Path = ROOT
|
|
requires: tuple = ()
|
|
env: dict = field(default_factory=dict)
|
|
skip: str = ""
|
|
measurement: dict = field(default_factory=dict)
|
|
checks: dict = field(default_factory=dict)
|
|
|
|
|
|
def plan(suites, native=False, rtt_ms=58, mib=8, platform=sys.platform, rtts=(0, 58, 100), repeats=3, policy=None, joint=None):
|
|
selected = set(suites)
|
|
if "all" in selected:
|
|
selected.update(("runner", "sdk", "engine", "client", "macos"))
|
|
steps = []
|
|
cargo = ["sh", "scripts/cargo-local.sh"]
|
|
go = ["sh", "scripts/go-local.sh"]
|
|
if "runner" in selected:
|
|
steps.append(Step("runner-tests", [sys.executable, "-m", "unittest", "discover", "-s", "scripts/tests", "-p", "*_test.py"]))
|
|
if "sdk" in selected:
|
|
steps.append(Step("sdk", cargo + ["test", "--workspace", "--release", "--locked"]))
|
|
if "engine" in selected:
|
|
steps.extend([
|
|
Step("go-race", go + ["test", "-race", "./..."]),
|
|
Step("go-vet", go + ["vet", "./..."]),
|
|
Step("go-bridge-build", ["sh", "scripts/build-singbox.sh"]),
|
|
Step("go-sdk-bridge", cargo + ["test", "-p", "meshport-socks", "--release", "--locked",
|
|
"--test", "bridge", "singbox_to_meshport_tcp_udp_and_withdrawal", "--", "--ignored"],
|
|
requires=("go-bridge-build",),
|
|
env={"MESHPORT_SINGBOX_BIN": str(ROOT / "target/debug/meshport-singbox")}),
|
|
])
|
|
if "client" in selected:
|
|
app = ROOT / "apps/meshport_lab"
|
|
steps.extend([
|
|
Step("flutter-dependencies", ["flutter", "pub", "get"], cwd=app),
|
|
Step("flutter-analyze", ["flutter", "analyze"], cwd=app, requires=("flutter-dependencies",)),
|
|
])
|
|
dependencies = ("flutter-dependencies",)
|
|
env = {}
|
|
if native:
|
|
if platform != "darwin":
|
|
raise ValueError("--native currently requires macOS (the native build script targets macOS)")
|
|
steps.extend([
|
|
Step("client-rust-library", cargo + ["build", "-p", "meshport-ffi", "--locked"]),
|
|
Step("client-go-library", ["sh", "scripts/build-singbox-library.sh", "macos"]),
|
|
])
|
|
dependencies += ("client-rust-library", "client-go-library")
|
|
env = {
|
|
"MESHPORT_LIBRARY": str(ROOT / "target/debug/libmeshport_ffi.dylib"),
|
|
"MESHPORT_PROXY_LIBRARY": str(ROOT / "target/debug/libmeshport_proxy.dylib"),
|
|
}
|
|
steps.append(Step("flutter-tests", ["flutter", "test"], cwd=app, requires=dependencies, env=env))
|
|
if "macos" in selected:
|
|
steps.append(Step("macos-profile-storage", ["sh", "scripts/test-macos-profile-storage.sh"],
|
|
skip="requires macOS and Xcode command-line tools" if platform != "darwin" else ""))
|
|
if "throughput" in selected and "speed" not in selected:
|
|
common = {"MESHPORT_TEST_DELAY_MS": str(rtt_ms // 2), "MESHPORT_TEST_MIB": str(mib)}
|
|
for mode in ("channel", "udp"):
|
|
steps.append(Step("tcp-stack-" + mode, cargo + ["test", "-p", "meshport-socks", "--release",
|
|
"--locked", "--test", "tcp_stack_diagnostic", "sdk_tcp_stack_diagnostic",
|
|
"--", "--ignored", "--nocapture"], env={**common, "MESHPORT_TEST_LINK": mode},
|
|
measurement=dict(mode=mode, target="stack", entries=["direct"], rtt_ms=rtt_ms, mib=mib)))
|
|
steps.append(Step("sdk-throughput", cargo + ["test", "-p", "meshport-socks", "--release", "--locked",
|
|
"--test", "bridge", "subnet_throughput_diagnostic", "--", "--ignored", "--nocapture"],
|
|
env={**common, "MESHPORT_TEST_TARGET": "sdk", "MESHPORT_TEST_ENTRIES": "sdk",
|
|
"MESHPORT_TEST_TRACE_TCP": "1"},
|
|
measurement=dict(mode="mesh", target="sdk", entries=["sdk"], rtt_ms=rtt_ms, mib=mib)))
|
|
if "speed" in selected:
|
|
build = "go-bridge-build" if "engine" in selected else "speed-engine-build"
|
|
if "engine" not in selected:
|
|
steps.append(Step(build, ["sh", "scripts/build-singbox.sh"]))
|
|
measured = []
|
|
for rtt in rtts:
|
|
for repeat in range(1, repeats + 1):
|
|
common = {"MESHPORT_TEST_DELAY_MS": str(rtt // 2), "MESHPORT_TEST_MIB": str(mib)}
|
|
for mode in ("channel", "udp", "mesh"):
|
|
name = f"speed-{mode}-{rtt}ms-{repeat}"
|
|
measured.append(name)
|
|
spec = dict(mode=mode, target="sdk" if mode == "mesh" else "stack",
|
|
entries=["sdk", "socks", "singbox"] if mode == "mesh" else ["direct"],
|
|
rtt_ms=rtt, mib=mib, repeat=repeat)
|
|
if mode == "mesh":
|
|
command = cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "bridge",
|
|
"subnet_throughput_diagnostic", "--", "--ignored", "--nocapture"]
|
|
env = {**common, "MESHPORT_TEST_TARGET": "sdk", "MESHPORT_TEST_ENTRIES": "sdk,socks,singbox",
|
|
"MESHPORT_TEST_TRACE_TCP": "1", "MESHPORT_SINGBOX_BIN": str(ROOT / "target/debug/meshport-singbox")}
|
|
requires = (build,)
|
|
else:
|
|
command = cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "tcp_stack_diagnostic",
|
|
"sdk_tcp_stack_diagnostic", "--", "--ignored", "--nocapture"]
|
|
env = {**common, "MESHPORT_TEST_LINK": mode}
|
|
requires = ()
|
|
steps.append(Step(name, command, requires=requires, env=env, measurement=spec))
|
|
steps.append(Step("speed-check", [], requires=tuple(measured), checks=policy or
|
|
dict(repeats=repeats, min_mbps=0, max_queue_drops=0, baseline=None, max_regression_percent=20)))
|
|
if "combined" in selected:
|
|
if platform != "darwin":
|
|
raise ValueError("combined currently requires macOS (native Go shared library build)")
|
|
build = "client-go-library" if "client" in selected and native else "combined-go-library"
|
|
if build == "combined-go-library":
|
|
steps.append(Step(build, ["sh", "scripts/build-singbox-library.sh", "macos"]))
|
|
spec = joint or dict(duration_seconds=2, repeats=3, proxy_limit_mbps=100, max_slowdown_percent=20)
|
|
spec = {**spec, "rtt_ms": rtt_ms, "mode": "combined"}
|
|
steps.append(Step("combined-coexistence", cargo + ["test", "-p", "meshport-socks", "--release", "--locked",
|
|
"--test", "combined", "combined_coexistence", "--", "--ignored", "--nocapture"],
|
|
requires=(build,), measurement=spec,
|
|
env={"MESHPORT_PROXY_LIBRARY": str(ROOT / "target/debug/libmeshport_proxy.dylib"),
|
|
"MESHPORT_TEST_DELAY_MS": str(rtt_ms // 2),
|
|
"MESHPORT_JOINT_SECONDS": str(spec["duration_seconds"]),
|
|
"MESHPORT_JOINT_REPEATS": str(spec["repeats"]),
|
|
"MESHPORT_JOINT_PROXY_MBPS": str(spec["proxy_limit_mbps"])}))
|
|
return steps
|
|
|
|
|
|
def test_environment(overrides):
|
|
# Do not inherit user profiles, drain fixtures, old native libraries or
|
|
# diagnostic knobs. Every optional native/diagnostic input is set explicitly.
|
|
return {**{key: value for key, value in os.environ.items() if not key.startswith("MESHPORT_")}, **overrides}
|
|
|
|
|
|
def stop_process(process):
|
|
if process.poll() is not None:
|
|
return
|
|
try:
|
|
if os.name == "posix":
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
else:
|
|
process.terminate()
|
|
except ProcessLookupError:
|
|
return
|
|
try:
|
|
process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
if os.name == "posix":
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
else:
|
|
process.kill()
|
|
process.wait()
|
|
|
|
|
|
def execute(step, directory, timeout):
|
|
log = directory / (step.name + ".log")
|
|
start = time.monotonic()
|
|
result = {"name": step.name, "command": step.command, "cwd": str(step.cwd), "log": str(log)}
|
|
print("RUN " + step.name + " " + shlex.join(step.command), flush=True)
|
|
process = None
|
|
try:
|
|
with log.open("w", encoding="utf-8") as output:
|
|
process = subprocess.Popen(step.command, cwd=step.cwd, env=test_environment(step.env),
|
|
stdout=output, stderr=subprocess.STDOUT, start_new_session=os.name == "posix")
|
|
try:
|
|
code = process.wait(timeout=timeout)
|
|
result.update(status="passed" if code == 0 else "failed", exit_code=code)
|
|
except subprocess.TimeoutExpired:
|
|
stop_process(process)
|
|
result.update(status="failed", exit_code=124, reason=f"timeout after {timeout} seconds")
|
|
except KeyboardInterrupt:
|
|
stop_process(process)
|
|
result.update(status="failed", exit_code=130, reason="interrupted")
|
|
except OSError as error:
|
|
log.write_text(str(error) + "\n", encoding="utf-8")
|
|
result.update(status="failed", exit_code=127, reason=str(error))
|
|
result["seconds"] = round(time.monotonic() - start, 3)
|
|
if step.measurement and result["status"] == "passed":
|
|
try:
|
|
content = log.read_text(encoding="utf-8", errors="replace")
|
|
if step.measurement.get("mode") == "combined":
|
|
result["joint_results"] = coexistence.parse(content, step.measurement)
|
|
if result["joint_results"]["issues"]:
|
|
result.update(status="failed", command_exit_code=0, exit_code=1,
|
|
reason="; ".join(result["joint_results"]["issues"]))
|
|
else:
|
|
result["measurements"] = performance.parse(content, step.measurement)
|
|
except (ValueError, KeyError, TypeError) as error:
|
|
result.update(status="failed", command_exit_code=0, exit_code=1, reason="invalid speed result: " + str(error))
|
|
print(("PASS " if result["status"] == "passed" else "FAIL ") + step.name + f" ({result['seconds']:.1f}s)", flush=True)
|
|
if result["status"] == "failed":
|
|
if "reason" in result:
|
|
print(result["reason"], flush=True)
|
|
print("\n".join(log.read_text(encoding="utf-8", errors="replace").splitlines()[-20:]), flush=True)
|
|
return result
|
|
|
|
|
|
def run(steps, directory, timeout=900, fail_fast=False):
|
|
directory.mkdir(parents=True, exist_ok=False)
|
|
results = []
|
|
stopped = False
|
|
for step in steps:
|
|
previous = {result["name"]: result["status"] for result in results}
|
|
blocked = [name for name in step.requires if previous.get(name) != "passed"]
|
|
reason = step.skip or ("run stopped after failure" if stopped else "")
|
|
if not reason and blocked:
|
|
reason = "prerequisite did not pass: " + ", ".join(blocked)
|
|
if reason:
|
|
results.append({"name": step.name, "status": "skipped", "reason": reason, "seconds": 0})
|
|
print("SKIP " + step.name + " " + reason, flush=True)
|
|
continue
|
|
if step.checks:
|
|
issues = performance.evaluate(performance.summarize(results), step.checks)
|
|
log = directory / (step.name + ".log")
|
|
log.write_text("\n".join(issues) + "\n" if issues else "All configured speed criteria passed.\n", encoding="utf-8")
|
|
result = dict(name=step.name, status="failed" if issues else "passed", exit_code=1 if issues else 0,
|
|
seconds=0, log=str(log), criteria=step.checks)
|
|
if issues:
|
|
result["reason"] = "; ".join(issues)
|
|
print(("FAIL " if issues else "PASS ") + step.name, flush=True)
|
|
for issue in issues:
|
|
print(issue, flush=True)
|
|
else:
|
|
result = execute(step, directory, timeout)
|
|
results.append(result)
|
|
stopped = stopped or result.get("exit_code") == 130 or (fail_fast and result["status"] == "failed")
|
|
write_reports(results, directory)
|
|
write_reports(results, directory)
|
|
return results
|
|
|
|
|
|
def write_reports(results, directory):
|
|
for result in results:
|
|
if "joint_results" in result:
|
|
(directory / "combined-metrics.json").write_text(json.dumps(result["joint_results"], indent=2) + "\n", encoding="utf-8")
|
|
(directory / "combined-summary.md").write_text(coexistence.markdown(result["joint_results"]), encoding="utf-8")
|
|
measurements = [record for result in results for record in result.get("measurements", [])]
|
|
if measurements:
|
|
summary = performance.summarize(results)
|
|
(directory / "metrics.json").write_text(json.dumps(dict(measurements=measurements, summary=summary), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
table = ["# Local synthetic speed results", "", "RTT is added round-trip delay; these are local fixtures, not WAN speed measurements.", "",
|
|
"| Path | RTT ms | MiB/direction | Direction | Samples | Median Mbps | Min | Max | Queue drops |",
|
|
"|---|---:|---:|---|---:|---:|---:|---:|---:|"]
|
|
for item in summary:
|
|
path = item["mode"] + "/" + item["target"] + "/" + item["entry"]
|
|
table.append(f"| {path} | {item['rtt_ms']} | {item['mib']} | {item['direction']} | {item['samples']} | {item['median_mbps']:.2f} | {item['min_mbps']:.2f} | {item['max_mbps']:.2f} | {item.get('queue_drops', 'n/a')} |")
|
|
(directory / "speed-summary.md").write_text("\n".join(table) + "\n", encoding="utf-8")
|
|
(directory / "report.json").write_text(json.dumps({"steps": results}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
suite = ET.Element("testsuite", name="meshport", tests=str(len(results)),
|
|
failures=str(sum(r["status"] == "failed" for r in results)),
|
|
skipped=str(sum(r["status"] == "skipped" for r in results)))
|
|
for result in results:
|
|
case = ET.SubElement(suite, "testcase", name=result["name"], classname="meshport.automation", time=str(result["seconds"]))
|
|
if result["status"] == "skipped":
|
|
ET.SubElement(case, "skipped", message=result["reason"])
|
|
elif result["status"] == "failed":
|
|
ET.SubElement(case, "failure", message=result.get("reason", f"exit code {result['exit_code']}"))
|
|
if "log" in result:
|
|
ET.SubElement(case, "system-out").text = result["log"]
|
|
ET.ElementTree(suite).write(directory / "junit.xml", encoding="utf-8", xml_declaration=True)
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("suites", nargs="*", metavar="SUITE", help=", ".join(SUITES) + " (default: all)")
|
|
parser.add_argument("--native", action="store_true", help="build macOS native libraries for Flutter tests")
|
|
parser.add_argument("--list", action="store_true", help="print commands without running them")
|
|
parser.add_argument("--fail-fast", action="store_true")
|
|
parser.add_argument("--timeout", type=int, default=900, help="timeout per command in seconds (default: 900)")
|
|
parser.add_argument("--rtt-ms", type=int, default=58, help="even simulated RTT, 0..1000, for throughput/combined")
|
|
parser.add_argument("--joint-seconds", type=int, default=2, help="combined duration per flow, 1..5")
|
|
parser.add_argument("--joint-repeat", type=int, default=3, help="combined repeats, 1..10")
|
|
parser.add_argument("--proxy-mbps", type=int, default=100, help="combined proxy fixture limit; 0 means unlimited")
|
|
parser.add_argument("--max-joint-slowdown-percent", type=float, default=20, help="maximum median slowdown between combined phases")
|
|
parser.add_argument("--mib", type=int, default=8, help="data per direction in MiB, 1..64, for throughput/speed")
|
|
parser.add_argument("--rtts-ms", default="0,58,100", help="comma-separated even RTTs for speed matrix")
|
|
parser.add_argument("--repeat", type=int, default=3, help="speed matrix repeats per RTT, 1..10")
|
|
parser.add_argument("--min-mbps", type=float, default=0, help="minimum group median for speed checks; default: no absolute floor")
|
|
parser.add_argument("--max-queue-drops", type=int, default=0, help="maximum queue drops per speed group across repeats")
|
|
parser.add_argument("--baseline", type=Path, help="previous metrics.json for matching speed groups")
|
|
parser.add_argument("--max-regression-percent", type=float, default=20, help="maximum median drop from baseline")
|
|
parser.add_argument("--log-dir", type=Path, help="new report directory; must not already exist")
|
|
args = parser.parse_args(argv)
|
|
args.suites = args.suites or ["all"]
|
|
if set(args.suites) - set(SUITES):
|
|
parser.error("unknown suite; choose from: " + ", ".join(SUITES))
|
|
if args.native and not set(args.suites).intersection({"all", "client"}):
|
|
parser.error("--native requires the client or all suite")
|
|
if args.timeout <= 0 or not 0 <= args.rtt_ms <= 1000 or args.rtt_ms % 2 or not 1 <= args.mib <= 64:
|
|
parser.error("timeout must be positive; RTT must be even and 0..1000; MiB must be 1..64")
|
|
try:
|
|
import math
|
|
if not 1 <= args.joint_seconds <= 5 or not 1 <= args.joint_repeat <= 10 or not 0 <= args.proxy_mbps <= 100000:
|
|
raise ValueError("joint seconds must be 1..5, repeats 1..10, proxy Mbps 0..100000")
|
|
if not math.isfinite(args.max_joint_slowdown_percent) or not 0 <= args.max_joint_slowdown_percent < 100:
|
|
raise ValueError("joint slowdown percent must be finite and in 0..<100")
|
|
if "combined" not in args.suites and (args.joint_seconds != 2 or args.joint_repeat != 3 or args.proxy_mbps != 100 or args.max_joint_slowdown_percent != 20):
|
|
raise ValueError("joint options require the combined suite")
|
|
rtts = tuple(int(value) for value in args.rtts_ms.split(","))
|
|
if not rtts or len(set(rtts)) != len(rtts) or any(not 0 <= value <= 1000 or value % 2 for value in rtts):
|
|
raise ValueError("speed RTTs must be distinct even integers in 0..1000")
|
|
if not 1 <= args.repeat <= 10 or not math.isfinite(args.min_mbps) or args.min_mbps < 0 or args.max_queue_drops < 0:
|
|
raise ValueError("repeat must be 1..10; speed and queue-drop limits must be nonnegative and finite")
|
|
if not math.isfinite(args.max_regression_percent) or not 0 <= args.max_regression_percent < 100:
|
|
raise ValueError("maximum regression percent must be finite and in 0..<100")
|
|
if "speed" not in args.suites and (args.min_mbps != 0 or args.max_queue_drops != 0 or args.repeat != 3 or
|
|
args.rtts_ms != "0,58,100" or args.max_regression_percent != 20):
|
|
raise ValueError("matrix/gate options require the speed suite")
|
|
if args.baseline and "speed" not in args.suites:
|
|
raise ValueError("--baseline requires the speed suite")
|
|
baseline = performance.validate_baseline(json.loads(args.baseline.read_text())) if args.baseline else None
|
|
policy = dict(repeats=args.repeat, min_mbps=args.min_mbps, max_queue_drops=args.max_queue_drops,
|
|
baseline=baseline, max_regression_percent=args.max_regression_percent)
|
|
joint = dict(duration_seconds=args.joint_seconds, repeats=args.joint_repeat,
|
|
proxy_limit_mbps=args.proxy_mbps, max_slowdown_percent=args.max_joint_slowdown_percent)
|
|
steps = plan(args.suites, args.native, args.rtt_ms, args.mib, rtts=rtts, repeats=args.repeat, policy=policy, joint=joint)
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
parser.error("cannot read baseline: " + str(error))
|
|
except ValueError as error:
|
|
parser.error(str(error))
|
|
if args.list:
|
|
for step in steps:
|
|
print(step.name + ": " + ("SKIP " + step.skip if step.skip else shlex.join(step.command) if step.command else "evaluate medians, drop counts and optional baseline"))
|
|
return 0
|
|
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
directory = (args.log_dir or ROOT / "outputs/validation/automated-tests" / (stamp + "-" + uuid.uuid4().hex[:8])).resolve()
|
|
if directory.exists():
|
|
parser.error("report directory already exists: " + str(directory))
|
|
print("Reports: " + str(directory), flush=True)
|
|
results = run(steps, directory, args.timeout, args.fail_fast)
|
|
counts = {state: sum(r["status"] == state for r in results) for state in ("passed", "failed", "skipped")}
|
|
print("Summary: " + ", ".join(f"{number} {state}" for state, number in counts.items()), flush=True)
|
|
print("Report: " + str(directory / "report.json"), flush=True)
|
|
if any(r.get("exit_code") == 130 for r in results):
|
|
return 130
|
|
return 1 if counts["failed"] else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|