Files

351 lines
21 KiB
Python
Executable File

#!/usr/bin/env python3
"""Run MeshPort's local regression suites and write CI-friendly reports."""
import argparse
from dataclasses import dataclass, field
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import shlex
import signal
import subprocess
import sys
import time
import uuid
import xml.etree.ElementTree as ET
import performance
import coexistence
ROOT = Path(__file__).resolve().parent.parent
SUITES = ("all", "runner", "sdk", "engine", "client", "macos", "throughput", "speed", "combined")
@dataclass
class Step:
name: str
command: list
cwd: Path = ROOT
requires: tuple = ()
env: dict = field(default_factory=dict)
skip: str = ""
measurement: dict = field(default_factory=dict)
checks: dict = field(default_factory=dict)
def plan(suites, native=False, rtt_ms=58, mib=8, platform=sys.platform, rtts=(0, 58, 100), repeats=3, policy=None, joint=None):
selected = set(suites)
if "all" in selected:
selected.update(("runner", "sdk", "engine", "client", "macos"))
steps = []
cargo = ["sh", "scripts/cargo-local.sh"]
go = ["sh", "scripts/go-local.sh"]
if "runner" in selected:
steps.append(Step("runner-tests", [sys.executable, "-m", "unittest", "discover", "-s", "scripts/tests", "-p", "*_test.py"]))
if "sdk" in selected:
steps.append(Step("sdk", cargo + ["test", "--workspace", "--release", "--locked"]))
if "engine" in selected:
steps.extend([
Step("go-race", go + ["test", "-race", "./..."]),
Step("go-vet", go + ["vet", "./..."]),
Step("go-bridge-build", ["sh", "scripts/build-singbox.sh"]),
Step("go-sdk-bridge", cargo + ["test", "-p", "meshport-socks", "--release", "--locked",
"--test", "bridge", "singbox_to_meshport_tcp_udp_and_withdrawal", "--", "--ignored"],
requires=("go-bridge-build",),
env={"MESHPORT_SINGBOX_BIN": str(ROOT / "target/debug/meshport-singbox")}),
])
if "client" in selected:
app = ROOT / "apps/meshport_lab"
steps.extend([
Step("flutter-dependencies", ["flutter", "pub", "get"], cwd=app),
Step("flutter-analyze", ["flutter", "analyze"], cwd=app, requires=("flutter-dependencies",)),
])
dependencies = ("flutter-dependencies",)
env = {}
if native:
if platform != "darwin":
raise ValueError("--native currently requires macOS (the native build script targets macOS)")
steps.extend([
Step("client-rust-library", cargo + ["build", "-p", "meshport-ffi", "--locked"]),
Step("client-go-library", ["sh", "scripts/build-singbox-library.sh", "macos"]),
])
dependencies += ("client-rust-library", "client-go-library")
env = {
"MESHPORT_LIBRARY": str(ROOT / "target/debug/libmeshport_ffi.dylib"),
"MESHPORT_PROXY_LIBRARY": str(ROOT / "target/debug/libmeshport_proxy.dylib"),
}
steps.append(Step("flutter-tests", ["flutter", "test"], cwd=app, requires=dependencies, env=env))
if "macos" in selected:
steps.append(Step("macos-profile-storage", ["sh", "scripts/test-macos-profile-storage.sh"],
skip="requires macOS and Xcode command-line tools" if platform != "darwin" else ""))
if "throughput" in selected and "speed" not in selected:
common = {"MESHPORT_TEST_DELAY_MS": str(rtt_ms // 2), "MESHPORT_TEST_MIB": str(mib)}
for mode in ("channel", "udp"):
steps.append(Step("tcp-stack-" + mode, cargo + ["test", "-p", "meshport-socks", "--release",
"--locked", "--test", "tcp_stack_diagnostic", "sdk_tcp_stack_diagnostic",
"--", "--ignored", "--nocapture"], env={**common, "MESHPORT_TEST_LINK": mode},
measurement=dict(mode=mode, target="stack", entries=["direct"], rtt_ms=rtt_ms, mib=mib)))
steps.append(Step("sdk-throughput", cargo + ["test", "-p", "meshport-socks", "--release", "--locked",
"--test", "bridge", "subnet_throughput_diagnostic", "--", "--ignored", "--nocapture"],
env={**common, "MESHPORT_TEST_TARGET": "sdk", "MESHPORT_TEST_ENTRIES": "sdk",
"MESHPORT_TEST_TRACE_TCP": "1"},
measurement=dict(mode="mesh", target="sdk", entries=["sdk"], rtt_ms=rtt_ms, mib=mib)))
if "speed" in selected:
build = "go-bridge-build" if "engine" in selected else "speed-engine-build"
if "engine" not in selected:
steps.append(Step(build, ["sh", "scripts/build-singbox.sh"]))
measured = []
for rtt in rtts:
for repeat in range(1, repeats + 1):
common = {"MESHPORT_TEST_DELAY_MS": str(rtt // 2), "MESHPORT_TEST_MIB": str(mib)}
for mode in ("channel", "udp", "mesh"):
name = f"speed-{mode}-{rtt}ms-{repeat}"
measured.append(name)
spec = dict(mode=mode, target="sdk" if mode == "mesh" else "stack",
entries=["sdk", "socks", "singbox"] if mode == "mesh" else ["direct"],
rtt_ms=rtt, mib=mib, repeat=repeat)
if mode == "mesh":
command = cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "bridge",
"subnet_throughput_diagnostic", "--", "--ignored", "--nocapture"]
env = {**common, "MESHPORT_TEST_TARGET": "sdk", "MESHPORT_TEST_ENTRIES": "sdk,socks,singbox",
"MESHPORT_TEST_TRACE_TCP": "1", "MESHPORT_SINGBOX_BIN": str(ROOT / "target/debug/meshport-singbox")}
requires = (build,)
else:
command = cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "tcp_stack_diagnostic",
"sdk_tcp_stack_diagnostic", "--", "--ignored", "--nocapture"]
env = {**common, "MESHPORT_TEST_LINK": mode}
requires = ()
steps.append(Step(name, command, requires=requires, env=env, measurement=spec))
steps.append(Step("speed-check", [], requires=tuple(measured), checks=policy or
dict(repeats=repeats, min_mbps=0, max_queue_drops=0, baseline=None, max_regression_percent=20)))
if "combined" in selected:
if platform != "darwin":
raise ValueError("combined currently requires macOS (native Go shared library build)")
build = "client-go-library" if "client" in selected and native else "combined-go-library"
if build == "combined-go-library":
steps.append(Step(build, ["sh", "scripts/build-singbox-library.sh", "macos"]))
spec = joint or dict(duration_seconds=2, repeats=3, proxy_limit_mbps=100, max_slowdown_percent=20)
spec = {**spec, "rtt_ms": rtt_ms, "mode": "combined"}
steps.append(Step("combined-coexistence", cargo + ["test", "-p", "meshport-socks", "--release", "--locked",
"--test", "combined", "combined_coexistence", "--", "--ignored", "--nocapture"],
requires=(build,), measurement=spec,
env={"MESHPORT_PROXY_LIBRARY": str(ROOT / "target/debug/libmeshport_proxy.dylib"),
"MESHPORT_TEST_DELAY_MS": str(rtt_ms // 2),
"MESHPORT_JOINT_SECONDS": str(spec["duration_seconds"]),
"MESHPORT_JOINT_REPEATS": str(spec["repeats"]),
"MESHPORT_JOINT_PROXY_MBPS": str(spec["proxy_limit_mbps"])}))
return steps
def test_environment(overrides):
# Do not inherit user profiles, drain fixtures, old native libraries or
# diagnostic knobs. Every optional native/diagnostic input is set explicitly.
return {**{key: value for key, value in os.environ.items() if not key.startswith("MESHPORT_")}, **overrides}
def stop_process(process):
if process.poll() is not None:
return
try:
if os.name == "posix":
os.killpg(process.pid, signal.SIGTERM)
else:
process.terminate()
except ProcessLookupError:
return
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
if os.name == "posix":
os.killpg(process.pid, signal.SIGKILL)
else:
process.kill()
process.wait()
def execute(step, directory, timeout):
log = directory / (step.name + ".log")
start = time.monotonic()
result = {"name": step.name, "command": step.command, "cwd": str(step.cwd), "log": str(log)}
print("RUN " + step.name + " " + shlex.join(step.command), flush=True)
process = None
try:
with log.open("w", encoding="utf-8") as output:
process = subprocess.Popen(step.command, cwd=step.cwd, env=test_environment(step.env),
stdout=output, stderr=subprocess.STDOUT, start_new_session=os.name == "posix")
try:
code = process.wait(timeout=timeout)
result.update(status="passed" if code == 0 else "failed", exit_code=code)
except subprocess.TimeoutExpired:
stop_process(process)
result.update(status="failed", exit_code=124, reason=f"timeout after {timeout} seconds")
except KeyboardInterrupt:
stop_process(process)
result.update(status="failed", exit_code=130, reason="interrupted")
except OSError as error:
log.write_text(str(error) + "\n", encoding="utf-8")
result.update(status="failed", exit_code=127, reason=str(error))
result["seconds"] = round(time.monotonic() - start, 3)
if step.measurement and result["status"] == "passed":
try:
content = log.read_text(encoding="utf-8", errors="replace")
if step.measurement.get("mode") == "combined":
result["joint_results"] = coexistence.parse(content, step.measurement)
if result["joint_results"]["issues"]:
result.update(status="failed", command_exit_code=0, exit_code=1,
reason="; ".join(result["joint_results"]["issues"]))
else:
result["measurements"] = performance.parse(content, step.measurement)
except (ValueError, KeyError, TypeError) as error:
result.update(status="failed", command_exit_code=0, exit_code=1, reason="invalid speed result: " + str(error))
print(("PASS " if result["status"] == "passed" else "FAIL ") + step.name + f" ({result['seconds']:.1f}s)", flush=True)
if result["status"] == "failed":
if "reason" in result:
print(result["reason"], flush=True)
print("\n".join(log.read_text(encoding="utf-8", errors="replace").splitlines()[-20:]), flush=True)
return result
def run(steps, directory, timeout=900, fail_fast=False):
directory.mkdir(parents=True, exist_ok=False)
results = []
stopped = False
for step in steps:
previous = {result["name"]: result["status"] for result in results}
blocked = [name for name in step.requires if previous.get(name) != "passed"]
reason = step.skip or ("run stopped after failure" if stopped else "")
if not reason and blocked:
reason = "prerequisite did not pass: " + ", ".join(blocked)
if reason:
results.append({"name": step.name, "status": "skipped", "reason": reason, "seconds": 0})
print("SKIP " + step.name + " " + reason, flush=True)
continue
if step.checks:
issues = performance.evaluate(performance.summarize(results), step.checks)
log = directory / (step.name + ".log")
log.write_text("\n".join(issues) + "\n" if issues else "All configured speed criteria passed.\n", encoding="utf-8")
result = dict(name=step.name, status="failed" if issues else "passed", exit_code=1 if issues else 0,
seconds=0, log=str(log), criteria=step.checks)
if issues:
result["reason"] = "; ".join(issues)
print(("FAIL " if issues else "PASS ") + step.name, flush=True)
for issue in issues:
print(issue, flush=True)
else:
result = execute(step, directory, timeout)
results.append(result)
stopped = stopped or result.get("exit_code") == 130 or (fail_fast and result["status"] == "failed")
write_reports(results, directory)
write_reports(results, directory)
return results
def write_reports(results, directory):
for result in results:
if "joint_results" in result:
(directory / "combined-metrics.json").write_text(json.dumps(result["joint_results"], indent=2) + "\n", encoding="utf-8")
(directory / "combined-summary.md").write_text(coexistence.markdown(result["joint_results"]), encoding="utf-8")
measurements = [record for result in results for record in result.get("measurements", [])]
if measurements:
summary = performance.summarize(results)
(directory / "metrics.json").write_text(json.dumps(dict(measurements=measurements, summary=summary), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
table = ["# Local synthetic speed results", "", "RTT is added round-trip delay; these are local fixtures, not WAN speed measurements.", "",
"| Path | RTT ms | MiB/direction | Direction | Samples | Median Mbps | Min | Max | Queue drops |",
"|---|---:|---:|---|---:|---:|---:|---:|---:|"]
for item in summary:
path = item["mode"] + "/" + item["target"] + "/" + item["entry"]
table.append(f"| {path} | {item['rtt_ms']} | {item['mib']} | {item['direction']} | {item['samples']} | {item['median_mbps']:.2f} | {item['min_mbps']:.2f} | {item['max_mbps']:.2f} | {item.get('queue_drops', 'n/a')} |")
(directory / "speed-summary.md").write_text("\n".join(table) + "\n", encoding="utf-8")
(directory / "report.json").write_text(json.dumps({"steps": results}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
suite = ET.Element("testsuite", name="meshport", tests=str(len(results)),
failures=str(sum(r["status"] == "failed" for r in results)),
skipped=str(sum(r["status"] == "skipped" for r in results)))
for result in results:
case = ET.SubElement(suite, "testcase", name=result["name"], classname="meshport.automation", time=str(result["seconds"]))
if result["status"] == "skipped":
ET.SubElement(case, "skipped", message=result["reason"])
elif result["status"] == "failed":
ET.SubElement(case, "failure", message=result.get("reason", f"exit code {result['exit_code']}"))
if "log" in result:
ET.SubElement(case, "system-out").text = result["log"]
ET.ElementTree(suite).write(directory / "junit.xml", encoding="utf-8", xml_declaration=True)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("suites", nargs="*", metavar="SUITE", help=", ".join(SUITES) + " (default: all)")
parser.add_argument("--native", action="store_true", help="build macOS native libraries for Flutter tests")
parser.add_argument("--list", action="store_true", help="print commands without running them")
parser.add_argument("--fail-fast", action="store_true")
parser.add_argument("--timeout", type=int, default=900, help="timeout per command in seconds (default: 900)")
parser.add_argument("--rtt-ms", type=int, default=58, help="even simulated RTT, 0..1000, for throughput/combined")
parser.add_argument("--joint-seconds", type=int, default=2, help="combined duration per flow, 1..5")
parser.add_argument("--joint-repeat", type=int, default=3, help="combined repeats, 1..10")
parser.add_argument("--proxy-mbps", type=int, default=100, help="combined proxy fixture limit; 0 means unlimited")
parser.add_argument("--max-joint-slowdown-percent", type=float, default=20, help="maximum median slowdown between combined phases")
parser.add_argument("--mib", type=int, default=8, help="data per direction in MiB, 1..64, for throughput/speed")
parser.add_argument("--rtts-ms", default="0,58,100", help="comma-separated even RTTs for speed matrix")
parser.add_argument("--repeat", type=int, default=3, help="speed matrix repeats per RTT, 1..10")
parser.add_argument("--min-mbps", type=float, default=0, help="minimum group median for speed checks; default: no absolute floor")
parser.add_argument("--max-queue-drops", type=int, default=0, help="maximum queue drops per speed group across repeats")
parser.add_argument("--baseline", type=Path, help="previous metrics.json for matching speed groups")
parser.add_argument("--max-regression-percent", type=float, default=20, help="maximum median drop from baseline")
parser.add_argument("--log-dir", type=Path, help="new report directory; must not already exist")
args = parser.parse_args(argv)
args.suites = args.suites or ["all"]
if set(args.suites) - set(SUITES):
parser.error("unknown suite; choose from: " + ", ".join(SUITES))
if args.native and not set(args.suites).intersection({"all", "client"}):
parser.error("--native requires the client or all suite")
if args.timeout <= 0 or not 0 <= args.rtt_ms <= 1000 or args.rtt_ms % 2 or not 1 <= args.mib <= 64:
parser.error("timeout must be positive; RTT must be even and 0..1000; MiB must be 1..64")
try:
import math
if not 1 <= args.joint_seconds <= 5 or not 1 <= args.joint_repeat <= 10 or not 0 <= args.proxy_mbps <= 100000:
raise ValueError("joint seconds must be 1..5, repeats 1..10, proxy Mbps 0..100000")
if not math.isfinite(args.max_joint_slowdown_percent) or not 0 <= args.max_joint_slowdown_percent < 100:
raise ValueError("joint slowdown percent must be finite and in 0..<100")
if "combined" not in args.suites and (args.joint_seconds != 2 or args.joint_repeat != 3 or args.proxy_mbps != 100 or args.max_joint_slowdown_percent != 20):
raise ValueError("joint options require the combined suite")
rtts = tuple(int(value) for value in args.rtts_ms.split(","))
if not rtts or len(set(rtts)) != len(rtts) or any(not 0 <= value <= 1000 or value % 2 for value in rtts):
raise ValueError("speed RTTs must be distinct even integers in 0..1000")
if not 1 <= args.repeat <= 10 or not math.isfinite(args.min_mbps) or args.min_mbps < 0 or args.max_queue_drops < 0:
raise ValueError("repeat must be 1..10; speed and queue-drop limits must be nonnegative and finite")
if not math.isfinite(args.max_regression_percent) or not 0 <= args.max_regression_percent < 100:
raise ValueError("maximum regression percent must be finite and in 0..<100")
if "speed" not in args.suites and (args.min_mbps != 0 or args.max_queue_drops != 0 or args.repeat != 3 or
args.rtts_ms != "0,58,100" or args.max_regression_percent != 20):
raise ValueError("matrix/gate options require the speed suite")
if args.baseline and "speed" not in args.suites:
raise ValueError("--baseline requires the speed suite")
baseline = performance.validate_baseline(json.loads(args.baseline.read_text())) if args.baseline else None
policy = dict(repeats=args.repeat, min_mbps=args.min_mbps, max_queue_drops=args.max_queue_drops,
baseline=baseline, max_regression_percent=args.max_regression_percent)
joint = dict(duration_seconds=args.joint_seconds, repeats=args.joint_repeat,
proxy_limit_mbps=args.proxy_mbps, max_slowdown_percent=args.max_joint_slowdown_percent)
steps = plan(args.suites, args.native, args.rtt_ms, args.mib, rtts=rtts, repeats=args.repeat, policy=policy, joint=joint)
except (OSError, json.JSONDecodeError) as error:
parser.error("cannot read baseline: " + str(error))
except ValueError as error:
parser.error(str(error))
if args.list:
for step in steps:
print(step.name + ": " + ("SKIP " + step.skip if step.skip else shlex.join(step.command) if step.command else "evaluate medians, drop counts and optional baseline"))
return 0
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
directory = (args.log_dir or ROOT / "outputs/validation/automated-tests" / (stamp + "-" + uuid.uuid4().hex[:8])).resolve()
if directory.exists():
parser.error("report directory already exists: " + str(directory))
print("Reports: " + str(directory), flush=True)
results = run(steps, directory, args.timeout, args.fail_fast)
counts = {state: sum(r["status"] == state for r in results) for state in ("passed", "failed", "skipped")}
print("Summary: " + ", ".join(f"{number} {state}" for state, number in counts.items()), flush=True)
print("Report: " + str(directory / "report.json"), flush=True)
if any(r.get("exit_code") == 130 for r in results):
return 130
return 1 if counts["failed"] else 0
if __name__ == "__main__":
sys.exit(main())