#!/usr/bin/env python3 """Run MeshPort's local regression suites and write CI-friendly reports.""" import argparse from dataclasses import dataclass, field from datetime import datetime, timezone import json import os from pathlib import Path import shlex import signal import subprocess import sys import time import uuid import xml.etree.ElementTree as ET import performance import coexistence ROOT = Path(__file__).resolve().parent.parent SUITES = ("all", "runner", "sdk", "engine", "client", "macos", "throughput", "speed", "combined") @dataclass class Step: name: str command: list cwd: Path = ROOT requires: tuple = () env: dict = field(default_factory=dict) skip: str = "" measurement: dict = field(default_factory=dict) checks: dict = field(default_factory=dict) def plan(suites, native=False, rtt_ms=58, mib=8, platform=sys.platform, rtts=(0, 58, 100), repeats=3, policy=None, joint=None): selected = set(suites) if "all" in selected: selected.update(("runner", "sdk", "engine", "client", "macos")) steps = [] cargo = ["sh", "scripts/cargo-local.sh"] go = ["sh", "scripts/go-local.sh"] if "runner" in selected: steps.append(Step("runner-tests", [sys.executable, "-m", "unittest", "discover", "-s", "scripts/tests", "-p", "*_test.py"])) if "sdk" in selected: steps.append(Step("sdk", cargo + ["test", "--workspace", "--release", "--locked"])) if "engine" in selected: steps.extend([ Step("go-race", go + ["test", "-race", "./..."]), Step("go-vet", go + ["vet", "./..."]), Step("go-bridge-build", ["sh", "scripts/build-singbox.sh"]), Step("go-sdk-bridge", cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "bridge", "singbox_to_meshport_tcp_udp_and_withdrawal", "--", "--ignored"], requires=("go-bridge-build",), env={"MESHPORT_SINGBOX_BIN": str(ROOT / "target/debug/meshport-singbox")}), ]) if "client" in selected: app = ROOT / "apps/meshport_lab" steps.extend([ Step("flutter-dependencies", ["flutter", "pub", "get"], cwd=app), Step("flutter-analyze", ["flutter", "analyze"], cwd=app, requires=("flutter-dependencies",)), ]) dependencies = ("flutter-dependencies",) env = {} if native: if platform != "darwin": raise ValueError("--native currently requires macOS (the native build script targets macOS)") steps.extend([ Step("client-rust-library", cargo + ["build", "-p", "meshport-ffi", "--locked"]), Step("client-go-library", ["sh", "scripts/build-singbox-library.sh", "macos"]), ]) dependencies += ("client-rust-library", "client-go-library") env = { "MESHPORT_LIBRARY": str(ROOT / "target/debug/libmeshport_ffi.dylib"), "MESHPORT_PROXY_LIBRARY": str(ROOT / "target/debug/libmeshport_proxy.dylib"), } steps.append(Step("flutter-tests", ["flutter", "test"], cwd=app, requires=dependencies, env=env)) if "macos" in selected: steps.append(Step("macos-profile-storage", ["sh", "scripts/test-macos-profile-storage.sh"], skip="requires macOS and Xcode command-line tools" if platform != "darwin" else "")) if "throughput" in selected and "speed" not in selected: common = {"MESHPORT_TEST_DELAY_MS": str(rtt_ms // 2), "MESHPORT_TEST_MIB": str(mib)} for mode in ("channel", "udp"): steps.append(Step("tcp-stack-" + mode, cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "tcp_stack_diagnostic", "sdk_tcp_stack_diagnostic", "--", "--ignored", "--nocapture"], env={**common, "MESHPORT_TEST_LINK": mode}, measurement=dict(mode=mode, target="stack", entries=["direct"], rtt_ms=rtt_ms, mib=mib))) steps.append(Step("sdk-throughput", cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "bridge", "subnet_throughput_diagnostic", "--", "--ignored", "--nocapture"], env={**common, "MESHPORT_TEST_TARGET": "sdk", "MESHPORT_TEST_ENTRIES": "sdk", "MESHPORT_TEST_TRACE_TCP": "1"}, measurement=dict(mode="mesh", target="sdk", entries=["sdk"], rtt_ms=rtt_ms, mib=mib))) if "speed" in selected: build = "go-bridge-build" if "engine" in selected else "speed-engine-build" if "engine" not in selected: steps.append(Step(build, ["sh", "scripts/build-singbox.sh"])) measured = [] for rtt in rtts: for repeat in range(1, repeats + 1): common = {"MESHPORT_TEST_DELAY_MS": str(rtt // 2), "MESHPORT_TEST_MIB": str(mib)} for mode in ("channel", "udp", "mesh"): name = f"speed-{mode}-{rtt}ms-{repeat}" measured.append(name) spec = dict(mode=mode, target="sdk" if mode == "mesh" else "stack", entries=["sdk", "socks", "singbox"] if mode == "mesh" else ["direct"], rtt_ms=rtt, mib=mib, repeat=repeat) if mode == "mesh": command = cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "bridge", "subnet_throughput_diagnostic", "--", "--ignored", "--nocapture"] env = {**common, "MESHPORT_TEST_TARGET": "sdk", "MESHPORT_TEST_ENTRIES": "sdk,socks,singbox", "MESHPORT_TEST_TRACE_TCP": "1", "MESHPORT_SINGBOX_BIN": str(ROOT / "target/debug/meshport-singbox")} requires = (build,) else: command = cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "tcp_stack_diagnostic", "sdk_tcp_stack_diagnostic", "--", "--ignored", "--nocapture"] env = {**common, "MESHPORT_TEST_LINK": mode} requires = () steps.append(Step(name, command, requires=requires, env=env, measurement=spec)) steps.append(Step("speed-check", [], requires=tuple(measured), checks=policy or dict(repeats=repeats, min_mbps=0, max_queue_drops=0, baseline=None, max_regression_percent=20))) if "combined" in selected: if platform != "darwin": raise ValueError("combined currently requires macOS (native Go shared library build)") build = "client-go-library" if "client" in selected and native else "combined-go-library" if build == "combined-go-library": steps.append(Step(build, ["sh", "scripts/build-singbox-library.sh", "macos"])) spec = joint or dict(duration_seconds=2, repeats=3, proxy_limit_mbps=100, max_slowdown_percent=20) spec = {**spec, "rtt_ms": rtt_ms, "mode": "combined"} steps.append(Step("combined-coexistence", cargo + ["test", "-p", "meshport-socks", "--release", "--locked", "--test", "combined", "combined_coexistence", "--", "--ignored", "--nocapture"], requires=(build,), measurement=spec, env={"MESHPORT_PROXY_LIBRARY": str(ROOT / "target/debug/libmeshport_proxy.dylib"), "MESHPORT_TEST_DELAY_MS": str(rtt_ms // 2), "MESHPORT_JOINT_SECONDS": str(spec["duration_seconds"]), "MESHPORT_JOINT_REPEATS": str(spec["repeats"]), "MESHPORT_JOINT_PROXY_MBPS": str(spec["proxy_limit_mbps"])})) return steps def test_environment(overrides): # Do not inherit user profiles, drain fixtures, old native libraries or # diagnostic knobs. Every optional native/diagnostic input is set explicitly. return {**{key: value for key, value in os.environ.items() if not key.startswith("MESHPORT_")}, **overrides} def stop_process(process): if process.poll() is not None: return try: if os.name == "posix": os.killpg(process.pid, signal.SIGTERM) else: process.terminate() except ProcessLookupError: return try: process.wait(timeout=5) except subprocess.TimeoutExpired: if os.name == "posix": os.killpg(process.pid, signal.SIGKILL) else: process.kill() process.wait() def execute(step, directory, timeout): log = directory / (step.name + ".log") start = time.monotonic() result = {"name": step.name, "command": step.command, "cwd": str(step.cwd), "log": str(log)} print("RUN " + step.name + " " + shlex.join(step.command), flush=True) process = None try: with log.open("w", encoding="utf-8") as output: process = subprocess.Popen(step.command, cwd=step.cwd, env=test_environment(step.env), stdout=output, stderr=subprocess.STDOUT, start_new_session=os.name == "posix") try: code = process.wait(timeout=timeout) result.update(status="passed" if code == 0 else "failed", exit_code=code) except subprocess.TimeoutExpired: stop_process(process) result.update(status="failed", exit_code=124, reason=f"timeout after {timeout} seconds") except KeyboardInterrupt: stop_process(process) result.update(status="failed", exit_code=130, reason="interrupted") except OSError as error: log.write_text(str(error) + "\n", encoding="utf-8") result.update(status="failed", exit_code=127, reason=str(error)) result["seconds"] = round(time.monotonic() - start, 3) if step.measurement and result["status"] == "passed": try: content = log.read_text(encoding="utf-8", errors="replace") if step.measurement.get("mode") == "combined": result["joint_results"] = coexistence.parse(content, step.measurement) if result["joint_results"]["issues"]: result.update(status="failed", command_exit_code=0, exit_code=1, reason="; ".join(result["joint_results"]["issues"])) else: result["measurements"] = performance.parse(content, step.measurement) except (ValueError, KeyError, TypeError) as error: result.update(status="failed", command_exit_code=0, exit_code=1, reason="invalid speed result: " + str(error)) print(("PASS " if result["status"] == "passed" else "FAIL ") + step.name + f" ({result['seconds']:.1f}s)", flush=True) if result["status"] == "failed": if "reason" in result: print(result["reason"], flush=True) print("\n".join(log.read_text(encoding="utf-8", errors="replace").splitlines()[-20:]), flush=True) return result def run(steps, directory, timeout=900, fail_fast=False): directory.mkdir(parents=True, exist_ok=False) results = [] stopped = False for step in steps: previous = {result["name"]: result["status"] for result in results} blocked = [name for name in step.requires if previous.get(name) != "passed"] reason = step.skip or ("run stopped after failure" if stopped else "") if not reason and blocked: reason = "prerequisite did not pass: " + ", ".join(blocked) if reason: results.append({"name": step.name, "status": "skipped", "reason": reason, "seconds": 0}) print("SKIP " + step.name + " " + reason, flush=True) continue if step.checks: issues = performance.evaluate(performance.summarize(results), step.checks) log = directory / (step.name + ".log") log.write_text("\n".join(issues) + "\n" if issues else "All configured speed criteria passed.\n", encoding="utf-8") result = dict(name=step.name, status="failed" if issues else "passed", exit_code=1 if issues else 0, seconds=0, log=str(log), criteria=step.checks) if issues: result["reason"] = "; ".join(issues) print(("FAIL " if issues else "PASS ") + step.name, flush=True) for issue in issues: print(issue, flush=True) else: result = execute(step, directory, timeout) results.append(result) stopped = stopped or result.get("exit_code") == 130 or (fail_fast and result["status"] == "failed") write_reports(results, directory) write_reports(results, directory) return results def write_reports(results, directory): for result in results: if "joint_results" in result: (directory / "combined-metrics.json").write_text(json.dumps(result["joint_results"], indent=2) + "\n", encoding="utf-8") (directory / "combined-summary.md").write_text(coexistence.markdown(result["joint_results"]), encoding="utf-8") measurements = [record for result in results for record in result.get("measurements", [])] if measurements: summary = performance.summarize(results) (directory / "metrics.json").write_text(json.dumps(dict(measurements=measurements, summary=summary), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") table = ["# Local synthetic speed results", "", "RTT is added round-trip delay; these are local fixtures, not WAN speed measurements.", "", "| Path | RTT ms | MiB/direction | Direction | Samples | Median Mbps | Min | Max | Queue drops |", "|---|---:|---:|---|---:|---:|---:|---:|---:|"] for item in summary: path = item["mode"] + "/" + item["target"] + "/" + item["entry"] table.append(f"| {path} | {item['rtt_ms']} | {item['mib']} | {item['direction']} | {item['samples']} | {item['median_mbps']:.2f} | {item['min_mbps']:.2f} | {item['max_mbps']:.2f} | {item.get('queue_drops', 'n/a')} |") (directory / "speed-summary.md").write_text("\n".join(table) + "\n", encoding="utf-8") (directory / "report.json").write_text(json.dumps({"steps": results}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") suite = ET.Element("testsuite", name="meshport", tests=str(len(results)), failures=str(sum(r["status"] == "failed" for r in results)), skipped=str(sum(r["status"] == "skipped" for r in results))) for result in results: case = ET.SubElement(suite, "testcase", name=result["name"], classname="meshport.automation", time=str(result["seconds"])) if result["status"] == "skipped": ET.SubElement(case, "skipped", message=result["reason"]) elif result["status"] == "failed": ET.SubElement(case, "failure", message=result.get("reason", f"exit code {result['exit_code']}")) if "log" in result: ET.SubElement(case, "system-out").text = result["log"] ET.ElementTree(suite).write(directory / "junit.xml", encoding="utf-8", xml_declaration=True) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("suites", nargs="*", metavar="SUITE", help=", ".join(SUITES) + " (default: all)") parser.add_argument("--native", action="store_true", help="build macOS native libraries for Flutter tests") parser.add_argument("--list", action="store_true", help="print commands without running them") parser.add_argument("--fail-fast", action="store_true") parser.add_argument("--timeout", type=int, default=900, help="timeout per command in seconds (default: 900)") parser.add_argument("--rtt-ms", type=int, default=58, help="even simulated RTT, 0..1000, for throughput/combined") parser.add_argument("--joint-seconds", type=int, default=2, help="combined duration per flow, 1..5") parser.add_argument("--joint-repeat", type=int, default=3, help="combined repeats, 1..10") parser.add_argument("--proxy-mbps", type=int, default=100, help="combined proxy fixture limit; 0 means unlimited") parser.add_argument("--max-joint-slowdown-percent", type=float, default=20, help="maximum median slowdown between combined phases") parser.add_argument("--mib", type=int, default=8, help="data per direction in MiB, 1..64, for throughput/speed") parser.add_argument("--rtts-ms", default="0,58,100", help="comma-separated even RTTs for speed matrix") parser.add_argument("--repeat", type=int, default=3, help="speed matrix repeats per RTT, 1..10") parser.add_argument("--min-mbps", type=float, default=0, help="minimum group median for speed checks; default: no absolute floor") parser.add_argument("--max-queue-drops", type=int, default=0, help="maximum queue drops per speed group across repeats") parser.add_argument("--baseline", type=Path, help="previous metrics.json for matching speed groups") parser.add_argument("--max-regression-percent", type=float, default=20, help="maximum median drop from baseline") parser.add_argument("--log-dir", type=Path, help="new report directory; must not already exist") args = parser.parse_args(argv) args.suites = args.suites or ["all"] if set(args.suites) - set(SUITES): parser.error("unknown suite; choose from: " + ", ".join(SUITES)) if args.native and not set(args.suites).intersection({"all", "client"}): parser.error("--native requires the client or all suite") if args.timeout <= 0 or not 0 <= args.rtt_ms <= 1000 or args.rtt_ms % 2 or not 1 <= args.mib <= 64: parser.error("timeout must be positive; RTT must be even and 0..1000; MiB must be 1..64") try: import math if not 1 <= args.joint_seconds <= 5 or not 1 <= args.joint_repeat <= 10 or not 0 <= args.proxy_mbps <= 100000: raise ValueError("joint seconds must be 1..5, repeats 1..10, proxy Mbps 0..100000") if not math.isfinite(args.max_joint_slowdown_percent) or not 0 <= args.max_joint_slowdown_percent < 100: raise ValueError("joint slowdown percent must be finite and in 0..<100") if "combined" not in args.suites and (args.joint_seconds != 2 or args.joint_repeat != 3 or args.proxy_mbps != 100 or args.max_joint_slowdown_percent != 20): raise ValueError("joint options require the combined suite") rtts = tuple(int(value) for value in args.rtts_ms.split(",")) if not rtts or len(set(rtts)) != len(rtts) or any(not 0 <= value <= 1000 or value % 2 for value in rtts): raise ValueError("speed RTTs must be distinct even integers in 0..1000") if not 1 <= args.repeat <= 10 or not math.isfinite(args.min_mbps) or args.min_mbps < 0 or args.max_queue_drops < 0: raise ValueError("repeat must be 1..10; speed and queue-drop limits must be nonnegative and finite") if not math.isfinite(args.max_regression_percent) or not 0 <= args.max_regression_percent < 100: raise ValueError("maximum regression percent must be finite and in 0..<100") if "speed" not in args.suites and (args.min_mbps != 0 or args.max_queue_drops != 0 or args.repeat != 3 or args.rtts_ms != "0,58,100" or args.max_regression_percent != 20): raise ValueError("matrix/gate options require the speed suite") if args.baseline and "speed" not in args.suites: raise ValueError("--baseline requires the speed suite") baseline = performance.validate_baseline(json.loads(args.baseline.read_text())) if args.baseline else None policy = dict(repeats=args.repeat, min_mbps=args.min_mbps, max_queue_drops=args.max_queue_drops, baseline=baseline, max_regression_percent=args.max_regression_percent) joint = dict(duration_seconds=args.joint_seconds, repeats=args.joint_repeat, proxy_limit_mbps=args.proxy_mbps, max_slowdown_percent=args.max_joint_slowdown_percent) steps = plan(args.suites, args.native, args.rtt_ms, args.mib, rtts=rtts, repeats=args.repeat, policy=policy, joint=joint) except (OSError, json.JSONDecodeError) as error: parser.error("cannot read baseline: " + str(error)) except ValueError as error: parser.error(str(error)) if args.list: for step in steps: print(step.name + ": " + ("SKIP " + step.skip if step.skip else shlex.join(step.command) if step.command else "evaluate medians, drop counts and optional baseline")) return 0 stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") directory = (args.log_dir or ROOT / "outputs/validation/automated-tests" / (stamp + "-" + uuid.uuid4().hex[:8])).resolve() if directory.exists(): parser.error("report directory already exists: " + str(directory)) print("Reports: " + str(directory), flush=True) results = run(steps, directory, args.timeout, args.fail_fast) counts = {state: sum(r["status"] == state for r in results) for state in ("passed", "failed", "skipped")} print("Summary: " + ", ".join(f"{number} {state}" for state, number in counts.items()), flush=True) print("Report: " + str(directory / "report.json"), flush=True) if any(r.get("exit_code") == 130 for r in results): return 130 return 1 if counts["failed"] else 0 if __name__ == "__main__": sys.exit(main())