import contextlib import io import json from pathlib import Path import sys import tempfile import unittest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import performance import test as runner def fixture(speed=20, drops=0, entries=("sdk",)): lines = [] for entry in entries: for direction in ("download", "upload"): lines.append(f"sdk entry={entry} added_rtt_ms=58 direction={direction} mib=8 mbps={speed}") wire = dict(data_packets=100, repeated_sequences=2, max_data_gap_ms=60) lines.append(json.dumps(dict(mode="easytier_encrypted_relay", target="sdk", entry=entry, direction=direction, client_wire_at_relay=wire, server_wire_at_relay=wire, udp_session_queue_full_drops=drops))) return "\n".join(lines) SPEC = dict(mode="mesh", target="sdk", entries=["sdk"], rtt_ms=58, mib=8) class PerformanceTest(unittest.TestCase): def test_extracts_each_path_and_both_directions(self): entries = ["sdk", "socks", "singbox"] records = performance.parse(fixture(entries=entries), {**SPEC, "entries": entries}) self.assertEqual(len(records), 6) self.assertEqual({r["entry"] for r in records}, set(entries)) self.assertTrue(all(r["repeated_sequences"] == 2 and r["queue_drops"] == 0 for r in records)) def test_rejects_missing_duplicate_mismatched_and_zero_results(self): good = fixture() invalid = ["", "\n".join(good.splitlines()[:2]), good + "\n" + good, good.replace("added_rtt_ms=58", "added_rtt_ms=100"), fixture(speed=0), "\n".join(line for line in good.splitlines() if not line.startswith("{"))] for text in invalid: with self.subTest(text=text[:80]), self.assertRaises(ValueError): performance.parse(text, SPEC) def test_checks_group_median_drops_and_baseline(self): results = [dict(measurements=performance.parse(fixture(speed=v), SPEC)) for v in (10, 20, 90)] summary = performance.summarize(results) self.assertTrue(all(item["median_mbps"] == 20 and item["samples"] == 3 for item in summary)) policy = dict(repeats=3, min_mbps=15, max_queue_drops=0, baseline=None, max_regression_percent=20) self.assertEqual(performance.evaluate(summary, policy), []) self.assertTrue(performance.evaluate(summary, {**policy, "min_mbps": 25})) with_drops = [{**item, "queue_drops": 1} for item in summary] self.assertTrue(performance.evaluate(with_drops, policy)) baseline = [{**item, "median_mbps": 30} for item in summary] self.assertTrue(performance.evaluate(summary, {**policy, "baseline": baseline})) mismatch = [{**item, "mib": 2} for item in baseline] self.assertIn("comparison refused", " ".join(performance.evaluate(summary, {**policy, "baseline": mismatch}))) def test_validates_baseline_and_finite_values(self): records = performance.parse(fixture(), SPEC) summary = performance.summarize([dict(measurements=records)]) self.assertEqual(performance.validate_baseline(dict(summary=summary)), summary) for bad in ({}, dict(summary=[]), dict(summary=summary + summary), dict(summary=[{**summary[0], "mode": []}]), dict(summary=[{**summary[0], "median_mbps": float("nan") }])): with self.assertRaises(ValueError): performance.validate_baseline(bad) def test_successful_command_with_missing_measurements_is_a_failure(self): step = runner.Step("empty", [sys.executable, "-c", "print('success, but no measurements')"], measurement=SPEC) with tempfile.TemporaryDirectory() as temporary, contextlib.redirect_stdout(io.StringIO()): result = runner.execute(step, Path(temporary), 5) self.assertEqual(result["command_exit_code"], 0) self.assertEqual(result["status"], "failed") def test_speed_gate_failure_produces_reports_even_after_command_success(self): spec = dict(SPEC) steps = [runner.Step("measured", [sys.executable, "-c", "print(" + repr(fixture(speed=2)) + ")"], measurement=spec), runner.Step("speed-check", [], requires=("measured",), checks=dict(repeats=1, min_mbps=10, max_queue_drops=0, baseline=None, max_regression_percent=20))] with tempfile.TemporaryDirectory() as temporary, contextlib.redirect_stdout(io.StringIO()): directory = Path(temporary) / "run" results = runner.run(steps, directory) self.assertEqual([r["status"] for r in results], ["passed", "failed"]) metrics = json.loads((directory / "metrics.json").read_text()) self.assertEqual(metrics["summary"][0]["median_mbps"], 2) self.assertIn("below", (directory / "speed-check.log").read_text()) self.assertTrue((directory / "speed-summary.md").exists()) def test_speed_plan_covers_matrix_and_deduplicates_throughput(self): steps = runner.plan(["speed", "throughput"], rtts=(0, 58, 100), repeats=3) measured = [s for s in steps if s.measurement] self.assertEqual(len(measured), 27) self.assertEqual(steps[-1].name, "speed-check") self.assertEqual(len(steps[-1].requires), 27) def test_gate_options_cannot_silently_run_functional_tests_only(self): with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: runner.main(["all", "--min-mbps", "10", "--list"]) self.assertEqual(error.exception.code, 2) if __name__ == "__main__": unittest.main()