104 lines
5.5 KiB
Python
104 lines
5.5 KiB
Python
import contextlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
import performance
|
|
import test as runner
|
|
|
|
|
|
def fixture(speed=20, drops=0, entries=("sdk",)):
|
|
lines = []
|
|
for entry in entries:
|
|
for direction in ("download", "upload"):
|
|
lines.append(f"sdk entry={entry} added_rtt_ms=58 direction={direction} mib=8 mbps={speed}")
|
|
wire = dict(data_packets=100, repeated_sequences=2, max_data_gap_ms=60)
|
|
lines.append(json.dumps(dict(mode="easytier_encrypted_relay", target="sdk", entry=entry, direction=direction,
|
|
client_wire_at_relay=wire, server_wire_at_relay=wire, udp_session_queue_full_drops=drops)))
|
|
return "\n".join(lines)
|
|
|
|
|
|
SPEC = dict(mode="mesh", target="sdk", entries=["sdk"], rtt_ms=58, mib=8)
|
|
|
|
|
|
class PerformanceTest(unittest.TestCase):
|
|
def test_extracts_each_path_and_both_directions(self):
|
|
entries = ["sdk", "socks", "singbox"]
|
|
records = performance.parse(fixture(entries=entries), {**SPEC, "entries": entries})
|
|
self.assertEqual(len(records), 6)
|
|
self.assertEqual({r["entry"] for r in records}, set(entries))
|
|
self.assertTrue(all(r["repeated_sequences"] == 2 and r["queue_drops"] == 0 for r in records))
|
|
|
|
def test_rejects_missing_duplicate_mismatched_and_zero_results(self):
|
|
good = fixture()
|
|
invalid = ["", "\n".join(good.splitlines()[:2]), good + "\n" + good,
|
|
good.replace("added_rtt_ms=58", "added_rtt_ms=100"), fixture(speed=0),
|
|
"\n".join(line for line in good.splitlines() if not line.startswith("{"))]
|
|
for text in invalid:
|
|
with self.subTest(text=text[:80]), self.assertRaises(ValueError):
|
|
performance.parse(text, SPEC)
|
|
|
|
def test_checks_group_median_drops_and_baseline(self):
|
|
results = [dict(measurements=performance.parse(fixture(speed=v), SPEC)) for v in (10, 20, 90)]
|
|
summary = performance.summarize(results)
|
|
self.assertTrue(all(item["median_mbps"] == 20 and item["samples"] == 3 for item in summary))
|
|
policy = dict(repeats=3, min_mbps=15, max_queue_drops=0, baseline=None, max_regression_percent=20)
|
|
self.assertEqual(performance.evaluate(summary, policy), [])
|
|
self.assertTrue(performance.evaluate(summary, {**policy, "min_mbps": 25}))
|
|
with_drops = [{**item, "queue_drops": 1} for item in summary]
|
|
self.assertTrue(performance.evaluate(with_drops, policy))
|
|
baseline = [{**item, "median_mbps": 30} for item in summary]
|
|
self.assertTrue(performance.evaluate(summary, {**policy, "baseline": baseline}))
|
|
mismatch = [{**item, "mib": 2} for item in baseline]
|
|
self.assertIn("comparison refused", " ".join(performance.evaluate(summary, {**policy, "baseline": mismatch})))
|
|
|
|
def test_validates_baseline_and_finite_values(self):
|
|
records = performance.parse(fixture(), SPEC)
|
|
summary = performance.summarize([dict(measurements=records)])
|
|
self.assertEqual(performance.validate_baseline(dict(summary=summary)), summary)
|
|
for bad in ({}, dict(summary=[]), dict(summary=summary + summary), dict(summary=[{**summary[0], "mode": []}]),
|
|
dict(summary=[{**summary[0], "median_mbps": float("nan") }])):
|
|
with self.assertRaises(ValueError):
|
|
performance.validate_baseline(bad)
|
|
|
|
def test_successful_command_with_missing_measurements_is_a_failure(self):
|
|
step = runner.Step("empty", [sys.executable, "-c", "print('success, but no measurements')"], measurement=SPEC)
|
|
with tempfile.TemporaryDirectory() as temporary, contextlib.redirect_stdout(io.StringIO()):
|
|
result = runner.execute(step, Path(temporary), 5)
|
|
self.assertEqual(result["command_exit_code"], 0)
|
|
self.assertEqual(result["status"], "failed")
|
|
|
|
def test_speed_gate_failure_produces_reports_even_after_command_success(self):
|
|
spec = dict(SPEC)
|
|
steps = [runner.Step("measured", [sys.executable, "-c", "print(" + repr(fixture(speed=2)) + ")"], measurement=spec),
|
|
runner.Step("speed-check", [], requires=("measured",), checks=dict(repeats=1, min_mbps=10,
|
|
max_queue_drops=0, baseline=None, max_regression_percent=20))]
|
|
with tempfile.TemporaryDirectory() as temporary, contextlib.redirect_stdout(io.StringIO()):
|
|
directory = Path(temporary) / "run"
|
|
results = runner.run(steps, directory)
|
|
self.assertEqual([r["status"] for r in results], ["passed", "failed"])
|
|
metrics = json.loads((directory / "metrics.json").read_text())
|
|
self.assertEqual(metrics["summary"][0]["median_mbps"], 2)
|
|
self.assertIn("below", (directory / "speed-check.log").read_text())
|
|
self.assertTrue((directory / "speed-summary.md").exists())
|
|
|
|
def test_speed_plan_covers_matrix_and_deduplicates_throughput(self):
|
|
steps = runner.plan(["speed", "throughput"], rtts=(0, 58, 100), repeats=3)
|
|
measured = [s for s in steps if s.measurement]
|
|
self.assertEqual(len(measured), 27)
|
|
self.assertEqual(steps[-1].name, "speed-check")
|
|
self.assertEqual(len(steps[-1].requires), 27)
|
|
|
|
def test_gate_options_cannot_silently_run_functional_tests_only(self):
|
|
with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error:
|
|
runner.main(["all", "--min-mbps", "10", "--list"])
|
|
self.assertEqual(error.exception.code, 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|