Files

104 lines
5.5 KiB
Python

import contextlib
import io
import json
from pathlib import Path
import sys
import tempfile
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import performance
import test as runner
def fixture(speed=20, drops=0, entries=("sdk",)):
lines = []
for entry in entries:
for direction in ("download", "upload"):
lines.append(f"sdk entry={entry} added_rtt_ms=58 direction={direction} mib=8 mbps={speed}")
wire = dict(data_packets=100, repeated_sequences=2, max_data_gap_ms=60)
lines.append(json.dumps(dict(mode="easytier_encrypted_relay", target="sdk", entry=entry, direction=direction,
client_wire_at_relay=wire, server_wire_at_relay=wire, udp_session_queue_full_drops=drops)))
return "\n".join(lines)
SPEC = dict(mode="mesh", target="sdk", entries=["sdk"], rtt_ms=58, mib=8)
class PerformanceTest(unittest.TestCase):
def test_extracts_each_path_and_both_directions(self):
entries = ["sdk", "socks", "singbox"]
records = performance.parse(fixture(entries=entries), {**SPEC, "entries": entries})
self.assertEqual(len(records), 6)
self.assertEqual({r["entry"] for r in records}, set(entries))
self.assertTrue(all(r["repeated_sequences"] == 2 and r["queue_drops"] == 0 for r in records))
def test_rejects_missing_duplicate_mismatched_and_zero_results(self):
good = fixture()
invalid = ["", "\n".join(good.splitlines()[:2]), good + "\n" + good,
good.replace("added_rtt_ms=58", "added_rtt_ms=100"), fixture(speed=0),
"\n".join(line for line in good.splitlines() if not line.startswith("{"))]
for text in invalid:
with self.subTest(text=text[:80]), self.assertRaises(ValueError):
performance.parse(text, SPEC)
def test_checks_group_median_drops_and_baseline(self):
results = [dict(measurements=performance.parse(fixture(speed=v), SPEC)) for v in (10, 20, 90)]
summary = performance.summarize(results)
self.assertTrue(all(item["median_mbps"] == 20 and item["samples"] == 3 for item in summary))
policy = dict(repeats=3, min_mbps=15, max_queue_drops=0, baseline=None, max_regression_percent=20)
self.assertEqual(performance.evaluate(summary, policy), [])
self.assertTrue(performance.evaluate(summary, {**policy, "min_mbps": 25}))
with_drops = [{**item, "queue_drops": 1} for item in summary]
self.assertTrue(performance.evaluate(with_drops, policy))
baseline = [{**item, "median_mbps": 30} for item in summary]
self.assertTrue(performance.evaluate(summary, {**policy, "baseline": baseline}))
mismatch = [{**item, "mib": 2} for item in baseline]
self.assertIn("comparison refused", " ".join(performance.evaluate(summary, {**policy, "baseline": mismatch})))
def test_validates_baseline_and_finite_values(self):
records = performance.parse(fixture(), SPEC)
summary = performance.summarize([dict(measurements=records)])
self.assertEqual(performance.validate_baseline(dict(summary=summary)), summary)
for bad in ({}, dict(summary=[]), dict(summary=summary + summary), dict(summary=[{**summary[0], "mode": []}]),
dict(summary=[{**summary[0], "median_mbps": float("nan") }])):
with self.assertRaises(ValueError):
performance.validate_baseline(bad)
def test_successful_command_with_missing_measurements_is_a_failure(self):
step = runner.Step("empty", [sys.executable, "-c", "print('success, but no measurements')"], measurement=SPEC)
with tempfile.TemporaryDirectory() as temporary, contextlib.redirect_stdout(io.StringIO()):
result = runner.execute(step, Path(temporary), 5)
self.assertEqual(result["command_exit_code"], 0)
self.assertEqual(result["status"], "failed")
def test_speed_gate_failure_produces_reports_even_after_command_success(self):
spec = dict(SPEC)
steps = [runner.Step("measured", [sys.executable, "-c", "print(" + repr(fixture(speed=2)) + ")"], measurement=spec),
runner.Step("speed-check", [], requires=("measured",), checks=dict(repeats=1, min_mbps=10,
max_queue_drops=0, baseline=None, max_regression_percent=20))]
with tempfile.TemporaryDirectory() as temporary, contextlib.redirect_stdout(io.StringIO()):
directory = Path(temporary) / "run"
results = runner.run(steps, directory)
self.assertEqual([r["status"] for r in results], ["passed", "failed"])
metrics = json.loads((directory / "metrics.json").read_text())
self.assertEqual(metrics["summary"][0]["median_mbps"], 2)
self.assertIn("below", (directory / "speed-check.log").read_text())
self.assertTrue((directory / "speed-summary.md").exists())
def test_speed_plan_covers_matrix_and_deduplicates_throughput(self):
steps = runner.plan(["speed", "throughput"], rtts=(0, 58, 100), repeats=3)
measured = [s for s in steps if s.measurement]
self.assertEqual(len(measured), 27)
self.assertEqual(steps[-1].name, "speed-check")
self.assertEqual(len(steps[-1].requires), 27)
def test_gate_options_cannot_silently_run_functional_tests_only(self):
with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error:
runner.main(["all", "--min-mbps", "10", "--list"])
self.assertEqual(error.exception.code, 2)
if __name__ == "__main__":
unittest.main()