139 lines
7.1 KiB
Python
139 lines
7.1 KiB
Python
"""Parse synthetic speed fixtures and evaluate explicit performance criteria."""
|
|
import json
|
|
import math
|
|
import re
|
|
import statistics
|
|
|
|
KEY_FIELDS = ("mode", "target", "entry", "rtt_ms", "mib", "direction")
|
|
|
|
|
|
def positive_number(value):
|
|
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value > 0
|
|
|
|
|
|
def key(record):
|
|
return tuple(record[field] for field in KEY_FIELDS)
|
|
|
|
|
|
def parse(text, spec):
|
|
records = {}
|
|
for line in text.replace("\r", "\n").splitlines():
|
|
match = re.fullmatch(r"(sdk|subnet) entry=(sdk|socks|singbox) added_rtt_ms=(\d+) direction=(download|upload) mib=(\d+) mbps=([\d.eE+-]+)", line)
|
|
if match and spec["mode"] == "mesh":
|
|
target, entry, rtt, direction, mib, mbps = match.groups()
|
|
record = dict(mode="mesh", target=target, entry=entry, rtt_ms=int(rtt), mib=int(mib), direction=direction, mbps=float(mbps))
|
|
elif line.startswith("{"):
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if spec["mode"] == "mesh" and value.get("mode") == "easytier_encrypted_relay":
|
|
identity = ("mesh", value["target"], value["entry"], spec["rtt_ms"], spec["mib"], value["direction"])
|
|
if identity not in records or "wire" in records[identity]:
|
|
raise ValueError("missing or duplicate throughput sample for wire observation")
|
|
records[identity]["wire"] = value
|
|
continue
|
|
if value.get("mode") != spec["mode"] or spec["mode"] not in ("channel", "udp"):
|
|
continue
|
|
record = dict(mode=value["mode"], target="stack", entry="direct", rtt_ms=value["added_rtt_ms"],
|
|
mib=value["mib"], direction=value["direction"], mbps=value["mbps"], wire=value)
|
|
else:
|
|
continue
|
|
if not positive_number(record["mbps"]):
|
|
raise ValueError("speed must be finite and greater than zero")
|
|
if record["rtt_ms"] != spec["rtt_ms"] or record["mib"] != spec["mib"]:
|
|
raise ValueError("fixture RTT/data size differs from requested parameters")
|
|
identity = key(record)
|
|
if identity in records:
|
|
raise ValueError("duplicate speed sample")
|
|
record["repeat"] = spec.get("repeat", 1)
|
|
records[identity] = record
|
|
expected = {(spec["mode"], spec["target"], entry, spec["rtt_ms"], spec["mib"], direction)
|
|
for entry in spec["entries"] for direction in ("download", "upload")}
|
|
if set(records) != expected:
|
|
raise ValueError("missing/unexpected speed samples; both upload and download are required for every path")
|
|
for record in records.values():
|
|
wire = record.get("wire")
|
|
if wire is None:
|
|
raise ValueError("missing packet-level observation")
|
|
if record["mode"] == "mesh":
|
|
drops = wire.get("udp_session_queue_full_drops")
|
|
if not isinstance(drops, int) or isinstance(drops, bool) or drops < 0:
|
|
raise ValueError("missing/invalid UDP session queue-drop count")
|
|
record["queue_drops"] = drops
|
|
sender = wire.get("server_wire_at_relay" if record["direction"] == "download" else "client_wire_at_relay", {})
|
|
else:
|
|
sender = wire.get("server_wire" if record["direction"] == "download" else "client_wire", {})
|
|
for field in ("data_packets", "repeated_sequences", "max_data_gap_ms"):
|
|
number = sender.get(field)
|
|
if not isinstance(number, int) or isinstance(number, bool) or number < 0:
|
|
raise ValueError("missing/invalid wire statistic: " + field)
|
|
if sender["data_packets"] == 0:
|
|
raise ValueError("no TCP data packets observed")
|
|
record["repeated_sequences"] = sender["repeated_sequences"]
|
|
record["max_data_gap_ms"] = sender["max_data_gap_ms"]
|
|
return list(records.values())
|
|
|
|
|
|
def summarize(results):
|
|
groups = {}
|
|
for result in results:
|
|
for record in result.get("measurements", []):
|
|
groups.setdefault(key(record), []).append(record)
|
|
summary = []
|
|
for identity, records in sorted(groups.items()):
|
|
speeds = [record["mbps"] for record in records]
|
|
item = dict(zip(KEY_FIELDS, identity))
|
|
item.update(samples=len(records), median_mbps=statistics.median(speeds), min_mbps=min(speeds), max_mbps=max(speeds),
|
|
repeated_sequences=sum(r["repeated_sequences"] for r in records),
|
|
max_data_gap_ms=max(r["max_data_gap_ms"] for r in records))
|
|
if identity[0] == "mesh":
|
|
item["queue_drops"] = sum(r["queue_drops"] for r in records)
|
|
summary.append(item)
|
|
return summary
|
|
|
|
|
|
def validate_baseline(value):
|
|
if not isinstance(value, dict) or not isinstance(value.get("summary"), list) or not value["summary"]:
|
|
raise ValueError("baseline must be a metrics.json with a nonempty summary")
|
|
seen = set()
|
|
for item in value["summary"]:
|
|
try:
|
|
identity = key(item)
|
|
except (KeyError, TypeError) as error:
|
|
raise ValueError("baseline is missing measurement identity fields") from error
|
|
if any(not isinstance(item[field], str) or not item[field] for field in ("mode", "target", "entry", "direction")):
|
|
raise ValueError("baseline path fields must be nonempty strings")
|
|
if item["direction"] not in ("download", "upload") or any(not isinstance(item[field], int) or isinstance(item[field], bool) for field in ("rtt_ms", "mib")):
|
|
raise ValueError("baseline direction/RTT/size is invalid")
|
|
if identity in seen or not positive_number(item.get("median_mbps")):
|
|
raise ValueError("baseline contains duplicate groups or invalid speed")
|
|
seen.add(identity)
|
|
return value["summary"]
|
|
|
|
|
|
def evaluate(summary, policy):
|
|
issues = []
|
|
if not summary:
|
|
return ["no performance measurements"]
|
|
for item in summary:
|
|
label = "/".join(str(v) for v in key(item))
|
|
if item["samples"] != policy["repeats"]:
|
|
issues.append(label + ": incomplete repeat count")
|
|
if item["median_mbps"] < policy["min_mbps"]:
|
|
issues.append(f"{label}: median {item['median_mbps']:.2f} Mbps below {policy['min_mbps']:.2f}")
|
|
if item.get("queue_drops", 0) > policy["max_queue_drops"]:
|
|
issues.append(f"{label}: {item['queue_drops']} queue drops exceed {policy['max_queue_drops']}")
|
|
baseline = policy.get("baseline")
|
|
if baseline is not None:
|
|
before = {key(item): item for item in baseline}
|
|
after = {key(item): item for item in summary}
|
|
if set(before) != set(after):
|
|
issues.append("baseline groups differ (mode, target, entry, RTT, size or direction); comparison refused")
|
|
else:
|
|
for identity, item in after.items():
|
|
floor = before[identity]["median_mbps"] * (1 - policy["max_regression_percent"] / 100)
|
|
if item["median_mbps"] < floor:
|
|
issues.append(f"{identity}: median {item['median_mbps']:.2f} Mbps exceeds allowed baseline regression")
|
|
return issues
|