Files

139 lines
7.1 KiB
Python

"""Parse synthetic speed fixtures and evaluate explicit performance criteria."""
import json
import math
import re
import statistics
KEY_FIELDS = ("mode", "target", "entry", "rtt_ms", "mib", "direction")
def positive_number(value):
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value > 0
def key(record):
return tuple(record[field] for field in KEY_FIELDS)
def parse(text, spec):
records = {}
for line in text.replace("\r", "\n").splitlines():
match = re.fullmatch(r"(sdk|subnet) entry=(sdk|socks|singbox) added_rtt_ms=(\d+) direction=(download|upload) mib=(\d+) mbps=([\d.eE+-]+)", line)
if match and spec["mode"] == "mesh":
target, entry, rtt, direction, mib, mbps = match.groups()
record = dict(mode="mesh", target=target, entry=entry, rtt_ms=int(rtt), mib=int(mib), direction=direction, mbps=float(mbps))
elif line.startswith("{"):
try:
value = json.loads(line)
except json.JSONDecodeError:
continue
if spec["mode"] == "mesh" and value.get("mode") == "easytier_encrypted_relay":
identity = ("mesh", value["target"], value["entry"], spec["rtt_ms"], spec["mib"], value["direction"])
if identity not in records or "wire" in records[identity]:
raise ValueError("missing or duplicate throughput sample for wire observation")
records[identity]["wire"] = value
continue
if value.get("mode") != spec["mode"] or spec["mode"] not in ("channel", "udp"):
continue
record = dict(mode=value["mode"], target="stack", entry="direct", rtt_ms=value["added_rtt_ms"],
mib=value["mib"], direction=value["direction"], mbps=value["mbps"], wire=value)
else:
continue
if not positive_number(record["mbps"]):
raise ValueError("speed must be finite and greater than zero")
if record["rtt_ms"] != spec["rtt_ms"] or record["mib"] != spec["mib"]:
raise ValueError("fixture RTT/data size differs from requested parameters")
identity = key(record)
if identity in records:
raise ValueError("duplicate speed sample")
record["repeat"] = spec.get("repeat", 1)
records[identity] = record
expected = {(spec["mode"], spec["target"], entry, spec["rtt_ms"], spec["mib"], direction)
for entry in spec["entries"] for direction in ("download", "upload")}
if set(records) != expected:
raise ValueError("missing/unexpected speed samples; both upload and download are required for every path")
for record in records.values():
wire = record.get("wire")
if wire is None:
raise ValueError("missing packet-level observation")
if record["mode"] == "mesh":
drops = wire.get("udp_session_queue_full_drops")
if not isinstance(drops, int) or isinstance(drops, bool) or drops < 0:
raise ValueError("missing/invalid UDP session queue-drop count")
record["queue_drops"] = drops
sender = wire.get("server_wire_at_relay" if record["direction"] == "download" else "client_wire_at_relay", {})
else:
sender = wire.get("server_wire" if record["direction"] == "download" else "client_wire", {})
for field in ("data_packets", "repeated_sequences", "max_data_gap_ms"):
number = sender.get(field)
if not isinstance(number, int) or isinstance(number, bool) or number < 0:
raise ValueError("missing/invalid wire statistic: " + field)
if sender["data_packets"] == 0:
raise ValueError("no TCP data packets observed")
record["repeated_sequences"] = sender["repeated_sequences"]
record["max_data_gap_ms"] = sender["max_data_gap_ms"]
return list(records.values())
def summarize(results):
groups = {}
for result in results:
for record in result.get("measurements", []):
groups.setdefault(key(record), []).append(record)
summary = []
for identity, records in sorted(groups.items()):
speeds = [record["mbps"] for record in records]
item = dict(zip(KEY_FIELDS, identity))
item.update(samples=len(records), median_mbps=statistics.median(speeds), min_mbps=min(speeds), max_mbps=max(speeds),
repeated_sequences=sum(r["repeated_sequences"] for r in records),
max_data_gap_ms=max(r["max_data_gap_ms"] for r in records))
if identity[0] == "mesh":
item["queue_drops"] = sum(r["queue_drops"] for r in records)
summary.append(item)
return summary
def validate_baseline(value):
if not isinstance(value, dict) or not isinstance(value.get("summary"), list) or not value["summary"]:
raise ValueError("baseline must be a metrics.json with a nonempty summary")
seen = set()
for item in value["summary"]:
try:
identity = key(item)
except (KeyError, TypeError) as error:
raise ValueError("baseline is missing measurement identity fields") from error
if any(not isinstance(item[field], str) or not item[field] for field in ("mode", "target", "entry", "direction")):
raise ValueError("baseline path fields must be nonempty strings")
if item["direction"] not in ("download", "upload") or any(not isinstance(item[field], int) or isinstance(item[field], bool) for field in ("rtt_ms", "mib")):
raise ValueError("baseline direction/RTT/size is invalid")
if identity in seen or not positive_number(item.get("median_mbps")):
raise ValueError("baseline contains duplicate groups or invalid speed")
seen.add(identity)
return value["summary"]
def evaluate(summary, policy):
issues = []
if not summary:
return ["no performance measurements"]
for item in summary:
label = "/".join(str(v) for v in key(item))
if item["samples"] != policy["repeats"]:
issues.append(label + ": incomplete repeat count")
if item["median_mbps"] < policy["min_mbps"]:
issues.append(f"{label}: median {item['median_mbps']:.2f} Mbps below {policy['min_mbps']:.2f}")
if item.get("queue_drops", 0) > policy["max_queue_drops"]:
issues.append(f"{label}: {item['queue_drops']} queue drops exceed {policy['max_queue_drops']}")
baseline = policy.get("baseline")
if baseline is not None:
before = {key(item): item for item in baseline}
after = {key(item): item for item in summary}
if set(before) != set(after):
issues.append("baseline groups differ (mode, target, entry, RTT, size or direction); comparison refused")
else:
for identity, item in after.items():
floor = before[identity]["median_mbps"] * (1 - policy["max_regression_percent"] / 100)
if item["median_mbps"] < floor:
issues.append(f"{identity}: median {item['median_mbps']:.2f} Mbps exceeds allowed baseline regression")
return issues