mtgodot-poc/test/compare.py

#!/usr/bin/env python3
"""Phase-1 cross-check harness.

Skinning's numeric truth lives in libgr2 (shared by mtgodot and the retired
xrender-poc) and is oracle-verified against real Granny. mtgodot's default
CPU-skin path applies libgr2's `skin` matrices verbatim, so "mtgodot == Granny"
reduces to "libgr2 == Granny" (oracle suite) + "mtgodot ran libgr2 cleanly"
(0 NaN). It renders each case on the CPU path and, for animated cases, also on the
GPU-skin (SRC_SKIN vertex shader) path. No pixel diff vs bgfx-reference
(renderers/cameras differ) — the reference shot path is recorded for eyeballing.

  test/compare.py [--cases test/compare-cases.json] [--out test/compare-report.json]

Prereqs used if present:
  - test/m2-numeric.json  (from oracle/run-diff-suite.sh; libgr2 vs Granny)
  - godot on PATH, extension built (project/bin/*.dylib)
  - Pillow + numpy
"""
import json, os, re, subprocess, sys, time
from pathlib import Path
import numpy as np
from PIL import Image

REPO = Path(__file__).resolve().parent.parent
# Metin2 assets live in-tree at <repo>/assets (gitignored); override with MT_ASSETS.
ASSETS = Path(os.environ.get("MT_ASSETS", REPO / "assets")).resolve()
OUT_DIR = REPO / "test" / "compare-shots"
REF_DIR = REPO / "test" / "bgfx-reference"

DEFAULT_CASES = [
    # name, model (rel to assets/PC/ymir work), texdir, anim ('-' = bind), t
    ("warrior-bind",     "pc/warrior/warrior_cheongrin.gr2", "pc/warrior", "-", 0.0),
    ("warrior-dance-t7", "pc/warrior/warrior_cheongrin.gr2", "pc/warrior",
     "pc/warrior/action/dance_1.gr2", 7.0),
    ("warrior-dance-t19","pc/warrior/warrior_cheongrin.gr2", "pc/warrior",
     "pc/warrior/action/dance_1.gr2", 19.0),
    ("sura-bind",        "pc/sura/sura_lord.gr2", "pc/sura", "-", 0.0),
    ("assassin-bind",    "pc/assassin/assassin.gr2", "pc/assassin", "-", 0.0),
    ("shaman-bind",      "pc/shaman/shaman_lord.gr2", "pc/shaman", "-", 0.0),
]


def run_godot(env_extra, timeout=120):
    godot = subprocess.run(["which", "godot"], capture_output=True, text=True).stdout.strip()
    if not godot:
        return None, "godot not on PATH", None
    env = {**os.environ, **env_extra}
    try:
        p = subprocess.run([godot, "--path", str(REPO / "project"),
                            "res://main.tscn", "--rendering-driver", "metal",
                            "--quit-after", "45"],
                           capture_output=True, text=True, env=env, timeout=timeout)
    except subprocess.TimeoutExpired:
        return None, "timeout", None
    log = p.stdout + p.stderr
    if p.returncode != 0:
        return log, f"godot exited with status {p.returncode}", p.returncode
    return log, None, p.returncode


def case_env(c):
    _, model, texdir, anim, t = c
    e = {"MTGODOT_MODEL":  str(ASSETS / "PC" / "ymir work" / model),
         "MTGODOT_TEXDIR": str(ASSETS / "PC" / "ymir work" / texdir),
         "MTGODOT_YAW":    "0.7"}
    if anim == "-":
        e["MTGODOT_ANIM"] = "-"
    else:
        e["MTGODOT_ANIM"] = str(ASSETS / "PC" / "ymir work" / anim)
        e["MTGODOT_ANIM_T"] = str(t)
    return e


def _luma(path):
    im = np.asarray(Image.open(path).convert("RGB")).astype(np.float32)
    return 0.2126 * im[..., 0] + 0.7152 * im[..., 1] + 0.0722 * im[..., 2], im


def img_stats(path):
    l, _ = _luma(path)
    return {"luma_mean": round(float(l.mean()), 2), "luma_std": round(float(l.std()), 2)}


def img_diff(a, b):
    """Coarse diff between two same-camera renders (CPU vs GPU skinning).

    Advisory only — both paths feed the SAME libgr2 matrices with the full affine,
    so they match visually; residual changed_frac is AA / float-precision on the
    shaded surface, not a collapse. (The old Skeleton3D route DID collapse limbs;
    it's gone — replaced by the SRC_SKIN vertex shader.) Eyeball the .cpu/.gpu PNGs.
    """
    la, ia = _luma(a)
    lb, ib = _luma(b)
    if ia.shape != ib.shape:
        return {"error": "size mismatch"}
    d = np.abs(ia - ib).sum(axis=2)
    return {"changed_frac": round(float((d > 24).mean()), 4),
            "max_abs": int(d.max()), "mean_abs": round(float(d.mean()), 3)}


def parse(log, pat):
    m = re.search(pat, log or "")
    return m.group(1) if m else None


def report_path(path):
    try:
        return str(path.relative_to(REPO))
    except ValueError:
        return str(path)


def oracle_gate():
    js = REPO / "test" / "m2-numeric.json"
    if not js.exists():
        return {"status": "skipped", "reason": "test/m2-numeric.json absent — run oracle/run-diff-suite.sh"}
    d = json.loads(js.read_text())
    cases = d.get("cases", [])
    fails = [c for c in cases if str(c.get("status", "")).upper() != "PASS"]
    return {"status": "pass" if cases and not fails else ("fail" if fails else "empty"),
            "cases": len(cases), "failures": len(fails),
            "worst_mat": max((float(c.get("mat", 0)) for c in cases), default=0.0),
            "worst_vtx": max((float(c.get("vtx", 0)) for c in cases), default=0.0),
            "note": "libgr2 vs real Granny 2.9.12 — shared by mtgodot's CPU-skin path"}


def main():
    cases_path = out_path = None
    args = sys.argv[1:]
    for i, a in enumerate(args):
        if a == "--cases" and i + 1 < len(args): cases_path = Path(args[i + 1])
        if a == "--out" and i + 1 < len(args):   out_path = Path(args[i + 1])
    out_path = out_path or REPO / "test" / "compare-report.json"
    cases = json.loads(cases_path.read_text()) if cases_path and cases_path.exists() else DEFAULT_CASES
    OUT_DIR.mkdir(parents=True, exist_ok=True)

    report = {"generated": time.strftime("%Y-%m-%d %H:%M:%S"),
              "oracle_gate": oracle_gate(), "cases": []}
    fails = []

    for c in cases:
        name, anim = c[0], c[3]
        rec = {"name": name}
        env = case_env(c)

        shot = OUT_DIR / f"{name}.cpu.png"
        shot.unlink(missing_ok=True)  # stale output must never turn a failed run green
        log, err, returncode = run_godot({**env, "MTGODOT_AUTOSHOT": str(shot)})
        nan = parse(log, r"NaN/Inf=(\d+)")
        surf = parse(log, r"surfaces=(\d+)")
        cpu = {"nan_inf": int(nan) if nan is not None else None,
               "surfaces": int(surf) if surf else None,
               "shot": report_path(shot) if shot.exists() else None,
               "returncode": returncode, "error": err}
        if shot.exists():
            cpu.update(img_stats(shot))
        rec["cpu"] = cpu
        if err:
            fails.append(f"{name}: {err}")
        if not shot.exists():
            fails.append(f"{name}: screenshot missing")
        if cpu["surfaces"] is None or cpu["surfaces"] <= 0:
            fails.append(f"{name}: model did not report a non-empty surface set")
        if anim != "-" and cpu["nan_inf"] is None:
            fails.append(f"{name}: animation self-check result missing")
        elif cpu["nan_inf"] not in (0, None):
            fails.append(f"{name}: NaN/Inf={cpu['nan_inf']}")

        if anim != "-":
            gshot = OUT_DIR / f"{name}.gpu.png"
            gshot.unlink(missing_ok=True)
            glog, gerr, greturncode = run_godot(
                {**env, "MTGODOT_GPUSKIN": "1", "MTGODOT_AUTOSHOT": str(gshot)})
            gpu = {"shot": report_path(gshot) if gshot.exists() else None,
                   "returncode": greturncode, "error": gerr}
            if gshot.exists():
                gpu.update(img_stats(gshot))
                if shot.exists():
                    gpu["vs_cpu"] = img_diff(shot, gshot)
            rec["gpu_skin"] = gpu
            if gerr:
                fails.append(f"{name} GPU: {gerr}")
            if not gshot.exists():
                fails.append(f"{name} GPU: screenshot missing")

        ref = next((p for p in REF_DIR.glob(f"*{name.split('-')[0]}*")), None)
        rec["bgfx_reference"] = str(ref.relative_to(REPO)) if ref else None
        report["cases"].append(rec)
        g = rec.get("gpu_skin", {})
        print(f"  {name:18s} cpu[nan={cpu['nan_inf']} surf={cpu['surfaces']} luma={cpu.get('luma_mean')}]"
              + (f"  gpu[changed_vs_cpu={g.get('vs_cpu', {}).get('changed_frac')}]" if g else ""))

    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(json.dumps(report, indent=2))
    og = report["oracle_gate"]
    print(f"\noracle gate: {og['status']}"
          + (f"  ({og['cases']} cases, worst mat={og['worst_mat']:.1e} vtx={og['worst_vtx']:.1e})"
             if og.get("cases") else f"  — {og.get('reason', '')}"))
    print(f"cpu failures: {fails or 'none'}")
    print("gpu-skin (SRC_SKIN vertex shader) rendered for each anim case — eyeball .gpu.png vs .cpu.png")
    print(f"report -> {report_path(out_path)}")
    return 1 if (og["status"] == "fail" or fails) else 0


if __name__ == "__main__":
    sys.exit(main())