audit(2A): include-dependency tool, 2V0-2V3 slice unit lists and batch-2 order

port_deps.py resolves explicit #include edges plus the implicit edges each
40250 library's StdAfx.h supplies (symbol index restricted to StdAfx-visible
headers), writes cumulative slice lists and the P0 topological order.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
shenlei
2026-09-22 19:21:10 +09:00
co-authored by Claude Opus 5
parent ae68ec5944
commit 4521845a69
9 changed files with 7610 additions and 2 deletions
@@ -148,6 +148,9 @@ behavior verified from code inspection alone.
— function inventory (active `.vcxproj` sources + the 40250 Python root), progress per library and
per layer, and tag/port-map/layout consistency. Run `check` before every commit that touches
`audit/port-map/`.
- `scripts/port_deps.py closure <file> | slices [--write] | order [--write]` — `#include` graph with the
implicit edges each library's `StdAfx.h` supplies; writes the 2V0-2V3 unit lists and the batch-2
topological order to `audit/slices/`. Regenerate after changing a slice definition.
- `scripts/py_embed_spike.py` (Python 2.7) — runs the 40250 `system.py` bootstrap with stub native
modules; used by the embedded-Python evaluation.
- `scripts/audit_ledger.py refresh --write | report --write | validate` — run once per batch, not
@@ -0,0 +1,468 @@
#!/usr/bin/env python3
"""40250 `#include` dependency graph: slice unit lists and batch-2 port order.
40250 headers lean on each library's precompiled `StdAfx.h` for most of their types, so the
explicit `#include` graph alone is not a compile closure. This tool adds *implicit* edges: an
identifier a file uses but does not see through its explicit includes is resolved to the header
that declares it, restricted to the headers the file's own library `StdAfx.h` makes visible in
40250 (so an implicit edge is exactly one that a slim per-library StdAfx must supply).
port_deps.py closure <unit|header> [...] compile closure (explicit + implicit), with reasons
port_deps.py slices [--write] audit/slices/2V*.json: cumulative 2V0-2V3 unit lists
port_deps.py order [--write] audit/slices/batch2-order.json: topological P0 order
Header ownership: `Lib/X.h` belongs to `Lib/X.cpp` when that is an active unit; a header-only
file belongs to the first unit (in port order) whose closure needs it.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import defaultdict
from pathlib import Path
import refroot # script directory is on sys.path when run directly
from port_map import PY_ROOT, UNIT_DIRS, inventory, layer, priority
HEADER_SUFFIXES = {".h", ".hpp", ".inl"}
SOURCE_DIRS = UNIT_DIRS + ("Discord", "CWebBrowser")
# Global names that the index must not turn into edges (Win32/CRT/STL spellings every lib uses).
IGNORED_SYMBOLS = {"TRUE", "FALSE", "NULL", "BOOL", "BYTE", "WORD", "DWORD", "UINT", "LONG", "INT", "VOID"}
# Cumulative vertical slices from docs/PORT-PLAN.md section 4 (2V0-2V3). `cpp` globs are real
# units of the slice; `py` seeds are root scripts whose top-level import closure the slice runs.
SLICES = {
"2V0": {
"cpp": ["ScriptLib/PythonLauncher.cpp", "UserInterface/PythonPackModule.cpp",
"EterPythonLib/PythonWindow*.cpp", "EterPythonLib/PythonWindowManager*.cpp",
"EterPythonLib/PythonGraphic*Module.cpp", "UserInterface/PythonApplicationModule.cpp"],
"py": ["system.py", "prototype.py", "intrologo.py"],
},
"2V1": {
"cpp": ["UserInterface/PythonNetworkStream.cpp", "UserInterface/PythonNetworkStreamPhaseHandShake.cpp",
"UserInterface/PythonNetworkStreamPhaseLogin.cpp", "UserInterface/PythonNetworkStreamPhaseSelect.cpp",
"UserInterface/PythonNetworkStreamPhaseLoading.cpp", "UserInterface/PythonNetworkStreamModule.cpp",
"UserInterface/AccountConnector.cpp"],
"py": ["intrologin.py", "introempire.py", "introselect.py", "introcreate.py", "introloading.py"],
},
"2V2": {
"cpp": ["UserInterface/PythonNetworkStreamPhaseGame.cpp", "UserInterface/PythonCharacterManager.cpp",
"UserInterface/InstanceBase.cpp", "GameLib/ActorInstance.cpp"],
"py": ["game.py"],
},
"2V3": {
"cpp": ["UserInterface/PythonPlayer.cpp", "UserInterface/PythonPlayerInput*.cpp",
"UserInterface/PythonPlayerEventHandler.cpp", "UserInterface/InstanceBaseMovement.cpp",
"GameLib/ActorInstanceMotion.cpp", "GameLib/ActorInstanceEvent.cpp",
"GameLib/ActorInstancePosition.cpp", "GameLib/ActorInstanceRotation.cpp",
"GameLib/ActorInstanceCollisionDetection.cpp"],
"py": [],
},
}
_STRIP = re.compile(r"//[^\n]*|/\*.*?\*/|\"(?:\\.|[^\"\\\n])*\"|'(?:\\.|[^'\\\n])*'", re.S)
_INCLUDE = re.compile(r"^[ \t]*#[ \t]*include[ \t]*([<\"])([^>\"]+)[>\"]", re.M)
_IDENT = re.compile(r"\b[A-Za-z_]\w*\b")
_TYPE_DECL = re.compile(r"\b(class|struct|union|enum)\s+(?:__declspec\s*\([^)]*\)\s*)?([A-Za-z_]\w*)\s*"
r"(?:final\s*)?(?::[^;{()]*)?\{")
_ENUM_BODY = re.compile(r"\benum\b[^;{()]*\{")
_TYPEDEF = re.compile(r"\btypedef\b")
_DEFINE = re.compile(r"^[ \t]*#[ \t]*define[ \t]+([A-Za-z_]\w*)(?:\(|[ \t]+\S)", re.M)
_FREE_FN = re.compile(r"^(?:extern\s+|inline\s+|static\s+|__forceinline\s+)*(?!return\b|typedef\b|else\b|if\b|"
r"while\b|for\b|switch\b|case\b|class\b|struct\b|enum\b|template\b|using\b|friend\b)"
r"[A-Za-z_][\w:<>, \t\*&]*?[ \t\*&]([A-Za-z_]\w*)[ \t]*\(", re.M)
_EXTERN_VAR = re.compile(r"^extern\s+[^;()]*?\b([A-Za-z_]\w*)\s*(?:\[[^\]]*\])?\s*;", re.M)
_PY_IMPORT = re.compile(r"^(?:import\s+([\w, \t]+)|from\s+(\w+)\s+import\b)", re.M)
_PY_INIT = re.compile(r"Py_InitModule4?\(\s*\"(\w+)\"")
def text(path: Path, keep_strings: bool = False) -> str:
def repl(m: re.Match) -> str:
tok = m.group(0)
if tok.startswith("/*"):
return "\n" * tok.count("\n")
return tok if keep_strings and tok[0] in "\"'" else " "
return _STRIP.sub(repl, path.read_bytes().decode("latin-1"))
def matching_brace(src: str, start: int) -> int:
depth = 0
for i in range(start, len(src)):
if src[i] == "{":
depth += 1
elif src[i] == "}":
depth -= 1
if depth == 0:
return i
return len(src) - 1
def top_level(src: str) -> str:
"""`src` with every brace-enclosed body blanked (newlines kept), leaving namespace-scope text."""
out, depth = [], 0
for ch in src:
if ch == "{":
depth += 1
elif ch == "}":
depth = max(depth - 1, 0)
elif depth == 0 or ch == "\n":
out.append(ch)
continue
return "".join(out)
def declared(src: str) -> set[str]:
"""Names a header declares at namespace scope: types, typedefs, enumerators, macros, free functions."""
names = {m.group(2) for m in _TYPE_DECL.finditer(src)}
for m in _ENUM_BODY.finditer(src):
body = src[m.end():matching_brace(src, m.end() - 1)]
for item in body.split(","):
if ident := re.match(r"\s*([A-Za-z_]\w*)", item):
names.add(ident.group(1))
for m in _TYPEDEF.finditer(src):
rest = src[m.end():]
brace, semi = rest.find("{"), rest.find(";")
if 0 <= brace < semi:
end = matching_brace(rest, brace)
tail = rest[end + 1:rest.find(";", end)]
names.update(re.findall(r"[A-Za-z_]\w*", tail))
elif semi >= 0:
decl = rest[:semi]
fnptr = re.search(r"\(\s*\w*\s*\*\s*([A-Za-z_]\w*)\s*\)", decl)
last = re.findall(r"([A-Za-z_]\w*)\s*(?:\[[^\]]*\]\s*)*$", decl.strip())
if fnptr:
names.add(fnptr.group(1))
elif last:
names.add(last[0])
names.update(_DEFINE.findall(src))
top = top_level(src)
names.update(m.group(1) for m in _FREE_FN.finditer(top))
names.update(_EXTERN_VAR.findall(top))
return names - IGNORED_SYMBOLS
class Graph:
def __init__(self, ref: Path):
self.ref = ref
self.files: dict[str, str] = {} # lower-case rel path -> real rel path
for lib in SOURCE_DIRS:
base = ref / lib
if base.is_dir():
for p in base.rglob("*"):
if p.suffix.lower() in HEADER_SUFFIXES | {".cpp", ".c"}:
rel = p.relative_to(ref).as_posix()
self.files[rel.lower()] = rel
self._src: dict[str, str] = {}
self._ids: dict[str, set[str]] = {}
self._inc: dict[str, list[tuple[str, str]]] = {}
self._explicit: dict[str, tuple[list[str], list[str]]] = {}
self.decl: dict[str, set[str]] = {}
self.symbols: dict[str, set[str]] = defaultdict(set) # name -> declaring headers
for rel in self.files.values():
if Path(rel).suffix.lower() in HEADER_SUFFIXES:
self.decl[rel] = declared(self.src(rel))
for name in self.decl[rel]:
self.symbols[name].add(rel)
self._visible: dict[str, set[str]] = {}
def src(self, rel: str) -> str:
if rel not in self._src:
self._src[rel] = text(self.ref / rel)
return self._src[rel]
def idents(self, rel: str) -> set[str]:
if rel not in self._ids:
self._ids[rel] = set(_IDENT.findall(self.src(rel)))
return self._ids[rel]
def includes(self, rel: str) -> list[tuple[str, str]]:
if rel not in self._inc:
self._inc[rel] = _INCLUDE.findall(text(self.ref / rel, keep_strings=True))
return self._inc[rel]
def resolve(self, frm: str, inc: str) -> str | None:
inc = inc.replace("\\", "/")
for base in (Path(frm).parent, Path(".")):
parts: list[str] = []
for part in (base / inc).as_posix().split("/"):
if part == "..":
if parts:
parts.pop()
elif part not in ("", "."):
parts.append(part)
hit = self.files.get("/".join(parts).lower())
if hit:
return hit
return None
def explicit(self, rel: str) -> tuple[list[str], list[str]]:
"""(resolved project includes, system includes); StdAfx includes are dropped."""
if rel not in self._explicit:
local, system = [], []
for kind, inc in self.includes(rel):
if Path(inc).name.lower() == "stdafx.h" and kind == '"':
continue
hit = self.resolve(rel, inc) if kind == '"' else None
if hit:
local.append(hit)
elif kind == "<" or not hit:
system.append(inc.lower())
self._explicit[rel] = (local, system)
return self._explicit[rel]
def explicit_closure(self, roots: list[str]) -> set[str]:
seen: set[str] = set()
stack = list(roots)
while stack:
rel = stack.pop()
if rel in seen:
continue
seen.add(rel)
# StdAfx chains are walked here (unlike explicit()), since that is what they supply.
for kind, inc in self.includes(rel):
hit = self.resolve(rel, inc) if kind == '"' else None
if hit:
stack.append(hit)
return seen
def visible(self, rel: str) -> set[str]:
"""Headers the 40250 StdAfx of `rel`'s library makes visible."""
lib = rel.split("/")[0]
if lib not in self._visible:
stdafx = self.files.get(f"{lib}/stdafx.h".lower())
self._visible[lib] = self.explicit_closure([stdafx]) - {stdafx} if stdafx else set()
return self._visible[lib]
def closure(self, roots: list[str]) -> tuple[dict[str, str], set[str], dict[str, list[str]]]:
"""Compile closure of `roots`: {file: reason}, system includes, ambiguous symbols."""
reason: dict[str, str] = {r: "root" for r in roots}
system: set[str] = set()
ambiguous: dict[str, list[str]] = {}
queue = list(roots)
scanned: set[str] = set()
known: set[str] = set()
while queue:
# Explicit edges first, so implicit resolution sees the full explicit closure.
while queue:
rel = queue.pop()
known |= self.decl.get(rel, set())
local, sys_inc = self.explicit(rel)
system.update(sys_inc)
for hit in local:
if hit not in reason:
reason[hit] = f"include from {rel}"
queue.append(hit)
for rel in sorted(set(reason) - scanned):
scanned.add(rel)
vis = self.visible(rel)
lib = rel.split("/")[0]
for name in sorted(self.idents(rel) - known):
cands = sorted(self.symbols.get(name, set()) & vis)
if not cands:
continue
pick = [c for c in cands if c.split("/")[0] == lib] or cands
if len(pick) > 1:
ambiguous[name] = pick
hit = pick[0]
known |= self.decl[hit]
if hit not in reason:
reason[hit] = f"implicit {name} from {rel}"
queue.append(hit)
return reason, system, ambiguous
def header_layer(units: dict[str, Path], rel: str) -> str:
stem = rel.rsplit(".", 1)[0]
return layer(stem + ".cpp") if stem + ".cpp" in units else layer(rel)
def expand(units: dict[str, Path], globs: list[str]) -> list[str]:
import fnmatch
out: list[str] = []
for pat in globs:
hits = sorted(u for u in units if fnmatch.fnmatch(u, pat))
if not hits:
raise SystemExit(f"slice pattern {pat!r} matches no active 40250 unit")
out += [h for h in hits if h not in out]
return out
def py_closure(ref: Path, seeds: list[str], native: dict[str, str]) -> tuple[list[str], dict[str, str]]:
root = (ref / PY_ROOT).resolve()
files = {p.stem.lower(): p for p in root.glob("*.py")}
seen: list[str] = []
mods: dict[str, str] = {}
stack = [Path(s).stem.lower() for s in seeds]
while stack:
name = stack.pop()
if name in seen:
continue
seen.append(name)
for a, b in _PY_IMPORT.findall(files[name].read_bytes().decode("latin-1")):
for mod in ([m.strip() for m in a.split(",")] if a else [b]):
if not mod:
continue
if mod.lower() in files:
stack.append(mod.lower())
else:
mods[mod] = native.get(mod, "stdlib")
return sorted(files[n].name for n in seen), dict(sorted(mods.items()))
def native_modules(ref: Path, units: dict[str, Path]) -> dict[str, str]:
out = {}
for unit, path in units.items():
if unit.endswith(".cpp"):
for name in _PY_INIT.findall(path.read_bytes().decode("latin-1")):
out[name] = unit
return out
def owner_map(units: dict[str, Path], order: list[str], closures: dict[str, set[str]]) -> dict[str, str]:
owner = {}
for unit in units:
if unit.endswith(".cpp"):
for suffix in (".h", ".hpp", ".inl"):
owner[unit[:-4] + suffix] = unit
for unit in order:
for rel in sorted(closures[unit]):
owner.setdefault(rel, unit)
return owner
def build_slices(ref: Path, g: Graph, units: dict[str, Path]) -> dict[str, dict]:
native = native_modules(ref, units)
cpp: list[str] = []
py_seeds: list[str] = []
out = {}
for name, spec in SLICES.items():
new = [u for u in expand(units, spec["cpp"]) if u not in cpp]
cpp += new
py_seeds += spec["py"]
reason, system, ambiguous = g.closure(cpp)
headers = sorted(r for r in reason if Path(r).suffix.lower() in HEADER_SUFFIXES)
scripts, mods = py_closure(ref, py_seeds, native)
out[name] = {
"slice": name,
"generated_by": ".agents/skills/metin2-40250-parity-audit/scripts/port_deps.py slices --write",
"units": [{"unit": u, "layer": layer(u), "new": u in new} for u in cpp],
"headers": [{"header": h, "layer": header_layer(units, h), "reason": reason[h],
"owner_in_slice": (h.rsplit(".", 1)[0] + ".cpp") in cpp}
for h in headers],
"stdafx_supplied": sorted(h for h in headers if reason[h].startswith("implicit")),
"system_includes": sorted(system),
"python_scripts": scripts,
"native_modules": [{"module": m, "provider": p, "real": p in cpp,
"layer": layer(p) if p in units else "stdlib"} for m, p in mods.items()],
"ambiguous_symbols": ambiguous,
}
return out
def build_order(g: Graph, units: dict[str, Path]) -> dict:
p0 = [u for u in units if priority(u) == "P0" and layer(u) == "logic"]
closures = {u: set(g.closure([u])[0]) for u in p0}
stems = {u[:-4].lower(): u for u in units if u.endswith(".cpp")}
deps: dict[str, set[str]] = {}
for u in p0:
deps[u] = set()
for rel in closures[u]:
owner = stems.get(rel.rsplit(".", 1)[0].lower())
if owner and owner != u and owner in closures:
deps[u].add(owner)
# Tarjan SCC, then Kahn on the condensation; ties broken by the roadmap's candidate order.
index, low, stack, on, sccs = {}, {}, [], set(), []
def strong(v: str, counter=[0]) -> None:
index[v] = low[v] = counter[0]
counter[0] += 1
stack.append(v)
on.add(v)
for w in sorted(deps[v]):
if w not in index:
strong(w)
low[v] = min(low[v], low[w])
elif w in on:
low[v] = min(low[v], index[w])
if low[v] == index[v]:
comp = []
while True:
w = stack.pop()
on.discard(w)
comp.append(w)
if w == v:
break
sccs.append(sorted(comp))
sys.setrecursionlimit(10000)
for v in sorted(p0):
if v not in index:
strong(v)
comp_of = {u: i for i, c in enumerate(sccs) for u in c}
# Tarjan emits SCCs in reverse topological order of the "depends on" edges: dependencies first.
steps = []
for comp in sccs:
after = sorted({w for u in comp for d in deps[u] if comp_of[d] != comp_of[u] for w in sccs[comp_of[d]]})
steps.append({"units": comp, "serial_group": len(comp) > 1, "after": after})
return {
"generated_by": ".agents/skills/metin2-40250-parity-audit/scripts/port_deps.py order --write",
"note": "P0 logic units; a unit starts only after every unit in `after` has landed. `serial_group` units "
"include each other's headers and are ported as one serial group.",
"steps": steps,
}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="cmd", required=True)
c = sub.add_parser("closure")
c.add_argument("files", nargs="+")
for name in ("slices", "order"):
s = sub.add_parser(name)
s.add_argument("--write", action="store_true")
args = ap.parse_args()
repo = refroot.repo_root()
ref = refroot.reference_root(repo)
units = inventory(ref)
g = Graph(ref)
out_dir = repo / "audit" / "slices"
if args.cmd == "closure":
roots = []
for f in args.files:
hit = g.files.get(f.lower())
if not hit:
raise SystemExit(f"{f}: not a 40250 source file")
roots.append(hit)
reason, system, ambiguous = g.closure(roots)
for rel in sorted(reason):
print(f"{rel:55} {header_layer(units, rel):8} {reason[rel]}")
print(f"\nsystem includes: {' '.join(sorted(system))}")
for name, cands in sorted(ambiguous.items()):
print(f"ambiguous {name}: {', '.join(cands)}")
return 0
data = build_slices(ref, g, units) if args.cmd == "slices" else {"batch2-order": build_order(g, units)}
for name, payload in data.items():
body = json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
if args.write:
out_dir.mkdir(parents=True, exist_ok=True)
(out_dir / f"{name}.json").write_text(body, encoding="utf-8")
print(f"wrote audit/slices/{name}.json")
elif args.cmd == "order":
for i, step in enumerate(payload["steps"], 1):
tag = " (serial)" if step["serial_group"] else ""
print(f"{i:2}. {', '.join(step['units'])}{tag} after: {', '.join(step['after']) or '-'}")
else:
print(f"{name}: {len(payload['units'])} units, {len(payload['headers'])} headers "
f"({len(payload['stdafx_supplied'])} via StdAfx), {len(payload['python_scripts'])} scripts, "
f"{sum(m['real'] for m in payload['native_modules'])}/{len(payload['native_modules'])} native modules real")
return 0
if __name__ == "__main__":
raise SystemExit(main())