#!/usr/bin/env python3 """40250 `#include` dependency graph: slice unit lists and batch-2 port order. 40250 headers lean on each library's precompiled `StdAfx.h` for most of their types, so the explicit `#include` graph alone is not a compile closure. This tool adds *implicit* edges: an identifier a file uses but does not see through its explicit includes is resolved to the header that declares it, restricted to the headers the file's own library `StdAfx.h` makes visible in 40250 (so an implicit edge is exactly one that a slim per-library StdAfx must supply). port_deps.py closure [...] compile closure (explicit + implicit), with reasons port_deps.py slices [--write] audit/slices/2V*.json: cumulative 2V0-2V3 unit lists port_deps.py order [--write] audit/slices/batch2-order.json: topological P0 order Header ownership: `Lib/X.h` belongs to `Lib/X.cpp` when that is an active unit; a header-only file belongs to the first unit (in port order) whose closure needs it. """ from __future__ import annotations import argparse import json import re import sys from collections import defaultdict from pathlib import Path import refroot # script directory is on sys.path when run directly from port_map import PY_ROOT, UNIT_DIRS, inventory, layer, priority HEADER_SUFFIXES = {".h", ".hpp", ".inl"} SOURCE_DIRS = UNIT_DIRS + ("Discord", "CWebBrowser") # Global names that the index must not turn into edges (Win32/CRT/STL spellings every lib uses). IGNORED_SYMBOLS = {"TRUE", "FALSE", "NULL", "BOOL", "BYTE", "WORD", "DWORD", "UINT", "LONG", "INT", "VOID"} # Cumulative vertical slices from docs/PORT-PLAN.md section 4 (2V0-2V3). `cpp` globs are real # units of the slice; `py` seeds are root scripts whose top-level import closure the slice runs. SLICES = { "2V0": { "cpp": ["ScriptLib/PythonLauncher.cpp", "UserInterface/PythonPackModule.cpp", "EterPythonLib/PythonWindow*.cpp", "EterPythonLib/PythonWindowManager*.cpp", "EterPythonLib/PythonGraphic*Module.cpp", "UserInterface/PythonApplicationModule.cpp"], "py": ["system.py", "prototype.py", "intrologo.py"], }, "2V1": { "cpp": ["UserInterface/PythonNetworkStream.cpp", "UserInterface/PythonNetworkStreamPhaseHandShake.cpp", "UserInterface/PythonNetworkStreamPhaseLogin.cpp", "UserInterface/PythonNetworkStreamPhaseSelect.cpp", "UserInterface/PythonNetworkStreamPhaseLoading.cpp", "UserInterface/PythonNetworkStreamModule.cpp", "UserInterface/AccountConnector.cpp"], "py": ["intrologin.py", "introempire.py", "introselect.py", "introcreate.py", "introloading.py"], }, "2V2": { "cpp": ["UserInterface/PythonNetworkStreamPhaseGame.cpp", "UserInterface/PythonCharacterManager.cpp", "UserInterface/InstanceBase.cpp", "GameLib/ActorInstance.cpp"], "py": ["game.py"], }, "2V3": { "cpp": ["UserInterface/PythonPlayer.cpp", "UserInterface/PythonPlayerInput*.cpp", "UserInterface/PythonPlayerEventHandler.cpp", "UserInterface/InstanceBaseMovement.cpp", "GameLib/ActorInstanceMotion.cpp", "GameLib/ActorInstanceEvent.cpp", "GameLib/ActorInstancePosition.cpp", "GameLib/ActorInstanceRotation.cpp", "GameLib/ActorInstanceCollisionDetection.cpp"], "py": [], }, } _STRIP = re.compile(r"//[^\n]*|/\*.*?\*/|\"(?:\\.|[^\"\\\n])*\"|'(?:\\.|[^'\\\n])*'", re.S) _INCLUDE = re.compile(r"^[ \t]*#[ \t]*include[ \t]*([<\"])([^>\"]+)[>\"]", re.M) _IDENT = re.compile(r"\b[A-Za-z_]\w*\b") _TYPE_DECL = re.compile(r"\b(class|struct|union|enum)\s+(?:__declspec\s*\([^)]*\)\s*)?([A-Za-z_]\w*)\s*" r"(?:final\s*)?(?::[^;{()]*)?\{") _ENUM_BODY = re.compile(r"\benum\b[^;{()]*\{") _TYPEDEF = re.compile(r"\btypedef\b") _DEFINE = re.compile(r"^[ \t]*#[ \t]*define[ \t]+([A-Za-z_]\w*)(?:\(|[ \t]+\S)", re.M) _FREE_FN = re.compile(r"^(?:extern\s+|inline\s+|static\s+|__forceinline\s+)*(?!return\b|typedef\b|else\b|if\b|" r"while\b|for\b|switch\b|case\b|class\b|struct\b|enum\b|template\b|using\b|friend\b)" r"[A-Za-z_][\w:<>, \t\*&]*?[ \t\*&]([A-Za-z_]\w*)[ \t]*\(", re.M) _EXTERN_VAR = re.compile(r"^extern\s+[^;()]*?\b([A-Za-z_]\w*)\s*(?:\[[^\]]*\])?\s*;", re.M) _PY_IMPORT = re.compile(r"^(?:import\s+([\w, \t]+)|from\s+(\w+)\s+import\b)", re.M) _PY_INIT = re.compile(r"Py_InitModule4?\(\s*\"(\w+)\"") def text(path: Path, keep_strings: bool = False) -> str: def repl(m: re.Match) -> str: tok = m.group(0) if tok.startswith("/*"): return "\n" * tok.count("\n") return tok if keep_strings and tok[0] in "\"'" else " " return _STRIP.sub(repl, path.read_bytes().decode("latin-1")) def matching_brace(src: str, start: int) -> int: depth = 0 for i in range(start, len(src)): if src[i] == "{": depth += 1 elif src[i] == "}": depth -= 1 if depth == 0: return i return len(src) - 1 def top_level(src: str) -> str: """`src` with every brace-enclosed body blanked (newlines kept), leaving namespace-scope text.""" out, depth = [], 0 for ch in src: if ch == "{": depth += 1 elif ch == "}": depth = max(depth - 1, 0) elif depth == 0 or ch == "\n": out.append(ch) continue return "".join(out) def declared(src: str) -> set[str]: """Names a header declares at namespace scope: types, typedefs, enumerators, macros, free functions.""" names = {m.group(2) for m in _TYPE_DECL.finditer(src)} for m in _ENUM_BODY.finditer(src): body = src[m.end():matching_brace(src, m.end() - 1)] for item in body.split(","): if ident := re.match(r"\s*([A-Za-z_]\w*)", item): names.add(ident.group(1)) for m in _TYPEDEF.finditer(src): rest = src[m.end():] brace, semi = rest.find("{"), rest.find(";") if 0 <= brace < semi: end = matching_brace(rest, brace) tail = rest[end + 1:rest.find(";", end)] names.update(re.findall(r"[A-Za-z_]\w*", tail)) elif semi >= 0: decl = rest[:semi] fnptr = re.search(r"\(\s*\w*\s*\*\s*([A-Za-z_]\w*)\s*\)", decl) last = re.findall(r"([A-Za-z_]\w*)\s*(?:\[[^\]]*\]\s*)*$", decl.strip()) if fnptr: names.add(fnptr.group(1)) elif last: names.add(last[0]) names.update(_DEFINE.findall(src)) top = top_level(src) names.update(m.group(1) for m in _FREE_FN.finditer(top)) names.update(_EXTERN_VAR.findall(top)) return names - IGNORED_SYMBOLS class Graph: def __init__(self, ref: Path): self.ref = ref self.files: dict[str, str] = {} # lower-case rel path -> real rel path for lib in SOURCE_DIRS: base = ref / lib if base.is_dir(): for p in base.rglob("*"): if p.suffix.lower() in HEADER_SUFFIXES | {".cpp", ".c"}: rel = p.relative_to(ref).as_posix() self.files[rel.lower()] = rel self._src: dict[str, str] = {} self._ids: dict[str, set[str]] = {} self._inc: dict[str, list[tuple[str, str]]] = {} self._explicit: dict[str, tuple[list[str], list[str]]] = {} self.decl: dict[str, set[str]] = {} self.symbols: dict[str, set[str]] = defaultdict(set) # name -> declaring headers for rel in self.files.values(): if Path(rel).suffix.lower() in HEADER_SUFFIXES: self.decl[rel] = declared(self.src(rel)) for name in self.decl[rel]: self.symbols[name].add(rel) self._visible: dict[str, set[str]] = {} def src(self, rel: str) -> str: if rel not in self._src: self._src[rel] = text(self.ref / rel) return self._src[rel] def idents(self, rel: str) -> set[str]: if rel not in self._ids: self._ids[rel] = set(_IDENT.findall(self.src(rel))) return self._ids[rel] def includes(self, rel: str) -> list[tuple[str, str]]: if rel not in self._inc: self._inc[rel] = _INCLUDE.findall(text(self.ref / rel, keep_strings=True)) return self._inc[rel] def resolve(self, frm: str, inc: str) -> str | None: inc = inc.replace("\\", "/") for base in (Path(frm).parent, Path(".")): parts: list[str] = [] for part in (base / inc).as_posix().split("/"): if part == "..": if parts: parts.pop() elif part not in ("", "."): parts.append(part) hit = self.files.get("/".join(parts).lower()) if hit: return hit return None def explicit(self, rel: str) -> tuple[list[str], list[str]]: """(resolved project includes, system includes); StdAfx includes are dropped.""" if rel not in self._explicit: local, system = [], [] for kind, inc in self.includes(rel): if Path(inc).name.lower() == "stdafx.h" and kind == '"': continue hit = self.resolve(rel, inc) if kind == '"' else None if hit: local.append(hit) elif kind == "<" or not hit: system.append(inc.lower()) self._explicit[rel] = (local, system) return self._explicit[rel] def explicit_closure(self, roots: list[str]) -> set[str]: seen: set[str] = set() stack = list(roots) while stack: rel = stack.pop() if rel in seen: continue seen.add(rel) # StdAfx chains are walked here (unlike explicit()), since that is what they supply. for kind, inc in self.includes(rel): hit = self.resolve(rel, inc) if kind == '"' else None if hit: stack.append(hit) return seen def visible(self, rel: str) -> set[str]: """Headers the 40250 StdAfx of `rel`'s library makes visible.""" lib = rel.split("/")[0] if lib not in self._visible: stdafx = self.files.get(f"{lib}/stdafx.h".lower()) self._visible[lib] = self.explicit_closure([stdafx]) - {stdafx} if stdafx else set() return self._visible[lib] def closure(self, roots: list[str]) -> tuple[dict[str, str], set[str], dict[str, list[str]]]: """Compile closure of `roots`: {file: reason}, system includes, ambiguous symbols.""" reason: dict[str, str] = {r: "root" for r in roots} system: set[str] = set() ambiguous: dict[str, list[str]] = {} queue = list(roots) scanned: set[str] = set() known: set[str] = set() while queue: # Explicit edges first, so implicit resolution sees the full explicit closure. while queue: rel = queue.pop() known |= self.decl.get(rel, set()) local, sys_inc = self.explicit(rel) system.update(sys_inc) for hit in local: if hit not in reason: reason[hit] = f"include from {rel}" queue.append(hit) for rel in sorted(set(reason) - scanned): scanned.add(rel) vis = self.visible(rel) lib = rel.split("/")[0] for name in sorted(self.idents(rel) - known): cands = sorted(self.symbols.get(name, set()) & vis) if not cands: continue pick = [c for c in cands if c.split("/")[0] == lib] or cands if len(pick) > 1: ambiguous[name] = pick hit = pick[0] known |= self.decl[hit] if hit not in reason: reason[hit] = f"implicit {name} from {rel}" queue.append(hit) return reason, system, ambiguous def header_layer(units: dict[str, Path], rel: str) -> str: stem = rel.rsplit(".", 1)[0] return layer(stem + ".cpp") if stem + ".cpp" in units else layer(rel) def expand(units: dict[str, Path], globs: list[str]) -> list[str]: import fnmatch out: list[str] = [] for pat in globs: hits = sorted(u for u in units if fnmatch.fnmatch(u, pat)) if not hits: raise SystemExit(f"slice pattern {pat!r} matches no active 40250 unit") out += [h for h in hits if h not in out] return out def py_closure(ref: Path, seeds: list[str], native: dict[str, str]) -> tuple[list[str], dict[str, str]]: root = (ref / PY_ROOT).resolve() files = {p.stem.lower(): p for p in root.glob("*.py")} seen: list[str] = [] mods: dict[str, str] = {} stack = [Path(s).stem.lower() for s in seeds] while stack: name = stack.pop() if name in seen: continue seen.append(name) for a, b in _PY_IMPORT.findall(files[name].read_bytes().decode("latin-1")): for mod in ([m.strip() for m in a.split(",")] if a else [b]): if not mod: continue if mod.lower() in files: stack.append(mod.lower()) else: mods[mod] = native.get(mod, "stdlib") return sorted(files[n].name for n in seen), dict(sorted(mods.items())) def native_modules(ref: Path, units: dict[str, Path]) -> dict[str, str]: out = {} for unit, path in units.items(): if unit.endswith(".cpp"): for name in _PY_INIT.findall(path.read_bytes().decode("latin-1")): out[name] = unit return out def owner_map(units: dict[str, Path], order: list[str], closures: dict[str, set[str]]) -> dict[str, str]: owner = {} for unit in units: if unit.endswith(".cpp"): for suffix in (".h", ".hpp", ".inl"): owner[unit[:-4] + suffix] = unit for unit in order: for rel in sorted(closures[unit]): owner.setdefault(rel, unit) return owner def build_slices(ref: Path, g: Graph, units: dict[str, Path]) -> dict[str, dict]: native = native_modules(ref, units) cpp: list[str] = [] py_seeds: list[str] = [] out = {} for name, spec in SLICES.items(): new = [u for u in expand(units, spec["cpp"]) if u not in cpp] cpp += new py_seeds += spec["py"] reason, system, ambiguous = g.closure(cpp) headers = sorted(r for r in reason if Path(r).suffix.lower() in HEADER_SUFFIXES) scripts, mods = py_closure(ref, py_seeds, native) out[name] = { "slice": name, "generated_by": ".agents/skills/metin2-40250-parity-audit/scripts/port_deps.py slices --write", "units": [{"unit": u, "layer": layer(u), "new": u in new} for u in cpp], "headers": [{"header": h, "layer": header_layer(units, h), "reason": reason[h], "owner_in_slice": (h.rsplit(".", 1)[0] + ".cpp") in cpp} for h in headers], "stdafx_supplied": sorted(h for h in headers if reason[h].startswith("implicit")), "system_includes": sorted(system), "python_scripts": scripts, "native_modules": [{"module": m, "provider": p, "real": p in cpp, "layer": layer(p) if p in units else "stdlib"} for m, p in mods.items()], "ambiguous_symbols": ambiguous, } return out def build_order(g: Graph, units: dict[str, Path]) -> dict: p0 = [u for u in units if priority(u) == "P0" and layer(u) == "logic"] closures = {u: set(g.closure([u])[0]) for u in p0} stems = {u[:-4].lower(): u for u in units if u.endswith(".cpp")} deps: dict[str, set[str]] = {} for u in p0: deps[u] = set() for rel in closures[u]: owner = stems.get(rel.rsplit(".", 1)[0].lower()) if owner and owner != u and owner in closures: deps[u].add(owner) # Tarjan SCC, then Kahn on the condensation; ties broken by the roadmap's candidate order. index, low, stack, on, sccs = {}, {}, [], set(), [] def strong(v: str, counter=[0]) -> None: index[v] = low[v] = counter[0] counter[0] += 1 stack.append(v) on.add(v) for w in sorted(deps[v]): if w not in index: strong(w) low[v] = min(low[v], low[w]) elif w in on: low[v] = min(low[v], index[w]) if low[v] == index[v]: comp = [] while True: w = stack.pop() on.discard(w) comp.append(w) if w == v: break sccs.append(sorted(comp)) sys.setrecursionlimit(10000) for v in sorted(p0): if v not in index: strong(v) comp_of = {u: i for i, c in enumerate(sccs) for u in c} # Tarjan emits SCCs in reverse topological order of the "depends on" edges: dependencies first. steps = [] for comp in sccs: after = sorted({w for u in comp for d in deps[u] if comp_of[d] != comp_of[u] for w in sccs[comp_of[d]]}) steps.append({"units": comp, "serial_group": len(comp) > 1, "after": after}) return { "generated_by": ".agents/skills/metin2-40250-parity-audit/scripts/port_deps.py order --write", "note": "P0 logic units; a unit starts only after every unit in `after` has landed. `serial_group` units " "include each other's headers and are ported as one serial group.", "steps": steps, } def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) c = sub.add_parser("closure") c.add_argument("files", nargs="+") for name in ("slices", "order"): s = sub.add_parser(name) s.add_argument("--write", action="store_true") args = ap.parse_args() repo = refroot.repo_root() ref = refroot.reference_root(repo) units = inventory(ref) g = Graph(ref) out_dir = repo / "audit" / "slices" if args.cmd == "closure": roots = [] for f in args.files: hit = g.files.get(f.lower()) if not hit: raise SystemExit(f"{f}: not a 40250 source file") roots.append(hit) reason, system, ambiguous = g.closure(roots) for rel in sorted(reason): print(f"{rel:55} {header_layer(units, rel):8} {reason[rel]}") print(f"\nsystem includes: {' '.join(sorted(system))}") for name, cands in sorted(ambiguous.items()): print(f"ambiguous {name}: {', '.join(cands)}") return 0 data = build_slices(ref, g, units) if args.cmd == "slices" else {"batch2-order": build_order(g, units)} for name, payload in data.items(): body = json.dumps(payload, indent=2, ensure_ascii=False) + "\n" if args.write: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / f"{name}.json").write_text(body, encoding="utf-8") print(f"wrote audit/slices/{name}.json") elif args.cmd == "order": for i, step in enumerate(payload["steps"], 1): tag = " (serial)" if step["serial_group"] else "" print(f"{i:2}. {', '.join(step['units'])}{tag} after: {', '.join(step['after']) or '-'}") else: print(f"{name}: {len(payload['units'])} units, {len(payload['headers'])} headers " f"({len(payload['stdafx_supplied'])} via StdAfx), {len(payload['python_scripts'])} scripts, " f"{sum(m['real'] for m in payload['native_modules'])}/{len(payload['native_modules'])} native modules real") return 0 if __name__ == "__main__": raise SystemExit(main())