Files

224 lines
10 KiB
Python
Executable File

#!/usr/bin/env python3
"""Push the client root to the phone's app data directory (development builds).
The APK holds code only; on Android the client runs in its external files directory
(/sdcard/Android/data/com.metin2.ava2/files) and reads the flattened client root from there:
0. assets/loose the client-root asset tree
1. build/dev/test-rundir the pack-manifest client root (tools/make_test_rundir.py): what
the original client serves from its packs, so it wins over the
~1000 older models/textures assets/loose holds (zone/b tree
atlases at half size broke the SpeedTree UVs on the phone)
2. upstream/python-root scripts; win over same-named asset files (case-insensitive)
3. <pack_maker>/../svside loose client-root folder that is in no pack
4. python27.zip the embedded interpreter's stdlib
Later layers replace earlier files with the same (case-insensitive) path.
The merged tree is staged as hard links (no copy) in build/android-data/root, then sent with
`adb push --sync`, which skips files the device already has with the same timestamp, so later
runs only transfer what changed. The first run moves about 4 GB.
assets/loose keeps a few symlinks into the decoded pack_maker tree of the machine it was
committed on; they are remapped to --pack-maker (default ../AVA2/TEST/1. Client DEV/pack_maker).
usage: android/push_assets.py [--serial SERIAL] [--pack-maker DIR] [--stdlib ZIP] [--stage-only]
"""
import argparse
import glob
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
PACKAGE = "com.metin2.ava2"
DEVICE_DIR = f"/sdcard/Android/data/{PACKAGE}/files"
PACK_MAKER_MARK = "/pack_maker/"
LOOSE_DIRS = ["svside"]
RUNDIR_MARKER = ".ava2_test_rundir"
def find_stdlib():
candidates = glob.glob(str(ROOT / "android/app/.cxx/*/*/arm64-v8a/python27.zip"))
candidates += [str(p) for p in (ROOT / "build").glob("*/python27.zip")]
candidates = [c for c in candidates if os.path.isfile(c)]
return max(candidates, key=os.path.getmtime) if candidates else None
def ascii_lower(name):
"""Lower-cases A-Z only, like the client's ResourceManager::NormalizePath; legacy CP949 names
(stored as Latin-1 code points) must keep their bytes."""
return name.translate(ASCII_LOWER)
ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
class Stage:
"""Hard-link tree; a later source replaces an earlier file with the same lower-case path."""
def __init__(self, out, lowercase=False):
self.out = out
self.lowercase = lowercase # stage every name lower-case (case-sensitive targets, iOS)
self.files = {} # lower-case relative path -> staged path
self.dirs = {"": out}
def _dir(self, rel_parent):
key = ""
for part in rel_parent.parts:
parent = self.dirs[key]
key = f"{key}/{part.lower()}" if key else part.lower()
if key not in self.dirs:
self.dirs[key] = parent / (ascii_lower(part) if self.lowercase else part)
self.dirs[key].mkdir()
return self.dirs[key]
def add(self, rel, source):
key = str(rel).lower()
# Saved accounts are user data, never part of a development asset install.
if key == "lib/wa_ac.py":
return
old = self.files.pop(key, None)
if old is not None:
old.unlink()
target = self._dir(rel.parent) / (ascii_lower(rel.name) if self.lowercase else rel.name)
try:
os.link(source, target)
except OSError: # another volume
shutil.copy2(source, target)
self.files[key] = target
def walk(top, pack_maker, missing):
"""Yields (relative path, real file) under top, following and remapping symlinks."""
stack = [(Path(top), Path(), frozenset())]
while stack:
directory, rel, ancestors = stack.pop()
identity = directory.resolve()
if identity in ancestors:
continue
ancestors = ancestors | {identity}
for entry in sorted(os.scandir(directory), key=lambda e: e.name):
path, child = Path(entry.path), rel / entry.name
if entry.name == ".DS_Store":
continue
# This local convenience link points to the entire decoded pack tree;
# its runtime files are already supplied by the flattened rundir.
if Path(top) == ROOT / "upstream/python-root" and child == Path("pack_maker"):
continue
if entry.is_symlink() and not path.exists():
link = os.readlink(path)
mark = link.find(PACK_MAKER_MARK)
remapped = pack_maker / link[mark + len(PACK_MAKER_MARK):] if mark >= 0 else None
if remapped is None or not remapped.exists():
if os.path.isabs(link): # relative dead links (pack_maker's bgm/bgm -> bgm) are noise
missing.append(f"{child} -> {link}")
continue
path = remapped
if path.is_dir():
stack.append((path, child, ancestors))
elif path.is_file():
yield child, path
PUSH_BATCH = 20000
def count_files(path):
return sum(len(files) for _, _, files in os.walk(path))
def push_batches(stage, rel):
"""Yields stage-relative paths (directories or files) of at most PUSH_BATCH files each."""
path = stage / rel
if not path.is_dir() or count_files(path) <= PUSH_BATCH:
yield rel
return
for entry in sorted(path.iterdir()):
yield from push_batches(stage, rel / entry.name)
def adb(serial, *args, **kwargs):
command = ["adb"] + (["-s", serial] if serial else []) + list(args)
return subprocess.run(command, check=True, **kwargs)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--serial", default=os.environ.get("ADB_SERIAL"))
parser.add_argument("--pack-maker", type=Path, default=ROOT.parent / "AVA2/TEST/1. Client DEV/pack_maker")
parser.add_argument("--stdlib", help="python27.zip (default: newest Android/desktop build output)")
parser.add_argument("--stage", type=Path, default=ROOT / "build/android-data/root")
parser.add_argument("--rundir", type=Path, default=ROOT / "build/dev/test-rundir",
help="pack-manifest client root (tools/make_test_rundir.py), staged underneath")
parser.add_argument("--stage-only", action="store_true", help="build the staging tree, do not push")
parser.add_argument("--lowercase", action="store_true",
help="lower-case every staged name, for case-sensitive file systems (iOS)")
args = parser.parse_args()
stdlib = args.stdlib or find_stdlib()
if not stdlib:
sys.exit("python27.zip not found; run android/build.sh first or pass --stdlib")
if args.stage.exists():
shutil.rmtree(args.stage)
args.stage.mkdir(parents=True)
stage, missing = Stage(args.stage, args.lowercase), []
for rel, path in walk(ROOT / "assets/loose", args.pack_maker, missing):
stage.add(rel, path)
# The pack-manifest client root the desktop tests run against, over assets/loose: assets/loose
# alone lacks the pack files (Dopador_manual/*, the switcher art...) game.py needs, and where
# both have a file the pack version is the one the original client loads.
if args.rundir.is_dir():
for rel, path in walk(args.rundir, args.pack_maker, missing):
if rel.name != RUNDIR_MARKER:
stage.add(rel, path)
else:
print(f"warning: {args.rundir} not found (tools/make_test_rundir.py); the game phase will miss "
"pack files", file=sys.stderr)
for rel, path in walk(ROOT / "upstream/python-root", args.pack_maker, missing):
stage.add(rel, path)
# Loose client-root folders that are in neither assets/loose nor any pack (same as
# tools/make_test_rundir.py); svsideoi.py, built by game.py, needs svside/*.jpg.
for name in LOOSE_DIRS:
top = args.pack_maker.parent / name
if not top.is_dir():
missing.append(f"{name} -> {top}")
continue
for rel, path in walk(top, args.pack_maker, missing):
stage.add(Path(name) / rel, path)
stage.add(Path("python27.zip"), stdlib)
for line in missing:
print(f"warning: unresolved symlink {line}", file=sys.stderr)
size = sum(p.stat().st_size for p in stage.files.values())
print(f"staged {len(stage.files)} files, {size / 2**30:.2f} GiB in {args.stage}")
if args.stage_only:
return
adb(args.serial, "shell", "mkdir", "-p", DEVICE_DIR)
# One `adb push --sync` over all ~98k files aborts in file_sync_client ("protocol fault: failed
# to read stat response"), so push in batches of at most PUSH_BATCH files.
for rel in push_batches(args.stage, Path()):
source, target = args.stage / rel, f"{DEVICE_DIR}/{rel.as_posix()}" if rel.parts else DEVICE_DIR
if source.is_dir():
adb(args.serial, "shell", "mkdir", "-p", f"'{target}'")
# Trailing "/." pushes the directory's contents rather than the directory itself.
adb(args.serial, "push", "--sync", f"{source}/.", target)
else:
adb(args.serial, "push", "--sync", str(source), target)
# Recent Android versions (seen on Android 17) leave what adb creates under Android/data owned
# by `shell` with 2770 directories, which the app's uid cannot enter ("Permission denied" on
# the first model load). Open up what adb created; files the app wrote itself (channel.inf,
# the cfg files) and the files directory belong to the app and cannot be chmodded by shell.
adb(args.serial, "shell",
f"find '{DEVICE_DIR}' -mindepth 1 -user shell -type d ! -perm -005 -exec chmod 2777 {{}} + ; "
f"find '{DEVICE_DIR}' -user shell -type f ! -perm -004 -exec chmod 666 {{}} +")
print(f"pushed to {DEVICE_DIR}")
if __name__ == "__main__":
main()