mtgodot-poc/android/sync_client.py

#!/usr/bin/env python3
"""Bring the game files in the app's private files/Client (SDL_GetPrefPath, where src/host looks for
files/Client/pack/Index) up to date with a local client directory, sending only what changed.

The device keeps files/Client/.mt_sync, the list (size, mtime, path) of what the last sync wrote. A sync
compares it with the local files, streams the new and changed ones as one tar through `adb shell -T run-as`
(no copy in /data/local/tmp), removes the ones gone locally, then writes the new list. Only the game
resources are synced (pack/, sound/, BGM/, miles/, lib/, channel.inf, locale*.cfg); what the app writes
there itself (settings such as ui_height.cfg, perf logs) is never touched. mouse.cfg is only seeded on
the first sync.

Needs the debug APK installed (run-as). adb and the device come from android/adb_env.sh ($ADB,
$ANDROID_SERIAL); android/push-client.sh is the shell entry point.
"""

import argparse
import hashlib
import os
import subprocess
import sys
import tarfile
import time
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
PKG = "org.metin2port.client"
MANIFEST = ".mt_sync"
HEADER = "# mt-sync 1"
SYNC_DIRS = ("pack", "sound", "BGM", "miles", "lib")
SEED_FILES = ("mouse.cfg",)
ZH_OUTPUT = REPO / "build/zh-locale/Client"
ZH_STAMP = REPO / "build/zh-locale/.inputs.sha256"


def adb(*args, **kwargs):
    return subprocess.run([os.environ.get("ADB", "adb"), *args], **kwargs)


def run_as_command(script):
    # `adb shell -T` (shell protocol v2, Android 7+): no pty, so stdin/stdout stay binary-safe, and unlike
    # exec-in/exec-out it returns the remote exit status.
    return [os.environ.get("ADB", "adb"), "shell", "-T", f"run-as {PKG} sh -c '{script}'"]


def run_as(script, **kwargs):
    """Runs script with sh as the app user in its data directory."""
    return subprocess.run(run_as_command(script), **kwargs)


def zh_inputs_digest(client_base):
    h = hashlib.sha256()
    inputs = [REPO / "scripts/build_zh_locale.py", client_base / "pack/Index"]
    inputs += sorted(p for p in (REPO / "data/zh_40250").rglob("*") if p.is_file())
    for path in inputs:
        h.update(str(path.relative_to(REPO)).encode())
        h.update(path.read_bytes() if path.exists() else b"")
    return h.hexdigest()


def ensure_zh_locale():
    """Rebuilds build/zh-locale/Client when its inputs (translations, the builder, the base pack Index)
    changed since the last build."""
    digest = zh_inputs_digest(REPO / "Client")
    built = (ZH_OUTPUT / "pack/Index").is_file()
    if built and ZH_STAMP.is_file() and ZH_STAMP.read_text().strip() == digest:
        return
    print("zh locale: inputs changed, rebuilding build/zh-locale/Client ...")
    result = subprocess.run([sys.executable, str(REPO / "scripts/build_zh_locale.py")], cwd=REPO)
    if result.returncode == 0:
        ZH_STAMP.write_text(digest + "\n")
    elif built:
        print("zh locale: rebuild failed, syncing the previous build", file=sys.stderr)
    else:
        sys.exit("zh locale: build failed (scripts/build_zh_locale.py)")


def local_files(client, locale):
    """{device path: local path} of the synced resources; locale.cfg comes from locale_<locale>.cfg."""
    files = {}
    for top in SYNC_DIRS:
        root = client / top
        if not root.is_dir():
            continue
        for dirpath, dirnames, filenames in os.walk(root, followlinks=True):
            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
            for name in filenames:
                if not name.startswith("."):
                    path = Path(dirpath) / name
                    files[path.relative_to(client).as_posix()] = path
    for path in client.iterdir():
        if path.is_file() and (path.name == "channel.inf" or (path.name.startswith("locale") and path.suffix == ".cfg")):
            files[path.name] = path
    if locale:
        cfg = client / f"locale_{locale}.cfg"
        if not cfg.is_file():
            sys.exit(f"{cfg} not found")
        pack = cfg.read_text().split()[2]
        index = (client / "pack/Index").read_text(errors="replace").split()
        if f"locale_{pack}" not in index:
            sys.exit(f"locale_{pack} is not registered in {client}/pack/Index")
        files["locale.cfg"] = cfg
    return files


def stat_key(path):
    st = path.stat()
    return f"{st.st_size}\t{st.st_mtime_ns}"


def read_device_manifest():
    out = run_as(f"cat files/Client/{MANIFEST} 2>/dev/null", capture_output=True).stdout.decode(errors="replace")
    lines = out.splitlines()
    if not lines or lines[0] != HEADER:
        return None
    manifest = {}
    for line in lines[1:]:
        size, mtime, path = line.split("\t", 2)
        manifest[path] = f"{size}\t{mtime}"
    return manifest


class Progress:
    def __init__(self, total):
        self.total, self.done, self.last, self.start = total, 0, 0.0, time.monotonic()

    def add(self, n):
        self.done += n
        now = time.monotonic()
        if now - self.last >= 1 or self.done == self.total:
            self.last = now
            rate = self.done / max(now - self.start, 1e-3) / 2**20
            print(f"\r  {self.done / 2**20:8.1f} / {self.total / 2**20:.1f} MB  {rate:5.1f} MB/s", end="", flush=True)


def push(files):
    """Streams {device path: local path} into files/Client as one tar."""
    total = sum(p.stat().st_size for p in files.values())
    progress = Progress(total)
    proc = subprocess.Popen(run_as_command("mkdir -p files/Client && cd files/Client && tar -xf -"),
                            stdin=subprocess.PIPE)
    with tarfile.open(fileobj=proc.stdin, mode="w|", format=tarfile.GNU_FORMAT, dereference=True) as tar:
        for name, path in sorted(files.items()):
            info = tar.gettarinfo(str(path), arcname=name)  # dereference: the zh client links the original packs
            info.uid = info.gid = 0
            info.uname = info.gname = ""
            with open(path, "rb") as f:
                tar.addfile(info, f)
            progress.add(info.size)
    proc.stdin.close()
    if progress.total:
        print()
    if proc.wait() != 0:
        sys.exit("push failed (tar on the device)")


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("client", nargs="?", type=Path, help="client directory (default: Client, or the zh build for --locale zh)")
    parser.add_argument("--locale", help="write locale.cfg from locale_<LOCALE>.cfg; zh uses build/zh-locale/Client, rebuilt when stale")
    parser.add_argument("--full", action="store_true", help="delete the synced resources on the device and push everything")
    parser.add_argument("--dry-run", action="store_true", help="only print what would change")
    args = parser.parse_args()

    if args.locale == "zh" and args.client is None:
        ensure_zh_locale()
        client = ZH_OUTPUT
    else:
        client = args.client or REPO / "Client"
    client = client.absolute()
    if not (client / "pack/Index").is_file():
        sys.exit(f"{client}/pack/Index not found")

    if not adb("shell", "pm", "path", PKG, capture_output=True).stdout.strip():
        sys.exit(f"{PKG} is not installed (android/build.sh installs it)")
    if run_as("true", capture_output=True).returncode != 0:
        sys.exit(f"run-as {PKG} failed: the installed APK must be the debug build")

    files = local_files(client, args.locale)
    wanted = {name: stat_key(path) for name, path in files.items()}
    device = None if args.full else read_device_manifest()
    first = device is None
    if first:
        changed = sorted(wanted)
        removed = []
        # First sync (or --full): also seed the default settings the app would otherwise lack.
        for name in SEED_FILES:
            if (client / name).is_file():
                files[name] = client / name
                changed.append(name)
    else:
        changed = sorted(n for n, key in wanted.items() if device.get(n) != key)
        removed = sorted(n for n in device if n not in wanted)
    size = sum(files[n].stat().st_size for n in changed)
    print(f"sync {client} -> {PKG} files/Client: {len(changed)} files to push ({size / 2**20:.1f} MB), "
          f"{len(removed)} to delete, {len(wanted) - len(changed) if not first else 0} unchanged"
          + (" [full]" if args.full else " [first sync]" if first else ""))
    if not changed and not removed:
        return
    if args.dry_run:
        for n in changed[:50]:
            print("  +", n)
        for n in removed[:50]:
            print("  -", n)
        return

    if args.full:
        dirs = " ".join(SYNC_DIRS)
        run_as(f"mkdir -p files/Client && cd files/Client && rm -rf {dirs} {MANIFEST}", check=True)
    # Drop the list before touching files: an interrupted sync then starts over as a first sync.
    run_as(f"rm -f files/Client/{MANIFEST}", check=True)
    if removed:
        run_as('cd files/Client && while IFS= read -r f; do rm -f "$f"; done',
               input="".join(n + "\n" for n in removed).encode(), check=True)
    if changed:
        push({n: files[n] for n in changed})
    manifest = HEADER + "\n" + "".join(f"{wanted[n]}\t{n}\n" for n in sorted(wanted))
    run_as(f"cat > files/Client/{MANIFEST}", input=manifest.encode(), check=True)
    locale = run_as("cat files/Client/locale.cfg", capture_output=True).stdout.decode(errors="replace").strip()
    print(f"done: files/Client updated (locale.cfg: {locale})")


if __name__ == "__main__":
    main()