diff --git a/.gitignore b/.gitignore index 75162311..a6c680c1 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,10 @@ compile_commands.json /project/bin/*.wasm /project/bin/ios/ /project/bin/android/ +# The embedded interpreter's stdlib, copied here from the build tree so the exporter packs it +# (extension/CMakeLists.txt, 批次 2P step 3c). Derived — never committed. +/project/python27.zip +/project/python27.zip.sha256 # Android Gradle build template (~1.2 GB of regenerable Gradle output + Godot # .aar). Recreate per machine: # godot --headless --path project --install-android-build-template \ diff --git a/audit/history.jsonl b/audit/history.jsonl index 53e07de8..8a916758 100644 --- a/audit/history.jsonl +++ b/audit/history.jsonl @@ -495,3 +495,4 @@ {"time": "2026-09-23T15:30:00Z", "event": "third_party", "unit": "PORT-PLAN 批次 2P step 1: vendored CPython 2.7.18 as the mtpython static library", "source": "official Python-2.7.18.tgz sha256 b62c0e7937551d0cc02b8fd5cb0f544f9405bafc9a54d3808ed4594812edef43, trimmed to 25MB (Doc/Demo/Tools/Mac/PCbuild, Modules/{_ctypes,expat,zlib,_sqlite}, stdlib test trees, pre-VS2010 PC projects); Lib/ kept for step 3's python27.zip", "shared": "source list = the 133 objects of the reference libpython2.7.a; built-in table config/config.c = 39 entries (config/Setup.static)", "per_platform": "config//pyconfig.h from tools/py_embed/gen_pyconfig.sh; android undefs HAVE_LANGINFO_H (nl_langinfo needs API 26); ios is derived from macos (2.7 configure refuses a darwin cross build) and drops process control + sys/random.h, keeping HAVE_GETGROUPS because posixmodule.c defines MAX_GROUPS only inside it", "patches": ["configure + configure.ac: arm64 MACOSX_DEFAULT_ARCH", "Modules/posixmodule.c: undef HAVE_SYSTEM/POPEN/FORK/EXECV/WAIT under TARGET_OS_IPHONE, which its own 'all other compilers' block hard-defines past pyconfig.h"], "not_done": ["Linux x86_64: gen_pyconfig.sh linux needs a Linux host", "Windows x64: upstream supports MSVC + PC/pyconfig.h while the Windows gate is MinGW, where PC/pyconfig.h leaves HAVE_UNISTD_H unset and posixmodule.c/dynload_win.c collide with the MinGW headers; decided in step 4"], "ported": [], "deleted": [], "divergent": [], "needs_live": [], "status_note": "no port-map entry: this is third-party source, not a ported 40250 unit. Linking into libmtgodot is step 2 (ScriptLib/PythonLauncher.cpp)", "tests": ["ctest python.embed (macOS arm64): 39 builtin modules, all import, sys.version 2.7.18, utf-8/cp1252 codecs + pickle round-trip via the vendored Lib/ PASS", "android arm64 NDK API 24: libmtpython.a builds clean", "ios arm64 iphoneos: libmtpython.a = 133 objects, no undefined _system/_fork/_popen"], "commit": "2P-step1"} {"time": "2026-09-23T17:30:00Z", "event": "port_round", "unit": "PORT-PLAN 批次 2P step 2: ScriptLib/PythonLauncher.cpp into port_logic on the embedded interpreter", "ported": ["ScriptLib/PythonLauncher.cpp (10 of 12 fns)", "ScriptLib/PythonMarshal.cpp (4 of 11)"], "adapted": [], "deleted": [], "divergent": [], "needs_live": [], "status_note": "statuses set only for what port.python_launcher actually runs. PythonUtils.cpp is copied and compiled but nothing calls it yet (the binding modules are 2V0), so all 19 stay TODO; TraceFunc/SetTraceFunc are _DEBUG-only; in PythonMarshal only the small-.pyc path runs (r_long, _PyMarshal_ReadLongFromFile, getfilesize, _PyMarshal_ReadLastObjectFromFile) because that one delegates to CPython's own PyMarshal_ReadObjectFromString under 16KB, leaving the ported r_object reader unreached. Two // PORT: edits: `unsigned char(val)` -> a cast (MSVC extension) in PythonUtils.cpp x2, and PyObject_AsCharBuffer's Py_ssize_t* in TraceFunc. CPythonLauncher::GetError dereferences the NULL PyErr_Fetch yields when nothing is pending — kept verbatim, the test only calls it with an error set.", "shims": "extension/src/port/common/shim/sdk/Python-2.7/: 12 forwarding headers for 40250's #include ; symtable.h forward-declares mod_ty instead of including Python-ast.h, whose one-word macros (Delete, Module, Set, Print) collide with EterLib/Pool.h — 40250 sidestepped the same clash by commenting out PySymtable_Build", "tests": ["ctest port.python_launcher (macOS arm64): Create/RunLine/RunMemoryTextFile(CRLF)/RunCompiledFile(.pyc)/RunFile PASS; RunFile(\"system.py\") reads the real pack and stops at ImportError: No module named app (the 2V0 C++ modules)", "ctest macOS: 27/27 PASS", "android arm64 NDK API 24 + ios arm64 iphoneos: port_logic (incl. ScriptLib) compiles"], "commit": "2P-step2"} {"time": "2026-09-23T19:30:00Z", "event": "port_round", "unit": "PORT-PLAN 批次 2P step 3a/3b: python27.zip on sys.path (platform/ScriptLib/PythonHost) + the pack module", "ported": ["UserInterface/PythonPackModule.cpp (3 of 5 fns: packExist, packGet, initpack)"], "adapted": [], "deleted": [], "divergent": [], "needs_live": [], "stdlib": {"builder": "tools/py_embed/make_stdlib_zip.py , CMake target mtpython_stdlib -> /python27.zip", "contents": "397 modules, 6876547 bytes, ZIP_STORED (no zlib module, so zipimport cannot inflate); deterministic (sorted, fixed timestamp) so the sha256 identifies it", "sha256": "5fce8f974a89b66110823d7529be96e359b56dea05fc6875189a241d2d37a7d6", "skipped": "test/tests trees, lib-tk, idlelib, lib2to3, bsddb, ensurepip, curses, distutils, unittest, wsgiref, multiprocessing, compiler, plat-*", "host": "platform/ScriptLib/PythonHost.cpp: Configure() sets Py_NoSiteFlag/IgnoreEnvironment/Frozen/DontWriteBytecode + program name before Py_Initialize; InstallStdLib() does PySys_SetPath then re-registers encodings.search_function, because 2.7 initializes the codec registry once at startup and never retries"}, "status_note": "PythonPackModule's other two functions are CAccountConnector's login key builder, which happens to live in that unit; kept verbatim under #if 0 until the 2V1 login slice brings AccountConnector.h. port/UserInterface/StdAfx.h is a partial mirror: the original includes every library's StdAfx plus /, so it carries only what compiles today (ScriptLib/StdAfx.h, APP_NAME, the name length enum, initpack) and grows with the 2V slices.", "tests": ["ctest python.embed now runs against python27.zip itself (zipimport), not the loose Lib/", "ctest port.python_launcher: InstallStdLib + `import os, string, types, codecs` + cp1252 encode; initpack then pack.Exist/pack.Get for system.py (byte count equals CEterPackManager's), uiscript/selectcharacterwindow.py and locale/en/locale_game.txt; .tga and a missing file give None", "ctest macOS: 27/27 PASS", "android arm64 + ios arm64: port_platform (incl. PythonHost, PythonPackModule) compiles"], "not_done": ["3c: stage python27.zip out of the app resources into the sandbox on Android/iOS and check it against the manifest sha256; PythonHost::DefaultStdLibPath() only reads MT_PYTHON_STDLIB today"], "commit": "2P-step3ab"} +{"time": "2026-09-23T21:40:00Z", "event": "port_round", "unit": "PORT-PLAN 批次 2P step 3c: staging python27.zip into the mobile sandbox", "ported": [], "adapted": [], "deleted": [], "divergent": [], "needs_live": [], "staging": {"ships": "make_stdlib_zip.py also writes .sha256; target mtpython_stdlib_project copy_if_different's python27.zip + .sha256 into project/ (gitignored), and both export presets' include_filter lists them so they enter the PCK", "runtime": "extension/src/python_stdlib.cpp (GDScript: Metin2Python.stdlib_path(force_stage)) — desktop uses ProjectSettings.globalize_path(res://python27.zip) in place; exported builds read the bytes out of the PCK, write user://python27.zip.part, verify the sha256 of the written file, then rename to user://python27.zip. An existing sandbox copy is reused only when it hashes to the shipped digest, so a torn first start or a new build re-stages.", "handoff": "PythonHost::SetDefaultStdLibPath() — DefaultStdLibPath() now answers with the staged path; $MT_PYTHON_STDLIB still wins. python_stdlib.cpp is the only unit that knows res:// / user://; port_platform stays godot-free.", "build_fix": "the Windows gate (MinGW, no mtpython) was broken by step 3b and is fixed here: port/CMakeLists.txt now also excludes UserInterface/StdAfx.h (it includes ScriptLib/StdAfx.h, as the original PCH does) and UserInterface/PythonPackModule.cpp, and platform/CMakeLists.txt excludes platform/ScriptLib/ the way port_logic already excluded port/ScriptLib/."}, "status_note": "On-device staging is not verified: the macOS test drives the mobile path with force_stage=true, and Android/iOS user:// sandboxes come with step 4. The zip's presence in an exported pack was checked with --export-pack, not with an APK. Nothing in the app's boot path calls Metin2Python.stdlib_path() yet — the interpreter only starts in the process with 2V0; today the test is its only caller.", "tests": ["project/python_stdlib_test.gd (godot --headless --path project --script python_stdlib_test.gd) -> PASS: in-place path on desktop; force_stage copies into user://, sha256 matches the shipped digest, bytes equal res://, no .part left, ZIPReader opens it and finds encodings/__init__.py and os.py; a second call does not re-copy (mtime unchanged); a deliberately corrupted sandbox copy is re-staged", "godot --headless --export-pack macOS lists python27.zip (6876547) and python27.zip.sha256 (65)"], "not_done": "真机验证（Android/iOS 进程内落盘 + system.py -> prototype.RunApp()）随 2P step 4；Linux/Windows 见 step 4 表格", "commit": "2P-step3c"} diff --git a/docs/PORT-PLAN.md b/docs/PORT-PLAN.md index f4a21bee..ceb6dedc 100644 --- a/docs/PORT-PLAN.md +++ b/docs/PORT-PLAN.md @@ -312,8 +312,22 @@ extension/third_party/cpython-2.7.18/ # 静态库（2P 批次 `locale/en/locale_game.txt`，非 `.py/.pyc/.txt` 和不存在的文件返回 `None`。 `UserInterface/StdAfx.h` 只镜像了当前能编的部分（原文件 include 整个客户端和 DirectShow）， `PythonPackModule.cpp` 里属于 `CAccountConnector` 的登录密钥函数原样保留但编译时排除，随 2V1 回来。 - **未完成（3c）**：Android/iOS 从应用资源把 zip 复制进沙盒并校验 sha256；`PythonHost::DefaultStdLibPath()` - 现在只认 `MT_PYTHON_STDLIB`。 + **已完成（3c 落盘）**：`make_stdlib_zip.py` 额外写出 `python27.zip.sha256`；构建把两个文件 + `copy_if_different` 到 `project/`（目标 `mtpython_stdlib_project`，两文件 gitignore）， + 导出预设的 `include_filter` 列出它们，所以它们进 PCK——`--export-pack` 验证过两个条目都在。 + `extension/src/python_stdlib.{h,cpp}`（GDScript 侧 `Metin2Python.stdlib_path(force_stage)`）： + 桌面 `res://` 就是磁盘目录，`globalize_path` 得到真实路径直接用，不复制；导出后 `res://` 在 PCK 里， + 于是读出字节 → 写 `user://python27.zip.part` → 按随包 sha256 校验**落盘后的文件**（不是内存里的 buffer， + 磁盘满要在这里报，而不是三个 import 之后变成 zipimport 失败）→ rename 成 `user://python27.zip`。 + 已经在沙盒里且 sha 相符就直接复用；被杀在半路的半成品或换了一版构建的旧 zip 都会重新落盘。 + 路径交给 `PythonHost::SetDefaultStdLibPath()`，`DefaultStdLibPath()` 从此答得出（`MT_PYTHON_STDLIB` 仍优先）。 + 测试 `project/python_stdlib_test.gd`（`godot --headless --path project --script python_stdlib_test.gd`）在 + macOS 上用 `force_stage=true` 跑的就是移动端那条路径：落盘、sha 一致、字节与 `res://` 相同、`.part` 不残留、 + `ZIPReader` 能打开且含 `encodings/__init__.py`，重复调用不重复复制，人为弄坏副本后重新落盘。 + **未完成**：真机上跑（Android/iOS 的 `user://` 沙盒路径本身没验过），随 step 4 一起； + app 的启动流程还没有调用它——解释器要到 2V0 才在进程里起来，现在只有测试调。 + `port_platform` 现在在没有 `mtpython` 时也会排除 `platform/ScriptLib/`（与 `port_logic` 同一条规则）， + 否则 Windows 门禁会编到 `PythonHost.cpp`。 4. 五个平台分别验证：编译、链接无未定义符号、`Py_Initialize`、静态 C 模块逐个 `import`、在 app 进程里跑 `system.py` → `prototype.RunApp()`（对照结果：74 个模块中 66 个加载成功，引导期调用 33 个 C++ 函数）。 | 平台 | 状态 | diff --git a/docs/THIRD-PARTY.md b/docs/THIRD-PARTY.md index e9af1b44..b8bea059 100644 --- a/docs/THIRD-PARTY.md +++ b/docs/THIRD-PARTY.md @@ -104,7 +104,10 @@ one via a 1-line shim header, see below). packs it into `/python27.zip` with `tools/py_embed/make_stdlib_zip.py`, which is what the host puts on `sys.path` (batch 2P step 3). Entries are **stored, never deflated** — the `zlib` module is trimmed out above, so - `zipimport` has nothing to inflate with. + `zipimport` has nothing to inflate with. The zip and its `.sha256` are copied + into `project/` (gitignored) so the exporter packs them; on Android/iOS + `extension/src/python_stdlib.cpp` stages the zip out of the PCK into `user://` + and checks that digest, because CPython opens it with its own stdio. * **Why vendored:** 2.7 is end-of-life, so there is no package to fetch on the four target SDKs, and the build must keep working offline and on the cross-build machines. It is the version the 40250 scripts are written for diff --git a/extension/CMakeLists.txt b/extension/CMakeLists.txt index d653c38c..3aac9230 100644 --- a/extension/CMakeLists.txt +++ b/extension/CMakeLists.txt @@ -198,10 +198,29 @@ add_library(mtgodot ${MT_LIB_KIND} src/net/m2_client.cpp src/proto/proto_node.cpp src/pack40250_node.cpp + src/python_stdlib.cpp ) target_compile_features(mtgodot PRIVATE cxx_std_20) target_link_libraries(mtgodot PRIVATE godot::cpp xrender::libgr2 xrender::formats mtnet mtproto port_platform) +# The embedded interpreter's standard library (批次 2P step 3c). python27.zip and its digest are +# copied next to project.godot so the exporter packs them into the APK/IPA (the export presets' +# include_filter lists them); src/python_stdlib.cpp stages them out of the PCK at runtime. +if(TARGET mtpython_stdlib) + target_compile_definitions(mtgodot PRIVATE MTGODOT_HAVE_PYTHON) + # A target rather than a custom command on the two files: the command that produces them lives in + # third_party/, and a Makefile generator only resolves a generated-file dependency inside the + # directory that declared it. copy_if_different keeps the no-op build a compare. + add_custom_target(mtpython_stdlib_project ALL + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${MT_PYTHON_STDLIB_ZIP}" "${CMAKE_SOURCE_DIR}/project/python27.zip" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${MT_PYTHON_STDLIB_SHA}" "${CMAKE_SOURCE_DIR}/project/python27.zip.sha256" + COMMENT "Copying python27.zip into project/ (res://)") + add_dependencies(mtpython_stdlib_project mtpython_stdlib) + add_dependencies(mtgodot mtpython_stdlib_project) +endif() + # Godot's .gdextension expects, per platform: # project/bin/libmtgodot.macos.template_{debug,release}.dylib # project/bin/libmtgodot.ios.template_{debug,release}.a diff --git a/extension/src/platform/CMakeLists.txt b/extension/src/platform/CMakeLists.txt index 84258597..2051a95f 100644 --- a/extension/src/platform/CMakeLists.txt +++ b/extension/src/platform/CMakeLists.txt @@ -4,6 +4,11 @@ # so it shares the per-library definitions. file(GLOB_RECURSE MT_PLATFORM_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +# Same rule as port_logic: without the embedded interpreter there is no ScriptLib to back +# (PORT-PLAN 批次 2P step 4 — Windows). +if(NOT TARGET mtpython) + list(FILTER MT_PLATFORM_SOURCES EXCLUDE REGEX "/ScriptLib/") +endif() add_library(port_platform STATIC ${MT_PLATFORM_SOURCES}) target_link_libraries(port_platform PUBLIC port_logic) foreach(src IN LISTS MT_PLATFORM_SOURCES) diff --git a/extension/src/platform/ScriptLib/PythonHost.cpp b/extension/src/platform/ScriptLib/PythonHost.cpp index 8cbab142..fc2ab07d 100644 --- a/extension/src/platform/ScriptLib/PythonHost.cpp +++ b/extension/src/platform/ScriptLib/PythonHost.cpp @@ -94,13 +94,24 @@ bool InstallStdLib(const char* stdlib_path, std::string* error) return true; } +namespace +{ +std::string g_staged_path; +} + +void SetDefaultStdLibPath(const char* stdlib_path) +{ + g_staged_path = stdlib_path ? stdlib_path : ""; +} + std::string DefaultStdLibPath() { + // The environment still wins, so a developer can point a build at another stdlib without rebuilding. if (const char* env = getenv("MT_PYTHON_STDLIB")) return env; - // Step 3c stages python27.zip out of the packed assets on Android/iOS and answers here; until then - // the caller passes the path (the build tree's copy, for the native tests). - return ""; + // Whatever staged the zip for this platform (extension/src/python_stdlib.cpp inside Godot; nothing + // in the native tests, which pass the build tree's copy on the command line). + return g_staged_path; } } diff --git a/extension/src/platform/ScriptLib/PythonHost.h b/extension/src/platform/ScriptLib/PythonHost.h index 55efd63c..07b7f4fc 100644 --- a/extension/src/platform/ScriptLib/PythonHost.h +++ b/extension/src/platform/ScriptLib/PythonHost.h @@ -20,8 +20,9 @@ void Configure(const char* program_name = "eter.python"); // and re-registers the codec search function. False leaves *error set. bool InstallStdLib(const char* stdlib_path, std::string* error); -// The stdlib path this build was given: $MT_PYTHON_STDLIB when set, else the python27.zip next to -// the executable, else "". Mobile staging (copy out of the app resources, checked against the -// manifest sha256) is 批次 2P step 3c and will answer here too. +// Where the host put the stdlib: $MT_PYTHON_STDLIB when set, else whatever SetDefaultStdLibPath was +// given, else "". Inside Godot that is the staged python27.zip (extension/src/python_stdlib.h), which +// on Android/iOS is the sandbox copy of the one in the read-only bundle. std::string DefaultStdLibPath(); +void SetDefaultStdLibPath(const char* stdlib_path); } diff --git a/extension/src/port/CMakeLists.txt b/extension/src/port/CMakeLists.txt index 755ff443..ff7ab48e 100644 --- a/extension/src/port/CMakeLists.txt +++ b/extension/src/port/CMakeLists.txt @@ -11,9 +11,12 @@ if(WIN32) endif() # The script layer needs the embedded interpreter (mt3p::python), which has no Windows build yet # (PORT-PLAN 批次 2P step 4). Without it, ScriptLib and its shims are left out. +# The units outside ScriptLib that reach it: UserInterface/StdAfx.h includes ScriptLib/StdAfx.h (as the +# original PCH does), and PythonPackModule.cpp is a Python module. Both go with it. +set(MT_PORT_NEEDS_PYTHON "/ScriptLib/|/UserInterface/StdAfx\\.h|/UserInterface/PythonPackModule\\.cpp") if(NOT TARGET mtpython) - list(FILTER MT_PORT_SOURCES EXCLUDE REGEX "/ScriptLib/") - list(FILTER MT_PORT_HEADERS EXCLUDE REGEX "/ScriptLib/|/common/shim/sdk/Python-2\\.7/") + list(FILTER MT_PORT_SOURCES EXCLUDE REGEX "${MT_PORT_NEEDS_PYTHON}") + list(FILTER MT_PORT_HEADERS EXCLUDE REGEX "${MT_PORT_NEEDS_PYTHON}|/common/shim/sdk/Python-2\\.7/") endif() add_library(port_logic STATIC ${MT_PORT_SOURCES}) diff --git a/extension/src/python_stdlib.cpp b/extension/src/python_stdlib.cpp new file mode 100644 index 00000000..549dd07c --- /dev/null +++ b/extension/src/python_stdlib.cpp @@ -0,0 +1,127 @@ +#include "python_stdlib.h" + +#include +#include +#include +#include +#include + +#include + +#ifdef MTGODOT_HAVE_PYTHON +#include "platform/ScriptLib/PythonHost.h" +#endif + +using namespace godot; + +namespace mtgodot { + +namespace { + +// Both ship in the PCK; the build copies them next to project.godot (extension/CMakeLists.txt). +constexpr const char *ZIP_RES = "res://python27.zip"; +constexpr const char *SHA_RES = "res://python27.zip.sha256"; +constexpr const char *ZIP_USER = "user://python27.zip"; +constexpr const char *ZIP_USER_TMP = "user://python27.zip.part"; + +// std::string, not String: this lives at file scope and a godot::String's constructor would run at +// dlopen time, before the extension's bindings exist. +std::string g_last_error; + +String fail(String *error, const String &text) { + g_last_error = text.utf8().get_data(); + if (error) { + *error = text; + } + return String(); +} + +String expected_sha() { + if (!FileAccess::file_exists(SHA_RES)) { + return String(); + } + return FileAccess::get_file_as_string(SHA_RES).strip_edges(); +} + +} // namespace + +String stage_python_stdlib(String *error, bool force_stage) { + g_last_error.clear(); + if (error) { + *error = String(); + } + if (!FileAccess::file_exists(ZIP_RES)) { + return fail(error, String("no ") + ZIP_RES + + " (build with -DMTGODOT_EMBED_PYTHON=ON, which copies it into project/)"); + } + + ProjectSettings *settings = ProjectSettings::get_singleton(); + // Desktop and the editor: res:// is a directory on disk, so CPython can open the file in place and + // nothing is copied. In an exported game globalize_path() returns a path next to the executable + // that does not exist, which is exactly the case the staging below is for. + const String in_place = settings->globalize_path(ZIP_RES); + if (!force_stage && !in_place.begins_with("res://") && FileAccess::file_exists(in_place)) { + return in_place; + } + + const String want = expected_sha(); + if (want.is_empty()) { + return fail(error, String("no ") + SHA_RES + " beside the stdlib zip"); + } + + // An already-staged copy is reused only when it is the zip this build ships: a new version of the + // app brings a new digest, and a first start that was killed mid-copy leaves a short file. + if (FileAccess::file_exists(ZIP_USER) && FileAccess::get_sha256(ZIP_USER) == want) { + return settings->globalize_path(ZIP_USER); + } + + const PackedByteArray data = FileAccess::get_file_as_bytes(ZIP_RES); + if (data.is_empty()) { + return fail(error, String("cannot read ") + ZIP_RES); + } + { + Ref out = FileAccess::open(ZIP_USER_TMP, FileAccess::WRITE); + if (out.is_null()) { + return fail(error, String("cannot write ") + ZIP_USER_TMP + ": " + + itos((int) FileAccess::get_open_error())); + } + out->store_buffer(data); + out->close(); + } + // Hash the file as written, not the buffer: a full disk reports itself here and not later as a + // zipimport failure three imports into system.py. + if (FileAccess::get_sha256(ZIP_USER_TMP) != want) { + DirAccess::remove_absolute(ZIP_USER_TMP); + return fail(error, String("staged ") + ZIP_USER + " does not match " + want); + } + if (FileAccess::file_exists(ZIP_USER)) { + DirAccess::remove_absolute(ZIP_USER); + } + if (DirAccess::rename_absolute(ZIP_USER_TMP, ZIP_USER) != OK) { + return fail(error, String("cannot rename ") + ZIP_USER_TMP + " to " + ZIP_USER); + } + UtilityFunctions::print("[python27] staged ", ZIP_USER, " (", data.size(), " bytes, sha256 ", want, ")"); + return settings->globalize_path(ZIP_USER); +} + +String Metin2Python::stdlib_path(bool force_stage) { + const String path = stage_python_stdlib(nullptr, force_stage); +#ifdef MTGODOT_HAVE_PYTHON + if (!path.is_empty()) { + PythonHost::SetDefaultStdLibPath(path.utf8().get_data()); + } +#endif + return path; +} + +String Metin2Python::last_error() { + return String::utf8(g_last_error.c_str()); +} + +void Metin2Python::_bind_methods() { + ClassDB::bind_static_method("Metin2Python", D_METHOD("stdlib_path", "force_stage"), + &Metin2Python::stdlib_path, DEFVAL(false)); + ClassDB::bind_static_method("Metin2Python", D_METHOD("last_error"), &Metin2Python::last_error); +} + +} // namespace mtgodot diff --git a/extension/src/python_stdlib.h b/extension/src/python_stdlib.h new file mode 100644 index 00000000..53aa796d --- /dev/null +++ b/extension/src/python_stdlib.h @@ -0,0 +1,37 @@ +#pragma once +// python_stdlib — where the embedded interpreter's standard library is on this platform +// (docs/PORT-PLAN.md 批次 2P step 3c). +// +// CPython opens python27.zip with its own stdio, so it needs a real filesystem path. That is free on +// the desktop, where res:// is a directory on disk, but on Android and iOS res:// is inside the PCK, +// which nothing outside Godot can read. There the zip is copied once into the sandbox (user://) and +// the copy is accepted only when it hashes to the digest built beside the zip, so an interrupted +// first start or a zip replaced by a new build re-stages instead of feeding zipimport a torn file. +// +// Only this file knows about res:// / user://: PythonHost (platform/ScriptLib) stays godot-free and +// is told the path, through PythonHost::SetDefaultStdLibPath. + +#include +#include + +namespace mtgodot { + +// The real filesystem path of python27.zip, staging it out of res:// when res:// is not on disk. +// Empty when the zip is missing or the staged copy does not verify; the reason goes to *error. +// force_stage runs the mobile path even where res:// is readable, which is how it is tested. +godot::String stage_python_stdlib(godot::String *error = nullptr, bool force_stage = false); + +class Metin2Python : public godot::Object { + GDCLASS(Metin2Python, godot::Object) + +public: + // Returns the path and, when the embedded interpreter is built in, hands it to PythonHost so + // PythonHost::DefaultStdLibPath() answers with it from then on. + static godot::String stdlib_path(bool force_stage); + static godot::String last_error(); + +protected: + static void _bind_methods(); +}; + +} // namespace mtgodot diff --git a/extension/src/register_types.cpp b/extension/src/register_types.cpp index 5858f37f..67264410 100644 --- a/extension/src/register_types.cpp +++ b/extension/src/register_types.cpp @@ -12,6 +12,7 @@ #include "net/m2_client.h" #include "pack40250_node.h" #include "proto/proto_node.h" +#include "python_stdlib.h" #include "static_object.h" #include "terrain_splat.h" #include "tree_placeholder.h" @@ -45,6 +46,7 @@ void initialize_mtgodot_module(ModuleInitializationLevel p_level) { GDREGISTER_CLASS(mtgodot::M2Client); GDREGISTER_CLASS(mtgodot::Metin2Proto); GDREGISTER_CLASS(mtgodot::Metin2Pack); + GDREGISTER_CLASS(mtgodot::Metin2Python); } void uninitialize_mtgodot_module(ModuleInitializationLevel p_level) { diff --git a/extension/third_party/cpython-2.7.18/CMakeLists.txt b/extension/third_party/cpython-2.7.18/CMakeLists.txt index cf0a8ca2..210e6baf 100644 --- a/extension/third_party/cpython-2.7.18/CMakeLists.txt +++ b/extension/third_party/cpython-2.7.18/CMakeLists.txt @@ -215,10 +215,11 @@ add_library(mt3p::python ALIAS mtpython) # is where every consumer (the native tests now, the mobile staging in step 3c) picks it up. find_package(Python3 COMPONENTS Interpreter REQUIRED) set(MT_PYTHON_STDLIB_ZIP "${CMAKE_BINARY_DIR}/python27.zip" CACHE INTERNAL "stdlib zip for the embedded interpreter") +set(MT_PYTHON_STDLIB_SHA "${MT_PYTHON_STDLIB_ZIP}.sha256" CACHE INTERNAL "sha256 of MT_PYTHON_STDLIB_ZIP") file(GLOB_RECURSE MT_PYTHON_STDLIB_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/Lib/*.py") -add_custom_command(OUTPUT "${MT_PYTHON_STDLIB_ZIP}" +add_custom_command(OUTPUT "${MT_PYTHON_STDLIB_ZIP}" "${MT_PYTHON_STDLIB_SHA}" COMMAND ${Python3_EXECUTABLE} "${CMAKE_SOURCE_DIR}/tools/py_embed/make_stdlib_zip.py" "${CMAKE_CURRENT_SOURCE_DIR}/Lib" "${MT_PYTHON_STDLIB_ZIP}" DEPENDS "${CMAKE_SOURCE_DIR}/tools/py_embed/make_stdlib_zip.py" ${MT_PYTHON_STDLIB_SOURCES} COMMENT "Packing python27.zip from the vendored Lib/") -add_custom_target(mtpython_stdlib ALL DEPENDS "${MT_PYTHON_STDLIB_ZIP}") +add_custom_target(mtpython_stdlib ALL DEPENDS "${MT_PYTHON_STDLIB_ZIP}" "${MT_PYTHON_STDLIB_SHA}") diff --git a/project/export_presets.cfg b/project/export_presets.cfg index 225a6189..49c0da14 100644 --- a/project/export_presets.cfg +++ b/project/export_presets.cfg @@ -7,7 +7,7 @@ advanced_options=false dedicated_server=false custom_features="" export_filter="all_resources" -include_filter="" +include_filter="python27.zip,python27.zip.sha256" exclude_filter="" export_path="../build/export/mtgodot-poc.app" patches=PackedStringArray() @@ -69,7 +69,7 @@ advanced_options=false dedicated_server=false custom_features="" export_filter="all_resources" -include_filter="" +include_filter="python27.zip,python27.zip.sha256" exclude_filter="" export_path="../build/export/mtgodot-poc.apk" patches=PackedStringArray() diff --git a/project/python_stdlib_test.gd b/project/python_stdlib_test.gd new file mode 100644 index 00000000..56ecccd9 --- /dev/null +++ b/project/python_stdlib_test.gd @@ -0,0 +1,77 @@ +# python_stdlib_test —— 内嵌解释器标准库的落盘（docs/PORT-PLAN.md 批次 2P step 3c）。 +# 桌面上 res:// 就是磁盘目录，CPython 直接打开；Android/iOS 的 res:// 在 PCK 里，必须先复制进 +# user:// 沙箱并按随包发的 sha256 校验。force_stage=true 就是在桌面上跑那条移动端路径。 +# godot --headless --path project --script python_stdlib_test.gd +extends SceneTree + +const ZIP_USER := "user://python27.zip" +const ZIP_PART := "user://python27.zip.part" + +var _fail := 0 + +func _ck(ok: bool, message: String) -> void: + if not ok: + _fail += 1 + printerr("FAIL: " + message) + +func _init() -> void: + if not ClassDB.class_exists("Metin2Python"): + printerr("FAIL: Metin2Python extension is not registered") + quit(1) + return + if not FileAccess.file_exists("res://python27.zip"): + print("SKIP: python_stdlib_test（没有 res://python27.zip，需要 -DMTGODOT_EMBED_PYTHON=ON 的构建）") + quit(0) + return + + var want := FileAccess.get_file_as_string("res://python27.zip.sha256").strip_edges() + _ck(want.length() == 64, "res://python27.zip.sha256 是一行 sha256") + + # 桌面：res:// 在磁盘上，原地用，不复制。 + if DirAccess.dir_exists_absolute(ProjectSettings.globalize_path("res://")): + var in_place := Metin2Python.stdlib_path(false) + _ck(in_place == ProjectSettings.globalize_path("res://python27.zip"), + "res:// 可直接读时返回原路径，不落盘：" + Metin2Python.last_error()) + + # 移动端路径：复制进沙箱，校验 sha256，返回真实文件系统路径。 + if FileAccess.file_exists(ZIP_USER): + DirAccess.remove_absolute(ZIP_USER) + var staged := Metin2Python.stdlib_path(true) + _ck(staged != "", "落盘成功：" + Metin2Python.last_error()) + _ck(staged == ProjectSettings.globalize_path(ZIP_USER), "返回的是 user:// 的真实路径") + _ck(FileAccess.file_exists(staged), "返回的路径在文件系统上存在") + _ck(FileAccess.get_sha256(ZIP_USER) == want, "落盘的 zip 与随包的 sha256 一致") + _ck(not FileAccess.file_exists(ZIP_PART), "不留下半成品 .part") + _ck(FileAccess.get_file_as_bytes(ZIP_USER) == FileAccess.get_file_as_bytes("res://python27.zip"), + "落盘的字节与 res:// 里的一致") + # zipimport 读不了 deflate（没有 zlib 模块），所以内容必须是 STORED。 + var reader := ZIPReader.new() + var opened := reader.open(staged) + _ck(opened == OK, "落盘的 zip 能打开") + if opened == OK: + _ck(reader.file_exists("encodings/__init__.py"), "zip 里有 encodings 包") + _ck(reader.file_exists("os.py"), "zip 里有 os.py") + reader.close() + + # 第二次启动：已经在沙箱里且 sha 对得上，直接复用。 + var modified := FileAccess.get_modified_time(ProjectSettings.globalize_path(ZIP_USER)) + var again := Metin2Python.stdlib_path(true) + _ck(again == staged, "第二次返回同一路径") + _ck(FileAccess.get_modified_time(ProjectSettings.globalize_path(ZIP_USER)) == modified, + "sha 相符时不重新复制") + + # 上次启动被杀在半路 / 换了一版构建：沙箱里的副本对不上就重新落盘。 + var f := FileAccess.open(ZIP_USER, FileAccess.WRITE) + f.store_string("torn") + f.close() + _ck(FileAccess.get_sha256(ZIP_USER) != want, "已经把沙箱里的副本弄坏") + var restaged := Metin2Python.stdlib_path(true) + _ck(restaged == staged, "坏副本之后仍返回同一路径：" + Metin2Python.last_error()) + _ck(FileAccess.get_sha256(ZIP_USER) == want, "坏副本被重新落盘覆盖") + + if _fail == 0: + print("PASS: python_stdlib_test (res:// -> user:// staging + sha256)") + quit(0) + else: + printerr("%d check(s) failed" % _fail) + quit(1) diff --git a/project/python_stdlib_test.gd.uid b/project/python_stdlib_test.gd.uid new file mode 100644 index 00000000..eea6efcc --- /dev/null +++ b/project/python_stdlib_test.gd.uid @@ -0,0 +1 @@ +uid://cfmd68ffjkrtv diff --git a/tools/py_embed/make_stdlib_zip.py b/tools/py_embed/make_stdlib_zip.py index 6fe91273..0e3e2979 100644 --- a/tools/py_embed/make_stdlib_zip.py +++ b/tools/py_embed/make_stdlib_zip.py @@ -11,12 +11,15 @@ tools/py_embed_android/build-and-run.sh (PORT-PLAN 批次 2P step 3). Entries are STORED, never deflated: the zlib module is not in the vendored Modules/ (see docs/THIRD-PARTY.md), so zipimport could not inflate them. Only .py goes in; zipimport reads source as happily as bytecode, and .pyc would pin the magic number to the build host's interpreter. -Written deterministically (sorted, fixed timestamp) so the file's sha256 identifies its contents — -3c checks that hash when staging the zip into a mobile sandbox. +Written deterministically (sorted, fixed timestamp) so the file's sha256 identifies its contents. +The hash is written next to the zip as .sha256 and ships with it: on Android/iOS the zip lives +in the read-only bundle and has to be copied into the sandbox before CPython can open it, and that +copy is accepted only when it hashes to this value (extension/src/python_stdlib.cpp, step 3c). """ from __future__ import annotations +import hashlib import sys import zipfile from pathlib import Path @@ -51,7 +54,9 @@ def main() -> int: info.external_attr = 0o644 << 16 zf.writestr(info, path.read_bytes()) tmp.replace(out) - print(f"make_stdlib_zip: {out} ({len(members)} modules, {out.stat().st_size} bytes)") + digest = hashlib.sha256(out.read_bytes()).hexdigest() + out.with_name(out.name + ".sha256").write_text(digest + "\n") + print(f"make_stdlib_zip: {out} ({len(members)} modules, {out.stat().st_size} bytes, sha256 {digest})") return 0