From 43bb561551c269a2602450dd6cbd772bfb172442 Mon Sep 17 00:00:00 2001 From: shenlei Date: Wed, 23 Sep 2026 08:54:48 +0900 Subject: [PATCH] port(2P step 2): ScriptLib/PythonLauncher on the embedded interpreter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Link mtpython into port_logic and copy 40250's ScriptLib launcher across: PythonLauncher, PythonUtils and PythonMarshal, plus the 12 forwarding headers that make its `#include ` lines compile unchanged. symtable.h forward-declares mod_ty rather than including Python-ast.h, whose one-word macros (Delete, Module, Set, Print) collide with EterLib/Pool.h — the same clash 40250 sidestepped by commenting out PySymtable_Build. Two `// PORT:` edits: `unsigned char(val)` is an MSVC extension (PythonUtils.cpp, twice), and PyObject_AsCharBuffer wants a Py_ssize_t* (TraceFunc). port_python_launcher_test drives the launcher in the order UserInterface.cpp does (:241-434): Create(), __DEBUG__/__COMMAND_LINE__, TRUE/FALSE, a CRLF script through RunMemoryTextFile, a .pyc through RunCompiledFile, then RunFile reading system.py out of the real pack. It stops at `ImportError: No module named app`, i.e. the first C++ module RunMainScript registers — that is the 2V0 slice, and the stdlib behind it is step 3. The host flags (Py_NoSiteFlag and friends) sit in the test for now; step 3 moves them into the platform layer. port-map statuses are set only for what the test actually executes: 10 of 12 launcher functions and 4 of 11 marshal ones (the small-.pyc path delegates to CPython's own reader, so the ported r_object stays unreached). PythonUtils compiles but nothing calls it until the binding modules arrive, so it stays TODO. Windows has no mtpython yet, so port_logic drops ScriptLib and the shims there. Tests: macOS ctest 27/27 incl. port.python_launcher; port_logic (with ScriptLib) also compiles for android arm64 (NDK API 24) and ios arm64. Co-Authored-By: Claude Opus 5 --- audit/history.jsonl | 1 + .../ScriptLib/PythonLauncher.cpp.json | 84 +++ .../port-map/ScriptLib/PythonMarshal.cpp.json | 57 +++ audit/port-map/ScriptLib/PythonUtils.cpp.json | 65 +++ docs/PORT-PLAN.md | 16 +- extension/src/port/CMakeLists.txt | 22 +- .../src/port/ScriptLib/PythonLauncher.cpp | 282 +++++++++++ extension/src/port/ScriptLib/PythonLauncher.h | 25 + .../src/port/ScriptLib/PythonMarshal.cpp | 478 ++++++++++++++++++ extension/src/port/ScriptLib/PythonMarshal.h | 8 + extension/src/port/ScriptLib/PythonUtils.cpp | 425 ++++++++++++++++ extension/src/port/ScriptLib/PythonUtils.h | 28 + extension/src/port/ScriptLib/Resource.h | 24 + extension/src/port/ScriptLib/StdAfx.h | 54 ++ .../port/common/shim/sdk/Python-2.7/compile.h | 9 + .../port/common/shim/sdk/Python-2.7/errcode.h | 9 + .../port/common/shim/sdk/Python-2.7/eval.h | 9 + .../common/shim/sdk/Python-2.7/frameobject.h | 9 + .../port/common/shim/sdk/Python-2.7/grammar.h | 9 + .../common/shim/sdk/Python-2.7/longintrepr.h | 9 + .../port/common/shim/sdk/Python-2.7/marshal.h | 9 + .../port/common/shim/sdk/Python-2.7/node.h | 9 + .../common/shim/sdk/Python-2.7/parsetok.h | 13 + .../port/common/shim/sdk/Python-2.7/python.h | 7 + .../common/shim/sdk/Python-2.7/symtable.h | 16 + .../port/common/shim/sdk/Python-2.7/token.h | 9 + extension/tests/port_python_launcher_test.cpp | 188 +++++++ extension/third_party/CMakeLists.txt | 13 +- 28 files changed, 1879 insertions(+), 8 deletions(-) create mode 100644 audit/port-map/ScriptLib/PythonLauncher.cpp.json create mode 100644 audit/port-map/ScriptLib/PythonMarshal.cpp.json create mode 100644 audit/port-map/ScriptLib/PythonUtils.cpp.json create mode 100644 extension/src/port/ScriptLib/PythonLauncher.cpp create mode 100644 extension/src/port/ScriptLib/PythonLauncher.h create mode 100644 extension/src/port/ScriptLib/PythonMarshal.cpp create mode 100644 extension/src/port/ScriptLib/PythonMarshal.h create mode 100644 extension/src/port/ScriptLib/PythonUtils.cpp create mode 100644 extension/src/port/ScriptLib/PythonUtils.h create mode 100644 extension/src/port/ScriptLib/Resource.h create mode 100644 extension/src/port/ScriptLib/StdAfx.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/compile.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/errcode.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/eval.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/frameobject.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/grammar.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/longintrepr.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/marshal.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/node.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/parsetok.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/python.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/symtable.h create mode 100644 extension/src/port/common/shim/sdk/Python-2.7/token.h create mode 100644 extension/tests/port_python_launcher_test.cpp diff --git a/audit/history.jsonl b/audit/history.jsonl index 3e64132b..553a6f92 100644 --- a/audit/history.jsonl +++ b/audit/history.jsonl @@ -493,3 +493,4 @@ {"time":"2026-09-23T12:30:00Z","event":"port_round","unit":"proto reader -> 40250 item_proto/mob_proto via CLZO + pack://locale// (PORT-PLAN §5 step 2 + step 5 runtime switch)","ported":["EterPack/EterPack.cpp (16 fns)","EterPack/EterPackManager.cpp (14)","EterPack/EterPackPolicy_CSHybridCrypt.cpp (1)","EterBase/lzo.cpp (11)","EterBase/tea.cpp (2)","EterBase/Stl.cpp (2)"],"adapted":["UserInterface/UserInterface.cpp PackInitialize","EterBase/FileLoader.cpp (5)","EterBase/MappedFile.cpp (9)","EterBase/FileBase.cpp (5)","EterBase/CRC32.cpp GetCRC32"],"deleted":["extension/src/proto/proto.cpp m2dev reader: XChaCha20 + MCOZ, 236-byte item / 335-byte mob strides"],"divergent":[],"needs_live":[],"status_note":"statuses set only for the functions a clang coverage run of proto_test + port_eterpack_test hit; the rest stay TODO. SMobTable is a local #pragma pack(1) mirror in proto.cpp (static_assert 255) until batch 2P ports GameLib/PythonNonPlayer. name = cp1252(szLocaleName) as CItemData::GetName / CPythonNonPlayer::GetMonsterName","tests":["extension/tests/proto_test.cpp: 5743 items / 1334 mobs from pack://locale/en PASS","extension/tests/proto_item_layout_test.cpp 156/255 strides, 236/335 rejected PASS","godot netbridge/equip_model/remote_player/mob_view PASS"],"commit":"2D-proto"} {"time":"2026-09-23T13:30:00Z","event":"asset_gate","unit":"PORT-PLAN §5 steps 3-4: MT_ASSETS_STRICT + provenance manifest + script/run_40250_asset_gate.sh","manifest":"audit/assets/40250-client.json (pack/Index + 214 .eix/.epk + 8 Eternexus/root/*.msm = 223 files, 1407474045 bytes, sha256)","env":{"MT_40250_CLIENT":"40250 Client dir (native tests, AssetRoot.client_path, gate)","MT_ASSETS":"m2dev loose dev override only","M2_ASSETS":"no longer used by tests"},"fixes":["remote_player_test accepted only twohand_sword/wait.msa; WAIT picks wait/wait_1 by weight, so it failed ~1/3 of runs"],"tests":["script/run_40250_asset_gate.sh macOS PASS (manifest, ctest port.eterpack|proto.*, godot netbridge/equip_model/remote_player/mob_view)","gate with MT_40250_CLIENT=/nonexistent exits 1"],"commit":"2D-gate"} {"time": "2026-09-23T15:30:00Z", "event": "third_party", "unit": "PORT-PLAN 批次 2P step 1: vendored CPython 2.7.18 as the mtpython static library", "source": "official Python-2.7.18.tgz sha256 b62c0e7937551d0cc02b8fd5cb0f544f9405bafc9a54d3808ed4594812edef43, trimmed to 25MB (Doc/Demo/Tools/Mac/PCbuild, Modules/{_ctypes,expat,zlib,_sqlite}, stdlib test trees, pre-VS2010 PC projects); Lib/ kept for step 3's python27.zip", "shared": "source list = the 133 objects of the reference libpython2.7.a; built-in table config/config.c = 39 entries (config/Setup.static)", "per_platform": "config//pyconfig.h from tools/py_embed/gen_pyconfig.sh; android undefs HAVE_LANGINFO_H (nl_langinfo needs API 26); ios is derived from macos (2.7 configure refuses a darwin cross build) and drops process control + sys/random.h, keeping HAVE_GETGROUPS because posixmodule.c defines MAX_GROUPS only inside it", "patches": ["configure + configure.ac: arm64 MACOSX_DEFAULT_ARCH", "Modules/posixmodule.c: undef HAVE_SYSTEM/POPEN/FORK/EXECV/WAIT under TARGET_OS_IPHONE, which its own 'all other compilers' block hard-defines past pyconfig.h"], "not_done": ["Linux x86_64: gen_pyconfig.sh linux needs a Linux host", "Windows x64: upstream supports MSVC + PC/pyconfig.h while the Windows gate is MinGW, where PC/pyconfig.h leaves HAVE_UNISTD_H unset and posixmodule.c/dynload_win.c collide with the MinGW headers; decided in step 4"], "ported": [], "deleted": [], "divergent": [], "needs_live": [], "status_note": "no port-map entry: this is third-party source, not a ported 40250 unit. Linking into libmtgodot is step 2 (ScriptLib/PythonLauncher.cpp)", "tests": ["ctest python.embed (macOS arm64): 39 builtin modules, all import, sys.version 2.7.18, utf-8/cp1252 codecs + pickle round-trip via the vendored Lib/ PASS", "android arm64 NDK API 24: libmtpython.a builds clean", "ios arm64 iphoneos: libmtpython.a = 133 objects, no undefined _system/_fork/_popen"], "commit": "2P-step1"} +{"time": "2026-09-23T17:30:00Z", "event": "port_round", "unit": "PORT-PLAN 批次 2P step 2: ScriptLib/PythonLauncher.cpp into port_logic on the embedded interpreter", "ported": ["ScriptLib/PythonLauncher.cpp (10 of 12 fns)", "ScriptLib/PythonMarshal.cpp (4 of 11)"], "adapted": [], "deleted": [], "divergent": [], "needs_live": [], "status_note": "statuses set only for what port.python_launcher actually runs. PythonUtils.cpp is copied and compiled but nothing calls it yet (the binding modules are 2V0), so all 19 stay TODO; TraceFunc/SetTraceFunc are _DEBUG-only; in PythonMarshal only the small-.pyc path runs (r_long, _PyMarshal_ReadLongFromFile, getfilesize, _PyMarshal_ReadLastObjectFromFile) because that one delegates to CPython's own PyMarshal_ReadObjectFromString under 16KB, leaving the ported r_object reader unreached. Two // PORT: edits: `unsigned char(val)` -> a cast (MSVC extension) in PythonUtils.cpp x2, and PyObject_AsCharBuffer's Py_ssize_t* in TraceFunc. CPythonLauncher::GetError dereferences the NULL PyErr_Fetch yields when nothing is pending — kept verbatim, the test only calls it with an error set.", "shims": "extension/src/port/common/shim/sdk/Python-2.7/: 12 forwarding headers for 40250's #include ; symtable.h forward-declares mod_ty instead of including Python-ast.h, whose one-word macros (Delete, Module, Set, Print) collide with EterLib/Pool.h — 40250 sidestepped the same clash by commenting out PySymtable_Build", "tests": ["ctest port.python_launcher (macOS arm64): Create/RunLine/RunMemoryTextFile(CRLF)/RunCompiledFile(.pyc)/RunFile PASS; RunFile(\"system.py\") reads the real pack and stops at ImportError: No module named app (the 2V0 C++ modules)", "ctest macOS: 27/27 PASS", "android arm64 NDK API 24 + ios arm64 iphoneos: port_logic (incl. ScriptLib) compiles"], "commit": "2P-step2"} diff --git a/audit/port-map/ScriptLib/PythonLauncher.cpp.json b/audit/port-map/ScriptLib/PythonLauncher.cpp.json new file mode 100644 index 00000000..cc572fe7 --- /dev/null +++ b/audit/port-map/ScriptLib/PythonLauncher.cpp.json @@ -0,0 +1,84 @@ +{ + "reference": "ScriptLib/PythonLauncher.cpp", + "reference_sha256": "3dbaccfb21153cbbf201c2fe7252063455f6b9a9c79c1c55f9d03e8573bf65c8", + "priority": "P4", + "contracts": [], + "functions": { + "CPythonLauncher::CPythonLauncher": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::CPythonLauncher" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "CPythonLauncher::~CPythonLauncher": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::~CPythonLauncher" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "CPythonLauncher::Clear": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::Clear" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "Traceback": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:Traceback" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "TraceFunc": { + "status": "TODO" + }, + "CPythonLauncher::SetTraceFunc": { + "status": "TODO" + }, + "CPythonLauncher::Create": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::Create" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "CPythonLauncher::RunCompiledFile": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::RunCompiledFile" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "CPythonLauncher::RunMemoryTextFile": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::RunMemoryTextFile" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "CPythonLauncher::RunFile": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::RunFile" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "CPythonLauncher::RunLine": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::RunLine" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "CPythonLauncher::GetError": { + "impl": [ + "extension/src/port/ScriptLib/PythonLauncher.cpp:CPythonLauncher::GetError" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + } + } +} diff --git a/audit/port-map/ScriptLib/PythonMarshal.cpp.json b/audit/port-map/ScriptLib/PythonMarshal.cpp.json new file mode 100644 index 00000000..5f31ad0c --- /dev/null +++ b/audit/port-map/ScriptLib/PythonMarshal.cpp.json @@ -0,0 +1,57 @@ +{ + "reference": "ScriptLib/PythonMarshal.cpp", + "reference_sha256": "c12ee08f78c9070097b851a5d0a22be9999e12fc03e04c6da639c5f1bbd81f05", + "priority": "P4", + "contracts": [], + "functions": { + "r_string": { + "status": "TODO" + }, + "r_short": { + "status": "TODO" + }, + "r_long": { + "impl": [ + "extension/src/port/ScriptLib/PythonMarshal.cpp:r_long" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "r_long64": { + "status": "TODO" + }, + "r_object": { + "status": "TODO" + }, + "_PyMarshal_ReadShortFromFile": { + "status": "TODO" + }, + "_PyMarshal_ReadLongFromFile": { + "impl": [ + "extension/src/port/ScriptLib/PythonMarshal.cpp:_PyMarshal_ReadLongFromFile" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "getfilesize": { + "impl": [ + "extension/src/port/ScriptLib/PythonMarshal.cpp:getfilesize" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "_PyMarshal_ReadLastObjectFromFile": { + "impl": [ + "extension/src/port/ScriptLib/PythonMarshal.cpp:_PyMarshal_ReadLastObjectFromFile" + ], + "status": "PORTED", + "test": "extension/tests/port_python_launcher_test.cpp (port.python_launcher)" + }, + "_PyMarshal_ReadObjectFromFile": { + "status": "TODO" + }, + "_PyMarshal_ReadObjectFromString": { + "status": "TODO" + } + } +} diff --git a/audit/port-map/ScriptLib/PythonUtils.cpp.json b/audit/port-map/ScriptLib/PythonUtils.cpp.json new file mode 100644 index 00000000..723b9745 --- /dev/null +++ b/audit/port-map/ScriptLib/PythonUtils.cpp.json @@ -0,0 +1,65 @@ +{ + "reference": "ScriptLib/PythonUtils.cpp", + "reference_sha256": "ede046a6a16563cb65976c8c99199f401a29fe9614c07a32e577176c2827aaba", + "priority": "P4", + "contracts": [], + "functions": { + "Py_BadArgument": { + "status": "TODO" + }, + "Py_BuildException": { + "status": "TODO" + }, + "Py_BuildNone": { + "status": "TODO" + }, + "Py_ReleaseNone": { + "status": "TODO" + }, + "PyTuple_GetObject": { + "status": "TODO" + }, + "PyTuple_GetLong": { + "status": "TODO" + }, + "PyTuple_GetDouble": { + "status": "TODO" + }, + "PyTuple_GetFloat": { + "status": "TODO" + }, + "PyTuple_GetByte": { + "status": "TODO" + }, + "PyTuple_GetInteger": { + "status": "TODO" + }, + "PyTuple_GetUnsignedLong": { + "status": "TODO" + }, + "PyTuple_GetUnsignedInteger": { + "status": "TODO" + }, + "PyTuple_GetString": { + "status": "TODO" + }, + "PyTuple_GetBoolean": { + "status": "TODO" + }, + "PyCallClassMemberFunc": { + "status": "TODO" + }, + "PyCallClassMemberFunc_ByPyString": { + "status": "TODO" + }, + "__PyCallClassMemberFunc_ByCString": { + "status": "TODO" + }, + "__PyCallClassMemberFunc_ByPyString": { + "status": "TODO" + }, + "__PyCallClassMemberFunc": { + "status": "TODO" + } + } +} diff --git a/docs/PORT-PLAN.md b/docs/PORT-PLAN.md index d239a016..4a866fda 100644 --- a/docs/PORT-PLAN.md +++ b/docs/PORT-PLAN.md @@ -82,7 +82,7 @@ extension/third_party/cpython-2.7.18/ # 静态库（2P 批次 | — | 定下目录结构，评估内嵌 Python；Android 独立程序验证 | 完成（9d0e50de、b02c49bb） | | **2A** | 基础：Win32 类型层、参考公共头的最小闭包、`port_logic` CMake 目标、platform 接口骨架、头文件可编译门禁、port-map 重新基线 | **进行中**（剩 Python 库闭包头，等 2P） | | 2R | 资源包能力盘点：EPK 类型/密钥/覆盖顺序/路径大小写/移动端交付 | 完成：只有 NONE/COMPRESS/SECURITY，不需要服务器密钥；设备上直接读 EPK | -| **2P** | CPython 2.7.18 编进 libmtgodot，五个平台分别配置和验证 | **进行中**：step 1（静态库 + macOS/Android/iOS 配置）完成 | +| **2P** | CPython 2.7.18 编进 libmtgodot，五个平台分别配置和验证 | **进行中**：step 1（静态库 + macOS/Android/iOS 配置）、step 2（ScriptLib 进 `port_logic`）完成 | | 2D | 数据源切到 40250（msm 路径、proto、资源根、严格资源测试） | 完成（5a26e93f、232d0461、240d2827、a9b31dec）；资源根只切了 proto/pack 后端，其余按第 5 节由 2V 切片逐个切换 | | **2V0** | UI 壳：真实 `PythonLauncher/wndMgr/grp/app` + platform UI，其他玩法模块用桩，显示并操作 Logo/Popup | 需要 2A、2P 和 root/uiscript 子集 | | **2V1** | 登录/选角：真实 `net` phase、登录/选角脚本及所需模块，走到 Loading | 2V0 之后 | @@ -286,6 +286,14 @@ extension/third_party/cpython-2.7.18/ # 静态库（2P 批次 `extension/tests/py_embed_test.cpp`（ctest `python.embed`）。三处 vendor 补丁见 `docs/THIRD-PARTY.md`。 Linux/Windows 见下表。 2. 静态链接进 `libmtgodot`，照抄 `ScriptLib/PythonLauncher.cpp`。 + **已完成**：`ScriptLib/{PythonLauncher,PythonUtils,PythonMarshal}.{h,cpp}` 进入 `port_logic`（`mt3p::python`）， + `common/shim/sdk/Python-2.7/` 12 个转发头让 40250 的 `#include ` 原样编译 + （`symtable.h` 自带 `mod_ty` 前向声明，绕开 `Python-ast.h` 的 `Delete`/`Module` 等单词宏与 `EterLib/Pool.h` 冲突）。 + `extension/tests/port_python_launcher_test.cpp`（ctest `port.python_launcher`）按 `UserInterface.cpp:241-434` 的顺序跑： + `Create()` → `__DEBUG__`/`__COMMAND_LINE__` → `TRUE/FALSE` → `RunMemoryTextFile`（CRLF）→ `RunCompiledFile`（.pyc） + → `RunFile("system.py")` 从真 pack 里取脚本，停在 `ImportError: No module named app`，即 2V0 的 C++ 模块。 + `Py_NoSiteFlag` 等宿主标志目前在测试里设置，step 3 移进 platform 层。Windows 没有 `mtpython`， + `port_logic` 会自动排除 ScriptLib 和这些转发头。 3. `pack` 模块通过 `asset_io` 读取 40250 的 root/uiscript。标准库沿用已验证的 `python27.zip + sys.path`： - 桌面端把 zip 放在 CPython 可读的真实文件系统路径； - Android/iOS 首次启动从应用资源复制到应用沙盒，校验提交在资源清单中的 sha256，再加入 `sys.path`； @@ -295,9 +303,9 @@ extension/third_party/cpython-2.7.18/ # 静态库（2P 批次 | 平台 | 状态 | | --- | --- | - | macOS arm64 | step 1 完成：`mtpython` 静态库 + `python.embed` 测试（39 个内建模块全部 import，标准库 codec/pickle 通过）；`system.py` 在 app 进程内未做 | - | Android arm64 | step 1 完成：NDK API 24 交叉编译出 `libmtpython.a`；独立可执行文件在真机跑通（b02c49bb）；APK 进程内未做 | - | iOS arm64 | step 1 完成：iphoneos arm64 编出 133 个目标文件，无 `_system`/`_fork`/`_popen` 未定义符号；真机未跑 | + | macOS arm64 | step 1、2 完成：`mtpython` 静态库 + `python.embed`（39 个内建模块全部 import，标准库 codec/pickle 通过）+ `port.python_launcher`（`CPythonLauncher` 跑到 `system.py` 的第一个 import）；`system.py` 在 app 进程内未做 | + | Android arm64 | step 1 完成：NDK API 24 交叉编译出 `libmtpython.a`；独立可执行文件在真机跑通（b02c49bb）；APK 进程内未做。step 2 的 ScriptLib 只在 macOS 编译过 | + | iOS arm64 | step 1 完成：iphoneos arm64 编出 133 个目标文件，无 `_system`/`_fork`/`_popen` 未定义符号；真机未跑。step 2 的 ScriptLib 只在 macOS 编译过 | | Linux x86_64 | 未做（`gen_pyconfig.sh linux` 需要 Linux 主机，当前无） | | Windows x64 | 未做：上游 2.7 只支持 MSVC + `PC/pyconfig.h`，而本项目 Windows 门禁是 MinGW 交叉编译，`PC/pyconfig.h` 的 gnu-win32 分支不设 `HAVE_UNISTD_H`，`posixmodule.c`/`dynload_win.c` 与 MinGW 头文件冲突，需要一套补丁——在本步骤（step 4）决定走 MSVC 还是打补丁 | diff --git a/extension/src/port/CMakeLists.txt b/extension/src/port/CMakeLists.txt index a83f753e..6e71353d 100644 --- a/extension/src/port/CMakeLists.txt +++ b/extension/src/port/CMakeLists.txt @@ -9,6 +9,12 @@ if(WIN32) # Stand-ins for Windows SDK headers; on Windows the real ones are used. list(FILTER MT_PORT_HEADERS EXCLUDE REGEX "/common/shim/win32/") endif() +# The script layer needs the embedded interpreter (mt3p::python), which has no Windows build yet +# (PORT-PLAN 批次 2P step 4). Without it, ScriptLib and its shims are left out. +if(NOT TARGET mtpython) + list(FILTER MT_PORT_SOURCES EXCLUDE REGEX "/ScriptLib/") + list(FILTER MT_PORT_HEADERS EXCLUDE REGEX "/ScriptLib/|/common/shim/sdk/Python-2\\.7/") +endif() add_library(port_logic STATIC ${MT_PORT_SOURCES}) target_include_directories(port_logic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) @@ -20,8 +26,11 @@ if(NOT WIN32) endif() target_include_directories(port_logic PUBLIC ${MT_PORT_SHIMS}) # Libraries 40250 links that are vendored as-is: (EterBase/lzo.h), -# (EterBase/cipher.h). +# (EterBase/cipher.h), (ScriptLib). target_link_libraries(port_logic PUBLIC mt3p::minilzo mt3p::cryptopp) +if(TARGET mtpython) + target_link_libraries(port_logic PUBLIC mt3p::python) +endif() # 40250 sources carry MSVC `#pragma warning(...)` lines verbatim, and non-literal printf formats # (EterImageLib/StdAfx.h) that the Android toolchain turns into errors. set(MT_PORT_WARNING_FLAGS) @@ -123,6 +132,9 @@ endforeach() add_library(port_header_gate OBJECT ${MT_PORT_GATE_SOURCES}) target_include_directories(port_header_gate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${MT_PORT_SHIMS}) target_link_libraries(port_header_gate PRIVATE mt3p::minilzo mt3p::cryptopp) +if(TARGET mtpython) + target_link_libraries(port_header_gate PRIVATE mt3p::python) +endif() target_compile_features(port_header_gate PRIVATE cxx_std_20) target_compile_definitions(port_header_gate PRIVATE "NANOBEGIN=" "NANOEND=") if(MINGW) @@ -148,4 +160,12 @@ if(BUILD_TESTING AND CMAKE_SYSTEM_NAME STREQUAL CMAKE_HOST_SYSTEM_NAME) target_link_libraries(port_eterpack_test PRIVATE port_platform) add_test(NAME port.eterpack COMMAND $ ${MT_40250_CLIENT}) set_tests_properties(port.eterpack PROPERTIES SKIP_RETURN_CODE 77) + + if(TARGET mtpython) + add_executable(port_python_launcher_test ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/port_python_launcher_test.cpp) + target_link_libraries(port_python_launcher_test PRIVATE port_platform) + add_test(NAME port.python_launcher + COMMAND $ ${MT_40250_CLIENT}) + set_tests_properties(port.python_launcher PROPERTIES SKIP_RETURN_CODE 77) + endif() endif() diff --git a/extension/src/port/ScriptLib/PythonLauncher.cpp b/extension/src/port/ScriptLib/PythonLauncher.cpp new file mode 100644 index 00000000..d738c3b6 --- /dev/null +++ b/extension/src/port/ScriptLib/PythonLauncher.cpp @@ -0,0 +1,282 @@ +#include "StdAfx.h" +#include +#include "../EterPack/EterPackManager.h" + +#include "PythonLauncher.h" + +CPythonLauncher::CPythonLauncher() +{ + Py_Initialize(); +} + +CPythonLauncher::~CPythonLauncher() +{ + Clear(); +} + +void CPythonLauncher::Clear() +{ + Py_Finalize(); +} + +std::string g_stTraceBuffer[512]; +int g_nCurTraceN = 0; + +void Traceback() +{ + std::string str; + + for (int i = 0; i < g_nCurTraceN; ++i) + { + str.append(g_stTraceBuffer[i]); + str.append("\n"); + } + + PyObject * exc; + PyObject * v; + PyObject * tb; + const char * errStr; + + PyErr_Fetch(&exc, &v, &tb); + + if (PyString_Check(v)) + { + errStr = PyString_AS_STRING(v); + str.append("Error: "); + str.append(errStr); + + Tracef("%s\n", errStr); + } + Py_DECREF(exc); + Py_DECREF(v); + Py_DECREF(tb); + LogBoxf("Traceback:\n\n%s\n", str.c_str()); +} + +int TraceFunc(PyObject * obj, PyFrameObject * f, int what, PyObject *arg) +{ + const char * funcname; + char szTraceBuffer[128]; + + switch (what) + { + case PyTrace_CALL: + if (g_nCurTraceN >= 512) + return 0; + + if (Py_OptimizeFlag) + f->f_lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + + funcname = PyString_AsString(f->f_code->co_name); + + _snprintf(szTraceBuffer, sizeof(szTraceBuffer), "Call: File \"%s\", line %d, in %s", + PyString_AsString(f->f_code->co_filename), + f->f_lineno, + funcname); + + g_stTraceBuffer[g_nCurTraceN++]=szTraceBuffer; + break; + + case PyTrace_RETURN: + if (g_nCurTraceN > 0) + --g_nCurTraceN; + break; + + case PyTrace_EXCEPTION: + if (g_nCurTraceN >= 512) + return 0; + + PyObject * exc_type, * exc_value, * exc_traceback; + + PyTuple_GetObject(arg, 0, &exc_type); + PyTuple_GetObject(arg, 1, &exc_value); + PyTuple_GetObject(arg, 2, &exc_traceback); + + Py_ssize_t len; // PORT: PyObject_AsCharBuffer takes Py_ssize_t*, which is `int` only in a 32-bit build + const char * exc_str; + PyObject_AsCharBuffer(exc_type, &exc_str, &len); + + _snprintf(szTraceBuffer, sizeof(szTraceBuffer), "Exception: File \"%s\", line %d, in %s", + PyString_AS_STRING(f->f_code->co_filename), + f->f_lineno, + PyString_AS_STRING(f->f_code->co_name)); + + g_stTraceBuffer[g_nCurTraceN++]=szTraceBuffer; + + break; + } + return 0; +} + +void CPythonLauncher::SetTraceFunc(int (*pFunc)(PyObject * obj, PyFrameObject * f, int what, PyObject *arg)) +{ + PyEval_SetTrace(pFunc, NULL); +} + +bool CPythonLauncher::Create(const char* c_szProgramName) +{ + NANOBEGIN + Py_SetProgramName((char*)c_szProgramName); +#ifdef _DEBUG + PyEval_SetTrace(TraceFunc, NULL); +#endif + m_poModule = PyImport_AddModule((char *) "__main__"); + + if (!m_poModule) + return false; + + m_poDic = PyModule_GetDict(m_poModule); + + PyObject * builtins = PyImport_ImportModule("__builtin__"); + PyModule_AddIntConstant(builtins, "TRUE", 1); + PyModule_AddIntConstant(builtins, "FALSE", 0); + PyDict_SetItemString(m_poDic, "__builtins__", builtins); + Py_DECREF(builtins); + + if (!RunLine("import __main__")) + return false; + + if (!RunLine("import sys")) + return false; + + NANOEND + return true; +} + +bool CPythonLauncher::RunCompiledFile(const char* c_szFileName) +{ + NANOBEGIN + FILE * fp = fopen(c_szFileName, "rb"); + + if (!fp) + return false; + + PyCodeObject *co; + PyObject *v; + long magic; + long PyImport_GetMagicNumber(void); + + magic = _PyMarshal_ReadLongFromFile(fp); + + if (magic != PyImport_GetMagicNumber()) + { + PyErr_SetString(PyExc_RuntimeError, "Bad magic number in .pyc file"); + fclose(fp); + return false; + } + + _PyMarshal_ReadLongFromFile(fp); + v = _PyMarshal_ReadLastObjectFromFile(fp); + + fclose(fp); + + if (!v || !PyCode_Check(v)) + { + Py_XDECREF(v); + PyErr_SetString(PyExc_RuntimeError, "Bad code object in .pyc file"); + return false; + } + + co = (PyCodeObject *) v; + v = PyEval_EvalCode(co, m_poDic, m_poDic); +/* if (v && flags) + flags->cf_flags |= (co->co_flags & PyCF_MASK);*/ + Py_DECREF(co); + if (!v) + { + Traceback(); + return false; + } + + Py_DECREF(v); + if (Py_FlushLine()) + PyErr_Clear(); + + NANOEND + return true; +} + + +bool CPythonLauncher::RunMemoryTextFile(const char* c_szFileName, UINT uFileSize, const VOID* c_pvFileData) +{ + NANOBEGIN + const CHAR* c_pcFileData=(const CHAR*)c_pvFileData; + + std::string stConvFileData; + stConvFileData.reserve(uFileSize); + stConvFileData+="exec(compile('''"; + + // ConvertPythonTextFormat + { + for (UINT i=0; i + +#include "../EterBase/Singleton.h" + +class CPythonLauncher : public CSingleton +{ + public: + CPythonLauncher(); + virtual ~CPythonLauncher(); + + void Clear(); + + bool Create(const char* c_szProgramName="eter.python"); + void SetTraceFunc(int (*pFunc)(PyObject * obj, PyFrameObject * f, int what, PyObject *arg)); + bool RunLine(const char* c_szLine); + bool RunFile(const char* c_szFileName); + bool RunMemoryTextFile(const char* c_szFileName, UINT uFileSize, const VOID* c_pvFileData); + bool RunCompiledFile(const char* c_szFileName); + const char* GetError(); + + protected: + PyObject* m_poModule; + PyObject* m_poDic; +}; diff --git a/extension/src/port/ScriptLib/PythonMarshal.cpp b/extension/src/port/ScriptLib/PythonMarshal.cpp new file mode 100644 index 00000000..286093a9 --- /dev/null +++ b/extension/src/port/ScriptLib/PythonMarshal.cpp @@ -0,0 +1,478 @@ +/* Write Python objects to files and read them back. + This is intended for writing and reading compiled Python code only; + a true persistent storage facility would be much harder, since + it would have to take circular links and sharing into account. */ + +#include "StdAfx.h" +#include + +/* High water mark to determine when the marshalled object is dangerously deep + * and risks coring the interpreter. When the object stack gets this deep, + * raise an exception instead of continuing. + */ +#define MAX_MARSHAL_STACK_DEPTH 5000 + +#define TYPE_NULL '0' +#define TYPE_NONE 'N' +#define TYPE_STOPITER 'S' +#define TYPE_ELLIPSIS '.' +#define TYPE_INT 'i' +#define TYPE_INT64 'I' +#define TYPE_FLOAT 'f' +#define TYPE_COMPLEX 'x' +#define TYPE_LONG 'l' +#define TYPE_STRING 's' +#define TYPE_TUPLE '(' +#define TYPE_LIST '[' +#define TYPE_DICT '{' +#define TYPE_CODE 'c' +#define TYPE_UNICODE 'u' +#define TYPE_UNKNOWN '?' + +typedef struct +{ + FILE * fp; + int error; + int depth; + PyObject * str; + char * ptr; + char * end; +} WFILE; + +typedef WFILE RFILE; /* Same struct with different invariants */ + +#define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF) + +#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p)) + +static int r_string(char *s, int n, RFILE *p) +{ + if (p->fp != NULL) + return fread(s, 1, n, p->fp); + if (p->end - p->ptr < n) + n = p->end - p->ptr; + memcpy(s, p->ptr, n); + p->ptr += n; + return n; +} + +static int r_short(RFILE *p) +{ + short x; + x = (short) r_byte(p); + x |= (short) r_byte(p) << 8; + /* Sign-extension, in case short greater than 16 bits */ + x |= -(x & 0x8000); + return x; +} + +static long r_long(RFILE *p) +{ + long x; + FILE *fp = p->fp; + if (fp) { + x = getc(fp); + x |= (long)getc(fp) << 8; + x |= (long)getc(fp) << 16; + x |= (long)getc(fp) << 24; + } + else { + x = rs_byte(p); + x |= (long)rs_byte(p) << 8; + x |= (long)rs_byte(p) << 16; + x |= (long)rs_byte(p) << 24; + } +#if SIZEOF_LONG > 4 + /* Sign extension for 64-bit machines */ + x |= -(x & 0x80000000L); +#endif + return x; +} + +/* r_long64 deals with the TYPE_INT64 code. On a machine with + sizeof(long) > 4, it returns a Python int object, else a Python long + object. Note that w_long64 writes out TYPE_INT if 32 bits is enough, + so there's no inefficiency here in returning a PyLong on 32-bit boxes + for everything written via TYPE_INT64 (i.e., if an int is written via + TYPE_INT64, it *needs* more than 32 bits). +*/ +static PyObject * r_long64(RFILE *p) +{ + long lo4 = r_long(p); + long hi4 = r_long(p); +#if SIZEOF_LONG > 4 + long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL); + return PyInt_FromLong(x); +#else + unsigned char buf[8]; + int one = 1; + int is_little_endian = (int)*(char*)&one; + if (is_little_endian) { + memcpy(buf, &lo4, 4); + memcpy(buf+4, &hi4, 4); + } + else { + memcpy(buf, &hi4, 4); + memcpy(buf+4, &lo4, 4); + } + return _PyLong_FromByteArray(buf, 8, is_little_endian, 1); +#endif +} + +static PyObject * r_object(RFILE *p) +{ + PyObject *v, *v2; + long i, n; + int type = r_byte(p); + + switch (type) { + + case EOF: + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + return NULL; + + case TYPE_NULL: + return NULL; + + case TYPE_NONE: + Py_INCREF(Py_None); + return Py_None; + + case TYPE_STOPITER: + Py_INCREF(PyExc_StopIteration); + return PyExc_StopIteration; + + case TYPE_ELLIPSIS: + Py_INCREF(Py_Ellipsis); + return Py_Ellipsis; + + case TYPE_INT: + return PyInt_FromLong(r_long(p)); + + case TYPE_INT64: + return r_long64(p); + + case TYPE_LONG: + { + int size; + PyLongObject* ob; + n = r_long(p); + size = n<0 ? -n : n; + ob = _PyLong_New(size); + if (ob == NULL) + return NULL; + ob->ob_size = n; + for (i = 0; i < size; i++) + ob->ob_digit[i] = (short) r_short(p); + return (PyObject *) ob; + } + + case TYPE_FLOAT: + { + char buf[256]; + double dx; + n = r_byte(p); + if (r_string(buf, (int)n, p) != n) { + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + return NULL; + } + buf[n] = '\0'; + PyFPE_START_PROTECT("atof", return 0) + dx = atof(buf); + PyFPE_END_PROTECT(dx) + return PyFloat_FromDouble(dx); + } + +#ifndef WITHOUT_COMPLEX + case TYPE_COMPLEX: + { + char buf[256]; + Py_complex c; + n = r_byte(p); + if (r_string(buf, (int)n, p) != n) { + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + return NULL; + } + buf[n] = '\0'; + PyFPE_START_PROTECT("atof", return 0) + c.real = atof(buf); + PyFPE_END_PROTECT(c) + n = r_byte(p); + if (r_string(buf, (int)n, p) != n) { + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + return NULL; + } + buf[n] = '\0'; + PyFPE_START_PROTECT("atof", return 0) + c.imag = atof(buf); + PyFPE_END_PROTECT(c) + return PyComplex_FromCComplex(c); + } +#endif + + case TYPE_STRING: + n = r_long(p); + if (n < 0) { + PyErr_SetString(PyExc_ValueError, "bad marshal data"); + return NULL; + } + v = PyString_FromStringAndSize((char *)NULL, n); + if (v != NULL) { + if (r_string(PyString_AS_STRING(v), (int)n, p) != n) { + Py_DECREF(v); + v = NULL; + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + } + } + return v; + +#ifdef Py_USING_UNICODE + case TYPE_UNICODE: + { + char *buffer; + + n = r_long(p); + if (n < 0) { + PyErr_SetString(PyExc_ValueError, "bad marshal data"); + return NULL; + } + buffer = PyMem_NEW(char, n); + if (buffer == NULL) + return PyErr_NoMemory(); + if (r_string(buffer, (int)n, p) != n) { + PyMem_DEL(buffer); + PyErr_SetString(PyExc_EOFError, + "EOF read where object expected"); + return NULL; + } + v = PyUnicode_DecodeUTF8(buffer, n, NULL); + PyMem_DEL(buffer); + return v; + } +#endif + + case TYPE_TUPLE: + n = r_long(p); + if (n < 0) { + PyErr_SetString(PyExc_ValueError, "bad marshal data"); + return NULL; + } + v = PyTuple_New((int)n); + if (v == NULL) + return v; + for (i = 0; i < n; i++) { + v2 = r_object(p); + if ( v2 == NULL ) { + Py_DECREF(v); + v = NULL; + break; + } + PyTuple_SET_ITEM(v, (int)i, v2); + } + return v; + + case TYPE_LIST: + n = r_long(p); + if (n < 0) { + PyErr_SetString(PyExc_ValueError, "bad marshal data"); + return NULL; + } + v = PyList_New((int)n); + if (v == NULL) + return v; + for (i = 0; i < n; i++) { + v2 = r_object(p); + if ( v2 == NULL ) { + Py_DECREF(v); + v = NULL; + break; + } + PyList_SetItem(v, (int)i, v2); + } + return v; + + case TYPE_DICT: + v = PyDict_New(); + if (v == NULL) + return NULL; + for (;;) { + PyObject *key, *val; + key = r_object(p); + if (key == NULL) + break; /* XXX Assume TYPE_NULL, not an error */ + val = r_object(p); + if (val != NULL) + PyDict_SetItem(v, key, val); + Py_DECREF(key); + Py_XDECREF(val); + } + return v; + + case TYPE_CODE: + if (PyEval_GetRestricted()) { + PyErr_SetString(PyExc_RuntimeError, + "cannot unmarshal code objects in " + "restricted execution mode"); + return NULL; + } + else { + int argcount = r_short(p); + int nlocals = r_short(p); + int stacksize = r_short(p); + int flags = r_short(p); + PyObject *code = NULL; + PyObject *consts = NULL; + PyObject *names = NULL; + PyObject *varnames = NULL; + PyObject *freevars = NULL; + PyObject *cellvars = NULL; + PyObject *filename = NULL; + PyObject *name = NULL; + int firstlineno = 0; + PyObject *lnotab = NULL; + + code = r_object(p); + if (code) consts = r_object(p); + if (consts) names = r_object(p); + if (names) varnames = r_object(p); + if (varnames) freevars = r_object(p); + if (freevars) cellvars = r_object(p); + if (cellvars) filename = r_object(p); + if (filename) name = r_object(p); + if (name) { + firstlineno = r_short(p); + lnotab = r_object(p); + } + + if (!PyErr_Occurred()) { + v = (PyObject *) PyCode_New( + argcount, nlocals, stacksize, flags, + code, consts, names, varnames, + freevars, cellvars, filename, name, + firstlineno, lnotab); + } + else + v = NULL; + Py_XDECREF(code); + Py_XDECREF(consts); + Py_XDECREF(names); + Py_XDECREF(varnames); + Py_XDECREF(freevars); + Py_XDECREF(cellvars); + Py_XDECREF(filename); + Py_XDECREF(name); + Py_XDECREF(lnotab); + + } + return v; + + default: + /* Bogus data got written, which isn't ideal. + This will let you keep working and recover. */ + PyErr_SetString(PyExc_ValueError, "bad marshal data"); + return NULL; + } +} + +int _PyMarshal_ReadShortFromFile(FILE *fp) +{ + RFILE rf; + rf.fp = fp; + return r_short(&rf); +} + +long _PyMarshal_ReadLongFromFile(FILE *fp) +{ + RFILE rf; + rf.fp = fp; + return r_long(&rf); +} + +/* Return size of file in bytes; < 0 if unknown. */ +static off_t getfilesize(FILE *fp) +{ + struct stat st; + if (fstat(fileno(fp), &st) != 0) + return -1; + else + return st.st_size; +} + +/* If we can get the size of the file up-front, and it's reasonably small, + * read it in one gulp and delegate to ...FromString() instead. Much quicker + * than reading a byte at a time from file; speeds .pyc imports. + * CAUTION: since this may read the entire remainder of the file, don't + * call it unless you know you're done with the file. + */ +PyObject *_PyMarshal_ReadLastObjectFromFile(FILE *fp) +{ + /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT. + * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. + */ +#define SMALL_FILE_LIMIT (1L << 14) +#define REASONABLE_FILE_LIMIT (1L << 18) + + off_t filesize; + + if (PyErr_Occurred()) + { + fprintf(stderr, "XXX rd_object called with exception set\n"); + return NULL; + } + + filesize = getfilesize(fp); + if (filesize > 0) { + char buf[SMALL_FILE_LIMIT]; + char* pBuf = NULL; + if (filesize <= SMALL_FILE_LIMIT) + pBuf = buf; + else if (filesize <= REASONABLE_FILE_LIMIT) + pBuf = (char *)PyMem_MALLOC(filesize); + if (pBuf != NULL) { + PyObject* v; + size_t n = fread(pBuf, 1, filesize, fp); + v = PyMarshal_ReadObjectFromString(pBuf, n); + if (pBuf != buf) + PyMem_FREE(pBuf); + return v; + } + + } + + /* We don't have fstat, or we do but the file is larger than + * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time. + */ + return _PyMarshal_ReadObjectFromFile(fp); +#undef SMALL_FILE_LIMIT +#undef REASONABLE_FILE_LIMIT +} + +PyObject * _PyMarshal_ReadObjectFromFile(FILE *fp) +{ + RFILE rf; + if (PyErr_Occurred()) { + fprintf(stderr, "XXX rd_object called with exception set\n"); + return NULL; + } + rf.fp = fp; + return r_object(&rf); +} + +PyObject * _PyMarshal_ReadObjectFromString(char *str, int len) +{ + RFILE rf; + if (PyErr_Occurred()) { + fprintf(stderr, "XXX rds_object called with exception set\n"); + return NULL; + } + rf.fp = NULL; + rf.str = NULL; + rf.ptr = str; + rf.end = str + len; + return r_object(&rf); +} diff --git a/extension/src/port/ScriptLib/PythonMarshal.h b/extension/src/port/ScriptLib/PythonMarshal.h new file mode 100644 index 00000000..7ef6c1ce --- /dev/null +++ b/extension/src/port/ScriptLib/PythonMarshal.h @@ -0,0 +1,8 @@ +#ifndef __INC_ETERMARSHAL_H__ +#define __INC_ETERMARSHAL_H__ + +extern PyObject * _PyMarshal_ReadObjectFromFile(FILE* fp); +extern PyObject * _PyMarshal_ReadLastObjectFromFile(FILE* fp); +extern long _PyMarshal_ReadLongFromFile(FILE *fp); + +#endif diff --git a/extension/src/port/ScriptLib/PythonUtils.cpp b/extension/src/port/ScriptLib/PythonUtils.cpp new file mode 100644 index 00000000..424b989a --- /dev/null +++ b/extension/src/port/ScriptLib/PythonUtils.cpp @@ -0,0 +1,425 @@ +#include "StdAfx.h" +#include "PythonUtils.h" + +IPythonExceptionSender * g_pkExceptionSender = NULL; + +bool __PyCallClassMemberFunc_ByCString(PyObject* poClass, const char* c_szFunc, PyObject* poArgs, PyObject** poRet); +bool __PyCallClassMemberFunc_ByPyString(PyObject* poClass, PyObject* poFuncName, PyObject* poArgs, PyObject** poRet); +bool __PyCallClassMemberFunc(PyObject* poClass, PyObject* poFunc, PyObject* poArgs, PyObject** poRet); + +PyObject * Py_BadArgument() +{ + PyErr_BadArgument(); + return NULL; +} + +PyObject * Py_BuildException(const char * c_pszErr, ...) +{ + if (!c_pszErr) + PyErr_Clear(); + else + { + char szErrBuf[512+1]; + va_list args; + va_start(args, c_pszErr); + vsnprintf(szErrBuf, sizeof(szErrBuf), c_pszErr, args); + va_end(args); + + PyErr_SetString(PyExc_RuntimeError, szErrBuf); + } + + return Py_BuildNone(); + //return NULL; +} + +PyObject * Py_BuildNone() +{ + Py_INCREF(Py_None); + return Py_None; +} + +void Py_ReleaseNone() +{ + Py_DECREF(Py_None); +} + +bool PyTuple_GetObject(PyObject* poArgs, int pos, PyObject** ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject * poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = poItem; + return true; +} + +bool PyTuple_GetLong(PyObject* poArgs, int pos, long* ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject* poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = PyLong_AsLong(poItem); + return true; +} + +bool PyTuple_GetDouble(PyObject* poArgs, int pos, double* ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject* poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = PyFloat_AsDouble(poItem); + return true; +} + +bool PyTuple_GetFloat(PyObject* poArgs, int pos, float* ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject * poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = float(PyFloat_AsDouble(poItem)); + return true; +} + +bool PyTuple_GetByte(PyObject* poArgs, int pos, unsigned char* ret) +{ + int val; + bool result = PyTuple_GetInteger(poArgs,pos,&val); + *ret = (unsigned char) val; // PORT: `unsigned char(val)` is an MSVC extension; C++ has no multi-word function-style cast + return result; +} + +bool PyTuple_GetInteger(PyObject* poArgs, int pos, unsigned char* ret) +{ + int val; + bool result = PyTuple_GetInteger(poArgs,pos,&val); + *ret = (unsigned char) val; // PORT: `unsigned char(val)` is an MSVC extension; C++ has no multi-word function-style cast + return result; +} + +bool PyTuple_GetInteger(PyObject* poArgs, int pos, WORD* ret) +{ + int val; + bool result = PyTuple_GetInteger(poArgs,pos,&val); + *ret = WORD(val); + return result; +} + +bool PyTuple_GetInteger(PyObject* poArgs, int pos, int* ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject* poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = PyLong_AsLong(poItem); + return true; +} + +bool PyTuple_GetUnsignedLong(PyObject* poArgs, int pos, unsigned long* ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject * poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = PyLong_AsUnsignedLong(poItem); + return true; +} + +bool PyTuple_GetUnsignedInteger(PyObject* poArgs, int pos, unsigned int* ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject* poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = PyLong_AsUnsignedLong(poItem); + return true; +} + +bool PyTuple_GetString(PyObject* poArgs, int pos, char** ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject* poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + if (!PyString_Check(poItem)) + return false; + + *ret = PyString_AsString(poItem); + return true; +} + +bool PyTuple_GetBoolean(PyObject* poArgs, int pos, bool* ret) +{ + if (pos >= PyTuple_Size(poArgs)) + return false; + + PyObject* poItem = PyTuple_GetItem(poArgs, pos); + + if (!poItem) + return false; + + *ret = PyLong_AsLong(poItem) ? true : false; + return true; +} + +bool PyCallClassMemberFunc(PyObject* poClass, PyObject* poFunc, PyObject* poArgs) +{ + PyObject* poRet; + + // NOTE : NULL 체크 추가.. - [levites] + if (!poClass) + { + Py_XDECREF(poArgs); + return false; + } + + if (!__PyCallClassMemberFunc(poClass, poFunc, poArgs, &poRet)) + return false; + + Py_DECREF(poRet); + return true; +} + +bool PyCallClassMemberFunc(PyObject* poClass, const char* c_szFunc, PyObject* poArgs) +{ + PyObject* poRet; + + // NOTE : NULL 체크 추가.. - [levites] + if (!poClass) + { + Py_XDECREF(poArgs); + return false; + } + + if (!__PyCallClassMemberFunc_ByCString(poClass, c_szFunc, poArgs, &poRet)) + return false; + + Py_DECREF(poRet); + return true; +} + +bool PyCallClassMemberFunc_ByPyString(PyObject* poClass, PyObject* poFuncName, PyObject* poArgs) +{ + PyObject* poRet; + + // NOTE : NULL 체크 추가.. - [levites] + if (!poClass) + { + Py_XDECREF(poArgs); + return false; + } + + if (!__PyCallClassMemberFunc_ByPyString(poClass, poFuncName, poArgs, &poRet)) + return false; + + Py_DECREF(poRet); + return true; +} + +bool PyCallClassMemberFunc(PyObject* poClass, const char* c_szFunc, PyObject* poArgs, bool* pisRet) +{ + PyObject* poRet; + + if (!__PyCallClassMemberFunc_ByCString(poClass, c_szFunc, poArgs, &poRet)) + return false; + + if (PyNumber_Check(poRet)) + *pisRet = (PyLong_AsLong(poRet) != 0); + else + *pisRet = true; + + Py_DECREF(poRet); + return true; +} + +bool PyCallClassMemberFunc(PyObject* poClass, const char* c_szFunc, PyObject* poArgs, long * plRetValue) +{ + PyObject* poRet; + + if (!__PyCallClassMemberFunc_ByCString(poClass, c_szFunc, poArgs, &poRet)) + return false; + + if (PyNumber_Check(poRet)) + { + *plRetValue = PyLong_AsLong(poRet); + Py_DECREF(poRet); + return true; + } + + Py_DECREF(poRet); + return false; +} + +/* + * 이 함수를 직접 호출하지 않도록 한다. + * 부득이 하게 직접 호출할 경우에는 반드시 false 가 리턴 됐을 때 + * Py_DECREF(poArgs); 를 해준다. + */ +bool __PyCallClassMemberFunc_ByCString(PyObject* poClass, const char* c_szFunc, PyObject* poArgs, PyObject** ppoRet) +{ + if (!poClass) + { + Py_XDECREF(poArgs); + return false; + } + + PyObject * poFunc = PyObject_GetAttrString(poClass, (char *)c_szFunc); // New Reference + + if (!poFunc) + { + PyErr_Clear(); + Py_XDECREF(poArgs); + return false; + } + + if (!PyCallable_Check(poFunc)) + { + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return false; + } + + PyObject * poRet = PyObject_CallObject(poFunc, poArgs); // New Reference + + if (!poRet) + { + if (g_pkExceptionSender) + g_pkExceptionSender->Clear(); + + PyErr_Print(); + + if (g_pkExceptionSender) + g_pkExceptionSender->Send(); + + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return false; + } + + *ppoRet = poRet; + + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return true; +} + +bool __PyCallClassMemberFunc_ByPyString(PyObject* poClass, PyObject* poFuncName, PyObject* poArgs, PyObject** ppoRet) +{ + if (!poClass) + { + Py_XDECREF(poArgs); + return false; + } + + PyObject * poFunc = PyObject_GetAttr(poClass, poFuncName); // New Reference + + if (!poFunc) + { + PyErr_Clear(); + Py_XDECREF(poArgs); + return false; + } + + if (!PyCallable_Check(poFunc)) + { + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return false; + } + + PyObject * poRet = PyObject_CallObject(poFunc, poArgs); // New Reference + + if (!poRet) + { + if (g_pkExceptionSender) + g_pkExceptionSender->Clear(); + + PyErr_Print(); + + if (g_pkExceptionSender) + g_pkExceptionSender->Send(); + + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return false; + } + + *ppoRet = poRet; + + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return true; +} + +bool __PyCallClassMemberFunc(PyObject* poClass, PyObject * poFunc, PyObject* poArgs, PyObject** ppoRet) +{ + if (!poClass) + { + Py_XDECREF(poArgs); + return false; + } + + if (!poFunc) + { + PyErr_Clear(); + Py_XDECREF(poArgs); + return false; + } + + if (!PyCallable_Check(poFunc)) + { + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return false; + } + + PyObject * poRet = PyObject_CallObject(poFunc, poArgs); // New Reference + + if (!poRet) + { + PyErr_Print(); + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return false; + } + + *ppoRet = poRet; + + Py_DECREF(poFunc); + Py_XDECREF(poArgs); + return true; +} \ No newline at end of file diff --git a/extension/src/port/ScriptLib/PythonUtils.h b/extension/src/port/ScriptLib/PythonUtils.h new file mode 100644 index 00000000..4055a765 --- /dev/null +++ b/extension/src/port/ScriptLib/PythonUtils.h @@ -0,0 +1,28 @@ +#pragma once + +#define SET_EXCEPTION(x) PyErr_SetString(PyExc_RuntimeError, #x) + +bool PyTuple_GetString(PyObject* poArgs, int pos, char** ret); +bool PyTuple_GetInteger(PyObject* poArgs, int pos, unsigned char* ret); +bool PyTuple_GetInteger(PyObject* poArgs, int pos, int* ret); +bool PyTuple_GetInteger(PyObject* poArgs, int pos, WORD* ret); +bool PyTuple_GetByte(PyObject* poArgs, int pos, unsigned char* ret); +bool PyTuple_GetUnsignedInteger(PyObject* poArgs, int pos, unsigned int* ret); +bool PyTuple_GetLong(PyObject* poArgs, int pos, long* ret); +bool PyTuple_GetUnsignedLong(PyObject* poArgs, int pos, unsigned long* ret); +bool PyTuple_GetFloat(PyObject* poArgs, int pos, float* ret); +bool PyTuple_GetDouble(PyObject* poArgs, int pos, double* ret); +bool PyTuple_GetObject(PyObject* poArgs, int pos, PyObject** ret); +bool PyTuple_GetBoolean(PyObject* poArgs, int pos, bool* ret); + +bool PyCallClassMemberFunc(PyObject* poClass, const char* c_szFunc, PyObject* poArgs); +bool PyCallClassMemberFunc(PyObject* poClass, const char* c_szFunc, PyObject* poArgs, bool* pisRet); +bool PyCallClassMemberFunc(PyObject* poClass, const char* c_szFunc, PyObject* poArgs, long * plRetValue); + +bool PyCallClassMemberFunc_ByPyString(PyObject* poClass, PyObject* poFuncName, PyObject* poArgs); +bool PyCallClassMemberFunc(PyObject* poClass, PyObject* poFunc, PyObject* poArgs); + +PyObject * Py_BuildException(const char * c_pszErr = NULL, ...); +PyObject * Py_BadArgument(); +PyObject * Py_BuildNone(); +PyObject * Py_BuildEmptyTuple(); diff --git a/extension/src/port/ScriptLib/Resource.h b/extension/src/port/ScriptLib/Resource.h new file mode 100644 index 00000000..1bfecde8 --- /dev/null +++ b/extension/src/port/ScriptLib/Resource.h @@ -0,0 +1,24 @@ +#pragma once + +#include "../EffectLib/StdAfx.h" +#include "../EterLib/Resource.h" +#include "../EterLib/ResourceManager.h" + +enum EResourceTypes +{ + RES_TYPE_UNKNOWN, +}; + +class CPythonResource : public CSingleton +{ + public: + CPythonResource(); + virtual ~CPythonResource(); + + void Destroy(); + + void DumpFileList(const char * c_szFileName); + + protected: + CResourceManager m_resManager; +}; \ No newline at end of file diff --git a/extension/src/port/ScriptLib/StdAfx.h b/extension/src/port/ScriptLib/StdAfx.h new file mode 100644 index 00000000..aa228681 --- /dev/null +++ b/extension/src/port/ScriptLib/StdAfx.h @@ -0,0 +1,54 @@ +#pragma once + +#include "../EterLib/StdAfx.h" +#include "../EterGrnLib/StdAfx.h" + +//#include +#ifdef _DEBUG + #undef _DEBUG + #include + #define _DEBUG +#else + #include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "PythonUtils.h" +#include "PythonLauncher.h" +#include "PythonMarshal.h" +#include "Resource.h" + +void initdbg(); + +// PYTHON_EXCEPTION_SENDER +class IPythonExceptionSender +{ + public: + void Clear() + { + m_strExceptionString = ""; + } + + void RegisterExceptionString(const char * c_szString) + { + m_strExceptionString += c_szString; + } + + virtual void Send() = 0; + + protected: + std::string m_strExceptionString; +}; + +extern IPythonExceptionSender * g_pkExceptionSender; + +void SetExceptionSender(IPythonExceptionSender * pkExceptionSender); +// END_OF_PYTHON_EXCEPTION_SENDER diff --git a/extension/src/port/common/shim/sdk/Python-2.7/compile.h b/extension/src/port/common/shim/sdk/Python-2.7/compile.h new file mode 100644 index 00000000..3d9bb9ee --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/compile.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/errcode.h b/extension/src/port/common/shim/sdk/Python-2.7/errcode.h new file mode 100644 index 00000000..836f0099 --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/errcode.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/eval.h b/extension/src/port/common/shim/sdk/Python-2.7/eval.h new file mode 100644 index 00000000..d492c764 --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/eval.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/frameobject.h b/extension/src/port/common/shim/sdk/Python-2.7/frameobject.h new file mode 100644 index 00000000..d1970448 --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/frameobject.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/grammar.h b/extension/src/port/common/shim/sdk/Python-2.7/grammar.h new file mode 100644 index 00000000..6e1da91c --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/grammar.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/longintrepr.h b/extension/src/port/common/shim/sdk/Python-2.7/longintrepr.h new file mode 100644 index 00000000..42f66eba --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/longintrepr.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/marshal.h b/extension/src/port/common/shim/sdk/Python-2.7/marshal.h new file mode 100644 index 00000000..4a7e016c --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/marshal.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/node.h b/extension/src/port/common/shim/sdk/Python-2.7/node.h new file mode 100644 index 00000000..ca3dfa9c --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/node.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/parsetok.h b/extension/src/port/common/shim/sdk/Python-2.7/parsetok.h new file mode 100644 index 00000000..0200b308 --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/parsetok.h @@ -0,0 +1,13 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +// parsetok.h names `node` and `grammar` without including their headers; 40250's StdAfx includes +// node.h and grammar.h before it, in this order. +#include +#include +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/python.h b/extension/src/port/common/shim/sdk/Python-2.7/python.h new file mode 100644 index 00000000..af392901 --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/python.h @@ -0,0 +1,7 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of . +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/symtable.h b/extension/src/port/common/shim/sdk/Python-2.7/symtable.h new file mode 100644 index 00000000..cc083bc5 --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/symtable.h @@ -0,0 +1,16 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +// CPython's Include/symtable.h is not self-contained: it names mod_ty, which only Python-ast.h +// declares — and that header also defines one-word macros (Module, Delete, Set, Print, Num, ...) +// that collide with 40250 identifiers such as CPool::Delete. 40250's own extern/include copy +// comments the one declaration that needs mod_ty (PySymtable_Build) out; repeating Python-ast.h's +// typedef keeps the upstream header intact and the macros out. +struct _mod; +typedef struct _mod *mod_ty; +#include diff --git a/extension/src/port/common/shim/sdk/Python-2.7/token.h b/extension/src/port/common/shim/sdk/Python-2.7/token.h new file mode 100644 index 00000000..235917ba --- /dev/null +++ b/extension/src/port/common/shim/sdk/Python-2.7/token.h @@ -0,0 +1,9 @@ +#pragma once +// Shim for 40250's : the client includes the CPython headers under a +// Python-2.7/ prefix (its extern/include layout). The vendored interpreter +// (extension/third_party/cpython-2.7.18, target mtpython) installs them flat, so each header +// here forwards to the real one. comes first in 40250's StdAfx, exactly +// as CPython requires of ; this shim includes it first so the header also stands +// alone under the port header gate. +#include +#include diff --git a/extension/tests/port_python_launcher_test.cpp b/extension/tests/port_python_launcher_test.cpp new file mode 100644 index 00000000..29d8ce0e --- /dev/null +++ b/extension/tests/port_python_launcher_test.cpp @@ -0,0 +1,188 @@ +// port/ScriptLib CPythonLauncher on the embedded interpreter (批次 2P step 2), driven exactly as +// 40250 UserInterface.cpp does it: Main() creates the launcher, calls Create(), and RunMainScript() +// then sets __DEBUG__ and __COMMAND_LINE__ with RunLine before RunFile("system.py") reads the script +// out of the registered packs (UserInterface.cpp:241-353, 419-434). +// +// The script layer's own modules (initapp, initpack, initwndMgr, ...) arrive with the 2V0 slice and the +// stdlib with step 3, so system.py is expected to stop at its first import ("No module named app") — +// what this test proves is the launcher: the interpreter starts, RunLine/RunMemoryTextFile/ +// RunCompiledFile run code, and RunFile reaches the real script through CEterPackManager. +// +// port_python_launcher_test <40250 Client dir> +// +// Exit 77 (ctest SKIP) when the client is missing, unless MT_ASSETS_STRICT=1, which fails instead. +#include "ScriptLib/StdAfx.h" +#include "ScriptLib/PythonLauncher.h" +#include "EterPack/EterPackManager.h" +#include "../src/platform/UserInterface/UserInterface.h" + +#include +#include +#include +#include +#include + +#include + +static int g_failures = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "%s:%d: CHECK(%s)\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +// The dictionary the launcher runs everything in, and the globals it leaves behind there. +static PyObject* main_dict() +{ + return PyModule_GetDict(PyImport_AddModule((char*) "__main__")); +} + +static long main_long(const char* name) +{ + PyObject* main_module = PyImport_AddModule((char*) "__main__"); + PyObject* value = PyDict_GetItemString(PyModule_GetDict(main_module), name); + return value && PyInt_Check(value) ? PyInt_AsLong(value) : -1; +} + +static std::string main_string(const char* name) +{ + PyObject* main_module = PyImport_AddModule((char*) "__main__"); + PyObject* value = PyDict_GetItemString(PyModule_GetDict(main_module), name); + return value && PyString_Check(value) ? PyString_AS_STRING(value) : ""; +} + +int main(int argc, char** argv) +{ + const char* strict = std::getenv("MT_ASSETS_STRICT"); + const bool is_strict = strict && std::string(strict) == "1"; + const std::string client = argc > 1 ? argv[1] : ""; + if (client.empty() || chdir(client.c_str()) != 0 || access("pack/Index", 0) != 0) + { + std::fprintf(stderr, "port_python_launcher_test: no 40250 Client/pack at '%s'\n", client.c_str()); + return is_strict ? 1 : 77; + } + + PackSingletons(); + CHECK(PackInitialize("pack")); + + // Host policy, set before the launcher's ctor runs Py_Initialize: 40250 shipped a Python with a real + // prefix on disk, the embedded one has none, so `import site` would fail at startup. The flags are the + // ones tools/py_embed_android/main.c uses; step 3 moves them, and the stdlib on sys.path, into the + // platform layer that every host shares. + Py_NoSiteFlag = 1; + Py_IgnoreEnvironmentFlag = 1; + Py_FrozenFlag = 1; + Py_DontWriteBytecodeFlag = 1; + Py_SetProgramName((char*) "eter.python"); // CPythonLauncher::Create's default program name + + { + CPythonLauncher launcher; // ctor runs Py_Initialize, as 40250 Main() does on the stack + CHECK(Py_IsInitialized()); + CHECK(launcher.Create()); + + // RegisterDebugFlag / RegisterCommandLine, verbatim from RunMainScript. + CHECK(launcher.RunLine("__DEBUG__ = 0")); + CHECK(launcher.RunLine("__COMMAND_LINE__ = \"\"")); + CHECK(main_long("__DEBUG__") == 0); + CHECK(main_string("__COMMAND_LINE__") == ""); + + // Create() adds TRUE/FALSE to __builtin__, which the 40250 scripts use unqualified. + CHECK(launcher.RunLine("__ok__ = (TRUE, FALSE)")); + CHECK(launcher.RunLine("assert TRUE == 1 and FALSE == 0")); + + // A failing line reports through Traceback(), which fetches (and so clears) the exception. + CHECK(!launcher.RunLine("raise RuntimeError('port test')")); + CHECK(PyErr_Occurred() == NULL); + + // GetError() reads the pending exception's value with neither normalization nor a NULL check, + // so it is only defined while an error really is pending and was set with a string value — after + // RunLine it never is, because Traceback() got there first. Kept verbatim: it is what 40250 does. + PyErr_SetString(PyExc_RuntimeError, "port test error"); + CHECK(std::string(launcher.GetError()) == "port test error"); + PyErr_Clear(); + + // RunMemoryTextFile wraps the text in exec(compile(...)) after dropping CR, so a CRLF script + // from the pack compiles and the file name it is given reaches the code object. + const char crlf_script[] = "__mem__ = 1\r\ndef f():\r\n\treturn f.func_code.co_filename\r\n__mem__ = f()\r\n"; + CHECK(launcher.RunMemoryTextFile("crlf.py", sizeof(crlf_script) - 1, crlf_script)); + CHECK(main_string("__mem__") == "crlf.py"); + + // RunCompiledFile reads a .pyc through the ported PythonMarshal readers: magic, timestamp, + // then the code object. + const std::string pyc = std::string(std::getenv("TMPDIR") ? std::getenv("TMPDIR") : "/tmp") + + "/port_python_launcher_test.pyc"; + PyObject* code = Py_CompileString("__pyc__ = 7\n", "compiled.py", Py_file_input); + CHECK(code != NULL); + if (code) + { + FILE* fp = fopen(pyc.c_str(), "wb"); + CHECK(fp != NULL); + if (fp) + { + PyMarshal_WriteLongToFile(PyImport_GetMagicNumber(), fp, Py_MARSHAL_VERSION); + PyMarshal_WriteLongToFile(0, fp, Py_MARSHAL_VERSION); // mtime, ignored on read + PyMarshal_WriteObjectToFile(code, fp, Py_MARSHAL_VERSION); + fclose(fp); + CHECK(launcher.RunCompiledFile(pyc.c_str())); + CHECK(main_long("__pyc__") == 7); + } + Py_DECREF(code); + std::remove(pyc.c_str()); + } + CHECK(!launcher.RunCompiledFile("no/such.pyc")); + + // RunFile pulls the script out of the packs. system.py is the 40250 entry point; without the + // stdlib (step 3) and the C++ modules (2V0) it stops at its first import, which is still proof + // that the pack read path and the compile reached Python. + { + CMappedFile file; + const VOID* data = NULL; + CHECK(CEterPackManager::Instance().Get(file, "system.py", &data)); + CHECK(file.Size() > 0); + + // Where it stops, spelled out: RunFile reports through Traceback(), which fetches the + // exception and so leaves nothing behind, and LogBoxf is a platform stub. Running the same + // source here once names it: today "No module named app", i.e. system.py compiles and runs + // until the first of the C++ modules RunMainScript registers (initapp, 2V0 slice) — the + // stdlib imports behind it come with step 3. + std::string source((const char*) data, file.Size()); + source.erase(std::remove(source.begin(), source.end(), '\r'), source.end()); + PyObject* result = PyRun_String(source.c_str(), Py_file_input, main_dict(), main_dict()); + CHECK(result == NULL); + if (result) + Py_DECREF(result); + else + { + PyObject *exc = NULL, *value = NULL, *tb = NULL; + PyErr_Fetch(&exc, &value, &tb); + PyErr_NormalizeException(&exc, &value, &tb); + PyObject* text = value ? PyObject_Str(value) : NULL; + std::printf("system.py stops at %s: %s\n", + exc ? ((PyTypeObject*) exc)->tp_name : "?", + text ? PyString_AsString(text) : ""); + CHECK(exc == PyExc_ImportError); + Py_XDECREF(text); + Py_XDECREF(exc); + Py_XDECREF(value); + Py_XDECREF(tb); + } + } + // Whatever it stops at, RunLine's Traceback() has already reported and cleared it, so there is + // nothing left for GetError() to read here either. + const bool ran = launcher.RunFile("system.py"); + CHECK(!ran); + CHECK(PyErr_Occurred() == NULL); + CHECK(!launcher.RunFile("no/such/script.py")); // not in any pack: no file, no exception + + launcher.Clear(); // Main() calls Clear() explicitly before the launcher leaves scope + } + CHECK(!Py_IsInitialized()); + + if (g_failures) + std::fprintf(stderr, "%d failure(s)\n", g_failures); + else + std::printf("port_python_launcher_test: PASS\n"); + return g_failures ? 1 : 0; +} diff --git a/extension/third_party/CMakeLists.txt b/extension/third_party/CMakeLists.txt index 8c926ce0..bed07fb7 100644 --- a/extension/third_party/CMakeLists.txt +++ b/extension/third_party/CMakeLists.txt @@ -46,8 +46,15 @@ set_target_properties(minilzo PROPERTIES C_STANDARD 99) add_library(mt3p::minilzo ALIAS minilzo) # --- CPython 2.7.18 (embedded 40250 script layer, 批次 2P) ---------------------- -# Off by default: it only needs to build where the script layer is actually linked or tested. -option(MTGODOT_EMBED_PYTHON "Build the embedded CPython 2.7.18 static library" OFF) +# On everywhere the interpreter has a pyconfig.h: since step 2 the ported ScriptLib is part of +# port_logic. Windows has none yet (MinGW vs MSVC, see cpython-2.7.18/CMakeLists.txt), so the +# script layer is left out of that build until 批次 2P step 4 decides. +if(WIN32) + set(MT_EMBED_PYTHON_DEFAULT OFF) +else() + set(MT_EMBED_PYTHON_DEFAULT ON) +endif() +option(MTGODOT_EMBED_PYTHON "Build the embedded CPython 2.7.18 static library" ${MT_EMBED_PYTHON_DEFAULT}) if(MTGODOT_EMBED_PYTHON) - add_subdirectory(cpython-2.7.18 EXCLUDE_FROM_ALL) + add_subdirectory(cpython-2.7.18) endif()