port(2P step 3a/3b): stdlib on sys.path + the pack module

step 3a — 标准库:
  tools/py_embed/make_stdlib_zip.py packs the vendored Lib/ into
  <build>/python27.zip (397 modules, 6876547 bytes, ZIP_STORED — the zlib
  module is trimmed out of our CPython, so zipimport cannot inflate).
  The mtpython_stdlib target builds it; py_embed_test and the launcher test
  both run against it.
  platform/ScriptLib/PythonHost sets the embedding flags before
  Py_Initialize and, after it, puts the zip on sys.path and re-registers
  encodings.search_function with codecs — 2.7 builds the codec registry
  once, at startup, so replacing sys.path afterwards needs it back.

step 3b — pack 模块:
  UserInterface/PythonPackModule.cpp copied verbatim; packExist/packGet/
  initpack read root/ and uiscript/ through CEterPackManager, i.e. through
  asset_io. The CHINA_CRYPT_KEY block in that unit is CAccountConnector's
  and is kept #if 0 until the login slice (2V1). UserInterface/StdAfx.h is
  a documented partial mirror — the original is the whole client's PCH.

The launcher test now runs initpack() as RunMainScript does (its first
line), reads system.py / uiscript / locale text back out of the packs, and
confirms cp1252 decoding works off the zip. system.py still stops at
"No module named app" — the C++ modules are the 2V0 slice.

gates: port_map.py check 0 errors · key leak check 8/8 none · macOS ctest
27/27 · android + ios port_platform compile clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

extension/CMakeLists.txt

@@ -159,10 +159,9 @@ if(BUILD_TESTING AND MT_HOST_BUILD)
     if(TARGET mtpython)
         add_executable(py_embed_test tests/py_embed_test.cpp)
         target_link_libraries(py_embed_test PRIVATE mtpython)
-        # The stdlib the host puts on sys.path: here the vendored Lib/ directly (batch 2P step 3 ships
-        # it as python27.zip).
-        add_test(NAME python.embed COMMAND $<TARGET_FILE:py_embed_test>
-                 "${CMAKE_CURRENT_SOURCE_DIR}/third_party/cpython-2.7.18/Lib")
+        add_dependencies(py_embed_test mtpython_stdlib)
+        # The stdlib exactly as the host ships it: python27.zip, read through zipimport (批次 2P step 3).
+        add_test(NAME python.embed COMMAND $<TARGET_FILE:py_embed_test> "${MT_PYTHON_STDLIB_ZIP}")
     endif()
 endif()
 

extension/src/platform/ScriptLib/PythonHost.cpp

@@ -0,0 +1,106 @@
+// See PythonHost.h: the interpreter setup 40250 got from a real python27 installation.
+#include "ScriptLib/StdAfx.h"
+#include "PythonHost.h"
+
+#include <cstdio>
+#include <cstdlib>
+
+namespace
+{
+// The pending exception as text, so a failed bootstrap says what went wrong instead of leaving the
+// error for the first unrelated script to trip over.
+std::string take_error()
+{
+	if (!PyErr_Occurred())
+		return "no exception";
+
+	PyObject *type = NULL, *value = NULL, *tb = NULL;
+	PyErr_Fetch(&type, &value, &tb);
+	PyErr_NormalizeException(&type, &value, &tb);
+	std::string text;
+	if (PyObject* str = value ? PyObject_Str(value) : NULL)
+	{
+		if (PyString_Check(str))
+			text = PyString_AS_STRING(str);
+		Py_DECREF(str);
+	}
+	if (type && text.empty())
+		text = ((PyTypeObject*) type)->tp_name;
+	Py_XDECREF(type);
+	Py_XDECREF(value);
+	Py_XDECREF(tb);
+	return text.empty() ? "unknown error" : text;
+}
+
+bool fail(std::string* error, const std::string& text)
+{
+	if (error)
+		*error = text;
+	return false;
+}
+}
+
+namespace PythonHost
+{
+
+void Configure(const char* program_name)
+{
+	// site.py lives in a prefix this interpreter does not have, the environment must not be able to
+	// redirect a game's script layer (PYTHONPATH/PYTHONHOME), the "could not find platform libraries"
+	// probe has nothing to find, and the pack directories are read-only on the mobile targets.
+	Py_NoSiteFlag = 1;
+	Py_IgnoreEnvironmentFlag = 1;
+	Py_FrozenFlag = 1;
+	Py_DontWriteBytecodeFlag = 1;
+	Py_SetProgramName((char*) program_name);
+}
+
+bool InstallStdLib(const char* stdlib_path, std::string* error)
+{
+	if (!Py_IsInitialized())
+		return fail(error, "interpreter is not initialized");
+	if (!stdlib_path || !*stdlib_path)
+		return fail(error, "no stdlib path (set MT_PYTHON_STDLIB or pass python27.zip)");
+
+	// zipimport and the source importer both go through the C library, so the path has to be a real
+	// file: a pack entry or an Android asset must be staged out first (step 3c).
+	if (FILE* fp = fopen(stdlib_path, "rb"))
+		fclose(fp);
+	else if (FILE* dir = fopen((std::string(stdlib_path) + "/os.py").c_str(), "rb"))
+		fclose(dir);
+	else
+		return fail(error, std::string("cannot open ") + stdlib_path);
+
+	PySys_SetPath((char*) stdlib_path);
+
+	// Py_Initialize ran _PyCodecRegistry_Init against the then-empty path and 2.7 never retries it
+	// ("no codec search functions registered" for the rest of the process), so the search function is
+	// registered again now that encodings is importable.
+	PyObject* encodings = PyImport_ImportModule("encodings");
+	if (!encodings)
+		return fail(error, "import encodings: " + take_error());
+	PyObject* search = PyObject_GetAttrString(encodings, (char*) "search_function");
+	Py_DECREF(encodings);
+	if (!search)
+		return fail(error, "encodings.search_function: " + take_error());
+
+	PyObject* codecs = PyImport_ImportModule("codecs");
+	PyObject* registered = codecs ? PyObject_CallMethod(codecs, (char*) "register", (char*) "O", search) : NULL;
+	Py_DECREF(search);
+	Py_XDECREF(codecs);
+	if (!registered)
+		return fail(error, "codecs.register: " + take_error());
+	Py_DECREF(registered);
+	return true;
+}
+
+std::string DefaultStdLibPath()
+{
+	if (const char* env = getenv("MT_PYTHON_STDLIB"))
+		return env;
+	// Step 3c stages python27.zip out of the packed assets on Android/iOS and answers here; until then
+	// the caller passes the path (the build tree's copy, for the native tests).
+	return "";
+}
+
+}

extension/src/platform/ScriptLib/PythonHost.h

@@ -0,0 +1,27 @@
+#pragma once
+// The interpreter setup 40250 got for free from a real python27 installation.
+//
+// 40250's Main() only constructs CPythonLauncher: python27.dll found its own prefix, so `import site`,
+// the codec registry and every stdlib import worked before a single script ran. The embedded
+// interpreter has no prefix and no filesystem to probe, so the host does that part itself —
+// the flags before Py_Initialize, and the stdlib (python27.zip) on sys.path afterwards.
+// See docs/PORT-PLAN.md 批次 2P step 3.
+
+#include <string>
+
+namespace PythonHost
+{
+// Before the interpreter starts, i.e. before CPythonLauncher's constructor.
+// `program_name` is what CPythonLauncher::Create defaults to; it only names the interpreter.
+void Configure(const char* program_name = "eter.python");
+
+// After Py_Initialize: replaces sys.path with `stdlib_path` (a python27.zip or a stdlib directory
+// on a real filesystem — zipimport and the .py importer both need a path the C library can open)
+// and re-registers the codec search function. False leaves *error set.
+bool InstallStdLib(const char* stdlib_path, std::string* error);
+
+// The stdlib path this build was given: $MT_PYTHON_STDLIB when set, else the python27.zip next to
+// the executable, else "". Mobile staging (copy out of the app resources, checked against the
+// manifest sha256) is 批次 2P step 3c and will answer here too.
+std::string DefaultStdLibPath();
+}

extension/src/port/CMakeLists.txt

@@ -164,8 +164,9 @@ if(BUILD_TESTING AND CMAKE_SYSTEM_NAME STREQUAL CMAKE_HOST_SYSTEM_NAME)
     if(TARGET mtpython)
         add_executable(port_python_launcher_test ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/port_python_launcher_test.cpp)
         target_link_libraries(port_python_launcher_test PRIVATE port_platform)
+        add_dependencies(port_python_launcher_test mtpython_stdlib)
         add_test(NAME port.python_launcher
-                 COMMAND $<TARGET_FILE:port_python_launcher_test> ${MT_40250_CLIENT})
+                 COMMAND $<TARGET_FILE:port_python_launcher_test> ${MT_40250_CLIENT} ${MT_PYTHON_STDLIB_ZIP})
         set_tests_properties(port.python_launcher PROPERTIES SKIP_RETURN_CODE 77)
     endif()
 endif()

extension/src/port/UserInterface/PythonPackModule.cpp

@@ -0,0 +1,92 @@
+#include "StdAfx.h"
+#include "../EterPack/EterPackManager.h"
+#include "../EterBase/tea.h"
+
+// CHINA_CRYPT_KEY
+DWORD g_adwEncryptKey[4];
+DWORD g_adwDecryptKey[4];
+
+// PORT: the login cipher's key builder happens to live in this unit, but it is CAccountConnector's —
+// AccountConnector.h and the class come with the login slice (docs/PORT-PLAN.md 批次 2V1). Kept verbatim
+// and compiled out until then; the two keys above stay, they are what the network stream reads.
+#if 0
+#include "AccountConnector.h"
+
+inline const BYTE* GetKey_20050304Myevan()
+{
+	volatile static DWORD s_adwKey[1938];
+
+	volatile DWORD seed=1491971513;
+	for (UINT i=0; i<BYTE(seed); i++)
+	{
+		seed^=2148941891;
+		seed+=3592385981;
+		s_adwKey[i]=seed;
+	}
+
+	return (const BYTE*)s_adwKey;
+}
+
+//#include <eterCrypt.h>
+
+void CAccountConnector::__BuildClientKey_20050304Myevan()
+{
+	const BYTE * c_pszKey = GetKey_20050304Myevan();
+	memcpy(g_adwEncryptKey, c_pszKey+157, 16);
+
+	for (DWORD i = 0; i < 4; ++i)
+		g_adwEncryptKey[i] = random();
+
+	tea_encrypt((DWORD *) g_adwDecryptKey, (const DWORD *) g_adwEncryptKey, (const DWORD *) (c_pszKey+37), 16);
+//	TEA_Encrypt((DWORD *) g_adwDecryptKey, (const DWORD *) g_adwEncryptKey, (const DWORD *) (c_pszKey+37), 16);
+}
+#endif
+// END_OF_CHINA_CRYPT_KEY
+
+PyObject * packExist(PyObject * poSelf, PyObject * poArgs)
+{
+	char * strFileName;
+
+	if (!PyTuple_GetString(poArgs, 0, &strFileName))
+		return Py_BuildException();
+
+	return Py_BuildValue("i", CEterPackManager::Instance().isExist(strFileName)?1:0);
+}
+
+PyObject * packGet(PyObject * poSelf, PyObject * poArgs)
+{
+	char * strFileName;
+	
+	if (!PyTuple_GetString(poArgs, 0, &strFileName))
+		return Py_BuildException();
+
+	// 파이썬에서 읽어드리는 패킹 파일은 python 파일과 txt 파일에 한정한다
+	const char* pcExt = strrchr(strFileName, '.');
+	if (pcExt) // 확장자가 있고
+	{
+		if ((stricmp(pcExt, ".py") == 0) ||
+			(stricmp(pcExt, ".pyc") == 0) ||
+			(stricmp(pcExt, ".txt") == 0))
+		{
+			CMappedFile file;
+			const void * pData = NULL;
+
+			if (CEterPackManager::Instance().Get(file,strFileName,&pData))
+				return Py_BuildValue("s#",pData, file.Size());
+		}
+	}
+
+	return Py_BuildException();
+}
+
+void initpack()
+{
+	static PyMethodDef s_methods[] =
+	{
+		{ "Exist",		packExist,		METH_VARARGS },
+		{ "Get",		packGet,		METH_VARARGS },
+		{ NULL, NULL },		
+	};
+	
+	Py_InitModule("pack", s_methods);
+}

extension/src/port/UserInterface/StdAfx.h

@@ -0,0 +1,27 @@
+#pragma once
+// PORT: a partial mirror of 40250 UserInterface/StdAfx.h.
+//
+// The original is the precompiled header of the whole client: it includes eterLib, eterPythonLib,
+// gameLib, scriptLib, milesLib, EffectLib, PRTerrainLib and SpeedTreeLib's StdAfx.h, plus <dshow.h>
+// and <qedit.h>, Locale.h and GameType.h. Only scriptLib exists in the port tree today, so this file
+// carries the part the ported units actually use and grows with the 2V slices that bring the rest
+// (docs/PORT-PLAN.md 批次 2V0-2V3).
+
+#include "../ScriptLib/StdAfx.h"
+
+#define APP_NAME	"Metin 2"
+
+enum
+{
+	POINT_MAX_NUM = 255,
+	CHARACTER_NAME_MAX_LEN = 24,
+#if defined(LOCALE_SERVICE_JAPAN)
+	PLAYER_NAME_MAX_LEN = 16,
+#else
+	PLAYER_NAME_MAX_LEN = 12,
+#endif
+};
+
+// The Python module initializers RunMainScript calls, each defined in its own Python*Module.cpp.
+// Listed here as 40250 lists them; only the ported ones are defined.
+void initpack();

extension/tests/port_python_launcher_test.cpp

@@ -14,6 +14,8 @@
 #include "ScriptLib/StdAfx.h"
 #include "ScriptLib/PythonLauncher.h"
 #include "EterPack/EterPackManager.h"
+#include "UserInterface/StdAfx.h" // initpack
+#include "../src/platform/ScriptLib/PythonHost.h"
 #include "../src/platform/UserInterface/UserInterface.h"
 
 #include <algorithm>
@@ -58,6 +60,7 @@ int main(int argc, char** argv)
 	const char* strict = std::getenv("MT_ASSETS_STRICT");
 	const bool is_strict = strict && std::string(strict) == "1";
 	const std::string client = argc > 1 ? argv[1] : "";
+	const std::string stdlib = argc > 2 ? argv[2] : PythonHost::DefaultStdLibPath();
 	if (client.empty() || chdir(client.c_str()) != 0 || access("pack/Index", 0) != 0)
 	{
 		std::fprintf(stderr, "port_python_launcher_test: no 40250 Client/pack at '%s'\n", client.c_str());
@@ -67,21 +70,36 @@ int main(int argc, char** argv)
 	PackSingletons();
 	CHECK(PackInitialize("pack"));
 
-	// Host policy, set before the launcher's ctor runs Py_Initialize: 40250 shipped a Python with a real
-	// prefix on disk, the embedded one has none, so `import site` would fail at startup. The flags are the
-	// ones tools/py_embed_android/main.c uses; step 3 moves them, and the stdlib on sys.path, into the
-	// platform layer that every host shares.
-	Py_NoSiteFlag = 1;
-	Py_IgnoreEnvironmentFlag = 1;
-	Py_FrozenFlag = 1;
-	Py_DontWriteBytecodeFlag = 1;
-	Py_SetProgramName((char*) "eter.python"); // CPythonLauncher::Create's default program name
+	// What a real python27 installation did for 40250, the host does here: flags before Py_Initialize,
+	// stdlib on sys.path after it (platform/ScriptLib/PythonHost.cpp).
+	PythonHost::Configure();
 
 	{
 		CPythonLauncher launcher; // ctor runs Py_Initialize, as 40250 Main() does on the stack
 		CHECK(Py_IsInitialized());
 		CHECK(launcher.Create());
 
+		std::string stdlib_error;
+		CHECK(PythonHost::InstallStdLib(stdlib.c_str(), &stdlib_error));
+		if (!stdlib_error.empty())
+			std::fprintf(stderr, "InstallStdLib: %s\n", stdlib_error.c_str());
+		// The stdlib is what the script layer's imports and the CP949/cp1252 text in the packs need.
+		CHECK(launcher.RunLine("import os, string, types, codecs\nassert u'\\xe4'.encode('cp1252') == '\\xe4'"));
+
+		// RunMainScript's first line (UserInterface.cpp:243): the pack module, which is how system.py's
+		// import hook and the window system read root/ and uiscript/ out of the registered packs.
+		initpack();
+		CHECK(launcher.RunLine("import pack"));
+		CHECK(launcher.RunLine("assert pack.Exist('system.py') == 1"));
+		CHECK(launcher.RunLine("assert pack.Exist('no/such.py') == 0"));
+		CHECK(launcher.RunLine("__pack_len__ = len(pack.Get('system.py'))"));
+		CHECK(launcher.RunLine("assert len(pack.Get('uiscript/selectcharacterwindow.py')) > 0"));
+		CHECK(launcher.RunLine("assert len(pack.Get('locale/en/locale_game.txt')) > 0"));
+		// Only .py/.pyc/.txt come out of it, and a miss is None rather than an exception
+		// (packGet returns Py_BuildException(), which clears the error and builds None).
+		CHECK(launcher.RunLine("assert pack.Get('icon/item/00000.tga') is None"));
+		CHECK(launcher.RunLine("assert pack.Get('no/such.py') is None"));
+
 		// RegisterDebugFlag / RegisterCommandLine, verbatim from RunMainScript.
 		CHECK(launcher.RunLine("__DEBUG__ = 0"));
 		CHECK(launcher.RunLine("__COMMAND_LINE__ = \"\""));
@@ -141,6 +159,7 @@ int main(int argc, char** argv)
 			const VOID* data = NULL;
 			CHECK(CEterPackManager::Instance().Get(file, "system.py", &data));
 			CHECK(file.Size() > 0);
+			CHECK(main_long("__pack_len__") == (long) file.Size()); // pack.Get returned the same entry
 
 			// Where it stops, spelled out: RunFile reports through Traceback(), which fetches the
 			// exception and so leaves nothing behind, and LogBoxf is a platform stub. Running the same

extension/third_party/cpython-2.7.18/CMakeLists.txt

@@ -209,3 +209,16 @@ endif()
 find_package(Threads REQUIRED)
 target_link_libraries(mtpython PUBLIC Threads::Threads m)
 add_library(mt3p::python ALIAS mtpython)
+
+# --- the stdlib the host puts on sys.path (批次 2P step 3) ---
+# Built from the same vendored Lib/, so the interpreter and its library always match. MT_PYTHON_STDLIB_ZIP
+# is where every consumer (the native tests now, the mobile staging in step 3c) picks it up.
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
+set(MT_PYTHON_STDLIB_ZIP "${CMAKE_BINARY_DIR}/python27.zip" CACHE INTERNAL "stdlib zip for the embedded interpreter")
+file(GLOB_RECURSE MT_PYTHON_STDLIB_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/Lib/*.py")
+add_custom_command(OUTPUT "${MT_PYTHON_STDLIB_ZIP}"
+    COMMAND ${Python3_EXECUTABLE} "${CMAKE_SOURCE_DIR}/tools/py_embed/make_stdlib_zip.py"
+            "${CMAKE_CURRENT_SOURCE_DIR}/Lib" "${MT_PYTHON_STDLIB_ZIP}"
+    DEPENDS "${CMAKE_SOURCE_DIR}/tools/py_embed/make_stdlib_zip.py" ${MT_PYTHON_STDLIB_SOURCES}
+    COMMENT "Packing python27.zip from the vendored Lib/")
+add_custom_target(mtpython_stdlib ALL DEPENDS "${MT_PYTHON_STDLIB_ZIP}")