From 1f1004da79a64a1c078acc6dfa5c6e3731f44f25 Mon Sep 17 00:00:00 2001 From: shenlei Date: Wed, 23 Sep 2026 00:56:39 +0900 Subject: [PATCH] port 2D step 5a: EterPack/EterPackManager + EterBase file units against the real 40250 packs - Mirror copies: EterPack (EterPack, EterPackManager, EterPackCursor, CSHybridCrypt policy, Inline.h) and EterBase logic units tea/lzo/Timer/Stl/Random, with PORT-tagged width fixes (LONG index fields, static_assert(sizeof(TEterPackIndex) == 192), tea_word = Win32 unsigned long). - TEA pack keys are extracted from the reference EterPack.cpp at configure time into build/.../EterPackKeys.generated.h; nothing key-bearing is tracked. - Platform EterBase: FileBase (stdio, case-insensitive fallback), MappedFile (mmap / MapViewOfFile), CRC32, Debug, Utils StringPath/StringLowers. - Win32Crt: CreateDirectory/DeleteFile/_access/MAKEFOURCC; Win32MinMax.h min/max (force-included on MinGW, whose windows.h omits them in C++). - tests/port_eterpack_test: registers pack/Index like PackInitialize and checks 119 registrations / 103 packs + root / 54891 entries / 52609 paths / 2282 overrides with first-registered-wins, and that exactly the 4 short-layout SECURITY files fail to load. - 2R count corrected (136 -> 119 registrations, 103 distinct packs + root). Not runtime-reachable yet (asset_io pack backend is step 5b), so port-map statuses stay TODO. port_gate: macos/android/ios/windows PASS. Co-Authored-By: Claude Opus 5 --- audit/history.jsonl | 1 + docs/PORT-PLAN.md | 7 +- extension/src/platform/EterBase/CRC32.cpp | 146 +- extension/src/platform/EterBase/Debug.cpp | 59 +- extension/src/platform/EterBase/FileBase.cpp | 181 +- .../src/platform/EterBase/MappedFile.cpp | 263 ++- extension/src/platform/EterBase/Utils.cpp | 59 +- extension/src/port/CMakeLists.txt | 50 + extension/src/port/EterBase/Random.cpp | 49 + extension/src/port/EterBase/Stl.cpp | 92 + extension/src/port/EterBase/Timer.cpp | 156 ++ extension/src/port/EterBase/lzo.cpp | 411 +++++ extension/src/port/EterBase/tea.cpp | 101 ++ extension/src/port/EterBase/tea.h | 12 +- extension/src/port/EterPack/EterPack.cpp | 1605 +++++++++++++++++ extension/src/port/EterPack/EterPack.h | 13 +- .../src/port/EterPack/EterPackCursor.cpp | 59 + extension/src/port/EterPack/EterPackCursor.h | 25 + .../src/port/EterPack/EterPackManager.cpp | 621 +++++++ .../EterPack/EterPackPolicy_CSHybridCrypt.cpp | 503 ++++++ .../EterPack/EterPackPolicy_CSHybridCrypt.h | 94 + extension/src/port/EterPack/Inline.h | 34 + extension/src/port/EterPack/StdAfx.cpp | 2 + extension/src/port/common/Win32Crt.cpp | 23 + extension/src/port/common/Win32Crt.h | 14 + extension/src/port/common/Win32MinMax.h | 13 + .../shim/sdk/cryptopp/cryptoppLibLink.h | 3 + .../src/port/common/shim/sdk/lzo/lzoLibLink.h | 3 + extension/src/port/common/shim/win32/io.h | 3 + extension/tests/port_eterpack_test.cpp | 176 ++ script/port_gate.sh | 4 +- 31 files changed, 4640 insertions(+), 142 deletions(-) create mode 100644 extension/src/port/EterBase/Random.cpp create mode 100644 extension/src/port/EterBase/Stl.cpp create mode 100644 extension/src/port/EterBase/Timer.cpp create mode 100644 extension/src/port/EterBase/lzo.cpp create mode 100644 extension/src/port/EterBase/tea.cpp create mode 100644 extension/src/port/EterPack/EterPack.cpp create mode 100644 extension/src/port/EterPack/EterPackCursor.cpp create mode 100644 extension/src/port/EterPack/EterPackCursor.h create mode 100644 extension/src/port/EterPack/EterPackManager.cpp create mode 100644 extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.cpp create mode 100644 extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.h create mode 100644 extension/src/port/EterPack/Inline.h create mode 100644 extension/src/port/EterPack/StdAfx.cpp create mode 100644 extension/src/port/common/Win32MinMax.h create mode 100644 extension/src/port/common/shim/sdk/cryptopp/cryptoppLibLink.h create mode 100644 extension/src/port/common/shim/sdk/lzo/lzoLibLink.h create mode 100644 extension/src/port/common/shim/win32/io.h create mode 100644 extension/tests/port_eterpack_test.cpp diff --git a/audit/history.jsonl b/audit/history.jsonl index 2e9f1a8e..fc66cdf4 100644 --- a/audit/history.jsonl +++ b/audit/history.jsonl @@ -488,3 +488,4 @@ {"time":"2026-09-22T13:00:00Z","event":"port_map_rebaseline","unit":"UserInterface/PythonPlayerEventHandler.cpp","reset_to_todo":["CPythonPlayerEventHandler::GetSingleton","CPythonPlayerEventHandler::~CPythonPlayerEventHandler","CPythonPlayerEventHandler::OnMoving","CPythonPlayerEventHandler::OnMove","CPythonPlayerEventHandler::OnStop","CPythonPlayerEventHandler::CPythonPlayerEventHandler"],"reason":"new mirror architecture owns the CPythonPlayerEventHandler singleton in native code; prior GDScript counterparts are migration sources, not final mirror implementations","tests":["port_map.py check"]} {"time":"2026-09-22T15:00:00Z","event":"port_map_rebaseline","scope":"all units","totals":{"logic":3343,"python":5175,"platform":2091,"all":10609,"done":0,"divergent":1},"previous":{"logic":3285,"python":4890,"platform":2229,"all":10404},"changes":["cpp_functions: strip comments, accept indented qualified definitions inside namespace blocks, long parameter/initializer lists (40-line window), all-caps qualified ctors, skip constructor initializer entries","EterBase split by unit: tea/lzo/cipher/Random/Stl/Timer/Poly are logic, CPostIt/CRC32/Debug/FileBase/FileDir/FileLoader/MappedFile/TempFile/Utils/error stay platform; 8 obsolete platform stub files removed","audit/slices regenerated (layer fields; EterLib/Dimm.h now platform)"],"reason":"PORT-PLAN 2A step 6; inventory cross-checked against the clang-derived platform skeleton: every stub is inventoried except 22 declarations 40250 never defines and namespace/typedef spelling variants","tests":["port_map.py check","script/port_gate.sh (macos/android/ios/windows PASS, linux BLOCKED)"]} {"time":"2026-09-22T16:00:00Z","event":"pack_inventory","scope":"Client/pack (batch 2R)","tool":"tools/epk_scan/epk_scan.py","report":"audit/packs/2R-scan.json","totals":{"registered_packs":136,"entries":54891,"unique_paths":52609,"NONE":20727,"COMPRESS":29812,"SECURITY":4352,"PANAMA":0,"HYBRIDCRYPT":0,"decode_failures":0,"overrides":2282,"overrides_different":2194,"case_collisions":0},"findings":["first-registered pack wins (CEterFileDict multimap, VS2022 STL)","short-layout packer in metin2_patch_{sd,mineral,mundi}: 4 SECURITY files rejected by 40250 CRC check","no server-delivered pack keys needed"]} +{"time":"2026-09-23T10:00:00Z","event":"port_round","unit":"EterPack/*, EterBase/{tea,lzo,Timer,Stl,Random}.cpp + platform EterBase/{CRC32,FileBase,MappedFile,Debug,Utils}","ported":["EterPack/EterPack.cpp","EterPack/EterPackManager.cpp","EterPack/EterPackCursor.cpp","EterPack/EterPackPolicy_CSHybridCrypt.cpp","EterBase/tea.cpp","EterBase/lzo.cpp","EterBase/Timer.cpp","EterBase/Stl.cpp","EterBase/Random.cpp"],"status_note":"mirror copies + platform FileBase/MappedFile/CRC32/Debug/Utils; not runtime-reachable until the asset_io pack backend (PORT-PLAN §5 step 5), so port-map statuses stay TODO","divergent":[],"needs_live":[],"corrections":["2R registered_packs 136 was wrong: 124 Index pairs -> 119 RegisterPack successes, 103 distinct packs + root (PORT-PLAN, memory)"],"findings":["libc++ unordered_multimap also keeps first-registered-wins: 0 wrong winners over 2282 overrides","exactly the 4 short-layout SECURITY files fail CEterPackManager::Get","pack/Index spells Etc for ETC.eix: CEterPack::CreateIndexFile fopens directly, so the bootstrap must resolve the on-disk dbname"],"keys":"EterPack TEA keys generated at configure time into the build tree (EterPackKeys.generated.h), never committed","tests":["extension/tests/port_eterpack_test.cpp PASS (macOS, 52605 files / 2023459178 bytes)","script/port_gate.sh macos/android/ios/windows PASS"]} diff --git a/docs/PORT-PLAN.md b/docs/PORT-PLAN.md index 2c33bc4a..00a5e08c 100644 --- a/docs/PORT-PLAN.md +++ b/docs/PORT-PLAN.md @@ -209,7 +209,7 @@ extension/third_party/cpython-2.7.18/ # 静态库（2P 批次 （`pack/Index`）→ `CEterPackManager::RegisterPack`（每个包再注册 `_texcache`，最后 `RegisterRootPack("pack/root")`） → `CEterPack::__BuildIndex`（`.eix` 全部是 `MCOZ`：TEA + LZO 包着 `EPKD` v2，每条 192 字节）→ `Get2`。 -1. **类型统计**（Index 注册的 136 个包，54,891 条，52,609 个不同路径）： +1. **类型统计**（Index 注册到的 103 个包加 `root`，54,891 条，52,609 个不同路径）： | 类型 | 文件数 | 存储 | 解码后 | | --- | --- | --- | --- | @@ -225,7 +225,8 @@ extension/third_party/cpython-2.7.18/ # 静态库（2P 批次 移植后的 `EterPack.cpp` 需要这两组常量；按"密钥不进仓库"，由构建时从参考树生成一个 gitignore 的头文件，不直接写进源码。 3. **注册顺序与覆盖**： - `Index` 第 0 行是 `PACK`，后面是 (目录, 名字) 对，共 124 个名字。 - - 实际存在的包有 136 个：Index 里的包，加上两个 `metin2_patch_dragon_rock*_texcache`，再加 `root`。 + - 124 对里有 119 次 `RegisterPack` 找到了包（有几个名字重复出现），去重后 103 个包（含两个 + `metin2_patch_dragon_rock*_texcache`），再加 `root`。2D 用移植后的 `CEterPackManager` 复核过（原先写的 136 有误）。 - 缺失的包：106 个 `_texcache`，以及 `locale_{ae,bg,ca,mx,pa,uk,us}`。40250 遇到缺失的包只记日志、继续执行。 - 目录里有、但没注册的文件，40250 永远不会读到：`Outdoor`、`Psyco`（0 字节）、`outdoorempirebattle1`、两个 `.rdch`。 `.rdch` 在源码里没有任何引用。 @@ -333,7 +334,7 @@ extension/third_party/cpython-2.7.18/ # 静态库（2P 批次 | `mob_proto` | `proto.cpp` | **读不了**：每条 255 字节 vs 335 字节，加密同上 | 同上 | | `item_list.txt` | `item_list.gd` | 177 行不同（m2dev 有 dummy 条目等）；40250 放在 `locale/en/`，m2dev 在 `locale/common/` | 按 40250 路径读取 | | `locale_interface.txt`、`itemdesc.txt`、`skilldesc.txt`、`skilltable.txt` | UI、技能表 | 20 行 / 2 行 / 2 行 / 位置不同 | 按 40250 路径读取 | -| 模型、贴图、地图（PC、Monster、Outdoor 等） | 渲染 | 40250 是经典 `.eix/.epk`，136 个包、52,609 个路径，全部可以本地解码（2R） | 移植 `EterPack`/`EterPackManager` 接到 `asset_io`，请求改用 40250 路径（见 2R 第 6 条） | +| 模型、贴图、地图（PC、Monster、Outdoor 等） | 渲染 | 40250 是经典 `.eix/.epk`，103 个包加 `root`、52,609 个路径，全部可以本地解码（2R） | 移植 `EterPack`/`EterPackManager` 接到 `asset_io`，请求改用 40250 路径（见 2R 第 6 条） | 切换步骤（批次 2D）： diff --git a/extension/src/platform/EterBase/CRC32.cpp b/extension/src/platform/EterBase/CRC32.cpp index 4451571d..6ef12427 100644 --- a/extension/src/platform/EterBase/CRC32.cpp +++ b/extension/src/platform/EterBase/CRC32.cpp @@ -1,36 +1,146 @@ -// Platform skeleton for EterBase/CRC32.h (40250 EterBase/CRC32.cpp), generated by platform_stub.py. -// Every MT_PLATFORM_STUB() body is unimplemented: replace it with the platform implementation. +// Platform implementation of EterBase/CRC32.h (40250 EterBase/CRC32.cpp). GetCRC32/GetCaseCRC32 are the 40250 +// loops; CRCTable is the reflected IEEE 802.3 table (polynomial 0xEDB88320), which is what 40250 spells out +// literally (checked entry for entry). The file variants read through a CFileBase-style FILE* handle. #include "EterBase/StdAfx.h" #include "EterBase/CRC32.h" -#include "../PlatformStub.h" +#include +#include -auto GetCRC32(const char *, size_t) -> DWORD +namespace { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); +struct CrcTable +{ + DWORD v[256]; + CrcTable() + { + for (DWORD n = 0; n < 256; ++n) + { + DWORD c = n; + for (int k = 0; k < 8; ++k) + c = (c & 1) ? (0xEDB88320u ^ (c >> 1)) : (c >> 1); + v[n] = c; + } + } +}; +const CrcTable s_crcTable; } -auto GetCaseCRC32(const char *, size_t) -> DWORD +#define CRCTable (s_crcTable.v) + +#define DO1(buf, i) crc = CRCTable[(crc ^ buf[i]) & 0xff] ^ (crc >> 8) +#define DO2(buf, i) DO1(buf, i); DO1(buf, i + 1); +#define DO4(buf, i) DO2(buf, i); DO2(buf, i + 2); +#define DO8(buf, i) DO4(buf, i); DO4(buf, i + 4); +#define DO16(buf, i) DO8(buf, i); DO8(buf, i + 8); + +DWORD GetCRC32(const char * buf, size_t len) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + DWORD crc = 0xffffffff; + + if (len >= 16) + { + do + { + DO16(buf, 0); + + buf += 16; + len -= 16; + } while (len >= 16); + } + + if (len != 0) + { + do + { + DO1(buf, 0); + ++buf; + --len; + } while (len > 0); + } + + crc ^= 0xffffffff; + return crc; } -auto GetHFILECRC32(HANDLE) -> DWORD +#ifndef UPPER +#define UPPER(c) (((c)>='a' && (c) <= 'z') ? ((c)+('A'-'a')) : (c)) +#endif + +#define DO1CI(buf, i) crc = CRCTable[(crc ^ UPPER(buf[i])) & 0xff] ^ (crc >> 8) +#define DO2CI(buf, i) DO1CI(buf, i); DO1CI(buf, i + 1); +#define DO4CI(buf, i) DO2CI(buf, i); DO2CI(buf, i + 2); +#define DO8CI(buf, i) DO4CI(buf, i); DO4CI(buf, i + 4); +#define DO16CI(buf, i) DO8CI(buf, i); DO8CI(buf, i + 8); + +DWORD GetCaseCRC32(const char * buf, size_t len) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + DWORD crc = 0xffffffff; + + if (16 <= len) + { + do + { + DO16CI(buf, 0); + + buf += 16; + len -= 16; + } while (len >= 16); + } + + if (0 != len) + { + do + { + DO1CI(buf, 0); + + ++buf; + --len; + } while (len > 0); + } + + crc ^= 0xffffffff; + return crc; } -auto GetFileCRC32(const char *) -> DWORD +DWORD GetHFILECRC32(HANDLE hFile) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + // 40250 maps the whole file from offset 0 and CRCs it; the stream position is left where it was. + FILE* fp = static_cast(hFile); + const long pos = std::ftell(fp); + std::fseek(fp, 0, SEEK_END); + const long size = std::ftell(fp); + std::fseek(fp, 0, SEEK_SET); + std::vector data(size > 0 ? size : 0); + const size_t got = data.empty() ? 0 : std::fread(data.data(), 1, data.size(), fp); + std::fseek(fp, pos, SEEK_SET); + if (got != data.size() || data.empty()) + return 0; + return GetCRC32(data.data(), data.size()); } -auto GetFileSize(const char *) -> DWORD +DWORD GetFileCRC32(const char* c_szFileName) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + FILE* fp = std::fopen(c_szFileName, "rb"); + if (!fp) + return 0; + + DWORD dwRetCRC32=GetHFILECRC32(fp); + + std::fclose(fp); + + return dwRetCRC32; +} + +DWORD GetFileSize(const char* c_szFileName) +{ + FILE* fp = std::fopen(c_szFileName, "rb"); + if (!fp) + return 0; + + std::fseek(fp, 0, SEEK_END); + const long size = std::ftell(fp); + std::fclose(fp); + + return size < 0 ? 0 : static_cast(size); } diff --git a/extension/src/platform/EterBase/Debug.cpp b/extension/src/platform/EterBase/Debug.cpp index f2c4ad91..802514ae 100644 --- a/extension/src/platform/EterBase/Debug.cpp +++ b/extension/src/platform/EterBase/Debug.cpp @@ -1,9 +1,17 @@ // Platform skeleton for EterBase/Debug.h (40250 EterBase/Debug.cpp), generated by platform_stub.py. -// Every MT_PLATFORM_STUB() body is unimplemented: replace it with the platform implementation. +// Every MT_PLATFORM_STUB() body is unimplemented: replace it with the platform implementation. Implemented +// bodies follow the 40250 non-_DEBUG build (batch 2D: Trace*, TraceError to stderr). #include "EterBase/StdAfx.h" #include "EterBase/Debug.h" #include "../PlatformStub.h" +#include "EterBase/Timer.h" + +#include +#include +#include + +const DWORD DEBUG_STRING_MAX_LEN = 1024; auto SetLogLevel(UINT) -> void { @@ -30,29 +38,58 @@ auto Lognf(UINT, const char *, ...) -> void MT_PLATFORM_STUB(); } -auto Trace(const char *) -> void +void Trace(const char *) { - MT_PLATFORM_STUB(); + // Non-_DEBUG 40250: written only to the log file (OpenLogFile, not implemented yet). } -auto Tracen(const char *) -> void +void Tracen(const char *) { - MT_PLATFORM_STUB(); + // Non-_DEBUG 40250: written only to the log file (OpenLogFile, not implemented yet). } -auto Tracenf(const char *, ...) -> void +void Tracenf(const char *, ...) { - MT_PLATFORM_STUB(); + // Non-_DEBUG 40250: written only to the log file (OpenLogFile, not implemented yet). } -auto Tracef(const char *, ...) -> void +void Tracef(const char *, ...) { - MT_PLATFORM_STUB(); + // Non-_DEBUG 40250: written only to the log file (OpenLogFile, not implemented yet). } -auto TraceError(const char *, ...) -> void +void TraceError(const char* c_szFormat, ...) { - MT_PLATFORM_STUB(); +#ifndef _DISTRIBUTE + char szBuf[DEBUG_STRING_MAX_LEN+2]; + + strncpy(szBuf, "SYSERR: ", DEBUG_STRING_MAX_LEN); + int len = strlen(szBuf); + + va_list args; + va_start(args, c_szFormat); + len = _vsnprintf(szBuf + len, sizeof(szBuf) - (len + 1), c_szFormat, args) + len; + va_end(args); + + // PORT: _vsnprintf returns -1 on truncation; 40250 then writes the terminator before the buffer. + if (len < 8) + len = DEBUG_STRING_MAX_LEN; + + szBuf[len] = '\n'; + szBuf[len + 1] = '\0'; + + time_t ct = time(0); + struct tm ctm = *localtime(&ct); + + fprintf(stderr, "%02d%02d %02d:%02d:%05d :: %s", + ctm.tm_mon + 1, + ctm.tm_mday, + ctm.tm_hour, + ctm.tm_min, + ELTimer_GetMSec() % 60000, + szBuf + 8); + fflush(stderr); +#endif } auto TraceErrorWithoutEnter(const char *, ...) -> void diff --git a/extension/src/platform/EterBase/FileBase.cpp b/extension/src/platform/EterBase/FileBase.cpp index 05b2b496..7184870c 100644 --- a/extension/src/platform/EterBase/FileBase.cpp +++ b/extension/src/platform/EterBase/FileBase.cpp @@ -1,78 +1,189 @@ -// Platform skeleton for EterBase/FileBase.h (40250 EterBase/FileBase.cpp), generated by platform_stub.py. -// Every MT_PLATFORM_STUB() body is unimplemented: replace it with the platform implementation. +// Platform implementation of EterBase/FileBase.h (40250 EterBase/FileBase.cpp). 40250 wraps a Win32 file +// HANDLE; here m_hFile holds a stdio FILE* on every target, with the same open modes (read: existing file, +// shared for reading; write: read/write, created when missing, not truncated) and the same size bookkeeping. +// Names are case-insensitive as on NTFS: 40250's pack/Index says "Etc" for ETC.eix, which a case-sensitive +// file system (Android, Linux) only finds through the fallback below. #include "EterBase/StdAfx.h" #include "EterBase/FileBase.h" -#include "../PlatformStub.h" +#include +#include -CFileBase::CFileBase() +#if !defined(_WIN32) +#include +#include +#endif + +// fopen, then (POSIX) resolve each path component case-insensitively when the exact spelling is missing. +static FILE* open_nocase(const char* filename, const char* mode) +{ + FILE* fp = std::fopen(filename, mode); +#if !defined(_WIN32) + if (fp || !filename[0]) + return fp; + + std::string path(filename); + for (char& c : path) + if (c == '\\') + c = '/'; + + std::string resolved = path[0] == '/' ? "/" : ""; + size_t pos = path[0] == '/' ? 1 : 0; + while (pos <= path.size()) + { + size_t end = path.find('/', pos); + if (end == std::string::npos) + end = path.size(); + const std::string part = path.substr(pos, end - pos); + pos = end + 1; + if (part.empty() || part == "." || part == "..") + { + if (!part.empty()) + resolved += part + "/"; + continue; + } + + DIR* dir = opendir(resolved.empty() ? "." : resolved.c_str()); + if (!dir) + return NULL; + std::string match; + while (dirent* e = readdir(dir)) + { + if (strcasecmp(e->d_name, part.c_str()) == 0) + { + match = e->d_name; + if (match == part) + break; + } + } + closedir(dir); + if (match.empty()) + return NULL; + resolved += match; + if (end != path.size()) + resolved += "/"; + } + fp = std::fopen(resolved.c_str(), mode); +#endif + return fp; +} + +static FILE* file_of(HANDLE h) +{ + return static_cast(h); +} + +static DWORD stream_size(FILE* fp) +{ + const long pos = std::ftell(fp); + std::fseek(fp, 0, SEEK_END); + const long size = std::ftell(fp); + std::fseek(fp, pos, SEEK_SET); + return size < 0 ? 0 : static_cast(size); +} + +CFileBase::CFileBase() : m_hFile(NULL), m_dwSize(0) { - MT_PLATFORM_STUB(); } CFileBase::~CFileBase() { - MT_PLATFORM_STUB(); + Destroy(); } -auto CFileBase::Destroy() -> void +char * CFileBase::GetFileName() { - MT_PLATFORM_STUB(); + return m_filename; } -auto CFileBase::Close() -> void +void CFileBase::Destroy() { - MT_PLATFORM_STUB(); + Close(); + m_dwSize = 0; } -auto CFileBase::Create(const char *, EFileMode) -> BOOL +void CFileBase::Close() { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + if (m_hFile) + { + std::fclose(file_of(m_hFile)); + m_hFile = NULL; + } } -auto CFileBase::Size() -> DWORD +BOOL CFileBase::Create(const char* filename, EFileMode mode) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + Destroy(); + + strncpy(m_filename, filename, MAX_PATH); + + FILE* fp = NULL; + if (mode == FILEMODE_WRITE) + { + // OPEN_ALWAYS + GENERIC_READ|GENERIC_WRITE: open without truncating, create when missing. + fp = open_nocase(filename, "r+b"); + if (!fp) + fp = std::fopen(filename, "w+b"); + } + else + fp = open_nocase(filename, "rb"); + + if (fp) + { + m_hFile = fp; + m_dwSize = stream_size(fp); + m_mode = mode; + return true; + } + + m_hFile = NULL; + return false; } -auto CFileBase::SeekCur(DWORD) -> void +DWORD CFileBase::Size() { - MT_PLATFORM_STUB(); + return (m_dwSize); } -auto CFileBase::Seek(DWORD) -> void +void CFileBase::SeekCur(DWORD size) { - MT_PLATFORM_STUB(); + // SetFilePointer takes the low DWORD as a signed distance when the high part is NULL. + std::fseek(file_of(m_hFile), static_cast(size), SEEK_CUR); } -auto CFileBase::GetPosition() -> DWORD +void CFileBase::Seek(DWORD offset) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + if (offset > m_dwSize) + offset = m_dwSize; + + std::fseek(file_of(m_hFile), static_cast(offset), SEEK_SET); } -auto CFileBase::Write(const void *, int) -> BOOL +DWORD CFileBase::GetPosition() { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + return static_cast(std::ftell(file_of(m_hFile))); } -auto CFileBase::Read(void *, int) -> BOOL +BOOL CFileBase::Write(const void* src, int bytes) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + FILE* fp = file_of(m_hFile); + if (std::fwrite(src, 1, bytes, fp) != static_cast(bytes)) + return false; + + std::fflush(fp); + m_dwSize = stream_size(fp); + return true; } -auto CFileBase::GetFileName() -> char * +BOOL CFileBase::Read(void* dest, int bytes) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + // ReadFile succeeds on a short read (end of file); only an I/O error fails. + FILE* fp = file_of(m_hFile); + std::fread(dest, 1, bytes, fp); + return !std::ferror(fp); } -auto CFileBase::IsNull() -> BOOL +BOOL CFileBase::IsNull() { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + return !m_hFile ? true : false; } diff --git a/extension/src/platform/EterBase/MappedFile.cpp b/extension/src/platform/EterBase/MappedFile.cpp index b8087714..42e14e0e 100644 --- a/extension/src/platform/EterBase/MappedFile.cpp +++ b/extension/src/platform/EterBase/MappedFile.cpp @@ -1,107 +1,272 @@ -// Platform skeleton for EterBase/MappedFile.h (40250 EterBase/MappedFile.cpp), generated by platform_stub.py. -// Every MT_PLATFORM_STUB() body is unimplemented: replace it with the platform implementation. +// Platform implementation of EterBase/MappedFile.h (40250 EterBase/MappedFile.cpp). Everything but Map/Unmap +// is the 40250 code; the view is a POSIX mmap (CreateFileMapping/MapViewOfFile on Windows) of the CFileBase +// stream, aligned down to the allocation granularity exactly as 40250 does. #include "EterBase/StdAfx.h" #include "EterBase/MappedFile.h" +#include "EterBase/Debug.h" -#include "../PlatformStub.h" +#include -CMappedFile::CMappedFile() +#if defined(_WIN32) +#include +#else +#include +#include +#endif + +static DWORD allocation_granularity() +{ +#if defined(_WIN32) + SYSTEM_INFO SysInfo; + GetSystemInfo(&SysInfo); + return SysInfo.dwAllocationGranularity; +#else + return static_cast(sysconf(_SC_PAGESIZE)); +#endif +} + +CMappedFile::CMappedFile() : +m_hFM(NULL), +m_lpMapData(NULL), +m_dataOffset(0), +m_mapSize(0), +m_seekPosition(0), +m_pLZObj(NULL), +m_pbBufLinkData(NULL), +m_dwBufLinkSize(0), +m_pbAppendResultDataBlock(NULL), +m_dwAppendResultDataSize(0) { - MT_PLATFORM_STUB(); } CMappedFile::~CMappedFile() { - MT_PLATFORM_STUB(); + Destroy(); } -auto CMappedFile::Link(DWORD, const void *) -> void +BOOL CMappedFile::Create(const char * filename) { - MT_PLATFORM_STUB(); + Destroy(); + return CFileBase::Create(filename, FILEMODE_READ); } -auto CMappedFile::Create(const char *) -> BOOL +BOOL CMappedFile::Create(const char * filename, const void** dest, int offset, int size) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + if (!CMappedFile::Create(filename)) + return FALSE; + + int ret = Map(dest, offset, size); + return (ret) > 0; } -auto CMappedFile::Create(const char *, const void **, int, int) -> BOOL +LPCVOID CMappedFile::Get() { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + return m_lpData; } -auto CMappedFile::Get() -> LPCVOID +void CMappedFile::Link(DWORD dwBufSize, const void* c_pvBufData) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + m_dwBufLinkSize=dwBufSize; + m_pbBufLinkData=(BYTE*)c_pvBufData; } -auto CMappedFile::Destroy() -> void +void CMappedFile::BindLZObject(CLZObject * pLZObj) { - MT_PLATFORM_STUB(); + assert(m_pLZObj == NULL); + m_pLZObj = pLZObj; + + Link(m_pLZObj->GetSize(), m_pLZObj->GetBuffer()); } -auto CMappedFile::Seek(DWORD, int) -> int +void CMappedFile::BindLZObjectWithBufferedSize(CLZObject * pLZObj) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + assert(m_pLZObj == NULL); + m_pLZObj = pLZObj; + + Link(m_pLZObj->GetBufferSize(), m_pLZObj->GetBuffer()); } -auto CMappedFile::Map(const void **, int, int) -> int +BYTE* CMappedFile::AppendDataBlock( const void* pBlock, DWORD dwBlockSize ) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + if( m_pbAppendResultDataBlock ) + { + delete []m_pbAppendResultDataBlock; + } + + //realloc + m_dwAppendResultDataSize = m_dwBufLinkSize+dwBlockSize; + m_pbAppendResultDataBlock = new BYTE[m_dwAppendResultDataSize]; + + memcpy(m_pbAppendResultDataBlock, m_pbBufLinkData, m_dwBufLinkSize ); + memcpy(m_pbAppendResultDataBlock + m_dwBufLinkSize, pBlock, dwBlockSize ); + + //redirect + Link(m_dwAppendResultDataSize, m_pbAppendResultDataBlock); + + return m_pbAppendResultDataBlock; } -auto CMappedFile::Size() -> DWORD +void CMappedFile::Destroy() { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + if (m_pLZObj) + { + delete m_pLZObj; + m_pLZObj = NULL; + } + + if (NULL != m_lpMapData) + { + Unmap(m_lpMapData); + m_lpMapData = NULL; + } + +#if defined(_WIN32) + if (NULL != m_hFM) + { + CloseHandle(m_hFM); + m_hFM = NULL; + } +#endif + + if( m_pbAppendResultDataBlock ) + { + delete []m_pbAppendResultDataBlock; + m_pbAppendResultDataBlock = NULL; + } + + m_dwAppendResultDataSize = 0; + + m_pbBufLinkData = NULL; + m_dwBufLinkSize = 0; + + m_seekPosition = 0; + m_dataOffset = 0; + m_mapSize = 0; + + CFileBase::Destroy(); } -auto CMappedFile::GetPosition() -> DWORD +int CMappedFile::Seek(DWORD offset, int iSeekType) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + switch (iSeekType) + { + case SEEK_TYPE_BEGIN: + if (offset > m_dwSize) + offset = m_dwSize; + + m_seekPosition = offset; + break; + + case SEEK_TYPE_CURRENT: + m_seekPosition = min(m_seekPosition + offset, Size()); + break; + + case SEEK_TYPE_END: + m_seekPosition = max(0, Size() - offset); + break; + } + + return m_seekPosition; } -auto CMappedFile::Read(void *, int) -> BOOL +int CMappedFile::Map(const void **dest, int offset, int size) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + m_dataOffset = offset; + + if (size == 0) + m_mapSize = m_dwSize; + else + m_mapSize = size; + + if (m_dataOffset + m_mapSize > m_dwSize) + return 0; + + DWORD dwSysGran = allocation_granularity(); + DWORD dwFileMapStart = (m_dataOffset / dwSysGran) * dwSysGran; + DWORD dwMapViewSize = (m_dataOffset % dwSysGran) + m_mapSize; + INT iViewDelta = m_dataOffset - dwFileMapStart; + + FILE* fp = static_cast(m_hFile); +#if defined(_WIN32) + m_hFM = CreateFileMapping((HANDLE) _get_osfhandle(_fileno(fp)), NULL, PAGE_READONLY, 0, + m_dataOffset + m_mapSize, NULL); + if (!m_hFM) + { + OutputDebugString("CMappedFile::Map !m_hFM\n"); + return 0; + } + + m_lpMapData = MapViewOfFile(m_hFM, FILE_MAP_READ, 0, dwFileMapStart, dwMapViewSize); + if (!m_lpMapData) + { + TraceError("CMappedFile::Map !m_lpMapData %lu", GetLastError()); + return 0; + } +#else + // A zero-length view (empty file) is a valid 40250 map; mmap rejects length 0. + if (dwMapViewSize == 0) + dwMapViewSize = 1; + void* p = mmap(NULL, dwMapViewSize, PROT_READ, MAP_PRIVATE, fileno(fp), static_cast(dwFileMapStart)); + if (p == MAP_FAILED) + { + TraceError("CMappedFile::Map !m_lpMapData %s", GetFileName()); + return 0; + } + m_lpMapData = p; +#endif + + m_lpData = (char*) m_lpMapData + iViewDelta; + *dest = (char*) m_lpData; + m_seekPosition = 0; + + Link(m_mapSize, m_lpData); + + return (m_mapSize); } -auto CMappedFile::GetSeekPosition() -> DWORD +BYTE * CMappedFile::GetCurrentSeekPoint() { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + return m_pbBufLinkData+m_seekPosition; } -auto CMappedFile::BindLZObject(CLZObject *) -> void +DWORD CMappedFile::Size() { - MT_PLATFORM_STUB(); + return m_dwBufLinkSize; } -auto CMappedFile::BindLZObjectWithBufferedSize(CLZObject *) -> void +DWORD CMappedFile::GetPosition() { - MT_PLATFORM_STUB(); + return m_dataOffset; } -auto CMappedFile::AppendDataBlock(const void *, DWORD) -> BYTE * +BOOL CMappedFile::Read(void * dest, int bytes) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + if (m_seekPosition + bytes > Size()) + return FALSE; + + memcpy(dest, GetCurrentSeekPoint(), bytes); + m_seekPosition += bytes; + return TRUE; } -auto CMappedFile::GetCurrentSeekPoint() -> BYTE * +DWORD CMappedFile::GetSeekPosition(void) { - MT_PLATFORM_STUB(); - return mt_platform_stub_return(); + return m_seekPosition; } -auto CMappedFile::Unmap(LPCVOID) -> void +void CMappedFile::Unmap(LPCVOID data) { - MT_PLATFORM_STUB(); +#if defined(_WIN32) + if (!UnmapViewOfFile(data)) + TraceError("CMappedFile::Unmap - Error"); +#else + // Same view length Map computed from the (unchanged) offset and size. + const DWORD dwSysGran = allocation_granularity(); + DWORD dwMapViewSize = (m_dataOffset % dwSysGran) + m_mapSize; + if (dwMapViewSize == 0) + dwMapViewSize = 1; + if (munmap(const_cast(data), dwMapViewSize) != 0) + TraceError("CMappedFile::Unmap - Error"); +#endif + m_lpData = NULL; } diff --git a/extension/src/platform/EterBase/Utils.cpp b/extension/src/platform/EterBase/Utils.cpp index 3b211202..da50a105 100644 --- a/extension/src/platform/EterBase/Utils.cpp +++ b/extension/src/platform/EterBase/Utils.cpp @@ -1,7 +1,9 @@ // Platform skeleton for EterBase/Utils.h (40250 EterBase/Utils.cpp), generated by platform_stub.py. -// Every MT_PLATFORM_STUB() body is unimplemented: replace it with the platform implementation. +// Every MT_PLATFORM_STUB() body is unimplemented: replace it with the platform implementation. Implemented +// bodies are the 40250 code (batch 2D: the path helpers EterPack uses); stl_lowers lives in EterBase/Stl.cpp. #include "EterBase/StdAfx.h" #include "EterBase/Utils.h" +#include "EterBase/Stl.h" #include "../PlatformStub.h" @@ -36,11 +38,6 @@ auto GetIndexingName(char *, DWORD) -> void MT_PLATFORM_STUB(); } -auto stl_lowers(std::string &) -> void -{ - MT_PLATFORM_STUB(); -} - auto GetOnlyFileName(const char *, std::string &) -> void { MT_PLATFORM_STUB(); @@ -73,29 +70,59 @@ auto GetWorkingFolder(std::string &) -> void MT_PLATFORM_STUB(); } -auto StringLowers(char *) -> void +void StringLowers(char * String) { - MT_PLATFORM_STUB(); + for (DWORD i = 0; i < strlen(String); ++i) + { + String[i] = korean_tolower(String[i]); + } } -auto StringPath(std::string &) -> void +void StringPath(std::string & rString) { - MT_PLATFORM_STUB(); + for (DWORD i = 0; i < rString.length(); ++i) + { + if (rString[i] == '\\') + rString[i] = '/'; + else + rString[i] = korean_tolower(rString[i]); + } } -auto StringPath(char *) -> void +void StringPath(char * pString) { - MT_PLATFORM_STUB(); + for (DWORD i = 0; i < strlen(pString); ++i) + { + if (pString[i] == '\\') + pString[i] = '/'; + else + pString[i] = korean_tolower(pString[i]); + } } -auto StringPath(const char *, char *) -> void +void StringPath(const char * c_szSrc, char * szDest) { - MT_PLATFORM_STUB(); + for (DWORD i = 0; i < strlen(c_szSrc); ++i) + { + if (c_szSrc[i] == '\\') + szDest[i] = '/'; + else + szDest[i] = korean_tolower(c_szSrc[i]); + } } -auto StringPath(const char *, std::string &) -> void +void StringPath(const char * c_szSrc, std::string & rString) { - MT_PLATFORM_STUB(); + rString = ""; + rString.resize(strlen(c_szSrc)); + + for (DWORD i = 0; i < strlen(c_szSrc); ++i) + { + if (c_szSrc[i] == '\\') + rString[i] = '/'; + else + rString[i] = korean_tolower(c_szSrc[i]); + } } auto PrintAsciiData(const void *, int) -> void diff --git a/extension/src/port/CMakeLists.txt b/extension/src/port/CMakeLists.txt index bb303707..c84aebc1 100644 --- a/extension/src/port/CMakeLists.txt +++ b/extension/src/port/CMakeLists.txt @@ -30,6 +30,44 @@ if(NOT MSVC) endif() target_compile_options(port_logic PUBLIC ${MT_PORT_WARNING_FLAGS}) target_compile_features(port_logic PUBLIC cxx_std_20) +# Armadillo nanomite markers (StdAfx.h `__asm _emit` jumps over a signature): x86-32 MSVC inline asm that no +# target here accepts, and a no-op without the Armadillo wrapper, so they expand to nothing. +target_compile_definitions(port_logic PUBLIC "NANOBEGIN=" "NANOEND=") +if(MINGW) + target_compile_options(port_logic PUBLIC -include ${CMAKE_CURRENT_SOURCE_DIR}/common/Win32MinMax.h) +endif() + +# EterPack/EterPack.cpp includes EterPackKeys.generated.h in place of the two TEA key arrays: the keys are +# read from the reference EterPack.cpp at configure time and written into the build tree, never the repository. +# Without the reference the arrays are zero and every COMPRESS/SECURITY entry fails to decode. +set(MT_40250_SOURCE "$ENV{MT_40250_SOURCE}" CACHE PATH "40250 ClientVS22/source reference root") +if(NOT MT_40250_SOURCE) + set(MT_40250_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/../../../../40250/Server Client TMP4/ClientVS22/source") +endif() +set(MT_PORT_GENERATED ${CMAKE_CURRENT_BINARY_DIR}/generated) +set(mt_eterpack_ref "${MT_40250_SOURCE}/EterPack/EterPack.cpp") +set(mt_eterpack_keys "") +if(EXISTS "${mt_eterpack_ref}") + file(READ "${mt_eterpack_ref}" mt_ref_text) + foreach(arr s_adwEterPackKey s_adwEterPackSecurityKey) + string(REGEX MATCH "static DWORD ${arr}\\[\\][ \t\r\n]*=[ \t\r\n]*{[^}]*}" block "${mt_ref_text}") + string(REGEX MATCHALL "[0-9]+" values "${block}") + list(LENGTH values n) + if(NOT n EQUAL 4) + message(FATAL_ERROR "port: ${arr} not found in ${mt_eterpack_ref}") + endif() + list(JOIN values ", " values) + string(APPEND mt_eterpack_keys "static DWORD ${arr}[] = { ${values} };\n") + endforeach() + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${mt_eterpack_ref}") +else() + message(WARNING "port: ${mt_eterpack_ref} not found (set MT_40250_SOURCE); EterPack keys are zero") + string(APPEND mt_eterpack_keys "static DWORD s_adwEterPackKey[] = { 0, 0, 0, 0 };\n" + "static DWORD s_adwEterPackSecurityKey[] = { 0, 0, 0, 0 };\n") +endif() +file(CONFIGURE OUTPUT ${MT_PORT_GENERATED}/EterPackKeys.generated.h + CONTENT "// Generated from the 40250 reference EterPack.cpp by port/CMakeLists.txt; do not commit.\n#pragma once\n${mt_eterpack_keys}") +target_include_directories(port_logic PRIVATE ${MT_PORT_GENERATED}) if(WIN32) target_link_libraries(port_logic PUBLIC winmm) endif() @@ -86,6 +124,10 @@ add_library(port_header_gate OBJECT ${MT_PORT_GATE_SOURCES}) target_include_directories(port_header_gate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${MT_PORT_SHIMS}) target_link_libraries(port_header_gate PRIVATE mt3p::minilzo mt3p::cryptopp) target_compile_features(port_header_gate PRIVATE cxx_std_20) +target_compile_definitions(port_header_gate PRIVATE "NANOBEGIN=" "NANOEND=") +if(MINGW) + target_compile_options(port_header_gate PRIVATE -include ${CMAKE_CURRENT_SOURCE_DIR}/common/Win32MinMax.h) +endif() target_compile_options(port_header_gate PRIVATE ${MT_PORT_WARNING_FLAGS}) add_dependencies(port_logic port_header_gate) @@ -95,4 +137,12 @@ if(BUILD_TESTING AND CMAKE_SYSTEM_NAME STREQUAL CMAKE_HOST_SYSTEM_NAME) add_executable(port_common_test ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/port_common_test.cpp) target_link_libraries(port_common_test PRIVATE port_logic) add_test(NAME port.common COMMAND $) + + # Against the real 40250 Client/pack next to the reference source; skipped (77) when it is missing, + # failed instead under MT_ASSETS_STRICT=1. + set(MT_40250_CLIENT "${MT_40250_SOURCE}/../../Client" CACHE PATH "40250 Client directory (pack/, Eternexus/)") + add_executable(port_eterpack_test ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/port_eterpack_test.cpp) + target_link_libraries(port_eterpack_test PRIVATE port_platform) + add_test(NAME port.eterpack COMMAND $ ${MT_40250_CLIENT}) + set_tests_properties(port.eterpack PROPERTIES SKIP_RETURN_CODE 77) endif() diff --git a/extension/src/port/EterBase/Random.cpp b/extension/src/port/EterBase/Random.cpp new file mode 100644 index 00000000..d851cda1 --- /dev/null +++ b/extension/src/port/EterBase/Random.cpp @@ -0,0 +1,49 @@ +#include "StdAfx.h" +#include "Random.h" // PORT: the POSIX-safe names (Random.h) must be in scope for the definitions + +#include + +static unsigned long randseed = 1; + +void srandom(unsigned long seed) +{ + randseed = seed; +} + +/* + * Pseudo-random number generator for randomizing the profiling clock, + * and whatever else we might use it for. The result is uniform on + * [0, 2^31 - 1]. + */ +unsigned long random() +{ + long x, hi, lo, t; + + /* + * Compute x[n + 1] = (7^5 * x[n]) mod (2^31 - 1). + * From "Random number generators: good ones are hard to find", + * Park and Miller, Communications of the ACM, vol. 31, no. 10, + * October 1988, p. 1195. + */ + x = randseed; + hi = x / 127773; + lo = x % 127773; + t = 16807 * lo - 2836 * hi; + if (t <= 0) + t += 0x7fffffff; + randseed = t; + return (t); +} + + +float frandom(float flLow, float flHigh) +{ + float fl = float(random()) / float(2147483648.0f); // float in [0,1) + return (fl * (flHigh - flLow)) + flLow; // float in [low,high) +} + +long random_range(long from, long to) +{ + assert(from <= to); + return ((random() % (to - from + 1)) + from); +} diff --git a/extension/src/port/EterBase/Stl.cpp b/extension/src/port/EterBase/Stl.cpp new file mode 100644 index 00000000..6933baab --- /dev/null +++ b/extension/src/port/EterBase/Stl.cpp @@ -0,0 +1,92 @@ +#include "StdAfx.h" +#include "Stl.h" + +static std::list s_stList; + +char korean_tolower(const char c) +{ + char ret = c; + if (c >= 'A' && c <= 'Z') + ret = c - 'A' + 'a'; + + assert(ret == tolower(c)); + return ret; +} + +std::string& stl_static_string(const char * c_sz) +{ + std::string str; + str.assign(c_sz); + + s_stList.push_back(str); + return s_stList.back(); +} + +void stl_lowers(std::string& rstRet) +{ + for (size_t i = 0; i < rstRet.length(); ++i) + rstRet[i] = korean_tolower(rstRet[i]); +} + +int split_string(const std::string& input, const std::string& delimiter, std::vector& results, bool includeEmpties) +{ + int iPos = 0; + int newPos = -1; + UINT sizeS2 = delimiter.size(); + UINT isize = input.size(); + + if ((isize == 0) || (sizeS2 == 0)) + { + return 0; + } + + std::vector positions; + + newPos = input.find(delimiter, 0); + + if (newPos < 0) + return 0; + + int numFound = 0; + + while (newPos >= iPos) + { + numFound++; + positions.push_back(newPos); + iPos = newPos; + newPos = input.find(delimiter, iPos+sizeS2); + } + + if (numFound == 0) + return 0; + + for (UINT i = 0; i <= positions.size(); ++i) + { + std::string s(""); + if (i == 0) + { + s = input.substr(i, positions[i]); + } + else + { + UINT offset = positions[i-1] + sizeS2; + + if (offset < isize) + { + if (i == positions.size()) + { + s = input.substr(offset); + } + else if (i > 0) + { + s = input.substr(positions[i-1] + sizeS2, positions[i] - positions[i-1] - sizeS2); + } + } + } + if (includeEmpties || (s.size() > 0)) + { + results.push_back(s); + } + } + return numFound; +} diff --git a/extension/src/port/EterBase/Timer.cpp b/extension/src/port/EterBase/Timer.cpp new file mode 100644 index 00000000..923b8294 --- /dev/null +++ b/extension/src/port/EterBase/Timer.cpp @@ -0,0 +1,156 @@ +#include "StdAfx.h" +#include "Timer.h" + +static LARGE_INTEGER gs_liTickCountPerSec; +static DWORD gs_dwBaseTime=0; +static DWORD gs_dwServerTime=0; +static DWORD gs_dwClientTime=0; +static DWORD gs_dwFrameTime=0; + +#pragma comment(lib, "winmm.lib") + +BOOL ELTimer_Init() +{ + /* + gs_liTickCountPerSec.QuadPart=0; + + if (!QueryPerformanceFrequency(&gs_liTickCountPerSec)) + return 0; + + LARGE_INTEGER liTickCount; + QueryPerformanceCounter(&liTickCount); + gs_dwBaseTime= (liTickCount.QuadPart*1000 / gs_liTickCountPerSec.QuadPart); + */ + gs_dwBaseTime = timeGetTime(); + return 1; +} + +DWORD ELTimer_GetMSec() +{ + //assert(gs_dwBaseTime!=0 && "ELTimer_Init 를 먼저 실행하세요"); + //LARGE_INTEGER liTickCount; + //QueryPerformanceCounter(&liTickCount); + return timeGetTime() - gs_dwBaseTime; //(liTickCount.QuadPart*1000 / gs_liTickCountPerSec.QuadPart)-gs_dwBaseTime; +} + +VOID ELTimer_SetServerMSec(DWORD dwServerTime) +{ + NANOBEGIN + if (0 != dwServerTime) // nanomite를 위한 더미 if + { + gs_dwServerTime = dwServerTime; + gs_dwClientTime = CTimer::instance().GetCurrentMillisecond(); + } + NANOEND +} + +DWORD ELTimer_GetServerMSec() +{ + return CTimer::instance().GetCurrentMillisecond() - gs_dwClientTime + gs_dwServerTime; + //return ELTimer_GetMSec() - gs_dwClientTime + gs_dwServerTime; +} + +DWORD ELTimer_GetFrameMSec() +{ + return gs_dwFrameTime; +} + +DWORD ELTimer_GetServerFrameMSec() +{ + return ELTimer_GetFrameMSec() - gs_dwClientTime + gs_dwServerTime; +} + +VOID ELTimer_SetFrameMSec() +{ + gs_dwFrameTime = ELTimer_GetMSec(); +} + +CTimer::CTimer() +{ + ELTimer_Init(); + + NANOBEGIN + if (this) // nanomite를 위한 더미 if + { + m_dwCurrentTime = 0; + m_bUseRealTime = true; + m_index = 0; + + m_dwElapsedTime = 0; + + m_fCurrentTime = 0.0f; + } + NANOEND +} + +CTimer::~CTimer() +{ +} + +void CTimer::SetBaseTime() +{ + m_dwCurrentTime = 0; +} + +void CTimer::Advance() +{ + if (!m_bUseRealTime) + { + ++m_index; + + if (m_index == 1) + m_index = -1; + + m_dwCurrentTime += 16 + (m_index & 1); + m_fCurrentTime = m_dwCurrentTime / 1000.0f; + } + else + { + DWORD currentTime = ELTimer_GetMSec(); + + if (m_dwCurrentTime == 0) + m_dwCurrentTime = currentTime; + + m_dwElapsedTime = currentTime - m_dwCurrentTime; + m_dwCurrentTime = currentTime; + } +} + +void CTimer::Adjust(int iTimeGap) +{ + m_dwCurrentTime += iTimeGap; +} + +float CTimer::GetCurrentSecond() +{ + if (m_bUseRealTime) + return ELTimer_GetMSec() / 1000.0f; + + return m_fCurrentTime; +} + +DWORD CTimer::GetCurrentMillisecond() +{ + if (m_bUseRealTime) + return ELTimer_GetMSec(); + + return m_dwCurrentTime; +} + +float CTimer::GetElapsedSecond() +{ + return GetElapsedMilliecond() / 1000.0f; +} + +DWORD CTimer::GetElapsedMilliecond() +{ + if (!m_bUseRealTime) + return 16 + (m_index & 1); + + return m_dwElapsedTime; +} + +void CTimer::UseCustomTime() +{ + m_bUseRealTime = false; +} diff --git a/extension/src/port/EterBase/lzo.cpp b/extension/src/port/EterBase/lzo.cpp new file mode 100644 index 00000000..25056936 --- /dev/null +++ b/extension/src/port/EterBase/lzo.cpp @@ -0,0 +1,411 @@ +#include "StdAfx.h" +#include +#include + +#include "lzo.h" +#include "tea.h" +#include "Debug.h" + +#define dbg_printf + +static class LZOFreeMemoryMgr +{ +public: + enum + { + REUSING_CAPACITY = 64*1024, + }; +public: + ~LZOFreeMemoryMgr() + { + std::vector::iterator i; + for (i = m_freeVector.begin(); i != m_freeVector.end(); ++i) + delete *i; + + m_freeVector.clear(); + } + BYTE* Alloc(unsigned capacity) + { + assert(capacity > 0); + if (capacity < REUSING_CAPACITY) + { + if (!m_freeVector.empty()) + { + BYTE* freeMem = m_freeVector.back(); + m_freeVector.pop_back(); + + dbg_printf("lzo.reuse_alloc\t%p(%d) free\n", freeMem, capacity); + return freeMem; + } + BYTE* newMem = new BYTE[REUSING_CAPACITY]; + dbg_printf("lzo.reuse_alloc\t%p(%d) real\n", newMem, capacity); + return newMem; + } + BYTE* newMem = new BYTE[capacity]; + dbg_printf("lzo.real_alloc\t%p(%d)\n", newMem, capacity); + return newMem; + } + void Free(BYTE* ptr, unsigned capacity) + { + assert(ptr != NULL); + assert(capacity > 0); + if (capacity < REUSING_CAPACITY) + { + dbg_printf("lzo.reuse_free\t%p(%d)\n", ptr, capacity); + m_freeVector.push_back(ptr); + return; + } + + dbg_printf("lzo.real_free\t%p(%d)\n", ptr, capacity); + delete [] ptr; + } +private: + std::vector m_freeVector; +} gs_freeMemMgr; + + + +DWORD CLZObject::ms_dwFourCC = MAKEFOURCC('M', 'C', 'O', 'Z'); + +CLZObject::CLZObject() +{ + Initialize(); +} + +void CLZObject::Initialize() +{ + m_bInBuffer = false; + m_pbBuffer = NULL; + m_dwBufferSize = 0; + + m_pHeader = NULL; + m_pbIn = NULL; + m_bCompressed = false; +} + +void CLZObject::Clear() +{ + if (m_pbBuffer && !m_bInBuffer) + gs_freeMemMgr.Free(m_pbBuffer, m_dwBufferSize); + + if (m_dwBufferSize > 0) + { + dbg_printf("lzo.free %d\n", m_dwBufferSize); + } + + Initialize(); +} + +CLZObject::~CLZObject() +{ + Clear(); +} + +DWORD CLZObject::GetSize() +{ + assert(m_pHeader); + + if (m_bCompressed) + { + if (m_pHeader->dwEncryptSize) + return sizeof(THeader) + sizeof(DWORD) + m_pHeader->dwEncryptSize; + else + return sizeof(THeader) + sizeof(DWORD) + m_pHeader->dwCompressedSize; + } + else + return m_pHeader->dwRealSize; +} + +void CLZObject::BeginCompress(const void * pvIn, UINT uiInLen) +{ + m_pbIn = (const BYTE *) pvIn; + + // sizeof(SHeader) + + // 암호화를 위한 fourCC 4바이트 + // 압축된 후 만들어질 수 있는 최대 용량 + + // 암호화를 위한 8 바이트 + m_dwBufferSize = sizeof(THeader) + sizeof(DWORD) + (uiInLen + uiInLen / 64 + 16 + 3) + 8; + + m_pbBuffer = gs_freeMemMgr.Alloc(m_dwBufferSize); + memset(m_pbBuffer, 0, m_dwBufferSize); + + m_pHeader = (THeader *) m_pbBuffer; + m_pHeader->dwFourCC = ms_dwFourCC; + m_pHeader->dwEncryptSize = m_pHeader->dwCompressedSize = m_pHeader->dwRealSize = 0; + m_pHeader->dwRealSize = uiInLen; +} + +void CLZObject::BeginCompressInBuffer(const void * pvIn, UINT uiInLen, void * /*pvOut*/) +{ + m_pbIn = (const BYTE *) pvIn; + + // sizeof(SHeader) + + // 암호화를 위한 fourCC 4바이트 + // 압축된 후 만들어질 수 있는 최대 용량 + + // 암호화를 위한 8 바이트 + m_dwBufferSize = sizeof(THeader) + sizeof(DWORD) + (uiInLen + uiInLen / 64 + 16 + 3) + 8; + + m_pbBuffer = gs_freeMemMgr.Alloc(m_dwBufferSize); + memset(m_pbBuffer, 0, m_dwBufferSize); + + m_pHeader = (THeader *) m_pbBuffer; + m_pHeader->dwFourCC = ms_dwFourCC; + m_pHeader->dwEncryptSize = m_pHeader->dwCompressedSize = m_pHeader->dwRealSize = 0; + m_pHeader->dwRealSize = uiInLen; + m_bInBuffer = true; +} + +bool CLZObject::Compress() +{ + UINT iOutLen; + BYTE * pbBuffer; + + pbBuffer = m_pbBuffer + sizeof(THeader); + *(DWORD *) pbBuffer = ms_dwFourCC; + pbBuffer += sizeof(DWORD); + +#if defined( LZO1X_999_MEM_COMPRESS ) + int r = lzo1x_999_compress((BYTE *) m_pbIn, m_pHeader->dwRealSize, pbBuffer, (lzo_uint*) &iOutLen, CLZO::Instance().GetWorkMemory()); +#else + int r = lzo1x_1_compress((BYTE *) m_pbIn, m_pHeader->dwRealSize, pbBuffer, (lzo_uint*) &iOutLen, CLZO::Instance().GetWorkMemory()); +#endif + + if (LZO_E_OK != r) + { + TraceError("LZO: lzo1x_999_compress failed"); + return false; + } + + m_pHeader->dwCompressedSize = iOutLen; + m_bCompressed = true; + return true; +} + +bool CLZObject::BeginDecompress(const void * pvIn) +{ + THeader * pHeader = (THeader *) pvIn; + + if (pHeader->dwFourCC != ms_dwFourCC) + { + TraceError("LZObject: not a valid data"); + return false; + } + + m_pHeader = pHeader; + m_pbIn = (const BYTE *) pvIn + (sizeof(THeader) + sizeof(DWORD)); + + /* + static unsigned sum = 0; + static unsigned count = 0; + sum += pHeader->dwRealSize; + count++; + printf("decompress cur: %d, ave: %d\n", pHeader->dwRealSize, sum/count); + */ + m_dwBufferSize = pHeader->dwRealSize; + m_pbBuffer = gs_freeMemMgr.Alloc(m_dwBufferSize); + memset(m_pbBuffer, 0, pHeader->dwRealSize); + return true; +} + +class DecryptBuffer +{ +public: + enum + { + LOCAL_BUF_SIZE = 8 * 1024, + }; +public: + DecryptBuffer(unsigned size) + { + static unsigned count = 0; + static unsigned sum = 0; + static unsigned maxSize = 0; + + sum += size; + count++; + + maxSize = max(size, maxSize); + if (size >= LOCAL_BUF_SIZE) + { + m_buf = new char[size]; + dbg_printf("DecryptBuffer - AllocHeap %d max(%d) ave(%d)\n", size, maxSize/1024, sum/count); + } + else + { + dbg_printf("DecryptBuffer - AllocStack %d max(%d) ave(%d)\n", size, maxSize/1024, sum/count); + m_buf = m_local_buf; + } + } + ~DecryptBuffer() + { + if (m_local_buf != m_buf) + { + dbg_printf("DecruptBuffer - FreeHeap\n"); + delete [] m_buf; + } + else + { + dbg_printf("DecruptBuffer - FreeStack\n"); + } + } + void* GetBufferPtr() + { + return m_buf; + } + +private: + char* m_buf; + char m_local_buf[LOCAL_BUF_SIZE]; +}; + +bool CLZObject::Decompress(DWORD * pdwKey) +{ + UINT uiSize; + int r; + + if (m_pHeader->dwEncryptSize) + { + DecryptBuffer buf(m_pHeader->dwEncryptSize); + + BYTE* pbDecryptedBuffer = (BYTE*)buf.GetBufferPtr(); + + __Decrypt(pdwKey, pbDecryptedBuffer); + + if (*(DWORD *) pbDecryptedBuffer != ms_dwFourCC) + { + TraceError("LZObject: key incorrect"); + return false; + } + + if (LZO_E_OK != (r = lzo1x_decompress(pbDecryptedBuffer + sizeof(DWORD), m_pHeader->dwCompressedSize, m_pbBuffer, (lzo_uint*) &uiSize, NULL))) + { + TraceError("LZObject: Decompress failed(decrypt) ret %d\n", r); + return false; + } + } + else + { + uiSize = m_pHeader->dwRealSize; + + //if (LZO_E_OK != (r = lzo1x_decompress_safe(m_pbIn, m_pHeader->dwCompressedSize, m_pbBuffer, (lzo_uint*) &uiSize, NULL))) + if (LZO_E_OK != (r = lzo1x_decompress(m_pbIn, m_pHeader->dwCompressedSize, m_pbBuffer, (lzo_uint*) &uiSize, NULL))) + { + TraceError("LZObject: Decompress failed : ret %d, CompressedSize %d\n", r, m_pHeader->dwCompressedSize); + return false; + } + } + + if (uiSize != m_pHeader->dwRealSize) + { + TraceError("LZObject: Size differs"); + return false; + } + + return true; +} + +bool CLZObject::Encrypt(DWORD * pdwKey) +{ + if (!m_bCompressed) + { + assert(!"not compressed yet"); + return false; + } + + BYTE * pbBuffer = m_pbBuffer + sizeof(THeader); + m_pHeader->dwEncryptSize = tea_encrypt((DWORD *) pbBuffer, (const DWORD *) pbBuffer, pdwKey, m_pHeader->dwCompressedSize + 19); + return true; +} + +bool CLZObject::__Decrypt(DWORD * key, BYTE* data) +{ + assert(m_pbBuffer); + + tea_decrypt((DWORD *) data, (const DWORD *) (m_pbIn - sizeof(DWORD)), key, m_pHeader->dwEncryptSize); + return true; +} + +void CLZObject::AllocBuffer(DWORD dwSrcSize) +{ + if (m_pbBuffer && !m_bInBuffer) + gs_freeMemMgr.Free(m_pbBuffer, m_dwBufferSize); + + m_pbBuffer = gs_freeMemMgr.Alloc(dwSrcSize); + m_dwBufferSize = dwSrcSize; +} +/* +void CLZObject::CopyBuffer(const char* pbSrc, DWORD dwSrcSize) +{ + AllocBuffer(dwSrcSize); + memcpy(m_pbBuffer, pbSrc, dwSrcSize); +} +*/ + +CLZO::CLZO() : m_pWorkMem(NULL) +{ + if (lzo_init() != LZO_E_OK) + { + TraceError("LZO: cannot initialize"); + return; + } + +#if defined( LZO1X_999_MEM_COMPRESS ) + m_pWorkMem = (BYTE *) malloc(LZO1X_999_MEM_COMPRESS); +#else + m_pWorkMem = (BYTE *) malloc(LZO1X_1_MEM_COMPRESS); +#endif + + if (NULL == m_pWorkMem) + { + TraceError("LZO: cannot alloc memory"); + return; + } +} + +CLZO::~CLZO() +{ + if (m_pWorkMem) + { + free(m_pWorkMem); + m_pWorkMem = NULL; + } +} + +bool CLZO::CompressMemory(CLZObject & rObj, const void * pIn, UINT uiInLen) +{ + rObj.BeginCompress(pIn, uiInLen); + return rObj.Compress(); +} + +bool CLZO::CompressEncryptedMemory(CLZObject & rObj, const void * pIn, UINT uiInLen, DWORD * pdwKey) +{ + rObj.BeginCompress(pIn, uiInLen); + + if (rObj.Compress()) + { + if (rObj.Encrypt(pdwKey)) + return true; + + return false; + } + + return false; +} + +bool CLZO::Decompress(CLZObject & rObj, const BYTE * pbBuf, DWORD * pdwKey) +{ + if (!rObj.BeginDecompress(pbBuf)) + return false; + + if (!rObj.Decompress(pdwKey)) + return false; + + return true; +} + + +BYTE * CLZO::GetWorkMemory() +{ + return m_pWorkMem; +} + diff --git a/extension/src/port/EterBase/tea.cpp b/extension/src/port/EterBase/tea.cpp new file mode 100644 index 00000000..b88da540 --- /dev/null +++ b/extension/src/port/EterBase/tea.cpp @@ -0,0 +1,101 @@ +/* +* Filename: tea.c +* Description: TEA 암호화 모듈 +* +* Author: 김한주 (aka. 비엽, Cronan), 송영진 (aka. myevan, 빗자루) +*/ +#include "StdAfx.h" +#include "tea.h" +#include + +/* +* TEA Encryption Module Instruction +* Edited by 김한주 aka. 비엽, Cronan +* +* void tea_code(const unsigned long sz, const unsigned long sy, const unsigned long *key, unsigned long *dest) +* void tea_decode(const unsigned long sz, const unsigned long sy, const unsigned long *key, unsigned long *dest) +* 8바이트를 암호/복호화 할때 사용된다. key 는 16 바이트여야 한다. +* sz, sy 는 8바이트의 역순으로 대입한다. +* +* int tea_decrypt(unsigned long *dest, const unsigned long *src, const unsigned long *key, int size); +* int tea_encrypt(unsigned long *dest, const unsigned long *src, const unsigned long *key, int size); +* 한꺼번에 8 바이트 이상을 암호/복호화 할때 사용한다. 만약 size 가 +* 8의 배수가 아니면 8의 배수로 크기를 "늘려서" 암호화 한다. +* +* ex. tea_code(pdwSrc[1], pdwSrc[0], pdwKey, pdwDest); +* tea_decrypt(pdwDest, pdwSrc, pdwKey, nSize); +*/ + +#define TEA_ROUND 32 // 32 를 권장하며, 높을 수록 결과가 난해해 진다. +#define DELTA 0x9E3779B9 // DELTA 값 바꾸지 말것. + +void tea_code(const tea_word sz, const tea_word sy, const tea_word *key, tea_word *dest) // PORT: tea_word (32-bit Win32 unsigned long) +{ + tea_word y = sy, z = sz, sum = 0; // PORT: tea_word (32-bit Win32 unsigned long) + tea_word n = TEA_ROUND; // PORT: tea_word (32-bit Win32 unsigned long) + + while (n-- > 0) + { + y += ((z << 4 ^ z >> 5) + z) ^ (sum + key[sum & 3]); + sum += DELTA; + z += ((y << 4 ^ y >> 5) + y) ^ (sum + key[sum >> 11 & 3]); + } + + *(dest++) = y; + *dest = z; +} + +void tea_decode(const tea_word sz, const tea_word sy, const tea_word *key, tea_word *dest) // PORT: tea_word (32-bit Win32 unsigned long) +{ +#pragma warning(disable:4307) + tea_word y = sy, z = sz, sum = DELTA * TEA_ROUND; // PORT: tea_word (32-bit Win32 unsigned long) +#pragma warning(default:4307) + + tea_word n = TEA_ROUND; // PORT: tea_word (32-bit Win32 unsigned long) + + while (n-- > 0) + { + z -= ((y << 4 ^ y >> 5) + y) ^ (sum + key[sum >> 11 & 3]); + sum -= DELTA; + y -= ((z << 4 ^ z >> 5) + z) ^ (sum + key[sum & 3]); + } + + *(dest++) = y; + *dest = z; +} + +int tea_encrypt(tea_word *dest, const tea_word *src, const tea_word * key, int size) // PORT: tea_word (32-bit Win32 unsigned long) +{ + int i; + int resize; + + if (size % 8 != 0) + { + resize = size + 8 - (size % 8); + memset((char *) src + size, 0, resize - size); + } + else + resize = size; + + for (i = 0; i < resize >> 3; i++, dest += 2, src += 2) + tea_code(*(src + 1), *src, key, dest); + + return (resize); +} + +int tea_decrypt(tea_word *dest, const tea_word *src, const tea_word * key, int size) // PORT: tea_word (32-bit Win32 unsigned long) +{ + int i; + int resize; + + if (size % 8 != 0) + resize = size + 8 - (size % 8); + else + resize = size; + + for (i = 0; i < resize >> 3; i++, dest += 2, src += 2) + tea_decode(*(src + 1), *src, key, dest); + + return (resize); +} + diff --git a/extension/src/port/EterBase/tea.h b/extension/src/port/EterBase/tea.h index c93f5036..ca516a4a 100644 --- a/extension/src/port/EterBase/tea.h +++ b/extension/src/port/EterBase/tea.h @@ -1,3 +1,11 @@ +// PORT: the 32-bit Win32 unsigned long; on Windows it stays unsigned long so DWORD* callers match. +#if defined(_WIN32) +typedef unsigned long tea_word; +#else +#include +typedef uint32_t tea_word; +#endif + #ifdef __cplusplus extern "C" { #endif @@ -11,8 +19,8 @@ extern "C" { #define TEA_KEY_LENGTH 16 - int tea_encrypt(unsigned long *dest, const unsigned long *src, const unsigned long *key, int size); - int tea_decrypt(unsigned long *dest, const unsigned long *src, const unsigned long *key, int size); + int tea_encrypt(tea_word *dest, const tea_word *src, const tea_word *key, int size); // PORT: tea_word (32-bit) + int tea_decrypt(tea_word *dest, const tea_word *src, const tea_word *key, int size); // PORT: tea_word (32-bit) #ifdef __cplusplus }; diff --git a/extension/src/port/EterPack/EterPack.cpp b/extension/src/port/EterPack/EterPack.cpp new file mode 100644 index 00000000..65d66dd0 --- /dev/null +++ b/extension/src/port/EterPack/EterPack.cpp @@ -0,0 +1,1605 @@ +#include "StdAfx.h" + +#include +#include +#include +#include +#include // PORT: std::istream_iterator (CMakePackLog); MSVC/libc++ pull it in transitively + +#include "EterPack.h" +#include "Inline.h" +#include "EterPackPolicy_CSHybridCrypt.h" + +#pragma warning(push, 3) +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#pragma warning(pop) + +#include "../EterBase/Utils.h" +#include "../EterBase/Debug.h" +#include "../EterBase/CRC32.h" + +#ifdef __THEMIDA__ +#include +#endif + +#include +#include + +void CMakePackLog::SetFileName(const char* c_szFileName) +{ + m_stFileName=c_szFileName; + m_stFileName +=".log"; + m_stErrorFileName = c_szFileName; + m_stErrorFileName += ".err"; +} + +CMakePackLog& CMakePackLog::GetSingleton() +{ + static CMakePackLog s_kMakePackLog; + return s_kMakePackLog; +} + +CMakePackLog::CMakePackLog() +{ + m_fp=NULL; + m_fp_err = NULL; +} + +CMakePackLog::~CMakePackLog() +{ + if (NULL!=m_fp) + { + fclose(m_fp); + m_fp=NULL; + } + if (NULL != m_fp_err) + { + fclose(m_fp_err); + m_fp_err = NULL; + } +} + +bool CMakePackLog::__IsLogMode() +{ + if (m_stFileName.empty()) + return false; + + return true; +} + +void CMakePackLog::Writef(const char* c_szFormat, ...) +{ + if (!__IsLogMode()) + return; + + va_list args; + va_start(args, c_szFormat); + + char szBuf[1024]; + int nBufLen = _vsnprintf(szBuf, sizeof(szBuf), c_szFormat, args); + szBuf[nBufLen++] = '\0'; + __Write(szBuf, nBufLen); +} + +void CMakePackLog::Writenf(const char* c_szFormat, ...) +{ + if (!__IsLogMode()) + return; + + va_list args; + va_start(args, c_szFormat); + + char szBuf[1024+1]; + int nBufLen = _vsnprintf(szBuf, sizeof(szBuf)-1, c_szFormat, args); + if (nBufLen > 0) + { + szBuf[nBufLen++] = '\n'; + szBuf[nBufLen++] = '\0'; + } + __Write(szBuf, nBufLen); +} + +void CMakePackLog::Write(const char* c_szBuf) +{ + if (!__IsLogMode()) + return; + + __Write(c_szBuf, strlen(c_szBuf)+1); +} + +void CMakePackLog::__Write(const char* c_szBuf, int nBufLen) +{ + if (!__IsLogMode()) + return; + + if (NULL==m_fp) + m_fp=fopen(m_stFileName.c_str(), "w"); + + fwrite(c_szBuf, nBufLen, 1, m_fp); + + printf("%s", c_szBuf); +} + + + +void CMakePackLog::WriteErrorf(const char* c_szFormat, ...) +{ + if (!__IsLogMode()) + return; + + va_list args; + va_start(args, c_szFormat); + + char szBuf[1024]; + int nBufLen = _vsnprintf(szBuf, sizeof(szBuf), c_szFormat, args); + szBuf[nBufLen++] = '\0'; + __WriteError(szBuf, nBufLen); +} + +void CMakePackLog::WriteErrornf(const char* c_szFormat, ...) +{ + if (!__IsLogMode()) + return; + + va_list args; + va_start(args, c_szFormat); + + char szBuf[1024+1]; + int nBufLen = _vsnprintf(szBuf, sizeof(szBuf)-1, c_szFormat, args); + if (nBufLen > 0) + { + szBuf[nBufLen++] = '\n'; + szBuf[nBufLen++] = '\0'; + } + __WriteError(szBuf, nBufLen); +} + +void CMakePackLog::WriteError(const char* c_szBuf) +{ + if (!__IsLogMode()) + return; + + __WriteError(c_szBuf, strlen(c_szBuf)+1); +} + +void CMakePackLog::__WriteError(const char* c_szBuf, int nBufLen) +{ + if (!__IsLogMode()) + return; + + if (NULL==m_fp_err) + m_fp_err=fopen(m_stErrorFileName.c_str(), "w"); + + fwrite(c_szBuf, nBufLen, 1, m_fp_err); + + printf("Error: %s", c_szBuf); +} + +void CMakePackLog::FlushError() +{ + std::wifstream iFile(m_stErrorFileName.c_str()); + std::istream_iterator iit(iFile); + std::istream_iterator eos; + + std::vector vText; + + std::copy (iit, eos, std::back_inserter(vText)); + + std::ostream_iterator > oit(std::wcout); + + std::sort (vText.begin(), vText.end()); + std::copy (vText.begin(), vText.end(), oit); +} +#ifdef __MAKE_PACK__ +FILE * CEterPack::ms_PackLogFile = NULL; +#endif +/////////////////////////////////////////////////////////////////////////////// +CEterPack::CEterPack() : m_indexCount(0), m_indexData(NULL), m_FragmentSize(0), m_bEncrypted(false), m_bReadOnly(false), m_bDecrypedIV(false) +{ + m_pCSHybridCryptPolicy = new EterPackPolicy_CSHybridCrypt; + +} + +CEterPack::~CEterPack() +{ + Destroy(); + + delete m_pCSHybridCryptPolicy; + m_pCSHybridCryptPolicy = NULL; +} + +void CEterPack::Destroy() +{ + m_bReadOnly = false; + m_bEncrypted = false; + m_indexCount = 0; + m_DataPositionMap.clear(); + + for (int i = 0; i < FREE_INDEX_MAX_SIZE + 1; ++i) + m_FreeIndexList[i].clear(); + + SAFE_DELETE_ARRAY(m_indexData); + + m_FragmentSize = 0; + + memset(m_dbName, 0, sizeof(m_dbName)); + memset(m_indexFileName, 0, sizeof(m_indexFileName)); +} + +const std::string& CEterPack::GetPathName() +{ + return m_stPathName; +} + +bool CEterPack::Create(CEterFileDict& rkFileDict, const char * dbname, const char* pathName, bool bReadOnly, const BYTE* iv) +{ + if (iv) + { + m_stIV_Panama.assign((const char*) iv, 32); + m_bDecrypedIV = false; + } + + m_stPathName = pathName; + + strncpy(m_dbName, dbname, DBNAME_MAX_LEN); + + strncpy(m_indexFileName, dbname, MAX_PATH); + strcat(m_indexFileName, ".eix"); + + m_stDataFileName = dbname; + m_stDataFileName += ".epk"; + + m_bReadOnly = bReadOnly; + + // bReadOnly 모드가 아니고 데이터 베이스가 열린다면 생성 실패 + if (!CreateIndexFile()) + return false; + + if (!CreateDataFile()) + return false; + + bool bOverwrite = (iv != NULL); + __BuildIndex(rkFileDict, bOverwrite); + + if (m_bReadOnly) + { + //m_bIsDataLoaded = true; + //if (!m_file.Create(m_stDataFileName.c_str(), (const void**)&m_file_data, 0, 0)) + // return false; + } + else + { + DecryptIndexFile(); + } + + return true; +} + +bool CEterPack::DecryptIV(DWORD dwPanamaKey) +{ + if (m_stIV_Panama.length() != 32) + return false; + + if (m_bDecrypedIV) // 이미 암호화가 풀렸으면 다시 처리 안함 + return true; + + DWORD* ivs = (DWORD*)&m_stIV_Panama[0]; + for (int i = 0; i != m_stIV_Panama.length() / sizeof(DWORD); ++i) + { + ivs[i] ^= dwPanamaKey + i * 16777619; + } + + m_bDecrypedIV = true; + return true; +} + +bool CEterPack::DecryptIndexFile() +{ + if (!m_bEncrypted) + return true; + + CFileBase file; + + if (!file.Create(m_indexFileName, CFileBase::FILEMODE_WRITE)) + return false; + + file.Write(&eterpack::c_IndexCC, sizeof(DWORD)); + file.Write(&eterpack::c_Version, sizeof(DWORD)); + file.Write(&m_indexCount, sizeof(LONG)); // PORT: 32-bit Win32 long + file.Write(m_indexData, sizeof(TEterPackIndex) * m_indexCount); + + file.Close(); + + m_bEncrypted = false; + return true; +} + +// PORT: the two key arrays (s_adwEterPackKey, s_adwEterPackSecurityKey) are generated at configure time +// from the reference EterPack.cpp (port/EterPack/CMakeLists fragment in port/CMakeLists.txt); pack keys stay +// out of the repository. +#include "EterPackKeys.generated.h" + +bool CEterPack::EncryptIndexFile() +{ + CMappedFile file; + LPCVOID pvData; + + if (NULL == file.Create(m_indexFileName, &pvData, 0, 0)) + { + TraceError("EncryptIndex: Cannot open pack index file! %s", m_dbName); + return false; + } + + BYTE * pbData = new BYTE[file.Size()]; + memcpy(pbData, pvData, file.Size()); + + CLZObject zObj; + + if (!CLZO::Instance().CompressEncryptedMemory(zObj, pbData, file.Size(), s_adwEterPackKey)) + { + TraceError("EncryptIndex: Cannot encrypt! %s", m_dbName); + SAFE_DELETE_ARRAY(pbData); + return false; + } + + file.Destroy(); + + while (!DeleteFile(m_indexFileName)); + + FILE * fp; + + fp = fopen(m_indexFileName, "wb"); + + if (!fp) + { + TraceError("EncryptIndex: Cannot open file for writing! %s", m_dbName); + SAFE_DELETE_ARRAY(pbData); + return false; + } + + if (1 != fwrite(zObj.GetBuffer(), zObj.GetSize(), 1, fp)) + { + TraceError("Encryptindex: Cannot write to file! %s", m_indexFileName); + SAFE_DELETE_ARRAY(pbData); + fclose(fp); + return false; + } + + fclose(fp); + + m_bEncrypted = true; + delete [] pbData; + return true; +} + +bool CEterPack::__BuildIndex(CEterFileDict& rkFileDict, bool bOverwrite) +{ + //DWORD dwBeginTime = ELTimer_GetMSec(); + CMappedFile file; + LPCVOID pvData; + CLZObject zObj; + + if (NULL == file.Create(m_indexFileName, &pvData, 0, 0)) + { + TraceError("Cannot open pack index file! %s", m_dbName); + return false; + } + + if (file.Size() < eterpack::c_HeaderSize) + { + TraceError("Pack index file header error! %s", m_dbName); + return false; + } + + DWORD fourcc = *(DWORD *) pvData; + + BYTE * pbData; + UINT uiFileSize; + + if (fourcc == eterpack::c_IndexCC) + { + pbData = (BYTE *) pvData; + uiFileSize = file.Size(); + } + else if (fourcc == CLZObject::ms_dwFourCC) + { + m_bEncrypted = true; + + if (!CLZO::Instance().Decompress(zObj, (const BYTE *) pvData, s_adwEterPackKey)) + return false; + + if (zObj.GetSize() < eterpack::c_HeaderSize) + return false; + + pbData = zObj.GetBuffer(); + uiFileSize = zObj.GetSize(); + } + else + { + TraceError("Pack index file fourcc error! %s", m_dbName); + return false; + } + + pbData += sizeof(DWORD); + + DWORD ver = *(DWORD *) pbData; + pbData += sizeof(DWORD); + + if (ver != eterpack::c_Version) + { + TraceError("Pack index file version error! %s", m_dbName); + return false; + } + + m_indexCount = *(LONG *) pbData; // PORT: 32-bit Win32 long + pbData += sizeof(LONG); // PORT: 32-bit Win32 long + + if (uiFileSize < eterpack::c_HeaderSize + sizeof(TEterPackIndex) * m_indexCount) + { + TraceError("Pack index file size error! %s, indexCount %d", m_dbName, m_indexCount); + return false; + } + + //Tracef("Loading Pack file %s elements: %d ... ", m_dbName, m_indexCount); + + m_indexData = new TEterPackIndex[m_indexCount]; + memcpy(m_indexData, pbData, sizeof(TEterPackIndex) * m_indexCount); + + TEterPackIndex * index = m_indexData; + + for (int i = 0; i < m_indexCount; ++i, ++index) + { + if (!index->filename_crc) + { + PushFreeIndex(index); + } + else + { + if (index->real_data_size > index->data_size) + m_FragmentSize += index->real_data_size - index->data_size; + + m_DataPositionMap.insert(TDataPositionMap::value_type(index->filename_crc, index)); + + if (bOverwrite) // 서버 연동 패킹 파일은 나중에 들어오지만 최상위로 등록해야한다 + rkFileDict.UpdateItem(this, index); + else + rkFileDict.InsertItem(this, index); + } + } + + //Tracef("Done. (spent %dms)\n", ELTimer_GetMSec() - dwBeginTime); + return true; +} +// +//void CEterPack::UpdateLastAccessTime() +//{ +// m_tLastAccessTime = time(NULL); +//} +// +//void CEterPack::ClearDataMemoryMap() +//{ +// // m_file이 data file이다... +// m_file.Destroy(); +// m_tLastAccessTime = 0; +// m_bIsDataLoaded = false; +//} + +bool CEterPack::Get(CMappedFile& out_file, const char * filename, LPCVOID * data) +{ + TEterPackIndex * index = FindIndex(filename); + + if (!index) + { + return false; + } + + //UpdateLastAccessTime(); + //if (!m_bIsDataLoaded) + //{ + // if (!m_file.Create(m_stDataFileName.c_str(), (const void**)&m_file_data, 0, 0)) + // return false; + // + // m_bIsDataLoaded = true; + //} + + // 기존에는 CEterPack에서 epk를 memory map에 올려놓고, 요청이 오면 그 부분을 링크해서 넘겨 줬었는데, + // 이제는 요청이 오면, 필요한 부분만 memory map에 올리고, 요청이 끝나면 해제하게 함. + out_file.Create(m_stDataFileName.c_str(), data, index->data_position, index->data_size); + + bool bIsSecurityCheckRequired = ( index->compressed_type == COMPRESSED_TYPE_SECURITY || + index->compressed_type == COMPRESSED_TYPE_PANAMA ); + + if( bIsSecurityCheckRequired ) + { +#ifdef CHECKSUM_CHECK_MD5 + MD5_CTX context; + GenerateMD5Hash( (BYTE*)(*data), index->data_size, context ); + + if( memcmp( index->MD5Digest, context.digest, 16 ) != 0 ) + { + return false; + } +#else + DWORD dwCrc32 = GetCRC32((const char*)(*data), index->data_size); + + if( index->data_crc != dwCrc32 ) + { + return false; + } +#endif + } + + + if (COMPRESSED_TYPE_COMPRESS == index->compressed_type) + { + CLZObject * zObj = new CLZObject; + + if (!CLZO::Instance().Decompress(*zObj, static_cast(*data))) + { + TraceError("Failed to decompress : %s", filename); + delete zObj; + return false; + } + + out_file.BindLZObject(zObj); + *data = zObj->GetBuffer(); + } + else if (COMPRESSED_TYPE_SECURITY == index->compressed_type) + { + CLZObject * zObj = new CLZObject; + + if (!CLZO::Instance().Decompress(*zObj, static_cast(*data), s_adwEterPackSecurityKey)) + { + TraceError("Failed to encrypt : %s", filename); + delete zObj; + return false; + } + + out_file.BindLZObject(zObj); + *data = zObj->GetBuffer(); + } + else if (COMPRESSED_TYPE_PANAMA == index->compressed_type) + { + CLZObject* zObj = new CLZObject; + if (!__Decrypt_Panama(filename, static_cast(*data), index->data_size, *zObj)) + { + delete zObj; + return false; + } + + out_file.BindLZObjectWithBufferedSize(zObj); + *data = zObj->GetBuffer(); + } + else if (COMPRESSED_TYPE_HYBRIDCRYPT == index->compressed_type || COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) + { +#ifdef __THEMIDA__ + VM_START +#endif + + CLZObject * zObj = new CLZObject; + auto ssvv = std::string(filename); + if (!m_pCSHybridCryptPolicy->DecryptMemory(ssvv, static_cast(*data), index->data_size, *zObj)) + { + delete zObj; + return false; + } + + out_file.BindLZObjectWithBufferedSize(zObj); + if( COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) + { + BYTE* pSDBData; + int iSDBSize; + + if (!m_pCSHybridCryptPolicy->GetSupplementaryDataBlock(ssvv, pSDBData, iSDBSize)) + { + delete zObj; + return false; + } + + *data = out_file.AppendDataBlock( pSDBData, iSDBSize ); + } + else + { + *data = zObj->GetBuffer(); + } +#ifdef __THEMIDA__ + VM_END +#endif + } + return true; +} + +bool CEterPack::Get2(CMappedFile& out_file, const char * filename, TEterPackIndex * index, LPCVOID * data) +{ + if (!index) + { + return false; + } + + //UpdateLastAccessTime(); + //if (!m_bIsDataLoaded) + //{ + // if (!m_file.Create(m_stDataFileName.c_str(), (const void**)&m_file_data, 0, 0)) + // return false; + // + // m_bIsDataLoaded = true; + //} + out_file.Create(m_stDataFileName.c_str(), data, index->data_position, index->data_size); + + bool bIsSecurityCheckRequired = ( index->compressed_type == COMPRESSED_TYPE_SECURITY || + index->compressed_type == COMPRESSED_TYPE_PANAMA ); + + if( bIsSecurityCheckRequired ) + { +#ifdef CHECKSUM_CHECK_MD5 + MD5_CTX context; + GenerateMD5Hash( (BYTE*)(*data), index->data_size, context ); + + if( memcmp( index->MD5Digest, context.digest, 16 ) != 0 ) + { + return false; + } +#else + DWORD dwCrc32 = GetCRC32((const char*)(*data), index->data_size); + + if( index->data_crc != dwCrc32 ) + { + return false; + } +#endif + } + + + if (COMPRESSED_TYPE_COMPRESS == index->compressed_type) + { + CLZObject * zObj = new CLZObject; + + if (!CLZO::Instance().Decompress(*zObj, static_cast(*data))) + { + TraceError("Failed to decompress : %s", filename); + delete zObj; + return false; + } + + out_file.BindLZObject(zObj); + *data = zObj->GetBuffer(); + } + else if (COMPRESSED_TYPE_SECURITY == index->compressed_type) + { + CLZObject * zObj = new CLZObject; + + if (!CLZO::Instance().Decompress(*zObj, static_cast(*data), s_adwEterPackSecurityKey)) + { + TraceError("Failed to encrypt : %s", filename); + delete zObj; + return false; + } + + out_file.BindLZObject(zObj); + *data = zObj->GetBuffer(); + } + else if (COMPRESSED_TYPE_PANAMA == index->compressed_type) + { + CLZObject * zObj = new CLZObject; + __Decrypt_Panama(filename, static_cast(*data), index->data_size, *zObj); + out_file.BindLZObjectWithBufferedSize(zObj); + *data = zObj->GetBuffer(); + } + else if (COMPRESSED_TYPE_HYBRIDCRYPT == index->compressed_type || COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) + { +#ifdef __THEMIDA__ + VM_START +#endif + + CLZObject * zObj = new CLZObject; + auto ssvv = std::string(filename); + if( !m_pCSHybridCryptPolicy->DecryptMemory(ssvv, static_cast(*data), index->data_size, *zObj) ) + { + delete zObj; + return false; + } + + out_file.BindLZObjectWithBufferedSize(zObj); + + if( COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) + { + BYTE* pSDBData; + int iSDBSize; + + if( !m_pCSHybridCryptPolicy->GetSupplementaryDataBlock(ssvv, pSDBData, iSDBSize) ) + { + return false; + } + + *data = out_file.AppendDataBlock( pSDBData, iSDBSize ); + } + else + { + *data = zObj->GetBuffer(); + } +#ifdef __THEMIDA__ + VM_END +#endif + } + + return true; +} + + +bool CEterPack::Delete(TEterPackIndex * pIndex) +{ + CFileBase fileIndex; + + if (!fileIndex.Create(m_indexFileName, CFileBase::FILEMODE_WRITE)) + return false; + + PushFreeIndex(pIndex); + WriteIndex(fileIndex, pIndex); + return true; +} + +bool CEterPack::Delete(const char * filename) +{ + TEterPackIndex * pIndex = FindIndex(filename); + + if (!pIndex) + return false; + + return Delete(pIndex); +} + +bool CEterPack::Extract() +{ + CMappedFile dataMapFile; + LPCVOID data; + + if (!dataMapFile.Create(m_stDataFileName.c_str(), &data, 0, 0)) + return false; + + CLZObject zObj; + + for (TDataPositionMap::iterator i = m_DataPositionMap.begin(); + i != m_DataPositionMap.end(); + ++i) + { + TEterPackIndex* index = i->second; + CFileBase writeFile; + + inlinePathCreate(index->filename); + printf("%s\n", index->filename); + + writeFile.Create(index->filename, CFileBase::FILEMODE_WRITE); + + if (COMPRESSED_TYPE_COMPRESS == index->compressed_type) + { + if (!CLZO::Instance().Decompress(zObj, (const BYTE *) data + index->data_position)) + { + printf("cannot decompress"); + return false; + } + + writeFile.Write(zObj.GetBuffer(), zObj.GetSize()); + zObj.Clear(); + } + else if (COMPRESSED_TYPE_SECURITY == index->compressed_type) + { + if (!CLZO::Instance().Decompress(zObj, (const BYTE *) data + index->data_position, s_adwEterPackSecurityKey)) + { + printf("cannot decompress"); + return false; + } + + writeFile.Write(zObj.GetBuffer(), zObj.GetSize()); + zObj.Clear(); + } + else if (COMPRESSED_TYPE_PANAMA == index->compressed_type) + { + __Decrypt_Panama(index->filename, (const BYTE *) data + index->data_position, index->data_size, zObj); + writeFile.Write(zObj.GetBuffer(), zObj.GetBufferSize()); + zObj.Clear(); + } + else if (COMPRESSED_TYPE_HYBRIDCRYPT == index->compressed_type || COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) + { +#ifdef __THEMIDA__ + VM_START +#endif + auto ssvv = std::string(index->filename); + if( !m_pCSHybridCryptPolicy->DecryptMemory(ssvv, (const BYTE *) data + index->data_position, index->data_size, zObj) ) + return false; + + if( COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) + { + dataMapFile.BindLZObjectWithBufferedSize(&zObj); + + BYTE* pSDBData; + int iSDBSize; + + if( !m_pCSHybridCryptPolicy->GetSupplementaryDataBlock(ssvv, pSDBData, iSDBSize) ) + return false; + + dataMapFile.AppendDataBlock( pSDBData, iSDBSize ); + writeFile.Write(dataMapFile.AppendDataBlock( pSDBData, iSDBSize ),dataMapFile.Size()); + } + else + { + writeFile.Write(zObj.GetBuffer(), zObj.GetBufferSize()); + } + zObj.Clear(); +#ifdef __THEMIDA__ + VM_END +#endif + } + else if (COMPRESSED_TYPE_NONE == index->compressed_type) + writeFile.Write((const char *) data + index->data_position, index->data_size); + + writeFile.Destroy(); + } + return true; +} + +bool CEterPack::GetNames(std::vector* retNames) +{ + CMappedFile dataMapFile; + LPCVOID data; + + if (!dataMapFile.Create(m_stDataFileName.c_str(), &data, 0, 0)) + return false; + + CLZObject zObj; + + for (TDataPositionMap::iterator i = m_DataPositionMap.begin(); + i != m_DataPositionMap.end(); + ++i) + { + TEterPackIndex* index = i->second; + + inlinePathCreate(index->filename); + + retNames->push_back(index->filename); + } + return true; +} + +bool CEterPack::Put(const char * filename, const char * sourceFilename, BYTE packType, const std::string& strRelateMapName ) +{ + CMappedFile mapFile; + LPCVOID data; + + if (sourceFilename) + { + if (!mapFile.Create(sourceFilename, &data, 0, 0)) + { + return false; + } + } + else if (!mapFile.Create(filename, &data, 0, 0)) + { + return false; + } + + BYTE* pMappedData = (BYTE*)data; + int iMappedDataSize = mapFile.Size(); + + if( packType == COMPRESSED_TYPE_HYBRIDCRYPT || packType == COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB ) + { +#ifdef __THEMIDA__ + VM_START +#endif + auto ssvvv = std::string(filename); + m_pCSHybridCryptPolicy->GenerateCryptKey(ssvvv); + + if( packType == COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB ) + { + if( !m_pCSHybridCryptPolicy->GenerateSupplementaryDataBlock(ssvvv, strRelateMapName, (const BYTE*)data, mapFile.Size(), pMappedData, iMappedDataSize )) + { + return false; + } + } +#ifdef __THEMIDA__ + VM_END +#endif + } + + return Put(filename, pMappedData, iMappedDataSize, packType); +} + +#ifdef CHECKSUM_CHECK_MD5 +void CEterPack::GenerateMD5Hash( BYTE* pData, int nLength, IN OUT MD5_CTX& mdContext ) +{ + MD5Init (&mdContext); + + const int nBlockSize = 1024; + + int nLoopCnt = nLength / nBlockSize; + int nRemainder = nLength % nBlockSize; + int i; + + for(i = 0; i < nLoopCnt; ++i ) + { + MD5Update (&mdContext, reinterpret_cast(pData + i*nBlockSize), nBlockSize); + } + + if( nRemainder > 0 ) + MD5Update (&mdContext, reinterpret_cast(pData + i*nBlockSize), nRemainder); + + MD5Final (&mdContext); +} +#endif +bool CEterPack::Put(const char * filename, LPCVOID data, long len, BYTE packType) +{ + if (m_bEncrypted) + { + TraceError("EterPack::Put : Cannot put to encrypted pack (filename: %s, DB: %s)", filename, m_dbName); + return false; + } + + CFileBase fileIndex; + + if (!fileIndex.Create(m_indexFileName, CFileBase::FILEMODE_WRITE)) + { + return false; + } + + CFileBase fileData; + + if (!fileData.Create(m_stDataFileName.c_str(), CFileBase::FILEMODE_WRITE)) + { + return false; + } + + TEterPackIndex * pIndex; + pIndex = FindIndex(filename); + + CLZObject zObj; + std::string encryptStr; + + if (packType == COMPRESSED_TYPE_SECURITY || + packType == COMPRESSED_TYPE_COMPRESS) + { + if (packType == COMPRESSED_TYPE_SECURITY) + { + if (!CLZO::Instance().CompressEncryptedMemory(zObj, data, len, s_adwEterPackSecurityKey)) + { + return false; + } + } + else + { + if (!CLZO::Instance().CompressMemory(zObj, data, len)) + { + return false; + } + } + + data = zObj.GetBuffer(); + len = zObj.GetSize(); + } + else if (packType == COMPRESSED_TYPE_PANAMA) + { + if (!__Encrypt_Panama(filename, (const BYTE *) data, len, zObj)) + { + return false; + } + + data = zObj.GetBuffer(); + len = zObj.GetBufferSize(); + } + else if (packType == COMPRESSED_TYPE_HYBRIDCRYPT || packType == COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB) + { +#ifdef __THEMIDA__ + VM_START +#endif + + auto ssvv = std::string(filename); + if (!m_pCSHybridCryptPolicy->EncryptMemory(ssvv, (const BYTE*)data, len, zObj)) + { + return false; + } + + data = zObj.GetBuffer(); + len = zObj.GetBufferSize(); +#ifdef __THEMIDA__ + VM_END +#endif + + } + + +#ifdef CHECKSUM_CHECK_MD5 + MD5_CTX context; + GenerateMD5Hash( (BYTE*)(data), len, context ); +#else + DWORD data_crc; + data_crc = GetCRC32((const char *) data, len); +#endif + + // 기존 데이터가 있으면.. + if (pIndex) + { + // 기존 data 크기가 넣을 데이터 크기를 수용할 수 있다면 + if (pIndex->real_data_size >= len) + { + ++m_map_indexRefCount[pIndex->id]; + + // 길이가 틀리거나, checksum이 틀릴 때만 저장 한다. + if ( (pIndex->data_size != len) || +#ifdef CHECKSUM_CHECK_MD5 + (memcmp( pIndex->MD5Digest, context.digest, 16 ) != 0) ) +#else + (pIndex->data_crc != data_crc) ) +#endif + { +#ifdef __MAKE_PACK__ + if (ms_PackLogFile) + fprintf(ms_PackLogFile, "Overwrite[%d/%d] %s\n", bCompress, bSecurity, pIndex->filename); + printf("Overwrite[%d/%d] %s\n", bCompress, bSecurity, pIndex->filename); +#endif + pIndex->data_size = len; +#ifdef CHECKSUM_CHECK_MD5 + memcpy( pIndex->MD5Digest, context.digest, 16 ); + +#else + pIndex->data_crc = data_crc; +#endif + + pIndex->compressed_type = packType; + + CMakePackLog::GetSingleton().Writef("Overwrite[type:%u] %s\n", pIndex->compressed_type, pIndex->filename); + + WriteIndex(fileIndex, pIndex); + WriteData(fileData, pIndex, data); + } + + return true; + } + + // 기존 데이터 크기가 새로 들어갈 것 보다 적다면, 새로 인덱스를 할당해 + // 넣어야 한다. 원래 있던 인덱스는 비워 둔다. + PushFreeIndex(pIndex); + WriteIndex(fileIndex, pIndex); + } + + // 새 데이터 + pIndex = NewIndex(fileIndex, filename, len); + pIndex->data_size = len; + +#ifdef CHECKSUM_CHECK_MD5 + memcpy( pIndex->MD5Digest, context.digest, 16 ); +#else + pIndex->data_crc = data_crc; +#endif + + + pIndex->data_position = GetNewDataPosition(fileData); + pIndex->compressed_type = packType; + + WriteIndex(fileIndex, pIndex); + WriteNewData(fileData, pIndex, data); + + ++m_map_indexRefCount[pIndex->id]; + +#ifdef __MAKE_PACK__ + if (ms_PackLogFile) + fprintf(ms_PackLogFile, "Write[%d/%d] %s\n", bCompress, bSecurity, pIndex->filename); + printf("Write[%d/%d] %s, position %ld realsize %ld size %ld\n", + bCompress, bSecurity, pIndex->filename, pIndex->data_position, pIndex->real_data_size, pIndex->data_size); +#endif + CMakePackLog::GetSingleton().Writef("Write[type:%u] %s\n", pIndex->compressed_type, pIndex->filename); + + return true; +} + +long CEterPack::GetFragmentSize() +{ + return m_FragmentSize; +} + +// Private methods +bool CEterPack::CreateIndexFile() +{ + FILE * fp; + + if (NULL != (fp = fopen(m_indexFileName, "rb"))) + { + fclose(fp); + return true; + } + else if (m_bReadOnly) + return false; + + // + // 파일이 없으므로 새로 만든다. + // + fp = fopen(m_indexFileName, "wb"); + + if (!fp) + return false; + + fwrite(&eterpack::c_IndexCC, sizeof(DWORD), 1, fp); + fwrite(&eterpack::c_Version, sizeof(DWORD), 1, fp); + fwrite(&m_indexCount, sizeof(LONG), 1, fp); // PORT: 32-bit Win32 long + + fclose(fp); + return true; +} + + +void CEterPack::WriteIndex(CFileBase & file, TEterPackIndex * index) +{ + file.Seek(sizeof(DWORD) + sizeof(DWORD)); + file.Write(&m_indexCount, sizeof(LONG)); // PORT: 32-bit Win32 long + file.Seek(eterpack::c_HeaderSize + (index->id * sizeof(TEterPackIndex))); + + if (!file.Write(index, sizeof(TEterPackIndex))) + { + assert(!"WriteIndex: fwrite failed"); + return; + } +} + +/* + * Free Block 이란 데이터에서 지워진 부분을 말한다. + * Free Block 들은 각각 FREE_INDEX_BLOCK_SIZE (32768) 단위로 나누어져 + * 리스트로 관리된다. + * + * 예를 들어 128k 의 데이터는 + * 128 * 1024 / FREE_INDEX_BLOCK_SIZE = 4 이므로 + * 최종 적으로는 m_FreeIndexList[4] 에 들어간다. + * + * FREE_INDEX_BLOCK_SIZE 의 최대 값은 FREE_INDEX_MAX_SIZE(512) 이다. + * 따라서 16MB 이상의 데이터는 무조건 배열의 512 위치에 들어간다. + */ +int CEterPack::GetFreeBlockIndex(long size) +{ + return min(FREE_INDEX_MAX_SIZE, size / FREE_INDEX_BLOCK_SIZE); +} + +void CEterPack::PushFreeIndex(TEterPackIndex* index) +{ + if (index->filename_crc != 0) + { + TDataPositionMap::iterator i = m_DataPositionMap.find(index->filename_crc); + + if (i != m_DataPositionMap.end()) + m_DataPositionMap.erase(i); + + index->filename_crc = 0; + memset(index->filename, 0, sizeof(index->filename)); + } + + int blockidx = GetFreeBlockIndex(index->real_data_size); + m_FreeIndexList[blockidx].push_back(index); + m_FragmentSize += index->real_data_size; + //printf("FreeIndex: size %d: blockidx: %d\n", index->real_data_size, blockidx); +} + +long CEterPack::GetNewIndexPosition(CFileBase & file) +{ + long pos = (file.Size() - eterpack::c_HeaderSize) / sizeof(TEterPackIndex); + ++m_indexCount; + return (pos); +} + +TEterPackIndex* CEterPack::NewIndex(CFileBase& file, const char* filename, long size) +{ + TEterPackIndex* index = NULL; + int block_size = size + (DATA_BLOCK_SIZE - (size % DATA_BLOCK_SIZE)); +// if ((index = FindIndex(filename))) // 이미 인덱스가 존재하는지 확인 +// return index; + + int blockidx = GetFreeBlockIndex(block_size); + + for (TFreeIndexList::iterator i = m_FreeIndexList[blockidx].begin(); + i != m_FreeIndexList[blockidx].end(); + ++i) + { + if ((*i)->real_data_size >= size) + { + index = *i; + m_FreeIndexList[blockidx].erase(i); + + assert(index->filename_crc == 0); + break; + } + } + + if (!index) + { + index = new TEterPackIndex; + index->real_data_size = block_size; + index->id = GetNewIndexPosition(file); + } + + strncpy(index->filename, filename, FILENAME_MAX_LEN); + index->filename[FILENAME_MAX_LEN] = '\0'; + inlineConvertPackFilename(index->filename); + + index->filename_crc = GetCRC32(index->filename, strlen(index->filename)); + + m_DataPositionMap.insert(TDataPositionMap::value_type(index->filename_crc, index)); + return index; +} + +TEterPackIndex* CEterPack::FindIndex(const char * filename) +{ + static char tmpFilename[MAX_PATH + 1]; + strncpy(tmpFilename, filename, MAX_PATH); + inlineConvertPackFilename(tmpFilename); + + DWORD filename_crc = GetCRC32(tmpFilename, strlen(tmpFilename)); + TDataPositionMap::iterator i = m_DataPositionMap.find(filename_crc); + + if (i == m_DataPositionMap.end()) + return NULL; + + return (i->second); +} + +bool CEterPack::IsExist(const char * filename) +{ + return FindIndex(filename) ? true : false; +} + +bool CEterPack::CreateDataFile() +{ + FILE * fp; + + if (NULL != (fp = fopen(m_stDataFileName.c_str(), "rb"))) + { + fclose(fp); + return true; + } + else if (m_bReadOnly) + return false; + + fp = fopen(m_stDataFileName.c_str(), "wb"); + + if (!fp) + return false; + + fclose(fp); + return true; +} + +long CEterPack::GetNewDataPosition(CFileBase& file) +{ + return file.Size(); +} + +bool CEterPack::ReadData(CFileBase & file, TEterPackIndex* index, LPVOID data, long maxsize) +{ + if (index->data_size > maxsize) + return false; + + file.Seek(index->data_position); + file.Read(data, index->data_size); + return true; +} + +bool CEterPack::WriteData(CFileBase & file, TEterPackIndex* index, LPCVOID data) +{ + file.Seek(index->data_position); + + if (!file.Write(data, index->data_size)) + { + assert(!"WriteData: fwrite data failed"); + return false; + } + + return true; +} + +bool CEterPack::WriteNewData(CFileBase& file, TEterPackIndex* index, LPCVOID data) +{ + file.Seek(index->data_position); + + if (!file.Write(data, index->data_size)) + { + assert(!"WriteData: fwrite data failed"); + return false; + } + + int empty_size = index->real_data_size - index->data_size; + + if (empty_size < 0) + { + printf("SYSERR: WriteNewData(): CRITICAL ERROR: empty_size lower than 0!\n"); + exit(1); + } + + if (empty_size == 0) + return true; + + char * empty_buf = (char *) calloc(empty_size, sizeof(char)); + + if (!file.Write(empty_buf, empty_size)) + { + assert(!"WriteData: fwrite empty data failed"); + return false; + } + + free(empty_buf); + return true; +} + +TDataPositionMap & CEterPack::GetIndexMap() +{ + return m_DataPositionMap; +} + +DWORD CEterPack::DeleteUnreferencedData() +{ + TDataPositionMap::iterator i = m_DataPositionMap.begin(); + DWORD dwCount = 0; + + while (i != m_DataPositionMap.end()) + { + TEterPackIndex * pIndex = (i++)->second; + + if (0 == m_map_indexRefCount[pIndex->id]) + { + printf("Unref File %s\n", pIndex->filename); + Delete(pIndex); + ++dwCount; + } + } + + return dwCount; +} + +const char * CEterPack::GetDBName() +{ + return m_dbName; +} + +void CEterPack::__CreateFileNameKey_Panama(const char * filename, BYTE * key, unsigned int keySize) +{ + // 키 암호화 + if (keySize != 32) + return; + + std::string SrcStringForKey(filename); + unsigned int idx = GetCRC32(SrcStringForKey.c_str(), SrcStringForKey.length()) & 3; + + CryptoPP::HashTransformation* hm1 = NULL; + CryptoPP::HashTransformation* hm2 = NULL; + + static CryptoPP::Tiger tiger; + static CryptoPP::SHA1 sha1; + static CryptoPP::RIPEMD128 ripemd128; + static CryptoPP::Whirlpool whirlpool; + + switch (idx & 3) + { + case 0: + hm1 = &whirlpool; + break; + + case 1: + hm1 = &tiger; + break; + + case 2: + hm1 = &sha1; + break; + + case 3: + hm1 = &ripemd128; + break; + } + + CryptoPP::StringSource(SrcStringForKey, true, + new CryptoPP::HashFilter(*hm1, + //new CryptoPP::HexEncoder( + new CryptoPP::ArraySink(key, 16) + //) // HexEncoder + ) // HashFilter + ); // StringSource + + // 만들어진 키의 첫번째 4바이트로 다음 16바이트 키 생성 알고리즘 선택 + unsigned int idx2 = *(unsigned int*) key; + + switch (idx2 & 3) + { + case 0: + hm2 = &sha1; + break; + + case 1: + hm2 = &ripemd128; + break; + + case 2: + hm2 = &whirlpool; + break; + + case 3: + hm2 = &tiger; + break; + } + + CryptoPP::StringSource(SrcStringForKey, true, + new CryptoPP::HashFilter(*hm2, + //new CryptoPP::HexEncoder( + new CryptoPP::ArraySink(key + 16, 16) + //) // HexEncoder + ) // HashFilter + ); // StringSource + // 키 생성 완료 +} + +bool CEterPack::__Encrypt_Panama(const char* filename, const BYTE* data, SIZE_T dataSize, CLZObject& zObj) +{ + if (32 != m_stIV_Panama.length()) + { + // 해커가 이 메세지를 보면 힌트를 얻을까봐 디버그에서만 출력 +#ifdef _DEBUG + TraceError("IV not set (filename: %s)", filename); +#endif + return false; + } + + CryptoPP::PanamaCipher::Encryption Encryptor; + + if (dataSize < Encryptor.MandatoryBlockSize()) + { +#ifdef _DEBUG + TraceError("Type 3 pack file must be bigger than %u bytes (filename: %s)", Encryptor.MandatoryBlockSize(), filename); +#endif + return false; + } + + BYTE key[32]; + + __CreateFileNameKey_Panama(filename, key, sizeof(key)); + Encryptor.SetKeyWithIV(key, sizeof(key), (const BYTE*) m_stIV_Panama.c_str(), 32); + + // MandatoryBlockSize에 나누어 떨어지게 만들고 최대 2048 바이트만 + DWORD cryptSize = dataSize - (dataSize % Encryptor.MandatoryBlockSize()); + cryptSize = cryptSize > 2048 ? 2048 : cryptSize; + + std::string tmp; + + tmp.reserve(cryptSize); + + CryptoPP::ArraySource(data, cryptSize, true, + new CryptoPP::StreamTransformationFilter(Encryptor, + new CryptoPP::StringSink(tmp) + ) + ); + + if (tmp.length() != cryptSize) + { +#ifdef _DEBUG + TraceError("Type 3 pack crypt buffer size error (out %u should be %u)", tmp.length(), cryptSize); +#endif + return false; + } + + zObj.AllocBuffer(dataSize); + memcpy(zObj.GetBuffer(), tmp.c_str(), cryptSize); + + if (dataSize - cryptSize > 0) + memcpy(zObj.GetBuffer() + cryptSize, data + cryptSize, dataSize - cryptSize); + + return true; +} + +bool CEterPack::__Decrypt_Panama(const char* filename, const BYTE* data, SIZE_T dataSize, CLZObject& zObj) +{ + if (32 != m_stIV_Panama.length()) + { + // 해커가 이 메세지를 보면 힌트를 얻을까봐 디버그에서만 출력 +#ifdef _DEBUG + TraceError("IV not set (filename: %s)", filename); +#endif + return false; + } + + CryptoPP::PanamaCipher::Decryption Decryptor; + + BYTE key[32]; + + __CreateFileNameKey_Panama(filename, key, sizeof(key)); + Decryptor.SetKeyWithIV(key, sizeof(key), (const BYTE*) m_stIV_Panama.c_str(), 32); + + // MandatoryBlockSize에 나누어 떨어지게 만들고 최대 2048 바이트만 + DWORD cryptSize = dataSize - (dataSize % Decryptor.MandatoryBlockSize()); + cryptSize = cryptSize > 2048 ? 2048 : cryptSize; + + std::string tmp; + + tmp.reserve(cryptSize); + + CryptoPP::ArraySource(data, cryptSize, true, + new CryptoPP::StreamTransformationFilter(Decryptor, + new CryptoPP::StringSink(tmp) + ) + ); + + if (tmp.length() != cryptSize) + { +#ifdef _DEBUG + TraceError("Type 3 pack crypt buffer size error (out %u should be %u)", tmp.length(), cryptSize); +#endif + return false; + } + + zObj.AllocBuffer(dataSize); + memcpy(zObj.GetBuffer(), tmp.c_str(), cryptSize); + + if (dataSize - cryptSize > 0) + memcpy(zObj.GetBuffer() + cryptSize, data + cryptSize, dataSize - cryptSize); + + return true; +} + +EterPackPolicy_CSHybridCrypt* CEterPack::GetPackPolicy_HybridCrypt() const +{ + return m_pCSHybridCryptPolicy; +} + + +///////////////////////// + +void CEterFileDict::InsertItem(CEterPack* pkPack, TEterPackIndex* pkInfo) +{ + Item item; + + item.pkPack = pkPack; + item.pkInfo = pkInfo; + + m_dict.insert(TDict::value_type(pkInfo->filename_crc, item)); +} + +void CEterFileDict::UpdateItem(CEterPack* pkPack, TEterPackIndex* pkInfo) +{ + Item item; + + item.pkPack = pkPack; + item.pkInfo = pkInfo; + + TDict::iterator f = m_dict.find(pkInfo->filename_crc); + if (f == m_dict.end()) + m_dict.insert(TDict::value_type(pkInfo->filename_crc, item)); + else + { + if (strcmp(f->second.pkInfo->filename, item.pkInfo->filename) == 0) + { + f->second = item; + } + else + { + TraceError("NAME_COLLISION: OLD: %s NEW: %s", f->second.pkInfo->filename, item.pkInfo->filename); + } + + } +} + +CEterFileDict::Item* CEterFileDict::GetItem(DWORD dwFileNameHash, const char * c_pszFileName) +{ + std::pair iter_pair = m_dict.equal_range(dwFileNameHash); + + TDict::iterator iter = iter_pair.first; + + while (iter != iter_pair.second) + { + Item& item = iter->second; + + if (0 == strcmp(c_pszFileName, item.pkInfo->filename)) + return &item; + + ++iter; + } + + return NULL; +} diff --git a/extension/src/port/EterPack/EterPack.h b/extension/src/port/EterPack/EterPack.h index 502c7472..c4e62e99 100644 --- a/extension/src/port/EterPack/EterPack.h +++ b/extension/src/port/EterPack/EterPack.h @@ -23,7 +23,7 @@ namespace eterpack const DWORD c_IndexCC = MAKEFOURCC('E', 'P', 'K', 'D'); const DWORD c_Version = 2; // FourCC + Version + m_indexCount - const DWORD c_HeaderSize = sizeof(DWORD) + sizeof(DWORD) + sizeof(long); + const DWORD c_HeaderSize = sizeof(DWORD) + sizeof(DWORD) + sizeof(LONG); // PORT: LONG (32-bit) for Win32 long }; enum EEterPackTypes @@ -46,20 +46,21 @@ enum EEterPackTypes #pragma pack(push, 4) typedef struct SEterPackIndex { - long id; + LONG id; // PORT: fixed-width Win32 long char filename[FILENAME_MAX_LEN + 1]; DWORD filename_crc; - long real_data_size; - long data_size; + LONG real_data_size; // PORT: fixed-width Win32 long + LONG data_size; // PORT: fixed-width Win32 long #ifdef CHECKSUM_CHECK_MD5 BYTE MD5Digest[16]; #else DWORD data_crc; #endif - long data_position; + LONG data_position; // PORT: fixed-width Win32 long char compressed_type; } TEterPackIndex; #pragma pack(pop) +static_assert(sizeof(TEterPackIndex) == 192, "TEterPackIndex on-disk layout"); // PORT: on-disk struct typedef std::unordered_map TDataPositionMap; typedef std::list TFreeIndexList; @@ -170,7 +171,7 @@ class CEterPack char* m_file_data; unsigned m_file_size; - long m_indexCount; + LONG m_indexCount; // PORT: read as 32-bit from the index header bool m_bEncrypted; char m_dbName[DBNAME_MAX_LEN+1]; diff --git a/extension/src/port/EterPack/EterPackCursor.cpp b/extension/src/port/EterPack/EterPackCursor.cpp new file mode 100644 index 00000000..288a90cc --- /dev/null +++ b/extension/src/port/EterPack/EterPackCursor.cpp @@ -0,0 +1,59 @@ +#include "StdAfx.h" +#include "EterPackCursor.h" +#include "Inline.h" + +CEterPackCursor::CEterPackCursor(CEterPack* pack) : m_pPack(pack), m_pData(NULL), m_ReadPoint(0) +{ +} + +CEterPackCursor::~CEterPackCursor() +{ + Close(); +} + +bool CEterPackCursor::Open(const char* filename) +{ + assert(m_pPack != NULL); + + char tmpFilename[MAX_PATH + 1]; + strncpy(tmpFilename, filename, MAX_PATH); + inlineConvertPackFilename(tmpFilename); + + if (!m_pPack->Get(m_file, tmpFilename, &m_pData)) + return false; + + return true; +} + +void CEterPackCursor::Close() +{ + m_file.Destroy(); + m_pData = NULL; + m_ReadPoint = 0; +} + +void CEterPackCursor::Seek(long offset) +{ + m_ReadPoint = max(0, min(Size(), offset)); +} + +bool CEterPackCursor::Read(LPVOID data, long size) +{ + if (m_file.IsNull()) + return false; + + if (m_ReadPoint + size > Size()) + return false; + + memcpy(data, (char*) m_pData + m_ReadPoint, size); + m_ReadPoint += size; + return true; +} + +long CEterPackCursor::Size() +{ + if (m_file.IsNull()) + return 0; + + return m_file.Size(); +} diff --git a/extension/src/port/EterPack/EterPackCursor.h b/extension/src/port/EterPack/EterPackCursor.h new file mode 100644 index 00000000..5ba802a5 --- /dev/null +++ b/extension/src/port/EterPack/EterPackCursor.h @@ -0,0 +1,25 @@ +#ifndef __INC_ETERPACKCURSOR_H__ +#define __INC_ETERPACKCURSOR_H__ + +#include "EterPack.h" + +class CEterPackCursor +{ + public: + CEterPackCursor(CEterPack * pack); + ~CEterPackCursor(); + + bool Open(const char* filename); + void Close(); + void Seek(long offset); + bool Read(LPVOID data, long size); + long Size(); + + private: + CEterPack * m_pPack; + CMappedFile m_file; + LPCVOID m_pData; + long m_ReadPoint; +}; + +#endif diff --git a/extension/src/port/EterPack/EterPackManager.cpp b/extension/src/port/EterPack/EterPackManager.cpp new file mode 100644 index 00000000..8059d9ab --- /dev/null +++ b/extension/src/port/EterPack/EterPackManager.cpp @@ -0,0 +1,621 @@ +#include "StdAfx.h" + +#include +#include + +#include "EterPackManager.h" +#include "EterPackPolicy_CSHybridCrypt.h" + +#include "../EterBase/Debug.h" +#include "../EterBase/CRC32.h" + +#define PATH_ABSOLUTE_YMIRWORK1 "d:/ymir work/" +#define PATH_ABSOLUTE_YMIRWORK2 "d:\\ymir work\\" + +#ifdef __THEMIDA__ +#include +#endif + + +CEterPack* CEterPackManager::FindPack(const char* c_szPathName) +{ + std::string strFileName; + + if (0 == ConvertFileName(c_szPathName, strFileName)) + { + return &m_RootPack; + } + else + { + for (TEterPackMap::iterator itor = m_DirPackMap.begin(); itor != m_DirPackMap.end(); ++itor) + { + const std::string & c_rstrName = itor->first; + CEterPack * pEterPack = itor->second; + + if (CompareName(c_rstrName.c_str(), c_rstrName.length(), strFileName.c_str())) + { + return pEterPack; + } + } + } + + return NULL; +} + +void CEterPackManager::SetCacheMode() +{ + m_isCacheMode=true; +} + +void CEterPackManager::SetRelativePathMode() +{ + m_bTryRelativePath = true; +} + + +// StringPath std::string 버전 +int CEterPackManager::ConvertFileName(const char * c_szFileName, std::string & rstrFileName) +{ + rstrFileName = c_szFileName; + stl_lowers(rstrFileName); + + int iCount = 0; + + for (DWORD i = 0; i < rstrFileName.length(); ++i) + { + if (rstrFileName[i] == '/') + ++iCount; + else if (rstrFileName[i] == '\\') + { + rstrFileName[i] = '/'; + ++iCount; + } + } + + return iCount; +} + +bool CEterPackManager::CompareName(const char * c_szDirectoryName, DWORD /*dwLength*/, const char * c_szFileName) +{ + const char * c_pszSrc = c_szDirectoryName; + const char * c_pszCmp = c_szFileName; + + while (*c_pszSrc) + { + if (*(c_pszSrc++) != *(c_pszCmp++)) + return false; + + if (!*c_pszCmp) + return false; + } + + return true; +} + +void CEterPackManager::LoadStaticCache(const char* c_szFileName) +{ + if (!m_isCacheMode) + return; + + std::string strFileName; + if (0 == ConvertFileName(c_szFileName, strFileName)) + { + return; + } + + DWORD dwFileNameHash = GetCRC32(strFileName.c_str(), strFileName.length()); + + std::unordered_map::iterator f = m_kMap_dwNameKey_kCache.find(dwFileNameHash); + if (m_kMap_dwNameKey_kCache.end() != f) + return; + + CMappedFile kMapFile; + const void* c_pvData; + if (!Get(kMapFile, c_szFileName, &c_pvData)) + return; + + SCache kNewCache; + kNewCache.m_dwBufSize = kMapFile.Size(); + kNewCache.m_abBufData = new BYTE[kNewCache.m_dwBufSize]; + memcpy(kNewCache.m_abBufData, c_pvData, kNewCache.m_dwBufSize); + m_kMap_dwNameKey_kCache.insert(std::unordered_map::value_type(dwFileNameHash, kNewCache)); +} + +CEterPackManager::SCache* CEterPackManager::__FindCache(DWORD dwFileNameHash) +{ + std::unordered_map::iterator f=m_kMap_dwNameKey_kCache.find(dwFileNameHash); + if (m_kMap_dwNameKey_kCache.end()==f) + return NULL; + + return &f->second; +} + +void CEterPackManager::__ClearCacheMap() +{ + std::unordered_map::iterator i; + + for (i = m_kMap_dwNameKey_kCache.begin(); i != m_kMap_dwNameKey_kCache.end(); ++i) + delete [] i->second.m_abBufData; + + m_kMap_dwNameKey_kCache.clear(); +} + +struct TimeChecker +{ + TimeChecker(const char* name) : name(name) + { + baseTime = timeGetTime(); + } + ~TimeChecker() + { + printf("load %s (%d)\n", name, timeGetTime() - baseTime); + } + + const char* name; + DWORD baseTime; +}; + +bool CEterPackManager::Get(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData) +{ + //TimeChecker timeChecker(c_szFileName); + //Logf(1, "Load %s\n", c_szFileName); + + if (m_iSearchMode == SEARCH_FILE_FIRST) + { + if (GetFromFile(rMappedFile, c_szFileName, pData)) + { + return true; + } + + return GetFromPack(rMappedFile, c_szFileName, pData); + } + + if (GetFromPack(rMappedFile, c_szFileName, pData)) + { + return true; + } + + return GetFromFile(rMappedFile, c_szFileName, pData); +} + +struct FinderLock +{ + FinderLock(CRITICAL_SECTION& cs) : p_cs(&cs) + { + EnterCriticalSection(p_cs); + } + + ~FinderLock() + { + LeaveCriticalSection(p_cs); + } + + CRITICAL_SECTION* p_cs; +}; + +bool CEterPackManager::GetFromPack(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData) +{ + FinderLock lock(m_csFinder); + + static std::string strFileName; + + if (0 == ConvertFileName(c_szFileName, strFileName)) + { + return m_RootPack.Get(rMappedFile, strFileName.c_str(), pData); + } + else + { + DWORD dwFileNameHash = GetCRC32(strFileName.c_str(), strFileName.length()); + SCache* pkCache = __FindCache(dwFileNameHash); + + if (pkCache) + { + rMappedFile.Link(pkCache->m_dwBufSize, pkCache->m_abBufData); + return true; + } + + CEterFileDict::Item* pkFileItem = m_FileDict.GetItem(dwFileNameHash, strFileName.c_str()); + + if (pkFileItem) + if (pkFileItem->pkPack) + { + bool r = pkFileItem->pkPack->Get2(rMappedFile, strFileName.c_str(), pkFileItem->pkInfo, pData); + //pkFileItem->pkPack->ClearDataMemoryMap(); + return r; + } + } +#ifdef _DEBUG + TraceError("CANNOT_FIND_PACK_FILE [%s]", strFileName.c_str()); +#endif + + return false; +} + +const time_t g_tCachingInterval = 10; // 10초 +void CEterPackManager::ArrangeMemoryMappedPack() +{ + //time_t curTime = time(NULL); + //CEterFileDict::TDict dict = m_FileDict.GetDict(); + //for (CEterFileDict::TDict::iterator it = dict.begin(); it != dict.end(); ++it) + //{ + // CEterFileDict::Item &rFileItem = it->second; + // CEterPack* pkPack = rFileItem.pkPack; + // if (pkPack) + // { + // if (curTime - pkPack->GetLastAccessTime() > g_tCachingInterval) + // { + // pkPack->ClearDataMemoryMap(); + // } + // } + //} +} + +bool CEterPackManager::GetFromFile(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData) +{ +#ifndef _DEBUG + //const char *pcExt = strchr(c_szFileName, '.'); + //if (pcExt && + // _strnicmp(pcExt, ".py", 3) == 0 && // python 스크립트 중 + // stricmp(c_szFileName, "logininfo.py") != 0 && // 로그인 정보 파일이 아니고 + // strnicmp(c_szFileName, "locale", 6) != 0 + // ) + //{ + // return false; + //} +#endif + + //if(m_bTryRelativePath) { + // if (strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK1, strlen(PATH_ABSOLUTE_YMIRWORK1)) == 0 || strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK2, strlen(PATH_ABSOLUTE_YMIRWORK2)) == 0) { + // if(rMappedFile.Create(c_szFileName+strlen(PATH_ABSOLUTE_YMIRWORK1), pData, 0, 0)) + // { + // return true; + // } + // } + //} + + return rMappedFile.Create(c_szFileName, pData, 0, 0) ? true : false; +} + +bool CEterPackManager::isExistInPack(const char * c_szFileName) +{ + std::string strFileName; + + if (0 == ConvertFileName(c_szFileName, strFileName)) + { + return m_RootPack.IsExist(strFileName.c_str()); + } + else + { + DWORD dwFileNameHash = GetCRC32(strFileName.c_str(), strFileName.length()); + CEterFileDict::Item* pkFileItem = m_FileDict.GetItem(dwFileNameHash, strFileName.c_str()); + + if (pkFileItem) + if (pkFileItem->pkPack) + return pkFileItem->pkPack->IsExist(strFileName.c_str()); + } + + // NOTE : 매치 되는 팩이 없다면 false - [levites] + return false; +} + +bool CEterPackManager::isExist(const char * c_szFileName) +{ + if (m_iSearchMode == SEARCH_PACK_FIRST) + { + if (isExistInPack(c_szFileName)) + return true; + + return _access(c_szFileName, 0) == 0 ? true : false; + } + + //if(m_bTryRelativePath) { + // if (strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK1, strlen(PATH_ABSOLUTE_YMIRWORK1)) == 0 || strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK2, strlen(PATH_ABSOLUTE_YMIRWORK2)) == 0) { + // if(access(c_szFileName+strlen(PATH_ABSOLUTE_YMIRWORK1), 0) == 0) + // return true; + // } + //} + + if (_access(c_szFileName, 0) == 0) + return true; + + return isExistInPack(c_szFileName); +} + + +void CEterPackManager::RegisterRootPack(const char * c_szName) +{ + if (!m_RootPack.Create(m_FileDict, c_szName, "")) + { + TraceError("%s: Pack file does not exist", c_szName); + } +} + +const char * CEterPackManager::GetRootPackFileName() +{ + return m_RootPack.GetDBName(); +} + +bool CEterPackManager::DecryptPackIV(DWORD dwPanamaKey) +{ + TEterPackMap::iterator itor = m_PackMap.begin(); + while (itor != m_PackMap.end()) + { + itor->second->DecryptIV(dwPanamaKey); + itor++; + } + return true; +} + +bool CEterPackManager::RegisterPackWhenPackMaking(const char * c_szName, const char * c_szDirectory, CEterPack* pPack) +{ + m_PackMap.insert(TEterPackMap::value_type(c_szName, pPack)); + m_PackList.push_front(pPack); + + m_DirPackMap.insert(TEterPackMap::value_type(c_szDirectory, pPack)); + return true; +} + + +bool CEterPackManager::RegisterPack(const char * c_szName, const char * c_szDirectory, const BYTE* c_pbIV) +{ + CEterPack * pEterPack = NULL; + { + TEterPackMap::iterator itor = m_PackMap.find(c_szName); + + if (m_PackMap.end() == itor) + { + bool bReadOnly = true; + + pEterPack = new CEterPack; + if (pEterPack->Create(m_FileDict, c_szName, c_szDirectory, bReadOnly, c_pbIV)) + { + m_PackMap.insert(TEterPackMap::value_type(c_szName, pEterPack)); + } + else + { +#ifdef _DEBUG + Tracef("The eterpack doesn't exist [%s]\n", c_szName); +#endif + delete pEterPack; + pEterPack = NULL; + return false; + } + } + else + { + pEterPack = itor->second; + } + } + + if (c_szDirectory && c_szDirectory[0] != '*') + { + TEterPackMap::iterator itor = m_DirPackMap.find(c_szDirectory); + if (m_DirPackMap.end() == itor) + { + m_PackList.push_front(pEterPack); + m_DirPackMap.insert(TEterPackMap::value_type(c_szDirectory, pEterPack)); + } + } + + return true; +} + +void CEterPackManager::SetSearchMode(bool bPackFirst) +{ + m_iSearchMode = bPackFirst ? SEARCH_PACK_FIRST : SEARCH_FILE_FIRST; +} + +int CEterPackManager::GetSearchMode() +{ + return m_iSearchMode; +} + +CEterPackManager::CEterPackManager() : m_bTryRelativePath(false), m_iSearchMode(SEARCH_FILE_FIRST), m_isCacheMode(false) +{ + InitializeCriticalSection(&m_csFinder); +} + +CEterPackManager::~CEterPackManager() +{ + __ClearCacheMap(); + + TEterPackMap::iterator i = m_PackMap.begin(); + TEterPackMap::iterator e = m_PackMap.end(); + while (i != e) + { + delete i->second; + i++; + } + DeleteCriticalSection(&m_csFinder); +} + +void CEterPackManager::RetrieveHybridCryptPackKeys(const BYTE *pStream) +{ + ////dump file format + //total packagecnt (4byte) + // for packagecntpackage + // db name hash ( stl.h stringhash ) + // extension cnt( 4byte) + // for extension cnt + // ext hash ( stl.h stringhash ) + // key-16byte + // iv-16byte + int iMemOffset = 0; + + int iPackageCnt; + DWORD dwPackageNameHash; + + memcpy( &iPackageCnt, pStream + iMemOffset, sizeof(int) ); + iMemOffset += sizeof(iPackageCnt); + + for( int i = 0; i < iPackageCnt; ++i ) + { + int iRecvedCryptKeySize = 0; + memcpy( &iRecvedCryptKeySize, pStream + iMemOffset, sizeof(iRecvedCryptKeySize) ); + iRecvedCryptKeySize -= sizeof(dwPackageNameHash); // 서버에서 받은 key stream에는 filename hash가 포함되어 있으므로, hash 사이즈 만큼 배줌. + iMemOffset += sizeof(iRecvedCryptKeySize); + + memcpy( &dwPackageNameHash, pStream + iMemOffset, sizeof(dwPackageNameHash) ); + iMemOffset += sizeof(dwPackageNameHash); + + TEterPackMap::const_iterator cit; + for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) + { + auto ssvv = std::string(cit->first); + std::string noPathName = CFileNameHelper::NoPath(ssvv); + if( dwPackageNameHash == stringhash().GetHash(noPathName) ) + { + EterPackPolicy_CSHybridCrypt* pCryptPolicy = cit->second->GetPackPolicy_HybridCrypt(); + int iHavedCryptKeySize = pCryptPolicy->ReadCryptKeyInfoFromStream( pStream + iMemOffset ); + if (iRecvedCryptKeySize != iHavedCryptKeySize) + { + TraceError("CEterPackManager::RetrieveHybridCryptPackKeys cryptokey length of file(%s) is not matched. received(%d) != haved(%d)", noPathName.c_str(), iRecvedCryptKeySize, iHavedCryptKeySize); + } + break; + } + } + iMemOffset += iRecvedCryptKeySize; + } +} + +void CEterPackManager::RetrieveHybridCryptPackSDB( const BYTE* pStream ) +{ + //cnt + //for cnt + //DWORD dwPackageIdentifier; + //DWORD dwFileIdentifier; + //std::vector vecSDBStream; + int iReadOffset = 0; + int iSDBInfoCount = 0; + + memcpy( &iSDBInfoCount, pStream+iReadOffset, sizeof(int) ); + iReadOffset += sizeof(int); + + for( int i = 0; i < iSDBInfoCount; ++i ) + { + DWORD dwPackgeIdentifier; + memcpy( &dwPackgeIdentifier, pStream+iReadOffset, sizeof(DWORD) ); + iReadOffset += sizeof(DWORD); + + TEterPackMap::const_iterator cit; + for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) + { + auto ssvv = std::string(cit->first); + std::string noPathName = CFileNameHelper::NoPath(ssvv); + if( dwPackgeIdentifier == stringhash().GetHash(noPathName) ) + { + EterPackPolicy_CSHybridCrypt* pCryptPolicy = cit->second->GetPackPolicy_HybridCrypt(); + iReadOffset += pCryptPolicy->ReadSupplementatyDataBlockFromStream( pStream+iReadOffset ); + break; + } + } + } +} + + +void CEterPackManager::WriteHybridCryptPackInfo(const char* pFileName) +{ + //NOTE : this file format contains a little bit of redundant data. + //however it`s better for seperating cryptkey & supplementary data block. + + //dump file format + + //SDB data offset(4) + + // about cryptkey + //total packagecnt (4byte) + // for packagecnt + // db name hash 4byte( stl.h stringhash ) + // extension cnt( 4byte) + // for extension cnt + // ext hash ( stl.h stringhash ) + // key-16byte + // iv-16byte + + //about SDB data + //total packagecnt (4byte) + // for packagecnt + // db name hash 4byte( stl.h stringhash ) +child node size(4byte) + // sdb file cnt( 4byte ) + // for sdb file cnt + // filename hash ( stl.h stringhash ) + // related map name size(4), relate map name + // sdb block size( 1byte ) + // sdb blocks + + CFileBase keyFile; + + if( !keyFile.Create( pFileName, CFileBase::FILEMODE_WRITE) ) + { + //TODO : write log + return; + } + + int iKeyPackageCount = 0; + + //write later ( SDB Offset & PackageCnt for Key ) + keyFile.SeekCur(2*sizeof(int)); + + TEterPackMap::const_iterator cit; + for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) + { + EterPackPolicy_CSHybridCrypt* pPolicy = cit->second->GetPackPolicy_HybridCrypt(); + if( !pPolicy || !pPolicy->IsContainingCryptKey() ) + continue; + + iKeyPackageCount++; + auto ssvv = std::string(cit->first); + std::string noPathName = CFileNameHelper::NoPath(ssvv); + + DWORD dwPackNamehash = stringhash().GetHash(noPathName); + + CMakePackLog::GetSingleton().Writef("CEterPackManager::WriteHybridCryptPackInfo PackName : %s, Hash : %x", noPathName.c_str(), dwPackNamehash); + keyFile.Write( &dwPackNamehash, sizeof(DWORD) ); + + pPolicy->WriteCryptKeyToFile( keyFile ); + } + + //Write SDB Data + int iSDBDataOffset = keyFile.GetPosition(); + int iSDBPackageCnt = 0; + + //Write SDB PackageCnt Later + keyFile.SeekCur(sizeof(int)); + for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) + { + EterPackPolicy_CSHybridCrypt* pPolicy = cit->second->GetPackPolicy_HybridCrypt(); + if( !pPolicy || !pPolicy->IsContainingSDBFile() ) + continue; + + iSDBPackageCnt++; + + auto ssvv1 = std::string(cit->first); + auto noPathName = CFileNameHelper::NoPath(ssvv1); + + DWORD dwPackNamehash = stringhash().GetHash(noPathName); + keyFile.Write( &dwPackNamehash, sizeof(DWORD) ); + + int iSDBSizeWriteOffset = keyFile.GetPosition(); + keyFile.SeekCur(sizeof(int)); + + pPolicy->WriteSupplementaryDataBlockToFile( keyFile ); + int iSDBSizeAfterWrite = keyFile.GetPosition(); + + keyFile.Seek(iSDBSizeWriteOffset); + + int iSDBSize = iSDBSizeAfterWrite-(iSDBSizeWriteOffset+4); + keyFile.Write( &iSDBSize, sizeof(int) ); + + keyFile.Seek(iSDBSizeAfterWrite); + } + + //write sdb data start offset & package cnt + keyFile.Seek(0); + keyFile.Write( &iSDBDataOffset, sizeof(int)); + keyFile.Write( &iKeyPackageCount, sizeof(int)); + + keyFile.Seek(iSDBDataOffset); + keyFile.Write( &iSDBPackageCnt, sizeof(int)); + + keyFile.Close(); +} diff --git a/extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.cpp b/extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.cpp new file mode 100644 index 00000000..9c8c19fc --- /dev/null +++ b/extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.cpp @@ -0,0 +1,503 @@ +#include "StdAfx.h" +#include "EterPackPolicy_CSHybridCrypt.h" +#include "../EterBase/Stl.h" +#include "../EterBase/Filename.h" +#include "../EterBase/FileBase.h" +#include "../EterBase/CRC32.h" +#include "../EterBase/lzo.h" +#include "../EterBase/Random.h" +#include +#include + +using namespace CryptoPP; + +#define CIPHER_MODE CTR_Mode + +#ifdef __THEMIDA__ +#include +#endif +//Cipher +//Block Size +//Key Length +// +//Default Minimum Maximum +//AES(Rijndael) 16 16 16 32 +//Blowfish 8 16 0 56 +//Camellia 16 16 16 32 +//CAST-128 8 16 5 16 +//CAST-256 16 16 16 32 +//DES 8 8 8 8 +//DES-EDE2 8 16 16 16 +//DES-EDE3 8 24 24 24 +//DES-XEX3 8 24 24 24 +//GOST 8 32 32 32 +//IDEA 8 16 16 16 +//MARS 16 16 16 56 +//RC2 8 16 1 128 +//RC5 8 16 0 255 +//RC6 16 16 0 255 +//SAFER-K 8 16 8 16 +//SAFER-SK 8 16 8 16 +//Serpent 16 16 1 32 +//SHACAL-2 32 16 1 64 +//SHARK-E 8 16 1 16 +//SKIPJACK 8 10 1 10 +//3-Way 12 12 1 12 +//Twofish 16 16 0 32 +//XTEA 8 16 1 16 + +inline std::string GetFileExt( std::string& rfileName ) +{ + stl_lowers(rfileName); + return CFileNameHelper::GetExtension(rfileName); +} + +EterPackPolicy_CSHybridCrypt::~EterPackPolicy_CSHybridCrypt() +{ + m_mapHybridCryptKey.clear(); + m_mapSDBMap.clear(); +} + +bool EterPackPolicy_CSHybridCrypt::IsContainingCryptKey() const +{ + return (m_mapHybridCryptKey.size() > 0) ? true : false; +} + + +bool EterPackPolicy_CSHybridCrypt::GenerateCryptKey( std::string& rfileName ) +{ +#ifdef __THEMIDA__ + VM_START +#endif + + //make lower & extract ext + std::string extName = GetFileExt(rfileName); + stl_lowers(extName); + + DWORD dwExtHash = stringhash().GetHash(extName); + + TCSHybridCryptKeyMap::const_iterator cit = m_mapHybridCryptKey.find( dwExtHash ); + + if( cit != m_mapHybridCryptKey.end() ) + { + //TODO : log already registered + return false; + } + + static AutoSeededRandomPool rnd; + + TCSHybridCryptKey info; + { + rnd.GenerateBlock( &(info.uEncryptKey.key[0]), sizeof(info.uEncryptKey) ); + rnd.GenerateBlock( &(info.uEncryptIV.iv[0]), sizeof(info.uEncryptIV) ); + + //for test + /* memset( &info.uEncryptKey.key, 0x10, sizeof(info.uEncryptKey) ); + memset( &info.uEncryptIV.iv, 0x10, sizeof(info.uEncryptIV) ); */ + } + m_mapHybridCryptKey[dwExtHash] = info; + +#ifdef __THEMIDA__ + VM_END +#endif + + return true; +} + +bool EterPackPolicy_CSHybridCrypt::GetPerFileCryptKey( std::string& rfileName, eHybridCipherAlgorithm& eAlgorithm, TEncryptKey& key, TEncryptIV& iv ) +{ +#ifdef __THEMIDA__ + VM_START +#endif + + std::string fileNamelower = rfileName; + stl_lowers(fileNamelower); + + std::string extName = GetFileExt(fileNamelower); + + TCSHybridCryptKeyMap::const_iterator cit = m_mapHybridCryptKey.find( stringhash().GetHash(extName)); + + if( cit == m_mapHybridCryptKey.end() ) + { + //TODO : log no file ext info + return false; + } + + DWORD dwfileNameCrc = GetCRC32(fileNamelower.c_str(), fileNamelower.size()); + + //make file specific algorithm & key & iv + eAlgorithm = (eHybridCipherAlgorithm)(dwfileNameCrc % Num_Of_Ciphers); + + ::memcpy(key.key, cit->second.uEncryptKey.key, sizeof(key) ); + ::memcpy(iv.iv, cit->second.uEncryptIV.iv, sizeof(iv) ); + + + //Themida Warning + for( int i = 0; i < (sizeof(key)/sizeof(dwfileNameCrc)); ++i) + { + *((DWORD*)key.key + i) ^= dwfileNameCrc; + } + for( int i = 0; i < (sizeof(iv)/sizeof(dwfileNameCrc)); ++i) + { + *((DWORD*)iv.iv + i) ^= dwfileNameCrc; + } +#ifdef __THEMIDA__ + VM_END +#endif + + return true; +} + + +bool EterPackPolicy_CSHybridCrypt::EncryptMemory( std::string& rfileName, IN const BYTE* pSrcData, IN int iSrcLen, OUT CLZObject& zObj ) +{ +#ifdef __THEMIDA__ + VM_START +#endif + + eHybridCipherAlgorithm eAlgorithm; + TEncryptKey key; + TEncryptIV iv; + + if( !GetPerFileCryptKey( rfileName, eAlgorithm, key, iv ) ) + { + return false; + } + + // start cipher + std::string strCipher; + //NOTE : ciphered stream size could be different from original size if you choose diffrent cipher mode & algorithm + //( i.e ECB or CBC mode ) + strCipher.reserve(iSrcLen); + + if( eAlgorithm == e_Cipher_Camellia ) + { + // Encryptor + CIPHER_MODE::Encryption Encryptor; + Encryptor.SetKeyWithIV(key.keyCamellia, sizeof(key.keyCamellia), iv.ivCamellia, sizeof(iv.ivCamellia)); + + ArraySource(pSrcData, iSrcLen, true, + new StreamTransformationFilter(Encryptor, new CryptoPP::StringSink(strCipher))); + } + else if( eAlgorithm == e_Cipher_Twofish ) + { + // Encryptor + CIPHER_MODE::Encryption Encryptor; + Encryptor.SetKeyWithIV(key.keyTwofish, sizeof(key.keyTwofish), iv.ivTwofish, sizeof(iv.ivTwofish)); + + ArraySource(pSrcData, iSrcLen, true, + new StreamTransformationFilter(Encryptor, new CryptoPP::StringSink(strCipher))); + } + else if( eAlgorithm == e_Cipher_XTEA ) + { + // Encryptor + CIPHER_MODE::Encryption Encryptor; + Encryptor.SetKeyWithIV(key.keyXTEA, sizeof(key.keyXTEA), iv.ivXTEA, sizeof(iv.ivXTEA)); + + ArraySource(pSrcData, iSrcLen, true, + new StreamTransformationFilter(Encryptor, new CryptoPP::StringSink(strCipher))); + } + + if (strCipher.length() != iSrcLen) + { + //TODO: size error log + return false; + } + + zObj.AllocBuffer(iSrcLen); + memcpy(zObj.GetBuffer(), strCipher.c_str(), strCipher.length() ); + +#ifdef __THEMIDA__ + VM_END +#endif + + return true; +} + +bool EterPackPolicy_CSHybridCrypt::DecryptMemory( std::string& rfilename, IN const BYTE* pEncryptedData, IN int iEncryptedLen, OUT CLZObject& zObj ) +{ +#ifdef __THEMIDA__ + VM_START +#endif + + eHybridCipherAlgorithm eAlgorithm; + TEncryptKey key; + TEncryptIV iv; + + if( !GetPerFileCryptKey( rfilename, eAlgorithm, key, iv ) ) + { + return false; + } + + // start decipher + std::string strDecipher; + //NOTE : ciphered stream size could be different from original size if you choose diffrent cipher mode & algorithm + //( i.e ECB or CBC mode ) + strDecipher.reserve(iEncryptedLen); + + if( eAlgorithm == e_Cipher_Camellia ) + { + // Decryptor + CIPHER_MODE::Decryption Decryptor; + Decryptor.SetKeyWithIV(key.keyCamellia, sizeof(key.keyCamellia), iv.ivCamellia, sizeof(iv.ivCamellia)); + + ArraySource(pEncryptedData, iEncryptedLen, true, + new StreamTransformationFilter(Decryptor, new CryptoPP::StringSink(strDecipher))); + } + else if( eAlgorithm == e_Cipher_Twofish ) + { + // Decryptor + CIPHER_MODE::Decryption Decryptor; + Decryptor.SetKeyWithIV(key.keyTwofish, sizeof(key.keyTwofish), iv.ivTwofish, sizeof(iv.ivTwofish)); + + ArraySource(pEncryptedData, iEncryptedLen, true, + new StreamTransformationFilter(Decryptor, new CryptoPP::StringSink(strDecipher))); + } + else if( eAlgorithm == e_Cipher_XTEA ) + { + // Decryptor + CIPHER_MODE::Decryption Decryptor; + Decryptor.SetKeyWithIV(key.keyXTEA, sizeof(key.keyXTEA), iv.ivXTEA, sizeof(iv.ivXTEA)); + + ArraySource(pEncryptedData, iEncryptedLen, true, + new StreamTransformationFilter(Decryptor, new CryptoPP::StringSink(strDecipher))); + } + + + if (strDecipher.length() != iEncryptedLen) + { + //TODO: size error log + return false; + } + + zObj.AllocBuffer(iEncryptedLen); + memcpy(zObj.GetBuffer(), strDecipher.c_str(), strDecipher.length() ); + +#ifdef __THEMIDA__ + VM_END +#endif + + return true; +} + +void EterPackPolicy_CSHybridCrypt::WriteCryptKeyToFile( CFileBase& rFile ) +{ + // ext cnt 4byte + // for ext hash ( crc32 ) + // key-16byte + // iv-16byte + + DWORD dwCryptKeySize = m_mapHybridCryptKey.size(); + rFile.Write( &dwCryptKeySize, sizeof(DWORD) ); + + TCSHybridCryptKeyMap::const_iterator cit; + for( cit = m_mapHybridCryptKey.begin(); cit != m_mapHybridCryptKey.end(); ++cit ) + { + DWORD extNamehash = cit->first; + const TCSHybridCryptKey& CryptKey = cit->second; + + rFile.Write( &extNamehash, sizeof(DWORD) ); + rFile.Write( CryptKey.uEncryptKey.key, sizeof(TEncryptKey) ); + rFile.Write( CryptKey.uEncryptIV.iv, sizeof(TEncryptIV) ); + } +} + +int EterPackPolicy_CSHybridCrypt::ReadCryptKeyInfoFromStream( IN const BYTE* pStream ) +{ + int iStreamOffset = 0; + + DWORD dwCryptoInfoSize; + memcpy(&dwCryptoInfoSize, pStream, sizeof(DWORD) ); + iStreamOffset += sizeof(DWORD); + + DWORD dwExtHash; + + m_mapHybridCryptKey.clear(); + + for( int i = 0; i < dwCryptoInfoSize; ++i ) + { + memcpy(&dwExtHash, pStream + iStreamOffset, sizeof(DWORD) ); + iStreamOffset += sizeof(DWORD); + + TCSHybridCryptKey info; + { + memcpy(info.uEncryptKey.key, pStream + iStreamOffset, sizeof(TEncryptKey) ); + iStreamOffset += sizeof(TEncryptKey); + + memcpy(info.uEncryptIV.iv, pStream + iStreamOffset, sizeof(TEncryptIV) ); + iStreamOffset += sizeof(TEncryptIV); + } + + m_mapHybridCryptKey[dwExtHash] = info; + } + + return iStreamOffset; +} + +bool EterPackPolicy_CSHybridCrypt::GenerateSupplementaryDataBlock(std::string& rfilename, const std::string& strMapName, IN const BYTE* pSrcData, IN int iSrcLen, OUT LPBYTE& pDestData, OUT int& iDestLen ) +{ +#ifdef __THEMIDA__ + VM_START +#endif + + std::string fileNamelower = rfilename; + stl_lowers( fileNamelower ); + + DWORD dwFileNameHash = stringhash().GetHash(fileNamelower); + TSupplementaryDataBlockMap::const_iterator cit = m_mapSDBMap.find( dwFileNameHash ); + + if( cit != m_mapSDBMap.end() ) + { + //TODO : log already registered + return false; + } + + //TODO : Find Better Method for deciding SDB Postion & Size + + //prevent stream copy duplication + TSupplementaryDataBlockInfo info; + m_mapSDBMap[dwFileNameHash] = info; + + std::string& strRelatedMapName = m_mapSDBMap[dwFileNameHash].strRelatedMapName; + std::vector& sdbVector = m_mapSDBMap[dwFileNameHash].vecStream; + + + //fill the data!! + { + strRelatedMapName = strMapName; + + int iSDBSize = random_range( 64, 128 ); + + if( iSrcLen < iSDBSize ) + { + iSDBSize = iSrcLen - 1; + if( iSDBSize <= 0 ) + { + //TODO : is there 1byte file exist??? + return false; + } + } + + sdbVector.resize( iSDBSize ); + + iDestLen = iSrcLen - iSDBSize; + pDestData = (LPBYTE)pSrcData; + + memcpy( &sdbVector[0], pDestData + iDestLen, iSDBSize ); + } +#ifdef __THEMIDA__ + VM_END +#endif + return true; +} + + +bool EterPackPolicy_CSHybridCrypt::GetSupplementaryDataBlock( std::string& rfilename, OUT LPBYTE& pSDB, OUT int& iSDBSize ) +{ +#ifdef __THEMIDA__ + VM_START +#endif + + std::string fileNamelower = rfilename; + stl_lowers( fileNamelower ); + + DWORD dwFileNameHash = stringhash().GetHash(fileNamelower); + TSupplementaryDataBlockMap::const_iterator cit = m_mapSDBMap.find( dwFileNameHash ); + + if( cit == m_mapSDBMap.end() ) + { + //TODO : log already registered + return false; + } + + const std::vector& vecSDB = cit->second.vecStream; + + iSDBSize = vecSDB.size(); + + if(iSDBSize <= 0) + { + pSDB = NULL; + return false; + } + + pSDB = (BYTE*)&vecSDB[0]; +#ifdef __THEMIDA__ + VM_END +#endif + + return true; +} + +bool EterPackPolicy_CSHybridCrypt::IsContainingSDBFile() const +{ + return m_mapSDBMap.size() > 0 ? true : false; +} + +void EterPackPolicy_CSHybridCrypt::WriteSupplementaryDataBlockToFile( CFileBase& rFile ) +{ + //about SDB data + // sdb file cnt( 4byte ) + // for sdb file cnt + // filename hash ( stl.h stringhash ) + // related map name size(4), relate map name + // sdb block size( 1byte ) + // sdb blocks + + DWORD dwSDBMapSize = m_mapSDBMap.size(); + rFile.Write( &dwSDBMapSize, sizeof(DWORD) ); + + TSupplementaryDataBlockMap::const_iterator cit; + for( cit = m_mapSDBMap.begin(); cit != m_mapSDBMap.end(); ++cit ) + { + DWORD dwFileNamehash = cit->first; + rFile.Write( &dwFileNamehash, sizeof(DWORD) ); + + const std::string strRelatedMapName = cit->second.strRelatedMapName; + DWORD dwMapNameSize = strRelatedMapName.size(); + rFile.Write( &dwMapNameSize, sizeof(DWORD) ); + rFile.Write( strRelatedMapName.c_str(), dwMapNameSize ); + + const std::vector& sdbVector = cit->second.vecStream; + BYTE bSDBSize = (BYTE)(sdbVector.size()); + + rFile.Write( &bSDBSize, sizeof(bSDBSize) ); + if( bSDBSize > 0 ) + rFile.Write( &sdbVector[0], bSDBSize ); + } +} + +int EterPackPolicy_CSHybridCrypt::ReadSupplementatyDataBlockFromStream( IN const BYTE* pStream ) +{ +#ifdef __THEMIDA__ + VM_START +#endif + + //DWORD dwFileIdentifier; + //std::vector vecSDBStream; + + DWORD dwFileNameHash; + BYTE bSDBSize; + int iStreamOffset = 0; + + memcpy(&dwFileNameHash, pStream + iStreamOffset, sizeof(DWORD) ); + iStreamOffset += sizeof(DWORD); + + memcpy(&bSDBSize, pStream + iStreamOffset, sizeof(BYTE) ); + iStreamOffset += sizeof(BYTE); + + // NOTE : related map name isn`t required in client. so we don`t recv it from stream to reduce packet size. + TSupplementaryDataBlockInfo info; + { + info.vecStream.resize( bSDBSize ); + memcpy(&info.vecStream[0], pStream + iStreamOffset, bSDBSize ); + iStreamOffset += bSDBSize; + + m_mapSDBMap[dwFileNameHash] = info; + } + +#ifdef __THEMIDA__ + VM_END +#endif + + return iStreamOffset; +} diff --git a/extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.h b/extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.h new file mode 100644 index 00000000..63f76ab2 --- /dev/null +++ b/extension/src/port/EterPack/EterPackPolicy_CSHybridCrypt.h @@ -0,0 +1,94 @@ +#ifndef __INC_ETERPACKLIB_ETERPACKPOLICY_CSHYBRIDCRYPT_H__ +#define __INC_ETERPACKLIB_ETERPACKPOLICY_CSHYBRIDCRYPT_H__ + +#include +#include +#include +#include +#include + +enum eHybridCipherAlgorithm +{ + e_Cipher_Camellia, + e_Cipher_Twofish, + e_Cipher_XTEA, + Num_Of_Ciphers +}; + + +class CFileBase; +class CLZObject; + +//THEMIDA +class EterPackPolicy_CSHybridCrypt +{ +public: + ~EterPackPolicy_CSHybridCrypt(); + + bool GenerateCryptKey( std::string& rfileName ); + bool EncryptMemory( std::string& rfilename, IN const BYTE* pSrcData, IN int iSrcLen, OUT CLZObject& zObj ); + bool DecryptMemory( std::string& rfilename, IN const BYTE* pSrcData, IN int iSrcLen, OUT CLZObject& zObj ); + bool IsContainingCryptKey() const; + + //Supplementary Data Block (SDB) + bool GenerateSupplementaryDataBlock(std::string& rfilename, const std::string& strMapName, IN const BYTE* pSrcData, IN int iSrcLen, OUT LPBYTE& pDestData, OUT int& iDestLen ); + bool GetSupplementaryDataBlock( std::string& rfilename, OUT LPBYTE& pSDB, OUT int& iSDBSize ); + bool IsContainingSDBFile() const; + + // Read/Write IO + void WriteCryptKeyToFile( CFileBase& rFile ); + int ReadCryptKeyInfoFromStream( IN const BYTE* pStream ); + + void WriteSupplementaryDataBlockToFile( CFileBase& rFile ); + int ReadSupplementatyDataBlockFromStream( IN const BYTE* pStream ); + +protected: + + typedef union UEncryptKey + { + BYTE key[16]; + BYTE keyCamellia[ CryptoPP::Camellia::DEFAULT_KEYLENGTH]; + BYTE keyTwofish [ CryptoPP::Twofish::DEFAULT_KEYLENGTH]; + BYTE keyXTEA [ CryptoPP::XTEA::DEFAULT_KEYLENGTH]; + + } TEncryptKey; + + typedef union UEncryptIV + { + BYTE iv[16]; + BYTE ivCamellia [ CryptoPP::Camellia::BLOCKSIZE]; + BYTE ivTwofish [ CryptoPP::Twofish::BLOCKSIZE]; + BYTE ivXTEA [ CryptoPP::XTEA::BLOCKSIZE]; + + } TEncryptIV; + + + typedef struct SCSHybridCryptKey + { + TEncryptKey uEncryptKey; + TEncryptIV uEncryptIV; + + } TCSHybridCryptKey; + + typedef std::unordered_map TCSHybridCryptKeyMap; + TCSHybridCryptKeyMap m_mapHybridCryptKey; + + typedef struct SSupplementaryDataBlockInfo + { + std::string strRelatedMapName; + std::vector vecStream; + + } TSupplementaryDataBlockInfo; + + typedef std::unordered_map TSupplementaryDataBlockMap; //key filename hash + TSupplementaryDataBlockMap m_mapSDBMap; + +private: + bool GetPerFileCryptKey( std::string& rfileName, eHybridCipherAlgorithm& eAlgorithm, TEncryptKey& key, TEncryptIV& iv ); + + +}; + + + +#endif // __INC_ETERPACKLIB_ETERPACKPOLICY_CSHYBRIDCRYPT_H__ diff --git a/extension/src/port/EterPack/Inline.h b/extension/src/port/EterPack/Inline.h new file mode 100644 index 00000000..92dbb49d --- /dev/null +++ b/extension/src/port/EterPack/Inline.h @@ -0,0 +1,34 @@ +#ifndef __INC_ETERPACK_INLINE_H__ +#define __INC_ETERPACK_INLINE_H__ + +inline void inlinePathCreate(const char* path) +{ + char dir[64]; + const char* p, *k; + + p = path + 3; + + while (NULL != (k = strchr(p, '/'))) + { + memset(dir, 0, sizeof(dir)); + strncpy(dir, path, k - path); + CreateDirectory(dir, NULL); + p = k + 1; + } +} + +inline void inlineConvertPackFilename(char* name) +{ + char * p = name; + + while (*p) + { + if (*p == '\\') + *p = '/'; + else + *p = (int) tolower(*p); + p++; + } +} + +#endif diff --git a/extension/src/port/EterPack/StdAfx.cpp b/extension/src/port/EterPack/StdAfx.cpp new file mode 100644 index 00000000..45cf0e1b --- /dev/null +++ b/extension/src/port/EterPack/StdAfx.cpp @@ -0,0 +1,2 @@ +#include "StdAfx.h" + diff --git a/extension/src/port/common/Win32Crt.cpp b/extension/src/port/common/Win32Crt.cpp index 1d70aee6..ce4ce4c0 100644 --- a/extension/src/port/common/Win32Crt.cpp +++ b/extension/src/port/common/Win32Crt.cpp @@ -8,6 +8,9 @@ #include #include +#include +#include + static_assert(sizeof(std::recursive_mutex) <= sizeof(CRITICAL_SECTION::storage), "CRITICAL_SECTION storage"); static_assert(alignof(std::recursive_mutex) <= 16, "CRITICAL_SECTION alignment"); @@ -21,6 +24,26 @@ void DeleteCriticalSection(LPCRITICAL_SECTION cs) { cs_mutex(cs).~recursive_mute void EnterCriticalSection(LPCRITICAL_SECTION cs) { cs_mutex(cs).lock(); } void LeaveCriticalSection(LPCRITICAL_SECTION cs) { cs_mutex(cs).unlock(); } +BOOL CreateDirectory(LPCSTR path, void*) +{ + return mkdir(path, 0777) == 0 ? TRUE : FALSE; +} + +BOOL DeleteFile(LPCSTR path) +{ + return unlink(path) == 0 ? TRUE : FALSE; +} + +int _access(const char* path, int mode) +{ + int m = F_OK; + if (mode & 2) + m |= W_OK; + if (mode & 4) + m |= R_OK; + return access(path, m); +} + char* _strlwr(char* s) { for (char* p = s; *p; ++p) diff --git a/extension/src/port/common/Win32Crt.h b/extension/src/port/common/Win32Crt.h index 449c5c60..11fe1891 100644 --- a/extension/src/port/common/Win32Crt.h +++ b/extension/src/port/common/Win32Crt.h @@ -12,6 +12,14 @@ #if !defined(_WIN32) #include +#include "Win32MinMax.h" + +// mmsystem.h +#ifndef MAKEFOURCC +#define MAKEFOURCC(ch0, ch1, ch2, ch3) \ + ((DWORD)(BYTE)(ch0) | ((DWORD)(BYTE)(ch1) << 8) | ((DWORD)(BYTE)(ch2) << 16) | ((DWORD)(BYTE)(ch3) << 24)) +#endif + #define ZeroMemory(dst, len) std::memset((dst), 0, (len)) #define FillMemory(dst, len, fill) std::memset((dst), (fill), (len)) #define CopyMemory(dst, src, len) std::memcpy((dst), (src), (len)) @@ -41,6 +49,12 @@ int _snprintf(char* buf, size_t count, const char* fmt, ...) #endif ; +// kernel32 file-system calls (ANSI). CreateDirectory ignores the security attributes; _access takes +// the MSVC modes 0 (exists), 2 (write), 4 (read), 6 (read/write). +BOOL CreateDirectory(LPCSTR path, void* security_attributes); +BOOL DeleteFile(LPCSTR path); +int _access(const char* path, int mode); + // winmm `timeGetTime`: milliseconds since system start, wrapping at 2^32. DWORD timeGetTime(); DWORD GetTickCount(); diff --git a/extension/src/port/common/Win32MinMax.h b/extension/src/port/common/Win32MinMax.h new file mode 100644 index 00000000..c99b0a75 --- /dev/null +++ b/extension/src/port/common/Win32MinMax.h @@ -0,0 +1,13 @@ +#pragma once +// windef.h `min`/`max` macros (40250 builds without NOMINMAX), as functions so and +// numeric_limits::max() keep working: same usual-arithmetic-conversion result as the macro, and a +// same-typed call under `using namespace std` still resolves to the more specialized std::min/max. +// Included by Win32Crt.h off Windows, and force-included on MinGW, whose leaves them out in C++. +#include + +#if !defined(min) && !defined(max) +template +constexpr std::common_type_t min(A a, B b) { return (a < b) ? a : b; } +template +constexpr std::common_type_t max(A a, B b) { return (a > b) ? a : b; } +#endif diff --git a/extension/src/port/common/shim/sdk/cryptopp/cryptoppLibLink.h b/extension/src/port/common/shim/sdk/cryptopp/cryptoppLibLink.h new file mode 100644 index 00000000..cffe18a6 --- /dev/null +++ b/extension/src/port/common/shim/sdk/cryptopp/cryptoppLibLink.h @@ -0,0 +1,3 @@ +#pragma once +// Shim for 40250's , which only holds MSVC `#pragma comment(lib, ...)` auto-link +// lines; the vendored Crypto++ (mt3p::cryptopp) is linked by CMake. diff --git a/extension/src/port/common/shim/sdk/lzo/lzoLibLink.h b/extension/src/port/common/shim/sdk/lzo/lzoLibLink.h new file mode 100644 index 00000000..88479551 --- /dev/null +++ b/extension/src/port/common/shim/sdk/lzo/lzoLibLink.h @@ -0,0 +1,3 @@ +#pragma once +// Shim for 40250's , which only holds MSVC `#pragma comment(lib, ...)` auto-link lines; +// the vendored minilzo (mt3p::minilzo) is linked by CMake. diff --git a/extension/src/port/common/shim/win32/io.h b/extension/src/port/common/shim/win32/io.h new file mode 100644 index 00000000..fe3ed7d5 --- /dev/null +++ b/extension/src/port/common/shim/win32/io.h @@ -0,0 +1,3 @@ +#pragma once +// Shim for on non-Windows targets; what 40250 logic uses from it (_access) is in . +#include "windows.h" diff --git a/extension/tests/port_eterpack_test.cpp b/extension/tests/port_eterpack_test.cpp new file mode 100644 index 00000000..4d9c8009 --- /dev/null +++ b/extension/tests/port_eterpack_test.cpp @@ -0,0 +1,176 @@ +// port/EterPack against the real 40250 Client/pack (batch 2D). Registers every pack in pack/Index order the way +// UserInterface.cpp PackInitialize does, then checks the numbers batch 2R measured with tools/epk_scan +// (audit/packs/2R-scan.json): 103 packs + root / 54891 entries / 52609 paths, first-registered pack wins, and exactly +// the 4 short-layout SECURITY files 40250 rejects fail to load. +// +// port_eterpack_test <40250 Client dir> +// +// Exit 77 (ctest SKIP) when the client is missing, unless MT_ASSETS_STRICT=1, which turns that into a failure. +#include "EterPack/StdAfx.h" +#include "EterPack/EterPackManager.h" +#include "EterBase/CRC32.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static int g_failures = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "%s:%d: CHECK(%s)\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +// Index names are spelled as on the NTFS original ("Etc" for ETC.eix); find the on-disk spelling. +static std::string on_disk_name(const std::string& dir, const std::string& name) +{ + std::string found = name; + if (DIR* d = opendir(dir.c_str())) + { + const std::string want = name + ".eix"; + while (dirent* e = readdir(d)) + if (strcasecmp(e->d_name, want.c_str()) == 0) + found = std::string(e->d_name, want.size() - 4); + closedir(d); + } + return found; +} + +static std::vector read_lines(const std::string& path) +{ + std::vector lines; + std::ifstream in(path, std::ios::binary); + std::string line; + while (std::getline(in, line)) + { + while (!line.empty() && (line.back() == '\r' || line.back() == ' ' || line.back() == '\t')) + line.pop_back(); + lines.push_back(line); + } + return lines; +} + +int main(int argc, char** argv) +{ + const char* strict = std::getenv("MT_ASSETS_STRICT"); + const bool is_strict = strict && std::string(strict) == "1"; + const std::string client = argc > 1 ? argv[1] : ""; + if (client.empty() || chdir(client.c_str()) != 0 || access("pack/Index", 0) != 0) + { + std::fprintf(stderr, "port_eterpack_test: no 40250 Client/pack at '%s'\n", client.c_str()); + return is_strict ? 1 : 77; + } + + // PackInitialize("pack"): line 0 is PACK, then (folder, name) pairs; every name also gets a _texcache pack. + const std::vector lines = read_lines("pack/Index"); + CHECK(!lines.empty() && lines[0] == "PACK"); + + CEterPackManager manager; // CSingleton: the instance registers itself + CEterPackManager& mgr = CEterPackManager::Instance(); + mgr.SetCacheMode(); + mgr.SetSearchMode(true); + + std::vector registered; + for (size_t i = 1; i + 1 < lines.size(); i += 2) + { + const std::string& folder = lines[i]; + for (const std::string& name : {lines[i + 1], lines[i + 1] + "_texcache"}) + { + const std::string db = "pack/" + on_disk_name("pack", name); + if (mgr.RegisterPack(db.c_str(), folder.c_str())) + registered.push_back(db); + } + } + mgr.RegisterRootPack("pack/root"); + + // 124 Index pairs; 119 registrations find a pack (some names are listed twice), 103 distinct packs, + root. + CHECK(registered.size() == 119); + CHECK(std::set(registered.begin(), registered.end()).size() == 103); + + // The whole dict (one entry per pack index record, overrides included). + struct Access : CEterPackManager + { + static const CEterFileDict::TDict& dict(CEterPackManager& m) { return static_cast(m).m_FileDict.GetDict(); } + static CEterFileDict::Item* item(CEterPackManager& m, DWORD h, const char* n) { return static_cast(m).m_FileDict.GetItem(h, n); } + }; + const CEterFileDict::TDict& dict = Access::dict(mgr); + CHECK(dict.size() == 54891); + + std::map order; + for (size_t i = 0; i < registered.size(); ++i) + order[registered[i]] = static_cast(i); + + // First registration per path, and CRC32 of the name equal to the stored filename_crc. + std::map first; + for (const auto& kv : dict) + { + const std::string name = kv.second.pkInfo->filename; + const std::string pack = kv.second.pkPack->GetDBName(); + CHECK(GetCRC32(name.c_str(), name.size()) == kv.first); + auto it = first.find(name); + if (it == first.end() || order[pack] < order[it->second]) + first[name] = pack; + } + CHECK(first.size() == 52609); + + size_t overridden = 0, wrong_winner = 0; + std::map per_name; + for (const auto& kv : dict) + ++per_name[kv.second.pkInfo->filename]; + for (const auto& [name, count] : per_name) + { + if (count < 2) + continue; + overridden += count - 1; + CEterFileDict::Item* item = Access::item(mgr, GetCRC32(name.c_str(), name.size()), name.c_str()); + if (!item || first[name] != item->pkPack->GetDBName()) + { + if (wrong_winner < 5) + std::fprintf(stderr, "winner %s: %s, expected %s\n", name.c_str(), + item ? item->pkPack->GetDBName() : "(none)", first[name].c_str()); + ++wrong_winner; + } + } + CHECK(overridden == 2282); + CHECK(wrong_winner == 0); + + // Every path loads through CEterPackManager::Get, except the 4 short-layout SECURITY files. + const std::set rejected_expected = { + "d:/ymir work/npc/mineral2/wait.msa", + "d:/ymir work/effect/background/metinstone_loop_redblack.mse", + "d:/ymir work/monster2/spider_spawn/wait.msa", + "d:/ymir work/monster2/spider_spawn/line1.mse", + }; + std::set rejected; + unsigned long long bytes = 0; + for (const auto& [name, pack] : first) + { + CMappedFile file; + LPCVOID data = nullptr; + if (mgr.Get(file, name.c_str(), &data)) + bytes += file.Size(); + else + rejected.insert(name); + } + for (const std::string& name : rejected) + if (!rejected_expected.count(name)) + std::fprintf(stderr, "unexpected load failure: %s\n", name.c_str()); + CHECK(rejected == rejected_expected); + + std::printf("packs %zu, entries %zu, paths %zu, overrides %zu, loaded %zu (%llu bytes), rejected %zu\n", + registered.size(), dict.size(), first.size(), overridden, first.size() - rejected.size(), bytes, + rejected.size()); + if (g_failures) + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return g_failures ? 1 : 0; +} diff --git a/script/port_gate.sh b/script/port_gate.sh index d7c3c5c2..36fa52e4 100755 --- a/script/port_gate.sh +++ b/script/port_gate.sh @@ -33,7 +33,7 @@ for p in "${PLATFORMS[@]}"; do BUILD_EXTRA=() case "$p" in macos) - BUILD_EXTRA=(--target port_common_test) + BUILD_EXTRA=(--target port_common_test --target port_eterpack_test) run macos "$OUT/macos" "${PORT_ONLY[@]}" -DBUILD_TESTING=ON last=$((${#RESULTS[@]} - 1)) if [[ "${RESULTS[$last]}" == *PASS ]] && ! ctest --test-dir "$OUT/macos" -R '^port\.' --output-on-failure >>"$OUT/macos.log" 2>&1; then @@ -71,7 +71,7 @@ for p in "${PLATFORMS[@]}"; do ;; linux) if [ "$(uname -s)" = Linux ]; then - BUILD_EXTRA=(--target port_common_test) + BUILD_EXTRA=(--target port_common_test --target port_eterpack_test) run linux "$OUT/linux" "${PORT_ONLY[@]}" -DBUILD_TESTING=ON elif [ -n "${MT_LINUX_TOOLCHAIN:-}" ]; then run linux "$OUT/linux" "${PORT_ONLY[@]}" -DBUILD_TESTING=OFF -DCMAKE_TOOLCHAIN_FILE="$MT_LINUX_TOOLCHAIN"