codepage: replace iconv with bundled WindowsBestFit tables
mt_codepage implements code pages 874/932/936/949/950/1250-1258 from Microsoft's bestfit<cp>.txt sources (script/gen_codepage_tables.py, SHA-256 pinned). Win32Crt's MultiByteToWideChar/WideCharToMultiByte/ IsDBCSLeadByteEx and net/text_codec use it, so the Android API 24 build no longer depends on iconv. port.codepage checks every table against its source file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shenlei
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
07c101fdc8
commit
0b701d88c3
@@ -31,12 +31,12 @@ target_include_directories(port_logic PUBLIC ${MT_PORT_SHIMS})
|
||||
# Libraries 40250 links that are vendored as-is: <lzo/lzo1x.h> (EterBase/lzo.h), <cryptopp/*>
|
||||
# (EterBase/cipher.h), <Python-2.7/*> (ScriptLib).
|
||||
target_link_libraries(port_logic PUBLIC mt3p::minilzo mt3p::cryptopp)
|
||||
# common/Win32Crt.cpp converts non-UTF-8 code pages with iconv. macOS ships it as a separate library;
|
||||
# extension/CMakeLists.txt finds it for mtnet, but the port-only build (script/port_gate.sh) never reads that file.
|
||||
find_library(MT_ICONV_LIBRARY NAMES iconv)
|
||||
if(MT_ICONV_LIBRARY)
|
||||
target_link_libraries(port_logic PUBLIC ${MT_ICONV_LIBRARY})
|
||||
# common/Win32Crt.cpp converts ANSI code pages with mt_codepage. extension/CMakeLists.txt adds it for
|
||||
# mtnet; the port-only build (script/port_gate.sh, native_render) never reads that file.
|
||||
if(NOT TARGET mt_codepage)
|
||||
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../codepage ${CMAKE_CURRENT_BINARY_DIR}/../codepage)
|
||||
endif()
|
||||
target_link_libraries(port_logic PUBLIC mt_codepage)
|
||||
if(TARGET mtpython)
|
||||
target_link_libraries(port_logic PUBLIC mt3p::python)
|
||||
endif()
|
||||
@@ -173,6 +173,14 @@ if(BUILD_TESTING AND CMAKE_SYSTEM_NAME STREQUAL CMAKE_HOST_SYSTEM_NAME)
|
||||
add_test(NAME port.eterpack COMMAND $<TARGET_FILE:port_eterpack_test> ${MT_40250_CLIENT})
|
||||
set_tests_properties(port.eterpack PROPERTIES SKIP_RETURN_CODE 77)
|
||||
|
||||
# mt_codepage against Microsoft's bestfit<cp>.txt sources (script/gen_codepage_tables.py caches them
|
||||
# in build/bestfit); skipped (77) without them unless MT_ASSETS_STRICT=1.
|
||||
add_executable(port_codepage_test ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/codepage_test.cpp)
|
||||
target_link_libraries(port_codepage_test PRIVATE mt_codepage)
|
||||
add_test(NAME port.codepage COMMAND $<TARGET_FILE:port_codepage_test>
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../../build/bestfit)
|
||||
set_tests_properties(port.codepage PROPERTIES SKIP_RETURN_CODE 77)
|
||||
|
||||
# 批次 2V0-g: the 40250 font path over the FreeType GDI emulation; needs a system Tahoma.
|
||||
add_executable(port_text_test ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/port_text_test.cpp)
|
||||
target_link_libraries(port_text_test PRIVATE port_platform)
|
||||
|
||||
@@ -11,11 +11,11 @@
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#if __has_include(<iconv.h>)
|
||||
#include <iconv.h>
|
||||
#define MT_WIN32CRT_HAS_ICONV 1
|
||||
#endif
|
||||
#include <glob.h>
|
||||
// ANSI code pages come from Microsoft's own tables, not the host iconv (bionic has no GBK).
|
||||
#include "codepage.h"
|
||||
#include <algorithm>
|
||||
#include <dirent.h>
|
||||
#include <fnmatch.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
@@ -103,17 +103,30 @@ HANDLE FindFirstFile(const char* pattern, WIN32_FIND_DATA* data)
|
||||
std::string normalized(pattern);
|
||||
for (char& ch : normalized)
|
||||
if (ch == '\\') ch = '/';
|
||||
glob_t matches{};
|
||||
const int result = glob(normalized.c_str(), 0, nullptr, &matches);
|
||||
if (result != 0 || matches.gl_pathc == 0)
|
||||
// Win32 wildcards only appear in the last component; match it against the directory like
|
||||
// glob(3) would (sorted, '*' skips dot files). glob itself is API 28+ on Android.
|
||||
std::vector<std::string> paths;
|
||||
const size_t slash = normalized.rfind('/');
|
||||
const std::string dir = slash == std::string::npos ? std::string() : normalized.substr(0, slash + 1);
|
||||
const std::string name = normalized.substr(dir.size());
|
||||
if (name.find_first_of("*?[") == std::string::npos)
|
||||
{
|
||||
globfree(&matches);
|
||||
return INVALID_HANDLE_VALUE;
|
||||
struct stat st;
|
||||
if (::stat(normalized.c_str(), &st) == 0)
|
||||
paths.push_back(normalized);
|
||||
}
|
||||
else if (DIR* d = ::opendir(dir.empty() ? "." : dir.c_str()))
|
||||
{
|
||||
while (const dirent* entry = ::readdir(d))
|
||||
if (::fnmatch(name.c_str(), entry->d_name, FNM_PERIOD) == 0)
|
||||
paths.push_back(dir + entry->d_name);
|
||||
::closedir(d);
|
||||
std::sort(paths.begin(), paths.end());
|
||||
}
|
||||
if (paths.empty())
|
||||
return INVALID_HANDLE_VALUE;
|
||||
auto* state = new FindState;
|
||||
for (size_t i = 0; i < matches.gl_pathc; ++i)
|
||||
state->paths.emplace_back(matches.gl_pathv[i]);
|
||||
globfree(&matches);
|
||||
state->paths = std::move(paths);
|
||||
fill_find_data(state->paths[0], data);
|
||||
state->next = 1;
|
||||
return state;
|
||||
@@ -199,13 +212,8 @@ DWORD GetTickCount()
|
||||
|
||||
namespace {
|
||||
|
||||
// Windows-1252 0x80..0x9F; the other bytes map to the same code point (0 marks the undefined ones).
|
||||
const char32_t kCp1252High[32] = {
|
||||
0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017D, 0,
|
||||
0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E, 0x0178,
|
||||
};
|
||||
|
||||
bool is_cp1252(UINT cp) { return cp == CP_ACP || cp == 1252; }
|
||||
// CP_ACP is the Western system code page here, as on the 40250 target machines.
|
||||
UINT ansi(UINT cp) { return cp == CP_ACP ? 1252 : cp; }
|
||||
|
||||
bool decode_utf8(const unsigned char* p, size_t n, std::u32string& out)
|
||||
{
|
||||
@@ -237,108 +245,25 @@ void encode_utf8(char32_t cp, std::string& out)
|
||||
else { out.push_back(char(0xF0 | (cp >> 18))); out.push_back(char(0x80 | ((cp >> 12) & 0x3F))); out.push_back(char(0x80 | ((cp >> 6) & 0x3F))); out.push_back(char(0x80 | (cp & 0x3F))); }
|
||||
}
|
||||
|
||||
#if defined(MT_WIN32CRT_HAS_ICONV)
|
||||
std::string iconv_name(UINT cp)
|
||||
{
|
||||
switch (cp) {
|
||||
case 949: return "CP949";
|
||||
case 936: return "GBK";
|
||||
case 950: return "BIG5";
|
||||
case 932: return "SHIFT_JIS";
|
||||
case 874: return "CP874";
|
||||
default: return "CP" + std::to_string(cp);
|
||||
}
|
||||
}
|
||||
|
||||
// One character at a time, so an undecodable byte becomes one default char like Windows does.
|
||||
bool iconv_decode(UINT cp, const unsigned char* p, size_t n, std::u32string& out)
|
||||
{
|
||||
iconv_t cd = iconv_open("UTF-32LE", iconv_name(cp).c_str());
|
||||
if (cd == reinterpret_cast<iconv_t>(-1)) return false;
|
||||
for (size_t i = 0; i < n;) {
|
||||
unsigned char wide[16];
|
||||
bool done = false;
|
||||
for (size_t len = 1; len <= 4 && i + len <= n && !done; ++len) {
|
||||
iconv(cd, nullptr, nullptr, nullptr, nullptr);
|
||||
char* in = const_cast<char*>(reinterpret_cast<const char*>(p + i));
|
||||
size_t in_left = len;
|
||||
char* o = reinterpret_cast<char*>(wide);
|
||||
size_t o_left = sizeof(wide);
|
||||
if (iconv(cd, &in, &in_left, &o, &o_left) != size_t(-1) && in_left == 0 && o_left < sizeof(wide)) {
|
||||
for (size_t k = 0; k + 4 <= sizeof(wide) - o_left; k += 4)
|
||||
out.push_back(char32_t(wide[k] | (wide[k + 1] << 8) | (wide[k + 2] << 16) | (unsigned(wide[k + 3]) << 24)));
|
||||
i += len;
|
||||
done = true;
|
||||
}
|
||||
}
|
||||
if (!done) { out.push_back(0xFFFD); ++i; }
|
||||
}
|
||||
iconv_close(cd);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool iconv_encode(UINT cp, const std::u32string& in, std::string& out, bool* used_default)
|
||||
{
|
||||
iconv_t cd = iconv_open(iconv_name(cp).c_str(), "UTF-32LE");
|
||||
if (cd == reinterpret_cast<iconv_t>(-1)) return false;
|
||||
for (char32_t c : in) {
|
||||
unsigned char src[4] = {static_cast<unsigned char>(c), static_cast<unsigned char>(c >> 8), static_cast<unsigned char>(c >> 16), static_cast<unsigned char>(c >> 24)};
|
||||
char buf[16];
|
||||
char* ip = reinterpret_cast<char*>(src);
|
||||
size_t il = 4;
|
||||
char* op = buf;
|
||||
size_t ol = sizeof(buf);
|
||||
iconv(cd, nullptr, nullptr, nullptr, nullptr);
|
||||
if (iconv(cd, &ip, &il, &op, &ol) != size_t(-1) && il == 0) {
|
||||
out.append(buf, sizeof(buf) - ol);
|
||||
} else {
|
||||
out.push_back('?');
|
||||
*used_default = true;
|
||||
}
|
||||
}
|
||||
iconv_close(cd);
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool decode(UINT cp, const unsigned char* p, size_t n, std::u32string& out)
|
||||
{
|
||||
if (cp == CP_UTF8) return decode_utf8(p, n, out);
|
||||
if (is_cp1252(cp)) {
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
const unsigned char c = p[i];
|
||||
out.push_back(c >= 0x80 && c < 0xA0 ? (kCp1252High[c - 0x80] ? kCp1252High[c - 0x80] : char32_t(c)) : char32_t(c));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
#if defined(MT_WIN32CRT_HAS_ICONV)
|
||||
if (iconv_decode(cp, p, n, out)) return true;
|
||||
#endif
|
||||
// No converter for this code page: ASCII passes, the rest becomes U+FFFD.
|
||||
if (mt_codepage::supported(ansi(cp))) return mt_codepage::decode(ansi(cp), p, n, out);
|
||||
// No table for this code page: ASCII passes, the rest becomes U+FFFD.
|
||||
for (size_t i = 0; i < n; ++i) out.push_back(p[i] < 0x80 ? char32_t(p[i]) : char32_t(0xFFFD));
|
||||
return true;
|
||||
}
|
||||
|
||||
void encode(UINT cp, const std::u32string& in, std::string& out, bool* used_default)
|
||||
void encode(UINT cp, DWORD flags, const std::u32string& in, std::string& out, bool* used_default)
|
||||
{
|
||||
if (cp == CP_UTF8) {
|
||||
for (char32_t c : in) encode_utf8(c, out);
|
||||
return;
|
||||
}
|
||||
if (is_cp1252(cp)) {
|
||||
for (char32_t c : in) {
|
||||
if (c < 0x80 || (c >= 0xA0 && c < 0x100)) { out.push_back(char(c)); continue; }
|
||||
int hit = -1;
|
||||
for (int k = 0; k < 32; ++k)
|
||||
if (kCp1252High[k] == c) { hit = k; break; }
|
||||
if (hit >= 0) out.push_back(char(0x80 + hit));
|
||||
else { out.push_back('?'); *used_default = true; }
|
||||
}
|
||||
if (mt_codepage::supported(ansi(cp))) {
|
||||
mt_codepage::encode(ansi(cp), in, out, (flags & WC_NO_BEST_FIT_CHARS) == 0, used_default);
|
||||
return;
|
||||
}
|
||||
#if defined(MT_WIN32CRT_HAS_ICONV)
|
||||
if (iconv_encode(cp, in, out, used_default)) return;
|
||||
#endif
|
||||
for (char32_t c : in) {
|
||||
if (c < 0x80) out.push_back(char(c));
|
||||
else { out.push_back('?'); *used_default = true; }
|
||||
@@ -361,7 +286,7 @@ int MultiByteToWideChar(UINT CodePage, DWORD dwFlags, LPCSTR lpMultiByteStr, int
|
||||
return int(wide.size());
|
||||
}
|
||||
|
||||
int WideCharToMultiByte(UINT CodePage, DWORD, LPCWSTR lpWideCharStr, int cchWideChar,
|
||||
int WideCharToMultiByte(UINT CodePage, DWORD dwFlags, LPCWSTR lpWideCharStr, int cchWideChar,
|
||||
LPSTR lpMultiByteStr, int cbMultiByte, LPCSTR lpDefaultChar, LPBOOL lpUsedDefaultChar)
|
||||
{
|
||||
if (!lpWideCharStr || cchWideChar == 0 || cbMultiByte < 0) return 0;
|
||||
@@ -372,14 +297,14 @@ int WideCharToMultiByte(UINT CodePage, DWORD, LPCWSTR lpWideCharStr, int cchWide
|
||||
for (size_t i = 0; i < n; ++i) wide[i] = char32_t(lpWideCharStr[i]);
|
||||
std::string bytes;
|
||||
bool used_default = false;
|
||||
encode(CodePage, wide, bytes, &used_default);
|
||||
encode(CodePage, dwFlags, wide, bytes, &used_default);
|
||||
if (used_default && lpDefaultChar && CodePage != CP_UTF8) {
|
||||
// '?' was the placeholder; the caller's default char replaces it only where a fallback happened.
|
||||
std::string redo;
|
||||
for (char32_t c : wide) {
|
||||
std::string one;
|
||||
bool miss = false;
|
||||
encode(CodePage, std::u32string(1, c), one, &miss);
|
||||
encode(CodePage, dwFlags, std::u32string(1, c), one, &miss);
|
||||
redo += miss ? std::string(1, *lpDefaultChar) : one;
|
||||
}
|
||||
bytes.swap(redo);
|
||||
@@ -393,13 +318,7 @@ int WideCharToMultiByte(UINT CodePage, DWORD, LPCWSTR lpWideCharStr, int cchWide
|
||||
|
||||
BOOL IsDBCSLeadByteEx(UINT CodePage, BYTE TestChar)
|
||||
{
|
||||
switch (CodePage) {
|
||||
case 932: return (TestChar >= 0x81 && TestChar <= 0x9F) || (TestChar >= 0xE0 && TestChar <= 0xFC);
|
||||
case 936:
|
||||
case 949:
|
||||
case 950: return TestChar >= 0x81 && TestChar <= 0xFE;
|
||||
default: return FALSE;
|
||||
}
|
||||
return mt_codepage::is_lead_byte(ansi(CodePage), TestChar) ? TRUE : FALSE;
|
||||
}
|
||||
|
||||
LPSTR CharNextExA(WORD CodePage, LPCSTR lpCurrentChar, DWORD)
|
||||
|
||||
@@ -104,8 +104,8 @@ void DeleteCriticalSection(LPCRITICAL_SECTION cs);
|
||||
void EnterCriticalSection(LPCRITICAL_SECTION cs);
|
||||
void LeaveCriticalSection(LPCRITICAL_SECTION cs);
|
||||
|
||||
// stringapiset.h. WCHAR is 32-bit here, so wide strings hold UTF-32 code points. CP_UTF8 and
|
||||
// CP_ACP/1252 are converted directly; other code pages go through iconv where the platform has it.
|
||||
// stringapiset.h. WCHAR is 32-bit here, so wide strings hold UTF-32 code points. CP_UTF8 is converted
|
||||
// directly; CP_ACP (= 1252) and the other ANSI code pages use Microsoft's tables (codepage/codepage.h).
|
||||
#define CP_ACP 0
|
||||
#define CP_UTF8 65001
|
||||
#define MB_PRECOMPOSED 0x00000001
|
||||
|
||||
Reference in New Issue
Block a user