layout: restructure into src/ tests/ android/ scripts/ tools/
- extension/src/{port,platform,codepage} -> src/; native_render -> src/host
(+ dxt, shaders/); libgr2 -> src/gr2; extension/third_party -> third_party
- extension/tests -> tests/port, libgr2/tests -> tests/gr2
- android-native -> android (build.sh, push-client.sh moved in)
- script -> scripts; tools/40250 -> tools/server; oracle -> tools/granny-oracle;
perf tools -> tools/perf
- all build trees under build/ (native, release, android, port-gate)
- xrender:: CMake aliases -> mt::; port-map ledger impl paths rewritten
No code changes. ctest 15/15, port_gate macos+android PASS, port_map check 0 errors.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
70710477cf
commit
a46093104c
@@ -0,0 +1,8 @@
|
||||
# mt_codepage — Windows ANSI code pages (874/932/936/949/950/1250-1258) from Microsoft's WindowsBestFit
|
||||
# tables; codepage_tables.inc is generated by scripts/gen_codepage_tables.py. Used by port_logic
|
||||
# (Win32Crt MultiByteToWideChar / WideCharToMultiByte) and mtnet (40250 CP936 wire text), in place of
|
||||
# the host iconv.
|
||||
add_library(mt_codepage STATIC codepage.cpp)
|
||||
target_include_directories(mt_codepage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
target_compile_features(mt_codepage PUBLIC cxx_std_17)
|
||||
set_target_properties(mt_codepage PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||
@@ -0,0 +1,169 @@
|
||||
#include "codepage.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
namespace mt_codepage {
|
||||
namespace {
|
||||
|
||||
constexpr uint16_t kNone = 0xFFFF;
|
||||
|
||||
struct DbcsRow {
|
||||
uint8_t lead;
|
||||
uint8_t trail_lo;
|
||||
uint8_t trail_hi;
|
||||
uint32_t offset; // into the Dbcs array, trail_lo first
|
||||
};
|
||||
|
||||
// A WCTABLE entry the first-match inverse of the byte tables gets wrong.
|
||||
struct EncodeOverride {
|
||||
uint16_t wc;
|
||||
uint16_t mb;
|
||||
uint8_t kind; // 0 exact (round-trips), 1 best fit only, 2 not encodable
|
||||
};
|
||||
|
||||
struct CodePageTable {
|
||||
unsigned cp;
|
||||
uint16_t mb_default; // Unicode char for an unmapped byte / pair (CPINFO)
|
||||
const uint16_t* single;
|
||||
const uint8_t* leads; // lo, hi pairs
|
||||
size_t lead_count;
|
||||
const DbcsRow* rows;
|
||||
size_t row_count;
|
||||
const uint16_t* dbcs;
|
||||
const EncodeOverride* overrides;
|
||||
size_t override_count;
|
||||
};
|
||||
|
||||
#include "codepage_tables.inc"
|
||||
|
||||
constexpr size_t kTableCount = sizeof(kTables) / sizeof(kTables[0]);
|
||||
|
||||
const CodePageTable* find(unsigned cp)
|
||||
{
|
||||
for (const CodePageTable& t : kTables)
|
||||
if (t.cp == cp) return &t;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool lead(const CodePageTable& t, unsigned char b)
|
||||
{
|
||||
for (size_t i = 0; i < t.lead_count; ++i)
|
||||
if (b >= t.leads[i * 2] && b <= t.leads[i * 2 + 1]) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
uint16_t pair(const CodePageTable& t, unsigned char l, unsigned char trail)
|
||||
{
|
||||
for (size_t i = 0; i < t.row_count; ++i) {
|
||||
const DbcsRow& r = t.rows[i];
|
||||
if (r.lead != l) continue;
|
||||
if (trail < r.trail_lo || trail > r.trail_hi) return kNone;
|
||||
return t.dbcs[r.offset + (trail - r.trail_lo)];
|
||||
}
|
||||
return kNone;
|
||||
}
|
||||
|
||||
// Exact encoder (BMP code point -> bytes, kNone when unmapped), built on first use per code page.
|
||||
struct Encoder {
|
||||
std::once_flag once;
|
||||
std::unique_ptr<uint16_t[]> exact;
|
||||
};
|
||||
Encoder g_encoders[kTableCount];
|
||||
|
||||
const uint16_t* exact_encoder(const CodePageTable& t)
|
||||
{
|
||||
Encoder& e = g_encoders[&t - kTables];
|
||||
std::call_once(e.once, [&] {
|
||||
e.exact.reset(new uint16_t[0x10000]);
|
||||
std::fill_n(e.exact.get(), 0x10000, kNone);
|
||||
uint16_t* map = e.exact.get();
|
||||
// First match in ascending byte order, as the generator assumed.
|
||||
for (unsigned b = 0; b < 256; ++b)
|
||||
if (t.single[b] != kNone && map[t.single[b]] == kNone) map[t.single[b]] = uint16_t(b);
|
||||
for (size_t i = 0; i < t.row_count; ++i) {
|
||||
const DbcsRow& r = t.rows[i];
|
||||
for (unsigned trail = r.trail_lo; trail <= r.trail_hi; ++trail) {
|
||||
const uint16_t wc = t.dbcs[r.offset + (trail - r.trail_lo)];
|
||||
if (wc != kNone && map[wc] == kNone) map[wc] = uint16_t(r.lead << 8 | trail);
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < t.override_count; ++i) {
|
||||
const EncodeOverride& o = t.overrides[i];
|
||||
if (o.kind == 0) map[o.wc] = o.mb;
|
||||
else if (o.kind == 2) map[o.wc] = kNone;
|
||||
}
|
||||
});
|
||||
return e.exact.get();
|
||||
}
|
||||
|
||||
uint16_t best_fit(const CodePageTable& t, char32_t c)
|
||||
{
|
||||
const EncodeOverride* end = t.overrides + t.override_count;
|
||||
const EncodeOverride* it = std::lower_bound(t.overrides, end, c,
|
||||
[](const EncodeOverride& o, char32_t v) { return o.wc < v; });
|
||||
return it != end && it->wc == c && it->kind == 1 ? it->mb : kNone;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool supported(unsigned cp)
|
||||
{
|
||||
return find(cp) != nullptr;
|
||||
}
|
||||
|
||||
bool is_lead_byte(unsigned cp, unsigned char b)
|
||||
{
|
||||
const CodePageTable* t = find(cp);
|
||||
return t && lead(*t, b);
|
||||
}
|
||||
|
||||
bool decode(unsigned cp, const unsigned char* p, size_t n, std::u32string& out)
|
||||
{
|
||||
const CodePageTable* t = find(cp);
|
||||
if (!t) return false;
|
||||
bool ok = true;
|
||||
for (size_t i = 0; i < n;) {
|
||||
const unsigned char b = p[i];
|
||||
uint16_t wc = kNone;
|
||||
if (lead(*t, b) && i + 1 < n && p[i + 1] != 0) {
|
||||
wc = pair(*t, b, p[i + 1]);
|
||||
i += 2;
|
||||
} else {
|
||||
wc = t->single[b];
|
||||
++i;
|
||||
}
|
||||
if (wc == kNone) {
|
||||
wc = t->mb_default;
|
||||
ok = false;
|
||||
}
|
||||
out.push_back(wc);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
void encode(unsigned cp, std::u32string_view in, std::string& out, bool allow_best_fit, bool* used_default)
|
||||
{
|
||||
const CodePageTable* t = find(cp);
|
||||
const uint16_t* exact = t ? exact_encoder(*t) : nullptr;
|
||||
for (char32_t c : in) {
|
||||
uint16_t mb = kNone;
|
||||
if (exact && c < 0x10000) {
|
||||
mb = exact[c];
|
||||
if (mb == kNone && allow_best_fit) mb = best_fit(*t, c);
|
||||
}
|
||||
if (mb == kNone) {
|
||||
out.push_back('?');
|
||||
if (used_default) *used_default = true;
|
||||
} else if (mb > 0xFF) {
|
||||
out.push_back(char(mb >> 8));
|
||||
out.push_back(char(mb & 0xFF));
|
||||
} else {
|
||||
out.push_back(char(mb));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mt_codepage
|
||||
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
// Windows ANSI code pages without the host iconv: MultiByteToWideChar / WideCharToMultiByte semantics
|
||||
// driven by Microsoft's WindowsBestFit tables (scripts/gen_codepage_tables.py). One implementation for
|
||||
// macOS, Linux, Android and iOS, so every platform decodes the 40250 locale and wire text the same way.
|
||||
//
|
||||
// Covered: 874, 932, 936, 949, 950, 1250-1258 (every code page 40250 EterLib/Util.cpp names).
|
||||
// UTF-8 is not handled here.
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace mt_codepage {
|
||||
|
||||
bool supported(unsigned cp);
|
||||
|
||||
// Byte string -> code points, as MultiByteToWideChar. A lead byte takes the next non-NUL byte as its
|
||||
// trail; an unmapped byte or pair becomes the code page's default char (U+30FB for 932, else '?').
|
||||
// Returns false when a default char was substituted (MB_ERR_INVALID_CHARS would fail).
|
||||
bool decode(unsigned cp, const unsigned char* p, size_t n, std::u32string& out);
|
||||
|
||||
// Code points -> bytes, as WideCharToMultiByte. With best_fit (no WC_NO_BEST_FIT_CHARS) a character
|
||||
// without an exact mapping may take its WCTABLE best fit ("Ā" -> "A" on 1252); otherwise, and for
|
||||
// anything unmapped, '?' is written and *used_default set.
|
||||
void encode(unsigned cp, std::u32string_view in, std::string& out, bool best_fit, bool* used_default);
|
||||
|
||||
bool is_lead_byte(unsigned cp, unsigned char b);
|
||||
|
||||
} // namespace mt_codepage
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user