#include "codepage.h" #include #include #include #include namespace mt_codepage { namespace { constexpr uint16_t kNone = 0xFFFF; struct DbcsRow { uint8_t lead; uint8_t trail_lo; uint8_t trail_hi; uint32_t offset; // into the Dbcs array, trail_lo first }; // A WCTABLE entry the first-match inverse of the byte tables gets wrong. struct EncodeOverride { uint16_t wc; uint16_t mb; uint8_t kind; // 0 exact (round-trips), 1 best fit only, 2 not encodable }; struct CodePageTable { unsigned cp; uint16_t mb_default; // Unicode char for an unmapped byte / pair (CPINFO) const uint16_t* single; const uint8_t* leads; // lo, hi pairs size_t lead_count; const DbcsRow* rows; size_t row_count; const uint16_t* dbcs; const EncodeOverride* overrides; size_t override_count; }; #include "codepage_tables.inc" constexpr size_t kTableCount = sizeof(kTables) / sizeof(kTables[0]); const CodePageTable* find(unsigned cp) { for (const CodePageTable& t : kTables) if (t.cp == cp) return &t; return nullptr; } bool lead(const CodePageTable& t, unsigned char b) { for (size_t i = 0; i < t.lead_count; ++i) if (b >= t.leads[i * 2] && b <= t.leads[i * 2 + 1]) return true; return false; } uint16_t pair(const CodePageTable& t, unsigned char l, unsigned char trail) { for (size_t i = 0; i < t.row_count; ++i) { const DbcsRow& r = t.rows[i]; if (r.lead != l) continue; if (trail < r.trail_lo || trail > r.trail_hi) return kNone; return t.dbcs[r.offset + (trail - r.trail_lo)]; } return kNone; } // Exact encoder (BMP code point -> bytes, kNone when unmapped), built on first use per code page. struct Encoder { std::once_flag once; std::unique_ptr exact; }; Encoder g_encoders[kTableCount]; const uint16_t* exact_encoder(const CodePageTable& t) { Encoder& e = g_encoders[&t - kTables]; std::call_once(e.once, [&] { e.exact.reset(new uint16_t[0x10000]); std::fill_n(e.exact.get(), 0x10000, kNone); uint16_t* map = e.exact.get(); // First match in ascending byte order, as the generator assumed. for (unsigned b = 0; b < 256; ++b) if (t.single[b] != kNone && map[t.single[b]] == kNone) map[t.single[b]] = uint16_t(b); for (size_t i = 0; i < t.row_count; ++i) { const DbcsRow& r = t.rows[i]; for (unsigned trail = r.trail_lo; trail <= r.trail_hi; ++trail) { const uint16_t wc = t.dbcs[r.offset + (trail - r.trail_lo)]; if (wc != kNone && map[wc] == kNone) map[wc] = uint16_t(r.lead << 8 | trail); } } for (size_t i = 0; i < t.override_count; ++i) { const EncodeOverride& o = t.overrides[i]; if (o.kind == 0) map[o.wc] = o.mb; else if (o.kind == 2) map[o.wc] = kNone; } }); return e.exact.get(); } uint16_t best_fit(const CodePageTable& t, char32_t c) { const EncodeOverride* end = t.overrides + t.override_count; const EncodeOverride* it = std::lower_bound(t.overrides, end, c, [](const EncodeOverride& o, char32_t v) { return o.wc < v; }); return it != end && it->wc == c && it->kind == 1 ? it->mb : kNone; } } // namespace bool supported(unsigned cp) { return find(cp) != nullptr; } bool is_lead_byte(unsigned cp, unsigned char b) { const CodePageTable* t = find(cp); return t && lead(*t, b); } bool decode(unsigned cp, const unsigned char* p, size_t n, std::u32string& out) { const CodePageTable* t = find(cp); if (!t) return false; bool ok = true; for (size_t i = 0; i < n;) { const unsigned char b = p[i]; uint16_t wc = kNone; if (lead(*t, b) && i + 1 < n && p[i + 1] != 0) { wc = pair(*t, b, p[i + 1]); i += 2; } else { wc = t->single[b]; ++i; } if (wc == kNone) { wc = t->mb_default; ok = false; } out.push_back(wc); } return ok; } void encode(unsigned cp, std::u32string_view in, std::string& out, bool allow_best_fit, bool* used_default) { const CodePageTable* t = find(cp); const uint16_t* exact = t ? exact_encoder(*t) : nullptr; for (char32_t c : in) { uint16_t mb = kNone; if (exact && c < 0x10000) { mb = exact[c]; if (mb == kNone && allow_best_fit) mb = best_fit(*t, c); } if (mb == kNone) { out.push_back('?'); if (used_default) *used_default = true; } else if (mb > 0xFF) { out.push_back(char(mb >> 8)); out.push_back(char(mb & 0xFF)); } else { out.push_back(char(mb)); } } } } // namespace mt_codepage