const std = @import("std"); const windows1252 = @import("windows1252.zig"); // TODO: Parts of this comment block may be more relevant to string/NameOrOrdinal parsing // than it is to the stuff in this file. // // ‰ representations for context: // Win-1252 89 // UTF-8 E2 80 B0 // UTF-16 20 30 // // With code page 65001: // ‰ RCDATA { "‰" L"‰" } // File encoded as Windows-1252: // ‰ => as u16 // "‰" => 0x3F ('?') // L"‰" => as u16 // File encoded as UTF-8: // ‰ => as u16 // "‰" => 0x89 ('‰' encoded as Windows-1252) // L"‰" => as u16 // // With code page 1252: // ‰ RCDATA { "‰" L"‰" } // File encoded as Windows-1252: // ‰ => as u16 // "‰" => 0x89 ('‰' encoded as Windows-1252) // L"‰" => as u16 // File encoded as UTF-8: // ‰ => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16 // ^ first byte of utf8 representation // ^ second byte of UTF-8 representation (0x80), but interpretted as // Windows-1252 ('€') and then converted to UTF-16 () // ^ third byte of utf8 representation // "‰" => 0xE2, 0x80, 0xB0 (the bytes of the UTF-8 representation) // L"‰" => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16 (see '‰ =>' explanation) // // With code page 1252: // <0x90> RCDATA { "<0x90>" L"<0x90>" } // File encoded as Windows-1252: // <0x90> => 0x90 as u16 // "<0x90>" => 0x90 // L"<0x90>" => 0x90 as u16 // File encoded as UTF-8: // <0x90> => 0xC2 as u16, 0x90 as u16 // "<0x90>" => 0xC2, 0x90 (the bytes of the UTF-8 representation of ) // L"<0x90>" => 0xC2 as u16, 0x90 as u16 // // Within a raw data block, file encoded as Windows-1252 ( is <0xC2>): // "Âa" L"Âa" "\xC2ad" L"\xC2AD" // With code page 1252: // C2 61 C2 00 61 00 C2 61 64 AD C2 // Â^ a^ Â~~~^ a~~~^ .^ a^ d^ ^~~~~\xC2AD // \xC2~` // With code page 65001: // 3F 61 FD FF 61 00 C2 61 64 AD C2 // ^. a^ ^~~~. a~~~^ ^. a^ d^ ^~~~~\xC2AD // `. `. `~\xC2 // `. `.~<0xC2>a is not well-formed UTF-8 (0xC2 expects a continutation byte after it). // `. Because 'a' is a valid first byte of a UTF-8 sequence, it is not included in the // `. invalid sequence so only the <0xC2> gets converted to . // `~Same as ^ but converted to '?' instead. // // Within a raw data block, file encoded as Windows-1252 (ð is <0xF0>, € is <0x80>): // "ð€a" L"ð€a" // With code page 1252: // F0 80 61 F0 00 AC 20 61 00 // ð^ €^ a^ ð~~~^ €~~~^ a~~~^ // With code page 65001: // 3F 61 FD FF 61 00 // ^. a^ ^~~~. a~~~^ // `. `. // `. `.~<0xF0><0x80> is not well-formed UTF-8, and <0x80> is not a valid first byte, so // `. both bytes are considered an invalid sequence and get converted to '' // `~Same as ^ but converted to '?' instead. /// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers pub const CodePage = enum(u16) { // supported windows1252 = 1252, // windows-1252 ANSI Latin 1; Western European (Windows) utf8 = 65001, // utf-8 Unicode (UTF-8) // unsupported but valid ibm037 = 37, // IBM037 IBM EBCDIC US-Canada ibm437 = 437, // IBM437 OEM United States ibm500 = 500, // IBM500 IBM EBCDIC International asmo708 = 708, // ASMO-708 Arabic (ASMO 708) asmo449plus = 709, // Arabic (ASMO-449+, BCON V4) transparent_arabic = 710, // Arabic - Transparent Arabic dos720 = 720, // DOS-720 Arabic (Transparent ASMO); Arabic (DOS) ibm737 = 737, // ibm737 OEM Greek (formerly 437G); Greek (DOS) ibm775 = 775, // ibm775 OEM Baltic; Baltic (DOS) ibm850 = 850, // ibm850 OEM Multilingual Latin 1; Western European (DOS) ibm852 = 852, // ibm852 OEM Latin 2; Central European (DOS) ibm855 = 855, // IBM855 OEM Cyrillic (primarily Russian) ibm857 = 857, // ibm857 OEM Turkish; Turkish (DOS) ibm00858 = 858, // IBM00858 OEM Multilingual Latin 1 + Euro symbol ibm860 = 860, // IBM860 OEM Portuguese; Portuguese (DOS) ibm861 = 861, // ibm861 OEM Icelandic; Icelandic (DOS) dos862 = 862, // DOS-862 OEM Hebrew; Hebrew (DOS) ibm863 = 863, // IBM863 OEM French Canadian; French Canadian (DOS) ibm864 = 864, // IBM864 OEM Arabic; Arabic (864) ibm865 = 865, // IBM865 OEM Nordic; Nordic (DOS) cp866 = 866, // cp866 OEM Russian; Cyrillic (DOS) ibm869 = 869, // ibm869 OEM Modern Greek; Greek, Modern (DOS) ibm870 = 870, // IBM870 IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 windows874 = 874, // windows-874 Thai (Windows) cp875 = 875, // cp875 IBM EBCDIC Greek Modern shift_jis = 932, // shift_jis ANSI/OEM Japanese; Japanese (Shift-JIS) gb2312 = 936, // gb2312 ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) ks_c_5601_1987 = 949, // ks_c_5601-1987 ANSI/OEM Korean (Unified Hangul Code) big5 = 950, // big5 ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) ibm1026 = 1026, // IBM1026 IBM EBCDIC Turkish (Latin 5) ibm01047 = 1047, // IBM01047 IBM EBCDIC Latin 1/Open System ibm01140 = 1140, // IBM01140 IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro) ibm01141 = 1141, // IBM01141 IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro) ibm01142 = 1142, // IBM01142 IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro) ibm01143 = 1143, // IBM01143 IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro) ibm01144 = 1144, // IBM01144 IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro) ibm01145 = 1145, // IBM01145 IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro) ibm01146 = 1146, // IBM01146 IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro) ibm01147 = 1147, // IBM01147 IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro) ibm01148 = 1148, // IBM01148 IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro) ibm01149 = 1149, // IBM01149 IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro) utf16 = 1200, // utf-16 Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications utf16_fffe = 1201, // unicodeFFFE Unicode UTF-16, big endian byte order; available only to managed applications windows1250 = 1250, // windows-1250 ANSI Central European; Central European (Windows) windows1251 = 1251, // windows-1251 ANSI Cyrillic; Cyrillic (Windows) windows1253 = 1253, // windows-1253 ANSI Greek; Greek (Windows) windows1254 = 1254, // windows-1254 ANSI Turkish; Turkish (Windows) windows1255 = 1255, // windows-1255 ANSI Hebrew; Hebrew (Windows) windows1256 = 1256, // windows-1256 ANSI Arabic; Arabic (Windows) windows1257 = 1257, // windows-1257 ANSI Baltic; Baltic (Windows) windows1258 = 1258, // windows-1258 ANSI/OEM Vietnamese; Vietnamese (Windows) johab = 1361, // Johab Korean (Johab) macintosh = 10000, // macintosh MAC Roman; Western European (Mac) x_mac_japanese = 10001, // x-mac-japanese Japanese (Mac) x_mac_chinesetrad = 10002, // x-mac-chinesetrad MAC Traditional Chinese (Big5); Chinese Traditional (Mac) x_mac_korean = 10003, // x-mac-korean Korean (Mac) x_mac_arabic = 10004, // x-mac-arabic Arabic (Mac) x_mac_hebrew = 10005, // x-mac-hebrew Hebrew (Mac) x_mac_greek = 10006, // x-mac-greek Greek (Mac) x_mac_cyrillic = 10007, // x-mac-cyrillic Cyrillic (Mac) x_mac_chinesesimp = 10008, // x-mac-chinesesimp MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) x_mac_romanian = 10010, // x-mac-romanian Romanian (Mac) x_mac_ukranian = 10017, // x-mac-ukrainian Ukrainian (Mac) x_mac_thai = 10021, // x-mac-thai Thai (Mac) x_mac_ce = 10029, // x-mac-ce MAC Latin 2; Central European (Mac) x_mac_icelandic = 10079, // x-mac-icelandic Icelandic (Mac) x_mac_turkish = 10081, // x-mac-turkish Turkish (Mac) x_mac_croatian = 10082, // x-mac-croatian Croatian (Mac) utf32 = 12000, // utf-32 Unicode UTF-32, little endian byte order; available only to managed applications utf32_be = 12001, // utf-32BE Unicode UTF-32, big endian byte order; available only to managed applications x_chinese_cns = 20000, // x-Chinese_CNS CNS Taiwan; Chinese Traditional (CNS) x_cp20001 = 20001, // x-cp20001 TCA Taiwan x_chinese_eten = 20002, // x_Chinese-Eten Eten Taiwan; Chinese Traditional (Eten) x_cp20003 = 20003, // x-cp20003 IBM5550 Taiwan x_cp20004 = 20004, // x-cp20004 TeleText Taiwan x_cp20005 = 20005, // x-cp20005 Wang Taiwan x_ia5 = 20105, // x-IA5 IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) x_ia5_german = 20106, // x-IA5-German IA5 German (7-bit) x_ia5_swedish = 20107, // x-IA5-Swedish IA5 Swedish (7-bit) x_ia5_norwegian = 20108, // x-IA5-Norwegian IA5 Norwegian (7-bit) us_ascii = 20127, // us-ascii US-ASCII (7-bit) x_cp20261 = 20261, // x-cp20261 T.61 x_cp20269 = 20269, // x-cp20269 ISO 6937 Non-Spacing Accent ibm273 = 20273, // IBM273 IBM EBCDIC Germany ibm277 = 20277, // IBM277 IBM EBCDIC Denmark-Norway ibm278 = 20278, // IBM278 IBM EBCDIC Finland-Sweden ibm280 = 20280, // IBM280 IBM EBCDIC Italy ibm284 = 20284, // IBM284 IBM EBCDIC Latin America-Spain ibm285 = 20285, // IBM285 IBM EBCDIC United Kingdom ibm290 = 20290, // IBM290 IBM EBCDIC Japanese Katakana Extended ibm297 = 20297, // IBM297 IBM EBCDIC France ibm420 = 20420, // IBM420 IBM EBCDIC Arabic ibm423 = 20423, // IBM423 IBM EBCDIC Greek ibm424 = 20424, // IBM424 IBM EBCDIC Hebrew x_ebcdic_korean_extended = 20833, // x-EBCDIC-KoreanExtended IBM EBCDIC Korean Extended ibm_thai = 20838, // IBM-Thai IBM EBCDIC Thai koi8_r = 20866, // koi8-r Russian (KOI8-R); Cyrillic (KOI8-R) ibm871 = 20871, // IBM871 IBM EBCDIC Icelandic ibm880 = 20880, // IBM880 IBM EBCDIC Cyrillic Russian ibm905 = 20905, // IBM905 IBM EBCDIC Turkish ibm00924 = 20924, // IBM00924 IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) euc_jp_jis = 20932, // EUC-JP Japanese (JIS 0208-1990 and 0212-1990) x_cp20936 = 20936, // x-cp20936 Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) x_cp20949 = 20949, // x-cp20949 Korean Wansung cp1025 = 21025, // cp1025 IBM EBCDIC Cyrillic Serbian-Bulgarian // = 21027, // (deprecated) koi8_u = 21866, // koi8-u Ukrainian (KOI8-U); Cyrillic (KOI8-U) iso8859_1 = 28591, // iso-8859-1 ISO 8859-1 Latin 1; Western European (ISO) iso8859_2 = 28592, // iso-8859-2 ISO 8859-2 Central European; Central European (ISO) iso8859_3 = 28593, // iso-8859-3 ISO 8859-3 Latin 3 iso8859_4 = 28594, // iso-8859-4 ISO 8859-4 Baltic iso8859_5 = 28595, // iso-8859-5 ISO 8859-5 Cyrillic iso8859_6 = 28596, // iso-8859-6 ISO 8859-6 Arabic iso8859_7 = 28597, // iso-8859-7 ISO 8859-7 Greek iso8859_8 = 28598, // iso-8859-8 ISO 8859-8 Hebrew; Hebrew (ISO-Visual) iso8859_9 = 28599, // iso-8859-9 ISO 8859-9 Turkish iso8859_13 = 28603, // iso-8859-13 ISO 8859-13 Estonian iso8859_15 = 28605, // iso-8859-15 ISO 8859-15 Latin 9 x_europa = 29001, // x-Europa Europa 3 is8859_8_i = 38598, // iso-8859-8-i ISO 8859-8 Hebrew; Hebrew (ISO-Logical) iso2022_jp = 50220, // iso-2022-jp ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) cs_iso2022_jp = 50221, // csISO2022JP ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) iso2022_jp_jis_x = 50222, // iso-2022-jp ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) iso2022_kr = 50225, // iso-2022-kr ISO 2022 Korean x_cp50227 = 50227, // x-cp50227 ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) iso2022_chinesetrad = 50229, // ISO 2022 Traditional Chinese ebcdic_jp_katakana_extended = 50930, // EBCDIC Japanese (Katakana) Extended ebcdic_us_ca_jp = 50931, // EBCDIC US-Canada and Japanese ebcdic_kr_extended = 50933, // EBCDIC Korean Extended and Korean ebcdic_chinesesimp_extended = 50935, // EBCDIC Simplified Chinese Extended and Simplified Chinese ebcdic_chinesesimp = 50936, // EBCDIC Simplified Chinese ebcdic_us_ca_chinesetrad = 50937, // EBCDIC US-Canada and Traditional Chinese ebcdic_jp_latin_extended = 50939, // EBCDIC Japanese (Latin) Extended and Japanese euc_jp = 51932, // euc-jp EUC Japanese euc_cn = 51936, // EUC-CN EUC Simplified Chinese; Chinese Simplified (EUC) euc_kr = 51949, // euc-kr EUC Korean euc_chinesetrad = 51950, // EUC Traditional Chinese hz_gb2312 = 52936, // hz-gb-2312 HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) gb18030 = 54936, // GB18030 Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) x_iscii_de = 57002, // x-iscii-de ISCII Devanagari x_iscii_be = 57003, // x-iscii-be ISCII Bangla x_iscii_ta = 57004, // x-iscii-ta ISCII Tamil x_iscii_te = 57005, // x-iscii-te ISCII Telugu x_iscii_as = 57006, // x-iscii-as ISCII Assamese x_iscii_or = 57007, // x-iscii-or ISCII Odia x_iscii_ka = 57008, // x-iscii-ka ISCII Kannada x_iscii_ma = 57009, // x-iscii-ma ISCII Malayalam x_iscii_gu = 57010, // x-iscii-gu ISCII Gujarati x_iscii_pa = 57011, // x-iscii-pa ISCII Punjabi utf7 = 65000, // utf-7 Unicode (UTF-7) pub fn codepointAt(code_page: CodePage, index: usize, bytes: []const u8) ?Codepoint { if (index >= bytes.len) return null; switch (code_page) { .windows1252 => { // All byte values have a representation, so just convert the byte return Codepoint{ .value = windows1252.toCodepoint(bytes[index]), .byte_len = 1, }; }, .utf8 => { return Utf8.WellFormedDecoder.decode(bytes[index..]); }, else => unreachable, } } pub fn isSupported(code_page: CodePage) bool { return switch (code_page) { .windows1252, .utf8 => true, else => false, }; } pub fn getByIdentifier(identifier: u16) !CodePage { // There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but // this should be fine, especially since this function likely won't be called much. inline for (@typeInfo(CodePage).@"enum".fields) |enumField| { if (identifier == enumField.value) { return @field(CodePage, enumField.name); } } return error.InvalidCodePage; } pub fn getByIdentifierEnsureSupported(identifier: u16) !CodePage { const code_page = try getByIdentifier(identifier); switch (isSupported(code_page)) { true => return code_page, false => return error.UnsupportedCodePage, } } }; pub const Utf8 = struct { /// Implements decoding with rejection of ill-formed UTF-8 sequences based on section /// D92 of Chapter 3 of the Unicode standard (Table 3-7 specifically). /// /// Note: This does not match "U+FFFD Substitution of Maximal Subparts", but instead /// matches the behavior of the Windows RC compiler. pub const WellFormedDecoder = struct { /// Like std.unicode.utf8ByteSequenceLength, but: /// - Rejects non-well-formed first bytes, i.e. C0-C1, F5-FF /// - Returns an optional value instead of an error union pub fn sequenceLength(first_byte: u8) ?u3 { return switch (first_byte) { 0x00...0x7F => 1, 0xC2...0xDF => 2, 0xE0...0xEF => 3, 0xF0...0xF4 => 4, else => null, }; } fn isContinuationByte(byte: u8) bool { return switch (byte) { 0x80...0xBF => true, else => false, }; } pub fn decode(bytes: []const u8) Codepoint { std.debug.assert(bytes.len > 0); const first_byte = bytes[0]; const expected_len = sequenceLength(first_byte) orelse { return .{ .value = Codepoint.invalid, .byte_len = 1 }; }; if (expected_len == 1) return .{ .value = first_byte, .byte_len = 1 }; var value: u21 = first_byte & 0b00011111; var byte_index: u8 = 1; while (byte_index < @min(bytes.len, expected_len)) : (byte_index += 1) { const byte = bytes[byte_index]; // See Table 3-7 of D92 in Chapter 3 of the Unicode Standard const valid: bool = switch (byte_index) { 1 => switch (first_byte) { 0xE0 => switch (byte) { 0xA0...0xBF => true, else => false, }, 0xED => switch (byte) { 0x80...0x9F => true, else => false, }, 0xF0 => switch (byte) { 0x90...0xBF => true, else => false, }, 0xF4 => switch (byte) { 0x80...0x8F => true, else => false, }, else => switch (byte) { 0x80...0xBF => true, else => false, }, }, else => switch (byte) { 0x80...0xBF => true, else => false, }, }; if (!valid) { var len = byte_index; // Only include the byte in the invalid sequence if it's in the range // of a continuation byte. All other values should not be included in the // invalid sequence. if (isContinuationByte(byte)) len += 1; return .{ .value = Codepoint.invalid, .byte_len = len }; } value <<= 6; value |= byte & 0b00111111; } if (byte_index != expected_len) { return .{ .value = Codepoint.invalid, .byte_len = byte_index }; } return .{ .value = value, .byte_len = expected_len }; } }; }; test "Utf8.WellFormedDecoder" { const invalid_utf8 = "\xF0\x80"; const decoded = Utf8.WellFormedDecoder.decode(invalid_utf8); try std.testing.expectEqual(Codepoint.invalid, decoded.value); try std.testing.expectEqual(@as(usize, 2), decoded.byte_len); } test "codepointAt invalid utf8" { { const invalid_utf8 = "\xf0\xf0\x80\x80\x80"; try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(0, invalid_utf8).?); try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 2, }, CodePage.utf8.codepointAt(1, invalid_utf8).?); try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(3, invalid_utf8).?); try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(4, invalid_utf8).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(5, invalid_utf8)); } { const invalid_utf8 = "\xE1\xA0\xC0"; try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 2, }, CodePage.utf8.codepointAt(0, invalid_utf8).?); try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(2, invalid_utf8).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(3, invalid_utf8)); } { const invalid_utf8 = "\xD2"; try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(0, invalid_utf8).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, invalid_utf8)); } { const invalid_utf8 = "\xE1\xA0"; try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 2, }, CodePage.utf8.codepointAt(0, invalid_utf8).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8)); } { const invalid_utf8 = "\xC5\xFF"; try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(0, invalid_utf8).?); try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(1, invalid_utf8).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8)); } { // encoded high surrogate const invalid_utf8 = "\xED\xA0\xBD"; try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 2, }, CodePage.utf8.codepointAt(0, invalid_utf8).?); try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(2, invalid_utf8).?); } } test "codepointAt utf8 encoded" { const utf8_encoded = "²"; // with code page utf8 try std.testing.expectEqual(Codepoint{ .value = '²', .byte_len = 2, }, CodePage.utf8.codepointAt(0, utf8_encoded).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, utf8_encoded)); // with code page windows1252 try std.testing.expectEqual(Codepoint{ .value = '\xC2', .byte_len = 1, }, CodePage.windows1252.codepointAt(0, utf8_encoded).?); try std.testing.expectEqual(Codepoint{ .value = '\xB2', .byte_len = 1, }, CodePage.windows1252.codepointAt(1, utf8_encoded).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, utf8_encoded)); } test "codepointAt windows1252 encoded" { const windows1252_encoded = "\xB2"; // with code page utf8 try std.testing.expectEqual(Codepoint{ .value = Codepoint.invalid, .byte_len = 1, }, CodePage.utf8.codepointAt(0, windows1252_encoded).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, windows1252_encoded)); // with code page windows1252 try std.testing.expectEqual(Codepoint{ .value = '\xB2', .byte_len = 1, }, CodePage.windows1252.codepointAt(0, windows1252_encoded).?); try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, windows1252_encoded)); } pub const Codepoint = struct { value: u21, byte_len: usize, pub const invalid: u21 = std.math.maxInt(u21); };