feat(libSceCesCs): report the JIS X 0213 level of a cell as the console does

core/libs/prx/libSceCesCs/Export.cpp

@@ -806,6 +806,24 @@ bool isJisX0213Plane2Row(uint32_t row) {
     return std::find(std::begin(JIS_X0213_PLANE2_ROWS), std::end(JIS_X0213_PLANE2_ROWS), static_cast<uint8_t>(row)) != std::end(JIS_X0213_PLANE2_ROWS);
 }
 
+constexpr uint16_t JIS_X0213_TWO_CODE_POINT_CELLS[25] = {
+    0x0457, 0x0458, 0x0459, 0x045A, 0x045B, 0x0557, 0x0558, 0x0559, 0x055A, 0x055B, 0x055C, 0x055D, 0x055E,
+    0x0658, 0x0B24, 0x0B28, 0x0B29, 0x0B2A, 0x0B2B, 0x0B2C, 0x0B2D, 0x0B2E, 0x0B2F, 0x0B45, 0x0B46,
+};
+
+bool isJisX0213Assigned(uint32_t plane, uint32_t row, uint32_t cell) {
+    const uint32_t index = (row - 1) * 94 + (cell - 1);
+    if (plane == 2) return JisX0213::PLANE2[index] != 0;
+    if (JisX0213::PLANE1[index] != 0) return true;
+    return std::find(std::begin(JIS_X0213_TWO_CODE_POINT_CELLS), std::end(JIS_X0213_TWO_CODE_POINT_CELLS), static_cast<uint16_t>(row << 8 | cell)) != std::end(JIS_X0213_TWO_CODE_POINT_CELLS);
+}
+
+uint32_t jisLevel(uint32_t plane, uint32_t row, uint32_t cell) {
+    if (plane == 2) return 4;
+    if (!isCp932ExtensionRow(row) && JisX0208::TO_UCS2[(row - 1) * 94 + (cell - 1)] != 0) return row <= 47 ? 1 : 2;
+    return 3;
+}
+
 struct Jis2004Cell {
     uint32_t code;
     uint16_t cell;
@@ -2164,6 +2182,16 @@ int APS5_VABI sceCesSJisCodeToJisX0213(uint16_t sjisCode, uint8_t* men, uint8_t*
     return 0;
 }
 
+int APS5_VABI sceCesJisGetLevel(uint8_t plane, uint8_t row, uint8_t cell, uint32_t* level) {
+    if (plane < 1 || plane > 2 || row < 1 || row > 94 || cell < 1 || cell > 94) {
+        if (level) *level = 0;
+        return CES_ERROR_ILLEGAL_CODE;
+    }
+    if (!level) return CES_ERROR_INVALID_PARAMETER;
+    *level = jisLevel(plane, row, cell);
+    return isJisX0213Assigned(plane, row, cell) ? 0 : CES_ERROR_UNASSIGNED_CODE;
+}
+
 int APS5_VABI sceCesJisX0213ToSJisCode(uint8_t men, uint8_t ku, uint8_t ten, uint16_t* sjisCode) {
     if (!sjisCode) return CES_ERROR_INVALID_DST_BUFFER;
     if (men < 1 || men > 2 || ku < 1 || ku > 94 || ten < 1 || ten > 94) return CES_ERROR_INVALID_ENCODE;

core/libs/prx/libSceCesCs/PendingExports.cpp

@@ -38,11 +38,6 @@ int APS5_VABI sceCesIso2022JpUcsProfileSetSbcs(void) {
     return 0;
 }
 
-int APS5_VABI sceCesJisGetLevel(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
 int APS5_VABI sceCesJiscsToUcs(void) {
     NotImplemented_nid_no_patch(__func__);
     return 0;

core/libs/prx/libSceCesCs/include/Api.hpp

@@ -225,7 +225,7 @@ int APS5_VABI sceCesIso2022StrToUtf32Str(void* context, const uint8_t* iso, uint
 int APS5_VABI sceCesIso2022StrToUtf8Str(void* context, const uint8_t* iso, uint32_t isoMax, uint32_t* isoLen, uint8_t* utf8, uint32_t utf8max, uint32_t* utf8Len);
 int APS5_VABI sceCesIso2022UcsContextInit(void* context, const void* profile);
 int APS5_VABI sceCesIso2022UcsContextInitCopy(void* context, const void* source);
-int APS5_VABI sceCesJisGetLevel(void);
+int APS5_VABI sceCesJisGetLevel(uint8_t plane, uint8_t row, uint8_t cell, uint32_t* level);
 int APS5_VABI sceCesJiscsToUcs(void);
 int APS5_VABI sceCesMbcsUcsSetMbcsReplacementCharCode(void);
 int APS5_VABI sceCesMbcsUcsSetMbcsReplacementCharUCode(void);

core/libs/tests/GuestCesJis2004.cpp

@@ -11,6 +11,7 @@ namespace {
 constexpr int invalidParameter = static_cast<int>(0x805C0001);
 constexpr int srcBufferEnd = static_cast<int>(0x805C0011);
 constexpr int invalidEncode = static_cast<int>(0x805C0014);
+constexpr int outOfCodeSpace = static_cast<int>(0x805C0015);
 constexpr int unassignedCode = static_cast<int>(0x805C0020);
 
 void Require(bool value) { if (!value) std::abort(); }
@@ -143,7 +144,36 @@ void CheckShiftJis() {
 
 }
 
+void CheckLevels() {
+    std::uint32_t hash = 0x811C9DC5u;
+    std::uint32_t assigned = 0;
+    for (std::uint32_t plane = 0; plane < 4; ++plane) {
+        for (std::uint32_t row = 0; row < 256; ++row) {
+            for (std::uint32_t cell = 0; cell < 256; ++cell) {
+                std::uint32_t level = 0xEEEEEEEE;
+                const int result = sceCesJisGetLevel(static_cast<std::uint8_t>(plane), static_cast<std::uint8_t>(row), static_cast<std::uint8_t>(cell), &level);
+                hash = (hash ^ (static_cast<std::uint32_t>(result) & 0xFF)) * 0x01000193u;
+                hash = (hash ^ (level & 0xFF)) * 0x01000193u;
+                if (result == 0) ++assigned;
+            }
+        }
+    }
+    Require(hash == 0xD41EFACBu && assigned == 11233);
+    std::uint32_t level = 0;
+    Require(sceCesJisGetLevel(1, 4, 2, &level) == 0 && level == 1);
+    Require(sceCesJisGetLevel(1, 48, 1, &level) == 0 && level == 2);
+    Require(sceCesJisGetLevel(1, 13, 1, &level) == 0 && level == 3);
+    Require(sceCesJisGetLevel(1, 4, 87, &level) == 0 && level == 3);
+    Require(sceCesJisGetLevel(1, 4, 92, &level) == unassignedCode && level == 3);
+    Require(sceCesJisGetLevel(2, 1, 1, &level) == 0 && level == 4);
+    Require(sceCesJisGetLevel(2, 2, 1, &level) == unassignedCode && level == 4);
+    Require(sceCesJisGetLevel(3, 1, 1, &level) == outOfCodeSpace && level == 0);
+    Require(sceCesJisGetLevel(1, 95, 1, &level) == outOfCodeSpace && level == 0);
+    Require(sceCesJisGetLevel(1, 0x21, 0x21, nullptr) == invalidParameter);
+}
+
 int main() {
+    CheckLevels();
     CheckEuc();
     CheckShiftJis();
     return 0;

docs/dev/debt/libSceCesCs.md

@@ -22,9 +22,10 @@
 - [Generic MBCS character and Shift-JIS code helpers](../../../core/libs/prx/libSceCesCs/Export.cpp) (libSceCesCs) - UTF-8/UTF-16 buffer errors and size pointers are unverified. Measured on a PS5: the single-character encoders (`sceCesUtf32ToMbc` over U+0000-U+110001 per profile, surrogates, a one-byte target) give 0x805C0015 for a surrogate, 0x805C0014 past U+10FFFF, the needed length with `DST_BUFFER_END` and 0 otherwise; a rejected decoded character (`sceCesMbcTo*` and each encoding's `To{Ucs2,Utf8,Utf16,Utf32}`, not `sceCesEucJpToUcs2`/`ToUtf8`) stores its length (the bytes before a NUL after the lead, the needed length when truncated, 1 for a bad lead, the sequence for a bad or unassigned trail), a zero UCS-2/UTF-32 character and no UTF-8/UTF-16 units. `sceCesMbcTo*` and `sceCes*ToMbc` take any profile the MBCS context accepts. `sceCesSJisGetCode`/`PutCode` are unchecked; `sceCesSJisCodeToJisX0213` and back use JIS X 0213 annex 1. Signatures follow Orbital
 - [EUC-JIS-2004 and Shift_JIS-2004 profiles](../../../core/libs/prx/libSceCesCs/JisX0213.hpp) (libSceCesCs) - the errors follow the other JIS profiles, unmeasured; the tables were measured on a PS5 (`sceCesMbcToUtf32` over every one-, two- and `0x8F`-led three-byte sequence, `sceCesUtf32ToMbc` over U+0000-U+110001). Both planes follow Project X0213's JIS X 0213:2004 table with the 90 cells it notes as having a fullwidth form mapped to that form, and also cells 1-17, 1-79 and 2-15 to 2-18 (U+FFE3, U+FFE5, U+FF07, U+FF02, U+FF0D, U+FF5E); U+203E and U+00A5 still encode into 1-17 and 1-79. EUC-JIS-2004's SS3 reads JIS X 0212 in the rows plane 2 leaves empty, both ways, with plane 2 preferred for a code point in both. The 25 cells the standard maps to two code points are unassigned both ways on the console (`A4F7` gives `0x805C0020`; U+304B U+309A encodes U+304B, then fails). Bytes below `0x80` are ASCII. The `sceCesEucJp*` and `sceCesSJis*` scalar helpers keep refusing these two profiles.
 - [ISO-2022-JP profiles and string conversions](../../../core/libs/prx/libSceCesCs/Export.cpp) (libSceCesCs) - the signatures follow the MBCS string family by analogy (none in the type database). The UTF-16 and UCS-2 forms share the converters and were not measured separately. `sceCesIso2022UcsContextInit` with a null profile zeroes 256 context bytes and answers `INVALID_PARAMETER`, from a PC run of the firmware 13.42 module, not a console. `sceCesIso2022JpUcsProfileSetSbcs` still throws. Measured on a PS5 through the UTF-32 and UTF-8 forms (every BMP scalar, every designation then every byte and 94x94 pair, 46 strings per profile): RFC 1468 starts in JIS X 0201 Roman (`ESC ( B` and `ESC ( J` both select it) and uses the strict JIS X 0208 table; the code pages start in ASCII and use CP932's table plus the JIS forms. No profile has JIS X 0212. A trailing designation is not counted; one at the buffer end answers `SRC_BUFFER_END`. The encoder returns to the initial set only if it left it.
+- [sceCesJisGetLevel](../../../core/libs/prx/libSceCesCs/Export.cpp) (libSceCesCs) - measured on a PS5 over every plane, row and cell byte (0-3, 0-255, 0-255): level 1 and 2 for the JIS X 0208 cells of rows 1-47 and 48-84, 3 for the rest of plane 1 and 4 for plane 2, `UNASSIGNED_CODE` where JIS X 0213 has no character (its 25 two-code-point cells count as assigned), and `0x805C0015` with level 0 outside planes 1-2 or rows and cells 1-94. A null level pointer answers `INVALID_PARAMETER`, measured for an assigned cell only.
 - [Single-byte profiles](../../../core/libs/prx/libSceCesCs/SingleByte.hpp) (libSceCesCs) - the console's tables for these 27 profiles are not verified. Each returns a profile accepted by `sceCesSbcToUtf8`, `sceCesUtf8ToSbc` and the MBCS context conversions, with bytes 0x00-0x7F unchanged and the high half of a public table: ISO 8859-1, -2, -4, -5, -9, -10, -13, -14, -15 and -16 from [unicode.org](https://unicode.org/Public/MAPPINGS/ISO8859/), code pages 437, 737, 775, 850, 852, 855, 860-863, 865 and 866 from [Microsoft's PC tables](https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/), KOI8-R and KOI8-U from [unicode.org](https://unicode.org/Public/MAPPINGS/VENDORS/MISC/), and 1254, 1256 and 1258 from Microsoft's [best-fit tables](https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WindowsBestFit/). Their unassigned bytes are in the next entry. Profiles whose public table leaves bytes undefined or maps them to private use still throw
 - [Cp1254 and Cp1258 unassigned bytes, and surrogates in sceCesUtf32ToSbc](../../../core/libs/prx/libSceCesCs/SingleByte.hpp) (libSceCesCs) - bytes `81 8D 8E 8F 90 9D 9E` of 1254 and `81 8A 8D 8E 8F 90 9A 9D 9E` of 1258 answer `0x805C0020` both ways, as [rooklz](https://github.com/boykopovar/AnyPS5/pull/10183#issuecomment-6389941634) measured on a PS5 through `sceCesSbcToUtf32` and `sceCesUtf32ToSbc`; BunnyFl's [13.42 run](https://github.com/boykopovar/AnyPS5/pull/9912#issuecomment-6403897082) gives the same bytes. The MBCS context conversions were not measured with these profiles. The same sweep found `sceCesUtf32ToSbc` answering `0x805C0015` (`ILLEGAL_CODE`) for a surrogate with every single-byte profile, as it does here; the other UTF-32 and UCS-2 forms were not measured and keep `INVALID_ENCODE`.
 - [sceCesRefersUcsProfileCp858](../../../core/libs/prx/libSceCesCs/SingleByte.hpp) (libSceCesCs) - the console's table is not verified. The profile is the Cp850 one with byte 0xD5 mapped to U+20AC instead of U+0131, which is the whole high half of ICU's [ibm-858_P100-1997.ucm](https://github.com/unicode-org/icu-data/blob/main/charset/data/ucm/ibm-858_P100-1997.ucm); bytes 0x00-0x7F stay unchanged as in the other single-byte profiles, not swapped as in IBM's control mapping
-- [libSceCesCs firmware 3.20 pending exports](../../../core/libs/prx/libSceCesCs/PendingExports.cpp) - 26 symbol names are present in [the PS5 3.20 libSceCesCs export list](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceCesCs.c). Their signatures and console behaviour are unverified. The declarations use the existing no-parameter unknown-signature convention solely to resolve imports, and every call throws through `NotImplemented_nid_no_patch`.
+- [libSceCesCs firmware 3.20 pending exports](../../../core/libs/prx/libSceCesCs/PendingExports.cpp) - 25 symbol names are present in [the PS5 3.20 libSceCesCs export list](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceCesCs.c). Their signatures and console behaviour are unverified. The declarations use the existing no-parameter unknown-signature convention solely to resolve imports, and every call throws through `NotImplemented_nid_no_patch`.
 - [EUC-KR, UHC and Big5 conversions](../../../core/libs/prx/libSceCesCs/Export.cpp) (libSceCesCs) - the errors are assumed; the decode tables and the Big5 duplicate choices were measured on a PS5 (`sceCesMbcToUtf32` and each `ToUtf8` over every one- and two-byte sequence): UHC and Big5 match, EUC-KR apart from its single bytes. The tables are Microsoft's [CP949.TXT](https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP949.TXT), plus `0xA2E8` U+327E from KS X 1001:2002, and [CP950.TXT](https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP950.TXT); EUC-KR is CP949 limited to bytes `0xA1`-`0xFE`, below `0x80` ASCII; the KS X 1003 profile is not implemented. User-defined areas answer `UNASSIGNED_CODE`. Big5 encoding takes the lower of two cells except for U+5341 and U+5345, as a PS5's `sceCesUtf32ToMbc` does for every scalar (CP950 takes the higher cell of U+2550, U+255E, U+2561 and U+256A; the console does not). The profiles tag the caller's sheet like the EUC-JP profile
 - [ISO 646 variants and JIS X 0201 profiles](../../../core/libs/prx/libSceCesCs/SingleByte.hpp) (libSceCesCs) - the tables and errors were measured on a PS5 with `sceCesSbcToUtf32` over every byte and `sceCesUtf32ToSbc` over every BMP scalar; the other single-byte forms share them and were not measured separately. The 7-bit sets (ASCII, BS 4730, GB 1988-80, JIS X 0201 Roman and Katakana, KS X 1003 and their `Tilde0x7e` variants) answer `INVALID_ENCODE` for bytes `0x80`-`0xFF`. BS 4730 changes only `0x23` (£), GB 1988-80 `0x24` (¥), JIS X 0201 Roman `0x5C` (¥) and KS X 1003 `0x5C` (₩); the base GB, JIS and KS profiles also map `0x7E` to overline and the `Tilde0x7e` variants keep a tilde. The katakana-only profile maps `0x21`-`0x5F` to U+FF61-U+FF9F and leaves `0x60`-`0x7E` unassigned; the 8-bit katakana profiles use `0xA1`-`0xDF`.